[LNT] Python 3 support: subprocess IO as text

[thopre]

Summary

In Python 3, subprocess stdin, stdout and stderr are in binary mode by
default. Trying to print or parse those as text as is done thoughought
in LNT then causes byte/str errors. This commits sets the
universal_newlines parameter to True when invoking subprocess methods
involving input or output to request those to be in text mode. It also
adapt _check_output to work whether universal_newlines is set to True in
the caller or not. Finally, it cleans up 2 cases subprocess method call
where the output file objects are stored in a variable but not used.

[PrzemekWirkus]

LGTM

lnt/server/db/rules/rule_update_profile_stats.py

def update_profile_stats(session, ts, run_id):
     try:
         history = json.loads(open(history_path).read())
     except Exception:
         history = []
     age = []
 
     dt = time.time()
     blocks = subprocess.check_output("du -s -k %s" % profile_path,
-                                     shell=True).split('\t')[0]
+                                     shell=True,
+                                     universal_newlines=True).split('\t')[0]
     kb = float(blocks)  # 1024 byte blocks.
 
     history.append((dt, kb))
 
     for f in glob.glob('%s/*.lntprof' % profile_path):
         mtime = os.stat(f).st_mtime
         sz = os.stat(f).st_size // 1000
         age.append([mtime, sz])
 
     open(history_path, 'w').write(json.dumps(history))
     open(age_path, 'w').write(json.dumps(age))
 
 
 post_submission_hook = update_profile_stats

lnt/testing/util/commands.py

def capture_with_result(args, include_stderr=False):
     """capture_with_result(command) -> (output, exit code)
 
     Run the given command (or argv list) in a shell and return the standard
     output and exit code."""
     stderr = subprocess.PIPE
     if include_stderr:
         stderr = subprocess.STDOUT
     try:
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=stderr)
+        p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=stderr,
+                             universal_newlines=True)
     except OSError as e:
         if e.errno == errno.ENOENT:
             fatal('no such file or directory: %r when running %s.' %
                   (args[0], ' '.join(args)))
         raise
     out, _ = p.communicate()
     return out, p.wait()
 

lnt/tests/compile.py

def runN(args, N, cwd, preprocess_cmd=None, env=None, sample_mem=False,
 
     if opts.verbose:
         g_log.info("running: %s" % " ".join("'%s'" % arg for arg in cmd))
     p = subprocess.Popen(args=cmd,
                          stdin=None,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE,
                          env=env,
-                         cwd=cwd)
+                         cwd=cwd,
+                         universal_newlines=True)
     runn_stdout, runn_stderr = p.communicate()
     res = p.returncode
 
     # If the runN command failed, or it had stderr when we didn't expect it,
     # fail immediately and don't try to parse the output.
     if res != 0:
         g_log.error("runN command failed with stderr:\n--\n%s\n--" % runn_stderr.strip())
         return None
else:
 
         if archive_path.endswith(".tar.gz") or \
            archive_path.endswith(".tar.bz2") or \
            archive_path.endswith(".tar.lzma"):
             p = subprocess.Popen(args=['tar', '-xf', archive_path],
                                  stdin=None,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE,
-                                 cwd=source_path)
+                                 cwd=source_path,
+                                 universal_newlines=True)
         else:
             p = subprocess.Popen(args=['unzip', '-q', archive_path],
                                  stdin=None,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE,
-                                 cwd=source_path)
+                                 cwd=source_path,
+                                 universal_newlines=True)
         stdout, stderr = p.communicate()
         if p.wait() != 0:
             fatal(("unable to extract archive %r at %r\n"
                    "-- stdout --\n%s\n"
                    "-- stderr --\n%s\n") % (archive_path, source_path,
                                             stdout, stderr))
 
         # Apply the patch file, if necessary.
         patch_files = project.get('patch_files', [])
         for patch_file in patch_files:
             g_log.info('applying patch file %r for %r' % (patch_file, name))
             patch_file_path = get_input_path(opts, patch_file)
             p = subprocess.Popen(args=['patch', '-i', patch_file_path,
                                        '-p', '1'],
                                  stdin=None,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE,
-                                 cwd=source_path)
+                                 cwd=source_path,
+                                 universal_newlines=True)
             stdout, stderr = p.communicate()
             if p.wait() != 0:
                 fatal(("unable to apply patch file %r in %r\n"
                        "-- stdout --\n%s\n"
                        "-- stderr --\n%s\n") % (patch_file_path, source_path,
                                                 stdout, stderr))
 
         # Write the hash tag.

lnt/tests/nt.py

if report_dir is not None:
         logfile = subprocess.PIPE
         # Open a duplicated logfile at the global dir.
         _, logname = os.path.split(test_log.name)
         global_log_path = os.path.join(report_dir, logname)
         global_log = open(global_log_path, 'a+')
 
     p = subprocess.Popen(args=args, stdin=None, stdout=logfile,
                          stderr=subprocess.STDOUT, cwd=basedir,
-                         env=os.environ)
+                         env=os.environ, universal_newlines=True)
 
     if report_dir is not None:
         while p.poll() is None:
             line = p.stdout.readline()
             if len(line) > 0:
                 test_log.write(line)
                 global_log.write(line)
 

lnt/tests/test_suite.py

def _check_call(self, *args, **kwargs):
             logger.info('          (In %s)' % kwargs['cwd'])
         return subprocess.check_call(*args, **kwargs)
 
     def _check_output(self, *args, **kwargs):
         logger.info('Execute: %s' % ' '.join(args[0]))
         if 'cwd' in kwargs:
             logger.info('          (In %s)' % kwargs['cwd'])
         output = subprocess.check_output(*args, **kwargs)
+        if kwargs.get('universal_newlines', False):
             sys.stdout.write(output)
+        else:
+            sys.stdout.buffer.write(output)
         return output
 
     def _clean(self, path):
         make_cmd = self.opts.make
 
         subdir = path
         if self.opts.only_test:
             components = [path] + [self.opts.only_test[0]]
def _get_lnt_code(self, code):
             'UNRESOLVED': lnt.testing.FAIL,
             'XFAIL': lnt.testing.XFAIL,
             'XPASS': lnt.testing.FAIL,
         }[code]
 
     def _extract_cmake_vars_from_cache(self):
         assert self.configured is True
         cmake_lah_output = self._check_output(
-            [self.opts.cmake] + ['-LAH', '-N'] + [self._base_path])
+            [self.opts.cmake] + ['-LAH', '-N'] + [self._base_path],
+            universal_newlines=True)
         pattern2var = [
             (re.compile("^%s:[^=]*=(.*)$" % cmakevar), cmakevar)
             for cmakevar in (
                 "CMAKE_C_COMPILER",
                 "CMAKE_BUILD_TYPE",
                 "CMAKE_CXX_FLAGS",
                 "CMAKE_CXX_FLAGS_DEBUG",
                 "CMAKE_CXX_FLAGS_MINSIZEREL",
def diagnose(self):
         os.chdir(path)
 
         # Run with -save-temps
         cmd = self._configure(path, execute=False)
         cmd_temps = cmd + ['-DTEST_SUITE_DIAGNOSE_FLAGS=-save-temps']
 
         logger.info(' '.join(cmd_temps))
 
-        out = subprocess.check_output(cmd_temps)
+        out = subprocess.check_output(cmd_temps, universal_newlines=True)
         logger.info(out)
 
         # Figure out our test's target.
         make_cmd = [self.opts.make, "VERBOSE=1", 'help']
 
-        make_targets = subprocess.check_output(make_cmd)
+        make_targets = subprocess.check_output(make_cmd,
+                                               universal_newlines=True)
         matcher = re.compile(r"^\.\.\.\s{}$".format(short_name),
                              re.MULTILINE | re.IGNORECASE)
         if not matcher.search(make_targets):
             assert False, "did not find benchmark, nestsed? Unimplemented."
 
         local_path = os.path.join(path, bm_path)
 
         make_deps = [self.opts.make, "VERBOSE=1", "timeit-target",
                      "timeit-host", "fpcmp-host"]
         logger.info(" ".join(make_deps))
         p = subprocess.Popen(make_deps,
                              stdout=subprocess.PIPE,
-                             stderr=subprocess.STDOUT)
+                             stderr=subprocess.STDOUT,
+                             universal_newlines=True)
         std_out, std_err = p.communicate()
         logger.info(std_out)
 
         make_save_temps = [self.opts.make, "VERBOSE=1", short_name]
         logger.info(" ".join(make_save_temps))
         p = subprocess.Popen(make_save_temps,
                              stdout=subprocess.PIPE,
-                             stderr=subprocess.STDOUT)
+                             stderr=subprocess.STDOUT,
+                             universal_newlines=True)
         std_out, std_err = p.communicate()
         logger.info(std_out)
         with open(report_path + "/build.log", 'w') as f:
             f.write(std_out)
         # Executable(s) and test file:
         shutil.copy(os.path.join(local_path, short_name), report_path)
         shutil.copy(os.path.join(local_path, short_name + ".test"),
                     report_path)
         # Temp files are in:
         temp_files = os.path.join(local_path, "CMakeFiles",
                                   short_name + ".dir")
 
         save_temps_file = ["/*.s", "/*.ii", "/*.i", "/*.bc"]
         build_files = ["/*.o", "/*.time", "/*.cmake", "/*.make",
                        "/*.includecache", "/*.txt"]
         self._cp_artifacts(local_path, report_path, save_temps_file)
         self._cp_artifacts(temp_files, report_path, build_files)
 
         # Now lets do -ftime-report.
         cmd_time_report = cmd + ['-DTEST_SUITE_DIAGNOSE_FLAGS=-ftime-report']
 
         logger.info(' '.join(cmd_time_report))
 
-        out = subprocess.check_output(cmd_time_report)
+        out = subprocess.check_output(cmd_time_report, universal_newlines=True)
         logger.info(out)
 
         make_time_report = [self.opts.make, "VERBOSE=1", short_name]
         p = subprocess.Popen(make_time_report,
                              stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
+                             stderr=subprocess.PIPE,
+                             universal_newlines=True)
         std_out, std_err = p.communicate()
 
         with open(report_path + "/time-report.txt", 'w') as f:
             f.write(std_err)
         logger.info("Wrote: " + report_path + "/time-report.txt")
 
         # Now lets do -llvm -stats.
         cmd_stats_report = cmd + ['-DTEST_SUITE_DIAGNOSE_FLAGS=-mllvm -stats']
 
         logger.info(' '.join(cmd_stats_report))
 
-        out = subprocess.check_output(cmd_stats_report)
+        out = subprocess.check_output(cmd_stats_report,
+                                      universal_newlines=True)
         logger.info(out)
 
         make_stats_report = [self.opts.make, "VERBOSE=1", short_name]
         p = subprocess.Popen(make_stats_report,
                              stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
+                             stderr=subprocess.PIPE,
+                             universal_newlines=True)
         std_out, std_err = p.communicate()
 
         with open(report_path + "/stats-report.txt", 'w') as f:
             f.write(std_err)
         logger.info("Wrote: " + report_path + "/stats-report.txt")
 
         #  Collect Profile:
         if "Darwin" in platform.platform():
             # For testing and power users, lets allow overrides of how sudo
             # and iprofiler are called.
             sudo = os.getenv("SUDO_CMD", "sudo")
             if " " in sudo:
                 sudo = sudo.split(" ")
             if not sudo:
                 sudo = []
             else:
                 sudo = [sudo]
             iprofiler = os.getenv("IPROFILER_CMD",
                                   "iprofiler -timeprofiler -I 40u")
 
             cmd_iprofiler = cmd + ['-DTEST_SUITE_RUN_UNDER=' + iprofiler]
             print(' '.join(cmd_iprofiler))
 
-            out = subprocess.check_output(cmd_iprofiler)
+            subprocess.check_output(cmd_iprofiler, universal_newlines=True)
 
             os.chdir(local_path)
             make_iprofiler_temps = [self.opts.make, "VERBOSE=1", short_name]
             p = subprocess.Popen(make_iprofiler_temps,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE)
-            std_out, std_err = p.communicate()
+            p.communicate()
             logger.warning("Using sudo to collect execution trace.")
             make_save_temps = sudo + [self.opts.lit, short_name + ".test"]
             p = subprocess.Popen(make_save_temps,
                                  stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE)
+                                 stderr=subprocess.PIPE,
+                                 universal_newlines=True)
             std_out, std_err = p.communicate()
             sys.stdout.write(std_out)
             sys.stderr.write(std_err)
             logger.warning("Tests may fail because of iprofiler's output.")
             # The dtps file will be saved as root, make it so
             # that we can read it.
             chmod = sudo + ["chown", "-R", getpass.getuser(),
                             short_name + ".dtps"]