Index: utils/lit/lit/TestRunner.py =================================================================== --- utils/lit/lit/TestRunner.py +++ utils/lit/lit/TestRunner.py @@ -344,8 +344,10 @@ stderr = StringIO() exitCode = 0 for dir in args: + dir = lit.util.convertToLocalEncoding(dir) + cwd = lit.util.convertToLocalEncoding(cmd_shenv.cwd) if not os.path.isabs(dir): - dir = os.path.realpath(os.path.join(cmd_shenv.cwd, dir)) + dir = os.path.realpath(os.path.join(cwd, dir)) if parent: lit.util.mkdir_p(dir) else: @@ -598,8 +600,10 @@ stderr = StringIO() exitCode = 0 for path in args: + path = lit.util.convertToLocalEncoding(path) + cwd = lit.util.convertToLocalEncoding(cmd_shenv.cwd) if not os.path.isabs(path): - path = os.path.realpath(os.path.join(cmd_shenv.cwd, path)) + path = os.path.realpath(os.path.join(cwd, path)) if force and not os.path.exists(path): continue try: @@ -695,7 +699,7 @@ else: # Make sure relative paths are relative to the cwd. redir_filename = os.path.join(cmd_shenv.cwd, name) - fd = open(redir_filename, mode) + fd = open(lit.util.convertToLocalEncoding(redir_filename), mode) # Workaround a Win32 and/or subprocess bug when appending. # # FIXME: Actually, this is probably an instance of PR6753. @@ -1080,27 +1084,30 @@ script += '.bat' # Write script file - mode = 'w' - if litConfig.isWindows and not isWin32CMDEXE: - mode += 'b' # Avoid CRLFs when writing bash scripts. - f = open(script, mode) + f = open(script, 'wb') if isWin32CMDEXE: for i, ln in enumerate(commands): - commands[i] = re.sub(kPdbgRegex, "echo '\\1' > nul && ", ln) + commands[i] = re.sub(kPdbgRegex, b"echo '\\1' > nul && ", ln) if litConfig.echo_all_commands: - f.write('@echo on\n') + f.write(b'@echo on\n') else: - f.write('@echo off\n') - f.write('\n@if %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands)) + f.write(b'@echo off\n') + # In python 3, this needs to be converted to bytes + if not isinstance(commands[i], bytes): + commands = map(lambda s: bytes(s, 'utf-8'), commands) + f.write(b'\n@if %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands)) else: for i, ln in enumerate(commands): commands[i] = re.sub(kPdbgRegex, ": '\\1'; ", ln) if test.config.pipefail: - f.write('set -o pipefail;') + f.write(b'set -o pipefail;\n') if litConfig.echo_all_commands: - f.write('set -x;') - f.write('{ ' + '; } &&\n{ '.join(commands) + '; }') - f.write('\n') + f.write(b'set -x;\n') + # In python 3, the commands need to be converted to bytes + if not isinstance(commands[i], bytes): + commands = map(lambda s: bytes(s, 'utf-8'), commands) + f.write(b'{ ' + b'; } &&\n{ '.join(commands) + b'; }\n') + f.write(b'\n') f.close() if isWin32CMDEXE: Index: utils/lit/lit/util.py =================================================================== --- utils/lit/lit/util.py +++ utils/lit/lit/util.py @@ -424,3 +424,21 @@ psutilProc.kill() except psutil.NoSuchProcess: pass + +def convertToLocalEncoding(text): + """This function converts utf-8 text into a format which the local system prefers + to work with. + """ + if platform.system() == 'Windows': + if sys.version_info < (3,0): + # On Windows and Python2, we want to use 'unicode' so it gets converted + # to UTF-16 + return text.decode('utf-8') if isinstance(text, str) else text + else: + # On Windows and Python3, we want to use a unicode string so it gets + # converted to UTF-16 + return text.decode('utf-8') if isinstance(text, bytes) else text + + # On non Windows, we just want bytes so python don't try to + # convert it to ascii + return text if isinstance(text, bytes) else text.encode('utf-8') Index: utils/lit/tests/Inputs/shtest-shell/rm-unicode-0.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/shtest-shell/rm-unicode-0.txt @@ -0,0 +1,7 @@ +# Check creating and removing unicode files +# +# RUN: mkdir -p Output/中文 +# RUN: echo "" > Output/中文/你好.txt +# RUN: rm Output/中文/你好.txt +# RUN: echo "" > Output/中文/你好.txt +# RUN: rm -r Output/中文 Index: utils/lit/tests/shtest-shell.py =================================================================== --- utils/lit/tests/shtest-shell.py +++ utils/lit/tests/shtest-shell.py @@ -224,6 +224,7 @@ # CHECK: Exit Code: 1 # CHECK: *** +# CHECK: PASS: shtest-shell :: rm-unicode-0.txt # CHECK: PASS: shtest-shell :: sequencing-0.txt # CHECK: XFAIL: shtest-shell :: sequencing-1.txt # CHECK: PASS: shtest-shell :: valid-shell.txt