[Format] Work around current vim bugs in clang-format.py

[sammccall]

Summary

Do line/col to byte conversions on the python side rather than relying on vim.
Its calculations are off when text annotations are present:

• https://github.com/vim/vim/issues/5930
• https://github.com/vim/vim/issues/3718 (fixed, but vim 8.1 is still common)

[hokein]

thanks for tracing it down! looks good, I assume you have test it locally.

clang/tools/clang-format/clang-format.py

if platform.python_version_tuple()[0] == '3':
     return vim.current.buffer
   return [ line.decode(encoding) for line in vim.current.buffer ]
 
 def main():
   # Get the current text.
   encoding = vim.eval("&encoding")
   buf = get_buffer(encoding)
   # Join the buffer into a single string with a terminating newline
-  text = '\n'.join(buf) + '\n'
+  text = ('\n'.join(buf) + '\n').encode(encoding)
 
   # Determine range to format.
   if vim.eval('exists("l:lines")') == '1':
     lines = ['-lines', vim.eval('l:lines')]
   elif vim.eval('exists("l:formatdiff")') == '1':
     with open(vim.current.buffer.name, 'r') as f:
       ondisk = f.read().splitlines();
     sequence = difflib.SequenceMatcher(None, ondisk, vim.current.buffer)
     lines = []
     for op in reversed(sequence.get_opcodes()):
       if op[0] not in ['equal', 'delete']:
         lines += ['-lines', '%s:%s' % (op[3] + 1, op[4])]
     if lines == []:
       return
   else:
     lines = ['-lines', '%s:%s' % (vim.current.range.start + 1,
                                   vim.current.range.end + 1)]
 
-  # Determine the cursor position.
-  cursor = int(vim.eval('line2byte(line("."))+col(".")')) - 2
-  if cursor < 0:
+  # Convert cursor (line, col) to bytes.
+  # Don't use line2byte: https://github.com/vim/vim/issues/5930
+  _, cursor_line, cursor_col, _ = vim.eval('getpos(".")') # 1-based
+  cursor_byte = 0
+  for line in text.split(b'\n')[:int(cursor_line) - 1]:
+    cursor_byte += len(line) + 1
+  cursor_byte += int(cursor_col) - 1
+  if cursor_byte < 0:
     print('Couldn\'t determine cursor position. Is your file empty?')
     return
 
   # Avoid flashing an ugly, ugly cmd prompt on Windows when invoking clang-format.
   startupinfo = None
   if sys.platform.startswith('win32'):
     startupinfo = subprocess.STARTUPINFO()
     startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
     startupinfo.wShowWindow = subprocess.SW_HIDE
 
   # Call formatter.
-  command = [binary, '-cursor', str(cursor)]
+  command = [binary, '-cursor', str(cursor_byte)]
   if lines != ['-lines', 'all']:
     command += lines
   if style:
     command.extend(['-style', style])
   if fallback_style:
     command.extend(['-fallback-style', fallback_style])
   if vim.current.buffer.name:
     command.extend(['-assume-filename', vim.current.buffer.name])
   p = subprocess.Popen(command,
                        stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                        stdin=subprocess.PIPE, startupinfo=startupinfo)
-  stdout, stderr = p.communicate(input=text.encode(encoding))
+  stdout, stderr = p.communicate(input=text)
 
   # If successful, replace buffer contents.
   if stderr:
     print(stderr)
 
   if not stdout:
     print(
         'No output from clang-format (crashed?).\n'
         'Please report to bugs.llvm.org.'
     )
   else:
-    lines = stdout.decode(encoding).split('\n')
-    output = json.loads(lines[0])
+    header, content = stdout.split(b'\n', 1)
+    header = json.loads(header)
     # Strip off the trailing newline (added above).
     # This maintains trailing empty lines present in the buffer if
     # the -lines specification requests them to remain unchanged.
-    lines = lines[1:-1]
+    lines = content.decode(encoding).split('\n')[:-1]
     sequence = difflib.SequenceMatcher(None, buf, lines)
     for op in reversed(sequence.get_opcodes()):
       if op[0] != 'equal':
         vim.current.buffer[op[1]:op[2]] = lines[op[3]:op[4]]
-    if output.get('IncompleteFormat'):
+    if header.get('IncompleteFormat'):
       print('clang-format: incomplete (syntax errors)')
-    vim.command('goto %d' % (output['Cursor'] + 1))
+    # Convert cursor bytes to (line, col)
+    # Don't use goto: https://github.com/vim/vim/issues/5930
+    cursor_byte = int(header['Cursor'])
+    prefix = content[0:cursor_byte]
+    cursor_line = 1 + prefix.count(b'\n')
+    cursor_column = 1 + len(prefix.rsplit(b'\n', 1)[-1])
+    vim.command('call cursor(%d, %d)' % (cursor_line, cursor_column))
 
 main()