clang-format-vs : Fix Unicode formatting

[empty2fill]

Summary

Use UTF-8 for communication with clang-format and convert the replacements offset/length to characters position/count.

Internally VisualStudio.Text.Editor.IWpfTextView use sequence of Unicode characters encoded using UTF-16 and use characters position/count for manipulating text.

Resolved "Error while running clang-format: Specified argument was out of the range of valid values. Parameter name: replaceSpan".

[hans]

klimek, What do you think?

empty2fill: Do you have an example input that I could use to hit the "Specified argument was out of the range of valid values." error?

[klimek]

generally makes sense

[empty2fill]

In D70633#1758372, @hans wrote:

klimek, What do you think?

empty2fill: Do you have an example input that I could use to hit the "Specified argument was out of the range of valid values." error?

Main.cpp196 BDownload

Encoded UTF-8(with BOM) for the Korean language.

[hans]

empty2fill: Do you have an example input that I could use to hit the "Specified argument was out of the range of valid values." error?

Main.cpp196 BDownload

Encoded UTF-8(with BOM) for the Korean language.

I applied your patch locally and it seems to work :-) I'll commit it for you.

clang/tools/clang-format-vs/ClangFormat/ClangFormatPackage.cs

 using System;
 using System.Collections;
 using System.ComponentModel;
 using System.ComponentModel.Design;
 using System.IO;
 using System.Runtime.InteropServices;
 using System.Xml.Linq;
 using System.Linq;
+using System.Text;
 
 namespace LLVM.ClangFormat
 {
     [ClassInterface(ClassInterfaceType.AutoDual)]
     [CLSCompliant(false), ComVisible(true)]
     public class OptionPageGrid : DialogPage
     {
         private string assumeFilename = "";
public sealed class ClangFormatPackage : Package
         {
             IWpfTextView view = Vsix.GetCurrentView();
             if (view == null)
                 // We're not in a text view.
                 return;
             string text = view.TextBuffer.CurrentSnapshot.GetText();
             int start = view.Selection.Start.Position.GetContainingLine().Start.Position;
             int end = view.Selection.End.Position.GetContainingLine().End.Position;
-            int length = end - start;
-            
             // clang-format doesn't support formatting a range that starts at the end
             // of the file.
             if (start >= text.Length && text.Length > 0)
                 start = text.Length - 1;
             string path = Vsix.GetDocumentParent(view);
             string filePath = Vsix.GetDocumentPath(view);
 
-            RunClangFormatAndApplyReplacements(text, start, length, path, filePath, options, view);
+            RunClangFormatAndApplyReplacements(text, start, end, path, filePath, options, view);
         }
 
         /// <summary>
         /// Runs clang-format on the current document
         /// </summary>
         private void FormatDocument(OptionPageGrid options)
         {
             FormatView(Vsix.GetCurrentView(), options);
public sealed class ClangFormatPackage : Package
             {
                 view.TextBuffer.Insert(view.TextBuffer.CurrentSnapshot.Length, Environment.NewLine);
                 text += Environment.NewLine;
             }
 
             RunClangFormatAndApplyReplacements(text, 0, text.Length, path, filePath, options, view);
         }
 
-        private void RunClangFormatAndApplyReplacements(string text, int offset, int length, string path, string filePath, OptionPageGrid options, IWpfTextView view)
+        private void RunClangFormatAndApplyReplacements(string text, int start, int end, string path, string filePath, OptionPageGrid options, IWpfTextView view)
         {
             try
             {
-                string replacements = RunClangFormat(text, offset, length, path, filePath, options);
+                string replacements = RunClangFormat(text, start, end, path, filePath, options);
                 ApplyClangFormatReplacements(replacements, view);
             }
             catch (Exception e)
             {
                 var uiShell = (IVsUIShell)GetService(typeof(SVsUIShell));
                 var id = Guid.Empty;
                 int result;
                 uiShell.ShowMessageBox(
                         0, ref id,
                         "Error while running clang-format:",
                         e.Message,
                         string.Empty, 0,
                         OLEMSGBUTTON.OLEMSGBUTTON_OK,
                         OLEMSGDEFBUTTON.OLEMSGDEFBUTTON_FIRST,
                         OLEMSGICON.OLEMSGICON_INFO,
                         0, out result);
             }
         }
 
         /// <summary>
         /// Runs the given text through clang-format and returns the replacements as XML.
         /// 
-        /// Formats the text range starting at offset of the given length.
+        /// Formats the text in range start and end.
         /// </summary>
-        private static string RunClangFormat(string text, int offset, int length, string path, string filePath, OptionPageGrid options)
+        private static string RunClangFormat(string text, int start, int end, string path, string filePath, OptionPageGrid options)
         {
             string vsixPath = Path.GetDirectoryName(
                 typeof(ClangFormatPackage).Assembly.Location);
 
             System.Diagnostics.Process process = new System.Diagnostics.Process();
             process.StartInfo.UseShellExecute = false;
             process.StartInfo.FileName = vsixPath + "\\clang-format.exe";
+            char[] chars = text.ToCharArray();
+            int offset = Encoding.UTF8.GetByteCount(chars, 0, start);
+            int length = Encoding.UTF8.GetByteCount(chars, 0, end) - offset;
             // Poor man's escaping - this will not work when quotes are already escaped
             // in the input (but we don't need more).
             string style = options.Style.Replace("\"", "\\\"");
             string fallbackStyle = options.FallbackStyle.Replace("\"", "\\\"");
             process.StartInfo.Arguments = " -offset " + offset +
                                           " -length " + length +
                                           " -output-replacements-xml " +
                                           " -style \"" + style + "\"" +
public sealed class ClangFormatPackage : Package
             {
                 throw new Exception(
                     "Cannot execute " + process.StartInfo.FileName + ".\n\"" + 
                     e.Message + "\".\nPlease make sure it is on the PATH.");
             }
             // 2. We write everything to the standard output - this cannot block, as clang-format
             //    reads the full standard input before analyzing it without writing anything to the
             //    standard output.
-            process.StandardInput.Write(text);
+            StreamWriter utf8Writer = new StreamWriter(process.StandardInput.BaseStream, new UTF8Encoding(false));
+            utf8Writer.Write(text);
             // 3. We notify clang-format that the input is done - after this point clang-format
             //    will start analyzing the input and eventually write the output.
-            process.StandardInput.Close();
+            utf8Writer.Close();
             // 4. We must read clang-format's output before waiting for it to exit; clang-format
             //    will close the channel by exiting.
             string output = process.StandardOutput.ReadToEnd();
             // 5. clang-format is done, wait until it is fully shut down.
             process.WaitForExit();
             if (process.ExitCode != 0)
             {
                 // FIXME: If clang-format writes enough to the standard error stream to block,
                 // we will never reach this point; instead, read the standard error asynchronously.
                 throw new Exception(process.StandardError.ReadToEnd());
             }
             return output;
         }
 
         /// <summary>
         /// Applies the clang-format replacements (xml) to the current view
         /// </summary>
         private static void ApplyClangFormatReplacements(string replacements, IWpfTextView view)
         {
             // clang-format returns no replacements if input text is empty
             if (replacements.Length == 0)
                 return;
 
+            string text = view.TextBuffer.CurrentSnapshot.GetText();
+            byte[] bytes = Encoding.UTF8.GetBytes(text);
+
             var root = XElement.Parse(replacements);
             var edit = view.TextBuffer.CreateEdit();
             foreach (XElement replacement in root.Descendants("replacement"))
             {
+                int offset = int.Parse(replacement.Attribute("offset").Value);
+                int length = int.Parse(replacement.Attribute("length").Value);
                 var span = new Span(
-                    int.Parse(replacement.Attribute("offset").Value),
-                    int.Parse(replacement.Attribute("length").Value));
+                    Encoding.UTF8.GetCharCount(bytes, 0, offset),
+                    Encoding.UTF8.GetCharCount(bytes, offset, length));
                 edit.Replace(span, replacement.Value);
             }
             edit.Apply();
         }
     }
 }