Use UTF-8 for all communication with clang-format

[phst]

Summary

Instead of picking the buffer file coding system, always use utf-8-unix for communicating with clang-format. This is fine because clang-format never actually reads the file to be formatted, only standard input. This is a bit simpler (process coding system is now a constant) and potentially faster, as utf-8-unix is Emacs's internal coding system. Also add an end-to-end test that actually invokes clang-format.

[phst]

Fix comments

[klimek]

lg; thanks!

tools/clang-format/clang-format-test.el

(cl-letf* ((call-process-args nil)
               (lambda (&rest args)
                 (push args call-process-args)
                 (pcase-exhaustive args
                   (`(,_start ,_end ,_program ,_delete (,stdout ,_stderr)
                              ,_display . ,_args)
                    (with-current-buffer stdout
                      (insert "<?xml version='1.0'?>
 <replacements xml:space='preserve' incomplete_format='false'>
-<replacement offset='7' length='0'> </replacement>
-<replacement offset='14' length='0'> </replacement>
+<replacement offset='4' length='0'> </replacement>
+<replacement offset='10' length='0'> </replacement>
 </replacements>
 "))
                    0)))))
     (with-temp-buffer
       (let ((buffer-file-name "foo.cpp")
             (buffer-file-coding-system 'utf-8-with-signature-dos)
             (default-process-coding-system 'latin-1-unix))
         (insert "ä =ö;\nü= ß;\n")
(pcase-exhaustive call-process-args
       (`((,start ,end ,_program ,delete (,_stdout ,_stderr) ,display . ,args))
        (should-not start)
        (should-not end)
        (should-not delete)
        (should-not display)
        (should (equal args
                       '("-output-replacements-xml" "-assume-filename" "foo.cpp"
                         "-style" "file"
-                        ;; Length of the UTF-8 byte-order mark.
-                        "-offset" "3"
+                        ;; Beginning of buffer, no byte-order mark.
+                        "-offset" "0"
                         ;; We have two lines with 2×2!bytes for the umlauts,
-                        ;; 2!bytes for the line ending, and 3!bytes for the
+                        ;; 1!byte for the line ending, and 3!bytes for the
                         ;; other ASCII characters each.
-                        "-length" "18"
-                        ;; Length of a single line (without line ending) plus
-                        ;; BOM.
-                        "-cursor" "10")))))))
+                        "-length" "16"
+                        ;; Length of a single line (without line ending).
+                        "-cursor" "7")))))))
 
 (ert-deftest clang-format-buffer--process-encoding ()
   "Tests that text is sent to the clang-format process in the
 right encoding."
   (cl-letf* ((hexdump (executable-find "hexdump"))
              (original-call-process-region
               (symbol-function 'call-process-region))
              (call-process-inputs nil)
"))
     (with-temp-buffer
       (let ((buffer-file-name "foo.cpp")
             (buffer-file-coding-system 'utf-8-with-signature-dos)
             (default-process-coding-system 'latin-1-unix))
         (insert "ä\n")
         (clang-format-buffer))
       (should (equal (buffer-string) "ä\n"))
       (should (eobp)))
-    (should (equal call-process-inputs '("ef bb bf c3 a4 0d 0a ")))))
+    (should (equal call-process-inputs '("c3 a4 0a ")))))
+
+(ert-deftest clang-format-buffer--end-to-end ()
+  "End-to-end test for ‘clang-format-buffer’.
+Actually calls the clang-format binary."
+  (skip-unless (file-executable-p clang-format-executable))
+  (with-temp-buffer
+    (let ((buffer-file-name "foo.cpp")
+          (buffer-file-coding-system 'utf-8-with-signature-dos)
+          (default-process-coding-system 'latin-1-unix))
+      (insert "ä =ö;\nü= ß;\n")
+      (goto-char (point-min))
+      (end-of-line)
+      (clang-format-buffer))
+    (should (equal (buffer-string) "ä = ö;\nü = ß;\n"))
+    (should (eolp))
+    (should (equal (buffer-substring (point) (point-max))
+                   "\nü = ß;\n"))))
 
 ;;; clang-format-test.el ends here

tools/clang-format/clang-format.el

(setq replacements (sort (delq nil replacements)
                                    (and (= (car a) (car b))
                                         (> (cadr a) (cadr b)))))))
 
     (list replacements cursor (string= incomplete-format "true"))))
 
 (defun clang-format--replace (offset length &optional text)
   "Replace the region defined by OFFSET and LENGTH with TEXT.
 OFFSET and LENGTH are measured in bytes, not characters.  OFFSET
-is a zero-based file offset."
-  (let ((start (clang-format--filepos-to-bufferpos offset 'exact))
-        (end (clang-format--filepos-to-bufferpos (+ offset length) 'exact)))
+is a zero-based file offset, assuming ‘utf-8-unix’ coding."
+  (let ((start (clang-format--filepos-to-bufferpos offset 'exact 'utf-8-unix))
+        (end (clang-format--filepos-to-bufferpos (+ offset length) 'exact
+                                                 'utf-8-unix)))
     (goto-char start)
     (delete-region start end)
     (when text
       (insert text))))
 
 ;; ‘bufferpos-to-filepos’ and ‘filepos-to-bufferpos’ are new in Emacs 25.1.
 ;; Provide fallbacks for older versions.
 (defalias 'clang-format--bufferpos-to-filepos
is no active region.  If no style is given uses `clang-format-style'."
   (interactive
    (if (use-region-p)
        (list (region-beginning) (region-end))
      (list (point) (point))))
 
   (unless style
     (setq style clang-format-style))
 
-  (let ((file-start (clang-format--bufferpos-to-filepos start 'approximate))
-        (file-end (clang-format--bufferpos-to-filepos end 'approximate))
-        (cursor (clang-format--bufferpos-to-filepos (point) 'exact))
+  (let ((file-start (clang-format--bufferpos-to-filepos start 'approximate
+                                                        'utf-8-unix))
+        (file-end (clang-format--bufferpos-to-filepos end 'approximate
+                                                      'utf-8-unix))
+        (cursor (clang-format--bufferpos-to-filepos (point) 'exact 'utf-8-unix))
         (temp-buffer (generate-new-buffer " *clang-format-temp*"))
         (temp-file (make-temp-file "clang-format"))
-        (default-process-coding-system
-          ;; Output is XML, which is always UTF-8.  Input encoding should match
-          ;; the file encoding, otherwise the offsets calculated above are off.
-          (cons 'utf-8-unix buffer-file-coding-system)))
+        ;; the encoding used to convert between buffer and file positions,
+        ;; otherwise the offsets calculated above are off.  For simplicity, we
+        ;; always use ‘utf-8-unix’ and ignore the buffer coding system.
+        (default-process-coding-system '(utf-8-unix . utf-8-unix)))
     (unwind-protect
         (let ((status (call-process-region
                        nil nil clang-format-executable
                        nil `(,temp-buffer ,temp-file) nil
 
                        "-output-replacements-xml"
                        "-assume-filename" (or (buffer-file-name) "")
                        "-style" style
(unwind-protect
 
           (cl-destructuring-bind (replacements cursor incomplete-format)
               (with-current-buffer temp-buffer
                 (clang-format--extract (car (xml-parse-region))))
             (save-excursion
               (dolist (rpl replacements)
                 (apply #'clang-format--replace rpl)))
             (when cursor
-              (goto-char (clang-format--filepos-to-bufferpos cursor 'exact)))
+              (goto-char (clang-format--filepos-to-bufferpos cursor 'exact
+                                                             'utf-8-unix)))
             (if incomplete-format
                 (message "(clang-format: incomplete (syntax errors)%s)" stderr)
               (message "(clang-format: success%s)" stderr))))
       (delete-file temp-file)
       (when (buffer-name temp-buffer) (kill-buffer temp-buffer)))))
 
 ;;;###autoload
 (defun clang-format-buffer (&optional style)