Make sure that clang-format input is in the right encoding

[phst]

Summary

Add unit tests.

[klimek]

Adding Jens as reviewer, as this is quite a bit of elisp :)

[massberg]

looks good to me

tools/clang-format/clang-format-test.el

+;;; clang-format-test.el --- unit tests for clang-format.el  -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2017  Google Inc.
+
+;; Author: Philipp Stephani <[e-mail]>
+
+;; This file is distributed under the University of Illinois Open Source
+;; License.  See LICENSE.TXT for details.
+
+;;; Commentary:
+
+;; Unit tests for clang-format.el.
+
+;;; Code:
+
+(require 'clang-format)
+
+(require 'cl-lib)
+(require 'ert)
+(require 'pcase)
+
+(ert-deftest clang-format-buffer--buffer-encoding ()
+  "Tests that encoded text is handled properly."
+  (cl-letf* ((call-process-args nil)
+             ((symbol-function 'call-process-region)
+              (lambda (&rest args)
+                (push args call-process-args)
+                (pcase-exhaustive args
+                  (`(,_start ,_end ,_program ,_delete (,stdout ,_stderr)
+                             ,_display . ,_args)
+                   (with-current-buffer stdout
+                     (insert "<?xml version='1.0'?>
+<replacements xml:space='preserve' incomplete_format='false'>
+<replacement offset='7' length='0'> </replacement>
+<replacement offset='14' length='0'> </replacement>
+</replacements>
+"))
+                   0)))))
+    (with-temp-buffer
+      (let ((buffer-file-name "foo.cpp")
+            (buffer-file-coding-system 'utf-8-with-signature-dos)
+            (default-process-coding-system 'latin-1-unix))
+        (insert "ä =ö;\nü= ß;\n")
+        (goto-char (point-min))
+        (end-of-line)
+        (clang-format-buffer))
+      (should (equal (buffer-string) "ä = ö;\nü = ß;\n"))
+      (should (eolp))
+      (should (equal (buffer-substring (point) (point-max))
+                     "\nü = ß;\n")))
+    (should-not (cdr call-process-args))
+    (pcase-exhaustive call-process-args
+      (`((,start ,end ,_program ,delete (,_stdout ,_stderr) ,display . ,args))
+       (should-not start)
+       (should-not end)
+       (should-not delete)
+       (should-not display)
+       (should (equal args
+                      '("-output-replacements-xml" "-assume-filename" "foo.cpp"
+                        "-style" "file"
+                        ;; Length of the UTF-8 byte-order mark.
+                        "-offset" "3"
+                        ;; We have two lines with 2×2!bytes for the umlauts,
+                        ;; 2!bytes for the line ending, and 3!bytes for the
+                        ;; other ASCII characters each.
+                        "-length" "18"
+                        ;; Length of a single line (without line ending) plus
+                        ;; BOM.
+                        "-cursor" "10")))))))
+
+(ert-deftest clang-format-buffer--process-encoding ()
+  "Tests that text is sent to the clang-format process in the
+right encoding."
+  (cl-letf* ((hexdump (executable-find "hexdump"))
+             (original-call-process-region
+              (symbol-function 'call-process-region))
+             (call-process-inputs nil)
+             ;; We redirect the input to hexdump so that we have guaranteed
+             ;; ASCII output.
+             ((symbol-function 'call-process-region)
+              (lambda (&rest args)
+                (pcase-exhaustive args
+                  (`(,start ,end ,_program ,_delete (,stdout ,_stderr)
+                            ,_display . ,_args)
+                   (with-current-buffer stdout
+                     (insert "<?xml version='1.0'?>
+<replacements xml:space='preserve' incomplete_format='false'>
+</replacements>
+"))
+                   (let ((stdin (current-buffer)))
+                     (with-temp-buffer
+                       (prog1
+                           (let ((stdout (current-buffer)))
+                             (with-current-buffer stdin
+                               (funcall original-call-process-region
+                                        start end hexdump nil stdout nil
+                                        "-v" "-e" "/1 \"%02x \"")))
+                         (push (buffer-string) call-process-inputs)))))))))
+    (skip-unless hexdump)
+    (with-temp-buffer
+      (let ((buffer-file-name "foo.cpp")
+            (buffer-file-coding-system 'utf-8-with-signature-dos)
+            (default-process-coding-system 'latin-1-unix))
+        (insert "ä\n")
+        (clang-format-buffer))
+      (should (equal (buffer-string) "ä\n"))
+      (should (eobp)))
+    (should (equal call-process-inputs '("ef bb bf c3 a4 0d 0a ")))))
+
+;;; clang-format-test.el ends here

tools/clang-format/clang-format.el

is no active region.  If no style is given uses `clang-format-style'."
 
   (unless style
     (setq style clang-format-style))
 
   (let ((file-start (clang-format--bufferpos-to-filepos start 'approximate))
         (file-end (clang-format--bufferpos-to-filepos end 'approximate))
         (cursor (clang-format--bufferpos-to-filepos (point) 'exact))
         (temp-buffer (generate-new-buffer " *clang-format-temp*"))
-        (temp-file (make-temp-file "clang-format")))
+        (temp-file (make-temp-file "clang-format"))
+        (default-process-coding-system
+          ;; Output is XML, which is always UTF-8.  Input encoding should match
+          ;; the file encoding, otherwise the offsets calculated above are off.
+          (cons 'utf-8-unix buffer-file-coding-system)))
     (unwind-protect
         (let ((status (call-process-region
                        nil nil clang-format-executable
                        nil `(,temp-buffer ,temp-file) nil
 
                        "-output-replacements-xml"
                        "-assume-filename" (or (buffer-file-name) "")
                        "-style" style