tsan: fix XMM register corruption in hacky call

[dvyukov]

Summary

The compiler does not recognize HACKY_CALL as a call
(we intentionally hide it from the compiler so that it can
compile non-leaf functions as leaf functions).
To compensate for that hacky call thunk saves and restores
all caller-saved registers. However, it saves only
general-purposes registers and does not save XMM registers.
This is a latent bug that was masked up until a recent "NFC" commit
d736002e90 ("tsan: move memory access functions to a separate file"),
which allowed more inlining and exposed the 10-year bug.
Save and restore caller-saved XMM registers (all) as well.

Currently the bug manifests as e.g. frexp interceptor messes the
return value and the added test fails with:

i=8177 y=0.000000 exp=4

[melver]

Does this trick still buy significant performance gains? LLVM/Clang has changed a lot in 10 years, and modern CPUs also have gotten faster. If it doesn't buy more than 1% or such, the additional risk of doing this might not be worth it anymore.

[dvyukov]

In D113742#3126795, @melver wrote:

Does this trick still buy significant performance gains? LLVM/Clang has changed a lot in 10 years, and modern CPUs also have gotten faster. If it doesn't buy more than 1% or such, the additional risk of doing this might not be worth it anymore.

I am sure it's still worth it because non-leaf functions will contain stack frame and a bunch of spills.
But to make this 10-year old latent bug double-fun: https://reviews.llvm.org/D112604

[apinski-cavium]

This breaks TSAN on processors which don't have AVX support because vmovdqu is not valid for them.
Reported as https://github.com/google/sanitizers/issues/1472

[dvyukov]

In D113742#3160106, @apinski-cavium wrote:

This breaks TSAN on processors which don't have AVX support because vmovdqu is not valid for them.
Reported as https://github.com/google/sanitizers/issues/1472

I guess I needed to use MOVDQU, but this code is unused since https://github.com/llvm/llvm-project/commit/66d4ce7e26a5 and will be removed in https://reviews.llvm.org/D112604

compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S

ASM_SYMBOL(__tsan_trace_switch_thunk):
   CFI_ADJUST_CFA_OFFSET(8)
   CFI_REL_OFFSET(%r9, 0)
   push %r10
   CFI_ADJUST_CFA_OFFSET(8)
   CFI_REL_OFFSET(%r10, 0)
   push %r11
   CFI_ADJUST_CFA_OFFSET(8)
   CFI_REL_OFFSET(%r11, 0)
+  # All XMM registers are caller-saved.
+  sub $0x100, %rsp
+  CFI_ADJUST_CFA_OFFSET(0x100)
+  vmovdqu %xmm0, 0x0(%rsp)
+  vmovdqu %xmm1, 0x10(%rsp)
+  vmovdqu %xmm2, 0x20(%rsp)
+  vmovdqu %xmm3, 0x30(%rsp)
+  vmovdqu %xmm4, 0x40(%rsp)
+  vmovdqu %xmm5, 0x50(%rsp)
+  vmovdqu %xmm6, 0x60(%rsp)
+  vmovdqu %xmm7, 0x70(%rsp)
+  vmovdqu %xmm8, 0x80(%rsp)
+  vmovdqu %xmm9, 0x90(%rsp)
+  vmovdqu %xmm10, 0xa0(%rsp)
+  vmovdqu %xmm11, 0xb0(%rsp)
+  vmovdqu %xmm12, 0xc0(%rsp)
+  vmovdqu %xmm13, 0xd0(%rsp)
+  vmovdqu %xmm14, 0xe0(%rsp)
+  vmovdqu %xmm15, 0xf0(%rsp)
   # Align stack frame.
   push %rbx  # non-scratch
   CFI_ADJUST_CFA_OFFSET(8)
   CFI_REL_OFFSET(%rbx, 0)
   mov %rsp, %rbx  # save current rsp
   CFI_DEF_CFA_REGISTER(%rbx)
   shr $4, %rsp  # clear 4 lsb, align to 16
   shl $4, %rsp
 
   call ASM_SYMBOL(__tsan_trace_switch)
 
   # Unalign stack frame back.
   mov %rbx, %rsp  # restore the original rsp
   CFI_DEF_CFA_REGISTER(%rsp)
   pop %rbx
   CFI_ADJUST_CFA_OFFSET(-8)
   # Restore scratch registers.
+  vmovdqu 0x0(%rsp), %xmm0
+  vmovdqu 0x10(%rsp), %xmm1
+  vmovdqu 0x20(%rsp), %xmm2
+  vmovdqu 0x30(%rsp), %xmm3
+  vmovdqu 0x40(%rsp), %xmm4
+  vmovdqu 0x50(%rsp), %xmm5
+  vmovdqu 0x60(%rsp), %xmm6
+  vmovdqu 0x70(%rsp), %xmm7
+  vmovdqu 0x80(%rsp), %xmm8
+  vmovdqu 0x90(%rsp), %xmm9
+  vmovdqu 0xa0(%rsp), %xmm10
+  vmovdqu 0xb0(%rsp), %xmm11
+  vmovdqu 0xc0(%rsp), %xmm12
+  vmovdqu 0xd0(%rsp), %xmm13
+  vmovdqu 0xe0(%rsp), %xmm14
+  vmovdqu 0xf0(%rsp), %xmm15
+  add $0x100, %rsp
+  CFI_ADJUST_CFA_OFFSET(-0x100)
   pop %r11
   CFI_ADJUST_CFA_OFFSET(-8)
   pop %r10
   CFI_ADJUST_CFA_OFFSET(-8)
   pop %r9
   CFI_ADJUST_CFA_OFFSET(-8)
   pop %r8
   CFI_ADJUST_CFA_OFFSET(-8)
ASM_SYMBOL(__tsan_report_race_thunk):
   CFI_ADJUST_CFA_OFFSET(8)
   CFI_REL_OFFSET(%r9, 0)
   push %r10
   CFI_ADJUST_CFA_OFFSET(8)
   CFI_REL_OFFSET(%r10, 0)
   push %r11
   CFI_ADJUST_CFA_OFFSET(8)
   CFI_REL_OFFSET(%r11, 0)
+  # All XMM registers are caller-saved.
+  sub $0x100, %rsp
+  CFI_ADJUST_CFA_OFFSET(0x100)
+  vmovdqu %xmm0, 0x0(%rsp)
+  vmovdqu %xmm1, 0x10(%rsp)
+  vmovdqu %xmm2, 0x20(%rsp)
+  vmovdqu %xmm3, 0x30(%rsp)
+  vmovdqu %xmm4, 0x40(%rsp)
+  vmovdqu %xmm5, 0x50(%rsp)
+  vmovdqu %xmm6, 0x60(%rsp)
+  vmovdqu %xmm7, 0x70(%rsp)
+  vmovdqu %xmm8, 0x80(%rsp)
+  vmovdqu %xmm9, 0x90(%rsp)
+  vmovdqu %xmm10, 0xa0(%rsp)
+  vmovdqu %xmm11, 0xb0(%rsp)
+  vmovdqu %xmm12, 0xc0(%rsp)
+  vmovdqu %xmm13, 0xd0(%rsp)
+  vmovdqu %xmm14, 0xe0(%rsp)
+  vmovdqu %xmm15, 0xf0(%rsp)
   # Align stack frame.
   push %rbx  # non-scratch
   CFI_ADJUST_CFA_OFFSET(8)
   CFI_REL_OFFSET(%rbx, 0)
   mov %rsp, %rbx  # save current rsp
   CFI_DEF_CFA_REGISTER(%rbx)
   shr $4, %rsp  # clear 4 lsb, align to 16
   shl $4, %rsp
 
   call ASM_SYMBOL(__tsan_report_race)
 
   # Unalign stack frame back.
   mov %rbx, %rsp  # restore the original rsp
   CFI_DEF_CFA_REGISTER(%rsp)
   pop %rbx
   CFI_ADJUST_CFA_OFFSET(-8)
   # Restore scratch registers.
+  vmovdqu 0x0(%rsp), %xmm0
+  vmovdqu 0x10(%rsp), %xmm1
+  vmovdqu 0x20(%rsp), %xmm2
+  vmovdqu 0x30(%rsp), %xmm3
+  vmovdqu 0x40(%rsp), %xmm4
+  vmovdqu 0x50(%rsp), %xmm5
+  vmovdqu 0x60(%rsp), %xmm6
+  vmovdqu 0x70(%rsp), %xmm7
+  vmovdqu 0x80(%rsp), %xmm8
+  vmovdqu 0x90(%rsp), %xmm9
+  vmovdqu 0xa0(%rsp), %xmm10
+  vmovdqu 0xb0(%rsp), %xmm11
+  vmovdqu 0xc0(%rsp), %xmm12
+  vmovdqu 0xd0(%rsp), %xmm13
+  vmovdqu 0xe0(%rsp), %xmm14
+  vmovdqu 0xf0(%rsp), %xmm15
+  add $0x100, %rsp
+  CFI_ADJUST_CFA_OFFSET(-0x100)
   pop %r11
   CFI_ADJUST_CFA_OFFSET(-8)
   pop %r10
   CFI_ADJUST_CFA_OFFSET(-8)
   pop %r9
   CFI_ADJUST_CFA_OFFSET(-8)
   pop %r8
   CFI_ADJUST_CFA_OFFSET(-8)

compiler-rt/test/sanitizer_common/TestCases/frexp.cpp

+// RUN: %clangxx -O2 %s -o %t && %run %t 2>&1 | FileCheck %s
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+  for (int i = 0; i < 10000; i++) {
+    volatile double x = 10;
+    int exp = 0;
+    double y = frexp(x, &exp);
+    if (y != 0.625 || exp != 4) {
+      printf("i=%d y=%lf exp=%d\n", i, y, exp);
+      exit(1);
+    }
+  }
+  fprintf(stderr, "DONE\n");
+  // CHECK: DONE
+  return 0;
+}