[compiler-rt] Refactor memintrinsic interceptors

[melver]

Summary

This moves memintrinsic interceptors (memcpy/memmove/memset) into a new
file sanitizer_common_interceptors_memintrinsics.inc.

This is in preparation of redefining builtins, however, we must be
careful to not redefine builtins in TUs that define interceptors of the
same name.

In all cases except for MSan, memintrinsic interceptors were moved to a
new TU $tool_interceptors_memintrinsics.cpp. In the case of MSan, it
turns out this is not yet necessary (as shown by the later patch
introducing memcpy tests).

NFC.

Depends on D151551

[dvyukov]

What's the failure mode if we accidentally include redefine into sanitizer_common_interceptors_memintrinsics.inc?

[melver]

In D151552#4375754, @dvyukov wrote:

What's the failure mode if we accidentally include redefine into sanitizer_common_interceptors_memintrinsics.inc?

I reworked the builtin redefinitions patch. I added a compile-time check in https://reviews.llvm.org/D151152.

[vitalybuka]

FYI @thurston

[haowei]

Hi

This patch breaks Fuchsia's clang build in building runtimes-aarch64-unknown-fuchsia step. Part of Error message:

FAILED: compiler-rt/lib/asan/CMakeFiles/RTAsan_dynamic.x86_64.dir/asan_interceptors_memintrinsics.cpp.obj 
/b/s/w/ir/x/w/staging/llvm_build/./bin/clang++ --target=x86_64-unknown-fuchsia --sysroot=/b/s/w/ir/x/w/sdk/arch/x64/sysroot -DASAN_DYNAMIC=1 -D_DEBUG -D_GLIBCXX_ASSERTIONS -D_LIBCPP_ENABLE_ASSERTIONS -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -I/b/s/w/ir/x/w/llvm-llvm-project/compiler-rt/lib/asan/.. --target=x86_64-unknown-fuchsia -I/b/s/w/ir/x/w/sdk/pkg/sync/include -I/b/s/w/ir/x/w/sdk/pkg/fdio/include -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wno-comment -Wstring-conversion -Wmisleading-indentation -Wctad-maybe-unsupported -ffunction-sections -fdata-sections -ffile-prefix-map=/b/s/w/ir/x/w/staging/llvm_build/runtimes/runtimes-x86_64-unknown-fuchsia-bins=../staging/llvm_build/runtimes/runtimes-x86_64-unknown-fuchsia-bins -ffile-prefix-map=/b/s/w/ir/x/w/llvm-llvm-project/= -no-canonical-prefixes -Wall -Wno-unused-parameter -O2 -g -DNDEBUG -std=c++17 -fPIC -fno-builtin -fno-exceptions -fomit-frame-pointer -funwind-tables -fno-stack-protector -fno-sanitize=safe-stack -fvisibility=hidden -fno-lto -Wthread-safety -Wthread-safety-reference -Wthread-safety-beta -O3 -gline-tables-only -Wno-gnu -Wno-variadic-macros -Wno-c99-extensions -nostdinc++ -fno-rtti -Wno-format -ftls-model=initial-exec -MD -MT compiler-rt/lib/asan/CMakeFiles/RTAsan_dynamic.x86_64.dir/asan_interceptors_memintrinsics.cpp.obj -MF compiler-rt/lib/asan/CMakeFiles/RTAsan_dynamic.x86_64.dir/asan_interceptors_memintrinsics.cpp.obj.d -o compiler-rt/lib/asan/CMakeFiles/RTAsan_dynamic.x86_64.dir/asan_interceptors_memintrinsics.cpp.obj -c /b/s/w/ir/x/w/llvm-llvm-project/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cpp
In file included from /b/s/w/ir/x/w/llvm-llvm-project/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cpp:78:
/b/s/w/ir/x/w/llvm-llvm-project/compiler-rt/lib/asan/../sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc:229:3: error: use of undeclared identifier 'ASAN_INTERCEPT_FUNC'
  INIT_MEMSET;
  ^

Full error can be found at https://logs.chromium.org/logs/fuchsia/buildbucket/cr-buildbucket/8779679021892159153/+/u/clang/build/stdout
Could you take a look? If it takes too long to fix forward, could you revert your change first? Thanks.

[melver]

In D151552#4381384, @haowei wrote:

Hi

This patch breaks Fuchsia's clang build in building runtimes-aarch64-unknown-fuchsia step. Part of Error message:

FAILED: compiler-rt/lib/asan/CMakeFiles/RTAsan_dynamic.x86_64.dir/asan_interceptors_memintrinsics.cpp.obj 
/b/s/w/ir/x/w/staging/llvm_build/./bin/clang++ --target=x86_64-unknown-fuchsia --sysroot=/b/s/w/ir/x/w/sdk/arch/x64/sysroot -DASAN_DYNAMIC=1 -D_DEBUG -D_GLIBCXX_ASSERTIONS -D_LIBCPP_ENABLE_ASSERTIONS -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -I/b/s/w/ir/x/w/llvm-llvm-project/compiler-rt/lib/asan/.. --target=x86_64-unknown-fuchsia -I/b/s/w/ir/x/w/sdk/pkg/sync/include -I/b/s/w/ir/x/w/sdk/pkg/fdio/include -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wno-comment -Wstring-conversion -Wmisleading-indentation -Wctad-maybe-unsupported -ffunction-sections -fdata-sections -ffile-prefix-map=/b/s/w/ir/x/w/staging/llvm_build/runtimes/runtimes-x86_64-unknown-fuchsia-bins=../staging/llvm_build/runtimes/runtimes-x86_64-unknown-fuchsia-bins -ffile-prefix-map=/b/s/w/ir/x/w/llvm-llvm-project/= -no-canonical-prefixes -Wall -Wno-unused-parameter -O2 -g -DNDEBUG -std=c++17 -fPIC -fno-builtin -fno-exceptions -fomit-frame-pointer -funwind-tables -fno-stack-protector -fno-sanitize=safe-stack -fvisibility=hidden -fno-lto -Wthread-safety -Wthread-safety-reference -Wthread-safety-beta -O3 -gline-tables-only -Wno-gnu -Wno-variadic-macros -Wno-c99-extensions -nostdinc++ -fno-rtti -Wno-format -ftls-model=initial-exec -MD -MT compiler-rt/lib/asan/CMakeFiles/RTAsan_dynamic.x86_64.dir/asan_interceptors_memintrinsics.cpp.obj -MF compiler-rt/lib/asan/CMakeFiles/RTAsan_dynamic.x86_64.dir/asan_interceptors_memintrinsics.cpp.obj.d -o compiler-rt/lib/asan/CMakeFiles/RTAsan_dynamic.x86_64.dir/asan_interceptors_memintrinsics.cpp.obj -c /b/s/w/ir/x/w/llvm-llvm-project/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cpp
In file included from /b/s/w/ir/x/w/llvm-llvm-project/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cpp:78:
/b/s/w/ir/x/w/llvm-llvm-project/compiler-rt/lib/asan/../sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc:229:3: error: use of undeclared identifier 'ASAN_INTERCEPT_FUNC'
  INIT_MEMSET;
  ^

Full error can be found at https://logs.chromium.org/logs/fuchsia/buildbucket/cr-buildbucket/8779679021892159153/+/u/clang/build/stdout
Could you take a look? If it takes too long to fix forward, could you revert your change first? Thanks.

https://reviews.llvm.org/rG8c5ad4a0e297 should fix it.

compiler-rt/lib/asan/asan_interceptors.h

 
 #  else
 // OS X interceptors don't need to be initialized with INTERCEPT_FUNCTION.
 #    define ASAN_INTERCEPT_FUNC(name)
 #  endif  // SANITIZER_APPLE
 
 #endif  // !SANITIZER_FUCHSIA
 
+#define ASAN_INTERCEPTOR_ENTER(ctx, func)                                      \
+  AsanInterceptorContext _ctx = {#func};                                       \
+  ctx = (void *)&_ctx;                                                         \
+  (void) ctx;
+#define COMMON_INTERCEPT_FUNCTION(name) ASAN_INTERCEPT_FUNC(name)
+
 #endif  // ASAN_INTERCEPTORS_H

compiler-rt/lib/asan/asan_interceptors.cpp

 } // namespace __asan
 
 // ---------------------- Wrappers ---------------- {{{1
 using namespace __asan;
 
 DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr)
 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
 
-#define ASAN_INTERCEPTOR_ENTER(ctx, func)                                      \
-  AsanInterceptorContext _ctx = {#func};                                       \
-  ctx = (void *)&_ctx;                                                         \
-  (void) ctx;                                                                  \
-
-#define COMMON_INTERCEPT_FUNCTION(name) ASAN_INTERCEPT_FUNC(name)
 #define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \
   ASAN_INTERCEPT_FUNC_VER(name, ver)
 #define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \
   ASAN_INTERCEPT_FUNC_VER_UNVERSIONED_FALLBACK(name, ver)
 #define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \
   ASAN_WRITE_RANGE(ctx, ptr, size)
 #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \
   ASAN_READ_RANGE(ctx, ptr, size)

 #  define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \
     if (AsanThread *t = GetCurrentThread()) {          \
       *begin = t->tls_begin();                         \
       *end = t->tls_end();                             \
     } else {                                           \
       *begin = *end = 0;                               \
     }
 
-#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size) \
-  do {                                                       \
-    ASAN_INTERCEPTOR_ENTER(ctx, memmove);                    \
-    ASAN_MEMMOVE_IMPL(ctx, to, from, size);                  \
-  } while (false)
-
-#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size) \
-  do {                                                      \
-    ASAN_INTERCEPTOR_ENTER(ctx, memcpy);                    \
-    ASAN_MEMCPY_IMPL(ctx, to, from, size);                  \
-  } while (false)
-
-#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size) \
-  do {                                                      \
-    ASAN_INTERCEPTOR_ENTER(ctx, memset);                    \
-    ASAN_MEMSET_IMPL(ctx, block, c, size);                  \
-  } while (false)
-
 #if CAN_SANITIZE_LEAKS
 #define COMMON_INTERCEPTOR_STRERROR()                       \
   __lsan::ScopedInterceptorDisabler disabler
 #endif
 
 #include "sanitizer_common/sanitizer_common_interceptors.inc"
 #include "sanitizer_common/sanitizer_signal_interceptors.inc"
 

compiler-rt/lib/asan/asan_interceptors_memintrinsics.h

if (UNLIKELY(!QuickCheckForUnpoisonedRegion(__offset, __size)) &&     \
       }                                                                   \
       if (!suppressed) {                                                  \
         GET_CURRENT_PC_BP_SP;                                             \
         ReportGenericError(pc, bp, sp, __bad, isWrite, __size, 0, false); \
       }                                                                   \
     }                                                                     \
   } while (0)
 
-// memcpy is called during __asan_init() from the internals of printf(...).
-// We do not treat memcpy with to==from as a bug.
-// See http://llvm.org/bugs/show_bug.cgi?id=11763.
-#define ASAN_MEMCPY_IMPL(ctx, to, from, size)                 \
-  do {                                                        \
-    if (LIKELY(replace_intrin_cached)) {                      \
-      if (LIKELY(to != from)) {                               \
-        CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \
-      }                                                       \
-      ASAN_READ_RANGE(ctx, from, size);                       \
-      ASAN_WRITE_RANGE(ctx, to, size);                        \
-    } else if (UNLIKELY(!asan_inited)) {                      \
-      return internal_memcpy(to, from, size);                 \
-    }                                                         \
-    return REAL(memcpy)(to, from, size);                      \
-  } while (0)
-
-// memset is called inside Printf.
-#define ASAN_MEMSET_IMPL(ctx, block, c, size) \
-  do {                                        \
-    if (LIKELY(replace_intrin_cached)) {      \
-      ASAN_WRITE_RANGE(ctx, block, size);     \
-    } else if (UNLIKELY(!asan_inited)) {      \
-      return internal_memset(block, c, size); \
-    }                                         \
-    return REAL(memset)(block, c, size);      \
-  } while (0)
-
-#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \
-  do {                                         \
-    if (LIKELY(replace_intrin_cached)) {       \
-      ASAN_READ_RANGE(ctx, from, size);        \
-      ASAN_WRITE_RANGE(ctx, to, size);         \
-    }                                          \
-    return internal_memmove(to, from, size);   \
-  } while (0)
-
 #define ASAN_READ_RANGE(ctx, offset, size) \
   ACCESS_MEMORY_RANGE(ctx, offset, size, false)
 #define ASAN_WRITE_RANGE(ctx, offset, size) \
   ACCESS_MEMORY_RANGE(ctx, offset, size, true)
 
 // Behavior of functions like "memcpy" or "strcpy" is undefined
 // if memory intervals overlap. We report error in this case.
 // Macro is used to avoid creation of new frames.

compiler-rt/lib/asan/asan_interceptors_memintrinsics.cpp

 //===-- asan_interceptors_memintrinsics.cpp -------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
 // This file is a part of AddressSanitizer, an address sanity checker.
 //
 // ASan versions of memcpy, memmove, and memset.
 //===---------------------------------------------------------------------===//
 
 #include "asan_interceptors_memintrinsics.h"
+
+#include "asan_interceptors.h"
 #include "asan_report.h"
 #include "asan_stack.h"
 #include "asan_suppressions.h"
 
 using namespace __asan;
 
+// memcpy is called during __asan_init() from the internals of printf(...).
+// We do not treat memcpy with to==from as a bug.
+// See http://llvm.org/bugs/show_bug.cgi?id=11763.
+#define ASAN_MEMCPY_IMPL(ctx, to, from, size)                 \
+  do {                                                        \
+    if (LIKELY(replace_intrin_cached)) {                      \
+      if (LIKELY(to != from)) {                               \
+        CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \
+      }                                                       \
+      ASAN_READ_RANGE(ctx, from, size);                       \
+      ASAN_WRITE_RANGE(ctx, to, size);                        \
+    } else if (UNLIKELY(!asan_inited)) {                      \
+      return internal_memcpy(to, from, size);                 \
+    }                                                         \
+    return REAL(memcpy)(to, from, size);                      \
+  } while (0)
+
+// memset is called inside Printf.
+#define ASAN_MEMSET_IMPL(ctx, block, c, size) \
+  do {                                        \
+    if (LIKELY(replace_intrin_cached)) {      \
+      ASAN_WRITE_RANGE(ctx, block, size);     \
+    } else if (UNLIKELY(!asan_inited)) {      \
+      return internal_memset(block, c, size); \
+    }                                         \
+    return REAL(memset)(block, c, size);      \
+  } while (0)
+
+#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \
+  do {                                         \
+    if (LIKELY(replace_intrin_cached)) {       \
+      ASAN_READ_RANGE(ctx, from, size);        \
+      ASAN_WRITE_RANGE(ctx, to, size);         \
+    }                                          \
+    return internal_memmove(to, from, size);   \
+  } while (0)
+
+#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size) \
+  do {                                                       \
+    ASAN_INTERCEPTOR_ENTER(ctx, memmove);                    \
+    ASAN_MEMMOVE_IMPL(ctx, to, from, size);                  \
+  } while (false)
+
+#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size) \
+  do {                                                      \
+    ASAN_INTERCEPTOR_ENTER(ctx, memcpy);                    \
+    ASAN_MEMCPY_IMPL(ctx, to, from, size);                  \
+  } while (false)
+
+#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size) \
+  do {                                                      \
+    ASAN_INTERCEPTOR_ENTER(ctx, memset);                    \
+    ASAN_MEMSET_IMPL(ctx, block, c, size);                  \
+  } while (false)
+
+#include "sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc"
+
 void *__asan_memcpy(void *to, const void *from, uptr size) {
   ASAN_MEMCPY_IMPL(nullptr, to, from, size);
 }
 
 void *__asan_memset(void *block, int c, uptr size) {
   ASAN_MEMSET_IMPL(nullptr, block, c, size);
 }
 

compiler-rt/lib/hwasan/hwasan_interceptors.cpp

 
 #    define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, length, prot, flags, \
                                          fd, offset)                           \
       do {                                                                     \
         (void)(ctx);                                                           \
         return mmap_interceptor(REAL(mmap), addr, sz, prot, flags, fd, off);   \
       } while (false)
 
+#    include "sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc"
 #    include "sanitizer_common/sanitizer_common_interceptors.inc"
 
 struct ThreadStartArg {
   __sanitizer_sigset_t starting_sigset_;
 };
 
 static void *HwasanThreadStartFunc(void *arg) {
   __hwasan_thread_enter();

compiler-rt/lib/memprof/memprof_interceptors.h

#define MEMPROF_INTERCEPT_FUNC_VER(name, ver)                                  \
   } while (0)
 #define MEMPROF_INTERCEPT_FUNC_VER_UNVERSIONED_FALLBACK(name, ver)             \
   do {                                                                         \
     if (!INTERCEPT_FUNCTION_VER(name, ver) && !INTERCEPT_FUNCTION(name))       \
       VReport(1, "MemProfiler: failed to intercept '%s@@%s' or '%s'\n", #name, \
               ver, #name);                                                     \
   } while (0)
 
+#define MEMPROF_INTERCEPTOR_ENTER(ctx, func)                                   \
+  ctx = 0;                                                                     \
+  (void)ctx;
+
+#define COMMON_INTERCEPT_FUNCTION(name) MEMPROF_INTERCEPT_FUNC(name)
+
 #endif // MEMPROF_INTERCEPTORS_H

compiler-rt/lib/memprof/memprof_interceptors.cpp

 } // namespace __memprof
 
 // ---------------------- Wrappers ---------------- {{{1
 using namespace __memprof;
 
 DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr)
 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
 
-#define MEMPROF_INTERCEPTOR_ENTER(ctx, func)                                   \
-  ctx = 0;                                                                     \
-  (void)ctx;
-
-#define COMMON_INTERCEPT_FUNCTION(name) MEMPROF_INTERCEPT_FUNC(name)
 #define COMMON_INTERCEPT_FUNCTION_VER(name, ver)                               \
   MEMPROF_INTERCEPT_FUNC_VER(name, ver)
 #define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver)          \
   MEMPROF_INTERCEPT_FUNC_VER_UNVERSIONED_FALLBACK(name, ver)
 #define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size)                         \
   MEMPROF_WRITE_RANGE(ptr, size)
 #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size)                          \
   MEMPROF_READ_RANGE(ptr, size)

 #define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end)                           \
   if (MemprofThread *t = GetCurrentThread()) {                                 \
     *begin = t->tls_begin();                                                   \
     *end = t->tls_end();                                                       \
   } else {                                                                     \
     *begin = *end = 0;                                                         \
   }
 
-#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size)                   \
-  do {                                                                         \
-    MEMPROF_INTERCEPTOR_ENTER(ctx, memmove);                                   \
-    MEMPROF_MEMMOVE_IMPL(to, from, size);                                      \
-  } while (false)
-
-#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size)                    \
-  do {                                                                         \
-    MEMPROF_INTERCEPTOR_ENTER(ctx, memcpy);                                    \
-    MEMPROF_MEMCPY_IMPL(to, from, size);                                       \
-  } while (false)
-
-#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size)                    \
-  do {                                                                         \
-    MEMPROF_INTERCEPTOR_ENTER(ctx, memset);                                    \
-    MEMPROF_MEMSET_IMPL(block, c, size);                                       \
-  } while (false)
-
 #include "sanitizer_common/sanitizer_common_interceptors.inc"
 
 #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) MEMPROF_READ_RANGE(p, s)
 #define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) MEMPROF_WRITE_RANGE(p, s)
 #define COMMON_SYSCALL_POST_READ_RANGE(p, s)                                   \
   do {                                                                         \
     (void)(p);                                                                 \
     (void)(s);                                                                 \

compiler-rt/lib/memprof/memprof_interceptors_memintrinsics.h

 // and MEMPROF_WRITE_RANGE as macro instead of function so
 // that no extra frames are created, and stack trace contains
 // relevant information only.
 #define ACCESS_MEMORY_RANGE(offset, size)                                      \
   do {                                                                         \
     __memprof_record_access_range(offset, size);                               \
   } while (0)
 
-// memcpy is called during __memprof_init() from the internals of printf(...).
-// We do not treat memcpy with to==from as a bug.
-// See http://llvm.org/bugs/show_bug.cgi?id=11763.
-#define MEMPROF_MEMCPY_IMPL(to, from, size)                                    \
-  do {                                                                         \
-    if (UNLIKELY(!memprof_inited))                                             \
-      return internal_memcpy(to, from, size);                                  \
-    if (memprof_init_is_running) {                                             \
-      return REAL(memcpy)(to, from, size);                                     \
-    }                                                                          \
-    ENSURE_MEMPROF_INITED();                                                   \
-    MEMPROF_READ_RANGE(from, size);                                            \
-    MEMPROF_WRITE_RANGE(to, size);                                             \
-    return REAL(memcpy)(to, from, size);                                       \
-  } while (0)
-
-// memset is called inside Printf.
-#define MEMPROF_MEMSET_IMPL(block, c, size)                                    \
-  do {                                                                         \
-    if (UNLIKELY(!memprof_inited))                                             \
-      return internal_memset(block, c, size);                                  \
-    if (memprof_init_is_running) {                                             \
-      return REAL(memset)(block, c, size);                                     \
-    }                                                                          \
-    ENSURE_MEMPROF_INITED();                                                   \
-    MEMPROF_WRITE_RANGE(block, size);                                          \
-    return REAL(memset)(block, c, size);                                       \
-  } while (0)
-
-#define MEMPROF_MEMMOVE_IMPL(to, from, size)                                   \
-  do {                                                                         \
-    if (UNLIKELY(!memprof_inited))                                             \
-      return internal_memmove(to, from, size);                                 \
-    ENSURE_MEMPROF_INITED();                                                   \
-    MEMPROF_READ_RANGE(from, size);                                            \
-    MEMPROF_WRITE_RANGE(to, size);                                             \
-    return internal_memmove(to, from, size);                                   \
-  } while (0)
-
 #define MEMPROF_READ_RANGE(offset, size) ACCESS_MEMORY_RANGE(offset, size)
 #define MEMPROF_WRITE_RANGE(offset, size) ACCESS_MEMORY_RANGE(offset, size)
 
 } // namespace __memprof
 
 #endif // MEMPROF_MEMINTRIN_H

compiler-rt/lib/memprof/memprof_interceptors_memintrinsics.cpp

 //===-- memprof_interceptors_memintrinsics.cpp ---------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
 // This file is a part of MemProfiler, a memory profiler.
 //
 // MemProf versions of memcpy, memmove, and memset.
 //===---------------------------------------------------------------------===//
 
 #include "memprof_interceptors_memintrinsics.h"
+
+#include "memprof_interceptors.h"
 #include "memprof_stack.h"
 
 using namespace __memprof;
 
+// memcpy is called during __memprof_init() from the internals of printf(...).
+// We do not treat memcpy with to==from as a bug.
+// See http://llvm.org/bugs/show_bug.cgi?id=11763.
+#define MEMPROF_MEMCPY_IMPL(to, from, size)                                    \
+  do {                                                                         \
+    if (UNLIKELY(!memprof_inited))                                             \
+      return internal_memcpy(to, from, size);                                  \
+    if (memprof_init_is_running) {                                             \
+      return REAL(memcpy)(to, from, size);                                     \
+    }                                                                          \
+    ENSURE_MEMPROF_INITED();                                                   \
+    MEMPROF_READ_RANGE(from, size);                                            \
+    MEMPROF_WRITE_RANGE(to, size);                                             \
+    return REAL(memcpy)(to, from, size);                                       \
+  } while (0)
+
+// memset is called inside Printf.
+#define MEMPROF_MEMSET_IMPL(block, c, size)                                    \
+  do {                                                                         \
+    if (UNLIKELY(!memprof_inited))                                             \
+      return internal_memset(block, c, size);                                  \
+    if (memprof_init_is_running) {                                             \
+      return REAL(memset)(block, c, size);                                     \
+    }                                                                          \
+    ENSURE_MEMPROF_INITED();                                                   \
+    MEMPROF_WRITE_RANGE(block, size);                                          \
+    return REAL(memset)(block, c, size);                                       \
+  } while (0)
+
+#define MEMPROF_MEMMOVE_IMPL(to, from, size)                                   \
+  do {                                                                         \
+    if (UNLIKELY(!memprof_inited))                                             \
+      return internal_memmove(to, from, size);                                 \
+    ENSURE_MEMPROF_INITED();                                                   \
+    MEMPROF_READ_RANGE(from, size);                                            \
+    MEMPROF_WRITE_RANGE(to, size);                                             \
+    return internal_memmove(to, from, size);                                   \
+  } while (0)
+
+#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size)                   \
+  do {                                                                         \
+    MEMPROF_INTERCEPTOR_ENTER(ctx, memmove);                                   \
+    MEMPROF_MEMMOVE_IMPL(to, from, size);                                      \
+  } while (false)
+
+#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size)                    \
+  do {                                                                         \
+    MEMPROF_INTERCEPTOR_ENTER(ctx, memcpy);                                    \
+    MEMPROF_MEMCPY_IMPL(to, from, size);                                       \
+  } while (false)
+
+#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size)                    \
+  do {                                                                         \
+    MEMPROF_INTERCEPTOR_ENTER(ctx, memset);                                    \
+    MEMPROF_MEMSET_IMPL(block, c, size);                                       \
+  } while (false)
+
+#include "sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc"
+
 void *__memprof_memcpy(void *to, const void *from, uptr size) {
   MEMPROF_MEMCPY_IMPL(to, from, size);
 }
 
 void *__memprof_memset(void *block, int c, uptr size) {
   MEMPROF_MEMSET_IMPL(block, c, size);
 }
 
 void *__memprof_memmove(void *to, const void *from, uptr size) {
   MEMPROF_MEMMOVE_IMPL(to, from, size);
 }

compiler-rt/lib/msan/msan_interceptors.cpp

 
 #define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, length, prot, flags, fd, \
                                      offset)                                   \
   do {                                                                         \
     return mmap_interceptor(REAL(mmap), addr, sz, prot, flags, fd, off);       \
   } while (false)
 
 #include "sanitizer_common/sanitizer_platform_interceptors.h"
+#include "sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc"
 #include "sanitizer_common/sanitizer_common_interceptors.inc"
 
 static uptr signal_impl(int signo, uptr cb);
 static int sigaction_impl(int signo, const __sanitizer_sigaction *act,
                           __sanitizer_sigaction *oldact);
 
 #define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signo, act, oldact) \
   { return sigaction_impl(signo, act, oldact); }

compiler-rt/lib/sanitizer_common/CMakeLists.txt

set(SANITIZER_IMPL_HEADERS
   sanitizer_atomic_msvc.h
   sanitizer_bitvector.h
   sanitizer_bvgraph.h
   sanitizer_chained_origin_depot.h
   sanitizer_common.h
   sanitizer_common_interceptors.inc
   sanitizer_common_interceptors_format.inc
   sanitizer_common_interceptors_ioctl.inc
+  sanitizer_common_interceptors_memintrinsics.inc
   sanitizer_common_interface.inc
   sanitizer_common_interface_posix.inc
   sanitizer_common_syscalls.inc
   sanitizer_coverage_interface.inc
   sanitizer_dbghelp.h
   sanitizer_deadlock_detector.h
   sanitizer_deadlock_detector_interface.h
   sanitizer_dense_map.h

compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc

 //   COMMON_INTERCEPTOR_FD_RELEASE
 //   COMMON_INTERCEPTOR_FD_ACCESS
 //   COMMON_INTERCEPTOR_SET_THREAD_NAME
 //   COMMON_INTERCEPTOR_DLOPEN
 //   COMMON_INTERCEPTOR_ON_EXIT
 //   COMMON_INTERCEPTOR_SET_PTHREAD_NAME
 //   COMMON_INTERCEPTOR_HANDLE_RECVMSG
 //   COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED
-//   COMMON_INTERCEPTOR_MEMSET_IMPL
-//   COMMON_INTERCEPTOR_MEMMOVE_IMPL
-//   COMMON_INTERCEPTOR_MEMCPY_IMPL
 //   COMMON_INTERCEPTOR_MMAP_IMPL
 //   COMMON_INTERCEPTOR_COPY_STRING
 //   COMMON_INTERCEPTOR_STRNDUP_IMPL
 //   COMMON_INTERCEPTOR_STRERROR
 //===----------------------------------------------------------------------===//
 
 #include "interception/interception.h"
 #include "sanitizer_addrhashmap.h"

 #define time __time64
 #define utimensat __utimensat_time64
 #define utimes __utimes_time64
 #define utime __utime64
 #define wait3 __wait3_time64
 #define wait4 __wait4_time64
 #endif
 
-// Platform-specific options.
-#if SANITIZER_APPLE
-#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
-#elif SANITIZER_WINDOWS64
-#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
-#else
-#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1
-#endif  // SANITIZER_APPLE
-
 #ifndef COMMON_INTERCEPTOR_INITIALIZE_RANGE
 #define COMMON_INTERCEPTOR_INITIALIZE_RANGE(p, size) {}
 #endif
 
 #ifndef COMMON_INTERCEPTOR_UNPOISON_PARAM
 #define COMMON_INTERCEPTOR_UNPOISON_PARAM(count) {}
 #endif
 

 // See also https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98920.
 #define COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(fn, ver) \
   COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(fn, ver)
 #else
 #define COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(fn, ver) \
   COMMON_INTERCEPT_FUNCTION(fn)
 #endif
 
-#ifndef COMMON_INTERCEPTOR_MEMSET_IMPL
-#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \
-  {                                                       \
-    if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)        \
-      return internal_memset(dst, v, size);               \
-    COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size);  \
-    if (common_flags()->intercept_intrin)                 \
-      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size);     \
-    return REAL(memset)(dst, v, size);                    \
-  }
-#endif
-
-#ifndef COMMON_INTERCEPTOR_MEMMOVE_IMPL
-#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size) \
-  {                                                          \
-    if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)           \
-      return internal_memmove(dst, src, size);               \
-    COMMON_INTERCEPTOR_ENTER(ctx, memmove, dst, src, size);  \
-    if (common_flags()->intercept_intrin) {                  \
-      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size);        \
-      COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size);         \
-    }                                                        \
-    return REAL(memmove)(dst, src, size);                    \
-  }
-#endif
-
-#ifndef COMMON_INTERCEPTOR_MEMCPY_IMPL
-#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size) \
-  {                                                         \
-    if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) {        \
-      return internal_memmove(dst, src, size);              \
-    }                                                       \
-    COMMON_INTERCEPTOR_ENTER(ctx, memcpy, dst, src, size);  \
-    if (common_flags()->intercept_intrin) {                 \
-      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size);       \
-      COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size);        \
-    }                                                       \
-    return REAL(memcpy)(dst, src, size);                    \
-  }
-#endif
-
 #ifndef COMMON_INTERCEPTOR_MMAP_IMPL
 #define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \
                                      off)                                  \
   { return REAL(mmap)(addr, sz, prot, flags, fd, off); }
 #endif
 
 #ifndef COMMON_INTERCEPTOR_COPY_STRING
 #define COMMON_INTERCEPTOR_COPY_STRING(ctx, to, from, size) {}
INTERCEPTOR(char *, strpbrk, const char *s1, const char *s2) {
   return r;
 }
 
 #define INIT_STRPBRK COMMON_INTERCEPT_FUNCTION(strpbrk);
 #else
 #define INIT_STRPBRK
 #endif
 
-#if SANITIZER_INTERCEPT_MEMSET
-INTERCEPTOR(void *, memset, void *dst, int v, uptr size) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size);
-}
-
-#define INIT_MEMSET COMMON_INTERCEPT_FUNCTION(memset)
-#else
-#define INIT_MEMSET
-#endif
-
-#if SANITIZER_INTERCEPT_MEMMOVE
-INTERCEPTOR(void *, memmove, void *dst, const void *src, uptr size) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
-}
-
-#define INIT_MEMMOVE COMMON_INTERCEPT_FUNCTION(memmove)
-#else
-#define INIT_MEMMOVE
-#endif
-
-#if SANITIZER_INTERCEPT_MEMCPY
-INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) {
-  // On OS X, calling internal_memcpy here will cause memory corruptions,
-  // because memcpy and memmove are actually aliases of the same
-  // implementation.  We need to use internal_memmove here.
-  // N.B.: If we switch this to internal_ we'll have to use internal_memmove
-  // due to memcpy being an alias of memmove on OS X.
-  void *ctx;
-#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE
-    COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size);
-#else
-    COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
-#endif
-}
-
-#define INIT_MEMCPY                                  \
-  do {                                               \
-    if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) { \
-      COMMON_INTERCEPT_FUNCTION(memcpy);             \
-    } else {                                         \
-      ASSIGN_REAL(memcpy, memmove);                  \
-    }                                                \
-    CHECK(REAL(memcpy));                             \
-  } while (false)
-
-#else
-#define INIT_MEMCPY
-#endif
-
 #if SANITIZER_INTERCEPT_MEMCMP
 DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, uptr called_pc,
                               const void *s1, const void *s2, uptr n,
                               int result)
 
 // Common code for `memcmp` and `bcmp`.
 int MemcmpInterceptorCommon(void *ctx,
                             int (*real_fn)(const void *, const void *, uptr),

 }
 #define INIT_CAPGET                  \
   COMMON_INTERCEPT_FUNCTION(capget); \
   COMMON_INTERCEPT_FUNCTION(capset);
 #else
 #define INIT_CAPGET
 #endif
 
-#if SANITIZER_INTERCEPT_AEABI_MEM
-INTERCEPTOR(void *, __aeabi_memmove, void *to, const void *from, uptr size) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memmove4, void *to, const void *from, uptr size) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memmove8, void *to, const void *from, uptr size) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memcpy, void *to, const void *from, uptr size) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memcpy4, void *to, const void *from, uptr size) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memcpy8, void *to, const void *from, uptr size) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
-}
-
-// Note the argument order.
-INTERCEPTOR(void *, __aeabi_memset, void *block, uptr size, int c) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memset4, void *block, uptr size, int c) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memset8, void *block, uptr size, int c) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memclr, void *block, uptr size) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memclr4, void *block, uptr size) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memclr8, void *block, uptr size) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
-}
-
-#define INIT_AEABI_MEM                         \
-  COMMON_INTERCEPT_FUNCTION(__aeabi_memmove);  \
-  COMMON_INTERCEPT_FUNCTION(__aeabi_memmove4); \
-  COMMON_INTERCEPT_FUNCTION(__aeabi_memmove8); \
-  COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy);   \
-  COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy4);  \
-  COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy8);  \
-  COMMON_INTERCEPT_FUNCTION(__aeabi_memset);   \
-  COMMON_INTERCEPT_FUNCTION(__aeabi_memset4);  \
-  COMMON_INTERCEPT_FUNCTION(__aeabi_memset8);  \
-  COMMON_INTERCEPT_FUNCTION(__aeabi_memclr);   \
-  COMMON_INTERCEPT_FUNCTION(__aeabi_memclr4);  \
-  COMMON_INTERCEPT_FUNCTION(__aeabi_memclr8);
-#else
-#define INIT_AEABI_MEM
-#endif  // SANITIZER_INTERCEPT_AEABI_MEM
-
-#if SANITIZER_INTERCEPT___BZERO
-INTERCEPTOR(void *, __bzero, void *block, uptr size) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
-}
-#define INIT___BZERO COMMON_INTERCEPT_FUNCTION(__bzero);
-#else
-#define INIT___BZERO
-#endif  // SANITIZER_INTERCEPT___BZERO
-
-#if SANITIZER_INTERCEPT_BZERO
-INTERCEPTOR(void *, bzero, void *block, uptr size) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
-}
-#define INIT_BZERO COMMON_INTERCEPT_FUNCTION(bzero);
-#else
-#define INIT_BZERO
-#endif  // SANITIZER_INTERCEPT_BZERO
-
 #if SANITIZER_INTERCEPT_FTIME
 INTERCEPTOR(int, ftime, __sanitizer_timeb *tp) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, ftime, tp);
   // FIXME: under ASan the call below may write to freed memory and corrupt
   // its metadata. See
   // https://github.com/google/sanitizers/issues/321.
   int res = REAL(ftime)(tp);

 
 #define INIT_ARGP_PARSE COMMON_INTERCEPT_FUNCTION(argp_parse);
 #else
 #define INIT_ARGP_PARSE
 #endif
 
 #include "sanitizer_common_interceptors_netbsd_compat.inc"
 
+namespace __sanitizer {
+void InitializeMemintrinsicInterceptors();
+}  // namespace __sanitizer
+
 static void InitializeCommonInterceptors() {
 #if SI_POSIX
   static u64 metadata_mem[sizeof(MetadataHashMap) / sizeof(u64) + 1];
   interceptor_metadata_map = new ((void *)&metadata_mem) MetadataHashMap();
 #endif
 
+  __sanitizer::InitializeMemintrinsicInterceptors();
+
   INIT_MMAP;
   INIT_MMAP64;
   INIT_TEXTDOMAIN;
   INIT_STRLEN;
   INIT_STRNLEN;
   INIT_STRNDUP;
   INIT___STRNDUP;
   INIT_STRCMP;
   INIT_STRNCMP;
   INIT_STRCASECMP;
   INIT_STRNCASECMP;
   INIT_STRSTR;
   INIT_STRCASESTR;
   INIT_STRCHR;
   INIT_STRCHRNUL;
   INIT_STRRCHR;
   INIT_STRSPN;
   INIT_STRTOK;
   INIT_STRPBRK;
   INIT_STRXFRM;
   INIT___STRXFRM_L;
-  INIT_MEMSET;
-  INIT_MEMMOVE;
-  INIT_MEMCPY;
   INIT_MEMCHR;
   INIT_MEMCMP;
   INIT_BCMP;
   INIT_MEMRCHR;
   INIT_MEMMEM;
   INIT_READ;
   INIT_FREAD;
   INIT_PREAD;
#endif
   INIT_TIMES;
   INIT_TLS_GET_ADDR;
   INIT_LISTXATTR;
   INIT_GETXATTR;
   INIT_GETRESID;
   INIT_GETIFADDRS;
   INIT_IF_INDEXTONAME;
   INIT_CAPGET;
-  INIT_AEABI_MEM;
-  INIT___BZERO;
-  INIT_BZERO;
   INIT_FTIME;
   INIT_XDR;
   INIT_XDRREC_LINUX;
   INIT_TSEARCH;
   INIT_LIBIO_INTERNALS;
   INIT_FOPEN;
   INIT_FOPEN64;
   INIT_FLOPEN;

compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc

+//===-- sanitizer_common_interceptors_memintrinsics.inc ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Memintrinsic function interceptors for tools like AddressSanitizer,
+// ThreadSanitizer, MemorySanitizer, etc.
+//
+// This file should be included into the tool's memintrinsic interceptor file,
+// which has to define its own macros:
+//   COMMON_INTERCEPTOR_ENTER
+//   COMMON_INTERCEPTOR_READ_RANGE
+//   COMMON_INTERCEPTOR_WRITE_RANGE
+//   COMMON_INTERCEPTOR_MEMSET_IMPL
+//   COMMON_INTERCEPTOR_MEMMOVE_IMPL
+//   COMMON_INTERCEPTOR_MEMCPY_IMPL
+//   COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED
+//===----------------------------------------------------------------------===//
+
+#include "interception/interception.h"
+#include "sanitizer_platform_interceptors.h"
+
+// Platform-specific options.
+#if SANITIZER_APPLE
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
+#elif SANITIZER_WINDOWS64
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
+#else
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1
+#endif  // SANITIZER_APPLE
+
+#ifndef COMMON_INTERCEPTOR_MEMSET_IMPL
+#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \
+  {                                                       \
+    if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)        \
+      return internal_memset(dst, v, size);               \
+    COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size);  \
+    if (common_flags()->intercept_intrin)                 \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size);     \
+    return REAL(memset)(dst, v, size);                    \
+  }
+#endif
+
+#ifndef COMMON_INTERCEPTOR_MEMMOVE_IMPL
+#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size) \
+  {                                                          \
+    if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)           \
+      return internal_memmove(dst, src, size);               \
+    COMMON_INTERCEPTOR_ENTER(ctx, memmove, dst, src, size);  \
+    if (common_flags()->intercept_intrin) {                  \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size);        \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size);         \
+    }                                                        \
+    return REAL(memmove)(dst, src, size);                    \
+  }
+#endif
+
+#ifndef COMMON_INTERCEPTOR_MEMCPY_IMPL
+#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size) \
+  {                                                         \
+    if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) {        \
+      return internal_memmove(dst, src, size);              \
+    }                                                       \
+    COMMON_INTERCEPTOR_ENTER(ctx, memcpy, dst, src, size);  \
+    if (common_flags()->intercept_intrin) {                 \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size);       \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size);        \
+    }                                                       \
+    return REAL(memcpy)(dst, src, size);                    \
+  }
+#endif
+
+#if SANITIZER_INTERCEPT_MEMSET
+INTERCEPTOR(void *, memset, void *dst, int v, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size);
+}
+
+#define INIT_MEMSET COMMON_INTERCEPT_FUNCTION(memset)
+#else
+#define INIT_MEMSET
+#endif
+
+#if SANITIZER_INTERCEPT_MEMMOVE
+INTERCEPTOR(void *, memmove, void *dst, const void *src, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
+}
+
+#define INIT_MEMMOVE COMMON_INTERCEPT_FUNCTION(memmove)
+#else
+#define INIT_MEMMOVE
+#endif
+
+#if SANITIZER_INTERCEPT_MEMCPY
+INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) {
+  // On OS X, calling internal_memcpy here will cause memory corruptions,
+  // because memcpy and memmove are actually aliases of the same
+  // implementation.  We need to use internal_memmove here.
+  // N.B.: If we switch this to internal_ we'll have to use internal_memmove
+  // due to memcpy being an alias of memmove on OS X.
+  void *ctx;
+#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE
+    COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size);
+#else
+    COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
+#endif
+}
+
+#define INIT_MEMCPY                                  \
+  do {                                               \
+    if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) { \
+      COMMON_INTERCEPT_FUNCTION(memcpy);             \
+    } else {                                         \
+      ASSIGN_REAL(memcpy, memmove);                  \
+    }                                                \
+    CHECK(REAL(memcpy));                             \
+  } while (false)
+
+#else
+#define INIT_MEMCPY
+#endif
+
+#if SANITIZER_INTERCEPT_AEABI_MEM
+INTERCEPTOR(void *, __aeabi_memmove, void *to, const void *from, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memmove4, void *to, const void *from, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memmove8, void *to, const void *from, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memcpy, void *to, const void *from, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memcpy4, void *to, const void *from, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memcpy8, void *to, const void *from, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
+}
+
+// Note the argument order.
+INTERCEPTOR(void *, __aeabi_memset, void *block, uptr size, int c) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memset4, void *block, uptr size, int c) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memset8, void *block, uptr size, int c) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memclr, void *block, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memclr4, void *block, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memclr8, void *block, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
+}
+
+#define INIT_AEABI_MEM                         \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memmove);  \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memmove4); \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memmove8); \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy);   \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy4);  \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy8);  \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memset);   \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memset4);  \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memset8);  \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memclr);   \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memclr4);  \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memclr8);
+#else
+#define INIT_AEABI_MEM
+#endif  // SANITIZER_INTERCEPT_AEABI_MEM
+
+#if SANITIZER_INTERCEPT___BZERO
+INTERCEPTOR(void *, __bzero, void *block, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
+}
+#define INIT___BZERO COMMON_INTERCEPT_FUNCTION(__bzero);
+#else
+#define INIT___BZERO
+#endif  // SANITIZER_INTERCEPT___BZERO
+
+#if SANITIZER_INTERCEPT_BZERO
+INTERCEPTOR(void *, bzero, void *block, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
+}
+#define INIT_BZERO COMMON_INTERCEPT_FUNCTION(bzero);
+#else
+#define INIT_BZERO
+#endif  // SANITIZER_INTERCEPT_BZERO
+
+namespace __sanitizer {
+// This does not need to be called if InitializeCommonInterceptors() is called.
+void InitializeMemintrinsicInterceptors() {
+  INIT_MEMSET;
+  INIT_MEMMOVE;
+  INIT_MEMCPY;
+  INIT_AEABI_MEM;
+  INIT___BZERO;
+  INIT_BZERO;
+}
+}  // namespace __sanitizer

compiler-rt/lib/tsan/rtl/CMakeLists.txt

 append_list_if(COMPILER_RT_HAS_LIBPTHREAD pthread TSAN_DYNAMIC_LINK_LIBS)
 
 set(TSAN_SOURCES
   tsan_debugging.cpp
   tsan_external.cpp
   tsan_fd.cpp
   tsan_flags.cpp
   tsan_ignoreset.cpp
+  tsan_interceptors_memintrinsics.cpp
   tsan_interceptors_posix.cpp
   tsan_interface.cpp
   tsan_interface_ann.cpp
   tsan_interface_atomic.cpp
   tsan_interface_java.cpp
   tsan_malloc_mac.cpp
   tsan_md5.cpp
   tsan_mman.cpp

compiler-rt/lib/tsan/rtl/tsan_interceptors.h

private:
   bool in_ignored_lib_ = false;
   bool in_blocking_func_ = false;
   bool ignoring_ = false;
 
   void DisableIgnoresImpl();
   void EnableIgnoresImpl();
 };
 
+struct TsanInterceptorContext {
+  ThreadState *thr;
+  const uptr pc;
+};
+
 LibIgnore *libignore();
 
 #if !SANITIZER_GO
 inline bool in_symbolizer() {
   return UNLIKELY(cur_thread_init()->in_symbolizer);
 }
 #endif
 
# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...) \
   TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \
   ALIAS(WRAP(pthread_##func2));
 #else
 # define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...)
 # define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...)
 # define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...)
 #endif
 
+#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
+
+#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \
+  (!cur_thread_init()->is_inited)
+
+#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size)                    \
+  MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr,                 \
+                    ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \
+                    true)
+
+#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size)                       \
+  MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr,                  \
+                    ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \
+                    false)
+
+#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
+  SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__);    \
+  TsanInterceptorContext _ctx = {thr, pc};       \
+  ctx = (void *)&_ctx;                           \
+  (void)ctx;
+
 #endif  // TSAN_INTERCEPTORS_H

compiler-rt/lib/tsan/rtl/tsan_interceptors_memintrinsics.cpp

+//===-- tsan_interceptors_posix.cpp ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "tsan_interceptors.h"
+#include "tsan_interface.h"
+
+using namespace __tsan;
+
+#include "sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc"
+
+extern "C" {
+
+void *__tsan_memcpy(void *dst, const void *src, uptr size) {
+  void *ctx;
+#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE
+  COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size);
+#else
+  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
+#endif
+}
+
+void *__tsan_memset(void *dst, int c, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, c, size);
+}
+
+void *__tsan_memmove(void *dst, const void *src, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
+}
+
+}  // extern "C"

compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp

 #include "tsan_rtl.h"
 #include "tsan_mman.h"
 #include "tsan_fd.h"
 
 #include <stdarg.h>
 
 using namespace __tsan;
 
+DECLARE_REAL(void *, memcpy, void *to, const void *from, SIZE_T size)
+DECLARE_REAL(void *, memset, void *block, int c, SIZE_T size)
+
 #if SANITIZER_FREEBSD || SANITIZER_APPLE
 #define stdout __stdoutp
 #define stderr __stderrp
 #endif
 
 #if SANITIZER_NETBSD
 #define dirfd(dirp) (*(int *)(dirp))
 #define fileno_unlocked(fp)              \

 #elif defined(__mips__)
 const int SA_SIGINFO = 8;
 const int SIG_SETMASK = 3;
 #else
 const int SA_SIGINFO = 4;
 const int SIG_SETMASK = 2;
 #endif
 
-#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \
-  (!cur_thread_init()->is_inited)
-
 namespace __tsan {
 struct SignalDesc {
   bool armed;
   __sanitizer_siginfo siginfo;
   ucontext_t ctx;
 };
 
 struct ThreadSignalContext {

 #endif
 
 static int OnExit(ThreadState *thr) {
   int status = Finalize(thr);
   FlushStreams();
   return status;
 }
 
-struct TsanInterceptorContext {
-  ThreadState *thr;
-  const uptr pc;
-};
-
 #if !SANITIZER_APPLE
 static void HandleRecvmsg(ThreadState *thr, uptr pc,
     __sanitizer_msghdr *msg) {
   int fds[64];
   int cnt = ExtractRecvmsgFDs(msg, fds, ARRAY_SIZE(fds));
   for (int i = 0; i < cnt; i++)
     FdEventCreate(thr, pc, fds[i]);
 }
 #endif
 
 #include "sanitizer_common/sanitizer_platform_interceptors.h"
 // Causes interceptor recursion (getaddrinfo() and fopen())
 #undef SANITIZER_INTERCEPT_GETADDRINFO
 // We define our own.
 #if SANITIZER_INTERCEPT_TLS_GET_ADDR
 #define NEED_TLS_GET_ADDR
 #endif
 #undef SANITIZER_INTERCEPT_TLS_GET_ADDR
 #define SANITIZER_INTERCEPT_TLS_GET_OFFSET 1
 #undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK
 
-#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
 #define COMMON_INTERCEPT_FUNCTION_VER(name, ver)                          \
   INTERCEPT_FUNCTION_VER(name, ver)
 #define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \
   (INTERCEPT_FUNCTION_VER(name, ver) || INTERCEPT_FUNCTION(name))
 
-#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size)                    \
-  MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr,                 \
-                    ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \
-                    true)
-
-#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size)                       \
-  MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr,                  \
-                    ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \
-                    false)
-
-#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
-  SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__);    \
-  TsanInterceptorContext _ctx = {thr, pc};       \
-  ctx = (void *)&_ctx;                           \
-  (void)ctx;
-
 #define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \
   SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__);              \
   TsanInterceptorContext _ctx = {thr, pc};                \
   ctx = (void *)&_ctx;                                    \
   (void)ctx;
 
 #define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \
   if (path)                                           \
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_wait(
   for (;;) {
     u32 cur = atomic_load(barrier, memory_order_relaxed);
     if (barrier_epoch(cur) != old_epoch)
       return;
     FutexWait(barrier, cur);
   }
 }
 
-void *__tsan_memcpy(void *dst, const void *src, uptr size) {
-  void *ctx;
-#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE
-  COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size);
-#else
-  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
-#endif
-}
-
-void *__tsan_memset(void *dst, int c, uptr size) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, c, size);
-}
-
-void *__tsan_memmove(void *dst, const void *src, uptr size) {
-  void *ctx;
-  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
-}
-}
+}  // extern "C"

compiler-rt/lib/tsan/rtl/tsan_report.cpp

 
 static bool FrameIsInternal(const SymbolizedStack *frame) {
   if (frame == 0)
     return false;
   const char *file = frame->info.file;
   const char *module = frame->info.module;
   if (file != 0 &&
       (internal_strstr(file, "tsan_interceptors_posix.cpp") ||
+       internal_strstr(file, "tsan_interceptors_memintrinsics.cpp") ||
        internal_strstr(file, "sanitizer_common_interceptors.inc") ||
        internal_strstr(file, "tsan_interface_")))
     return true;
   if (module != 0 && (internal_strstr(module, "libclang_rt.tsan_")))
     return true;
   return false;
 }
 

llvm/utils/gn/secondary/compiler-rt/lib/tsan/rtl/BUILD.gn

sources = [
     "tsan_fd.h",
     "tsan_flags.cpp",
     "tsan_flags.h",
     "tsan_flags.inc",
     "tsan_ignoreset.cpp",
     "tsan_ignoreset.h",
     "tsan_ilist.h",
     "tsan_interceptors.h",
+    "tsan_interceptors_memintrinsics.cpp",
     "tsan_interceptors_posix.cpp",
     "tsan_interface.cpp",
     "tsan_interface.h",
     "tsan_interface.inc",
     "tsan_interface_ann.cpp",
     "tsan_interface_ann.h",
     "tsan_interface_atomic.cpp",
     "tsan_interface_java.cpp",