Index: CMakeLists.txt =================================================================== --- CMakeLists.txt +++ CMakeLists.txt @@ -37,6 +37,8 @@ mark_as_advanced(COMPILER_RT_BUILD_BUILTINS) option(COMPILER_RT_BUILD_SANITIZERS "Build sanitizers" ON) mark_as_advanced(COMPILER_RT_BUILD_SANITIZERS) +option(COMPILER_RT_BUILD_XRAY "Build xray" ON) +mark_as_advanced(COMPILER_RT_BUILD_XRAY) if (COMPILER_RT_STANDALONE_BUILD) if (NOT LLVM_CONFIG_PATH) Index: cmake/config-ix.cmake =================================================================== --- cmake/config-ix.cmake +++ cmake/config-ix.cmake @@ -161,6 +161,7 @@ set(ALL_CFI_SUPPORTED_ARCH ${X86} ${X86_64} ${MIPS64}) set(ALL_ESAN_SUPPORTED_ARCH ${X86_64}) set(ALL_SCUDO_SUPPORTED_ARCH ${X86_64}) +set(ALL_XRAY_SUPPORTED_ARCH ${X86_64}) if(APPLE) include(CompilerRTDarwinUtils) @@ -350,6 +351,9 @@ list_intersect(SCUDO_SUPPORTED_ARCH ALL_SCUDO_SUPPORTED_ARCH SANITIZER_COMMON_SUPPORTED_ARCH) + list_intersect(XRAY_SUPPORTED_ARCH + ALL_XRAY_SUPPORTED_ARCH + SANITIZER_COMMON_SUPPORTED_ARCH) else() # Architectures supported by compiler-rt libraries. filter_available_targets(SANITIZER_COMMON_SUPPORTED_ARCH @@ -373,6 +377,7 @@ filter_available_targets(ESAN_SUPPORTED_ARCH ${ALL_ESAN_SUPPORTED_ARCH}) filter_available_targets(SCUDO_SUPPORTED_ARCH ${ALL_SCUDO_SUPPORTED_ARCH}) + filter_available_targets(XRAY_SUPPORTED_ARCH ${ALL_XRAY_SUPPORTED_ARCH}) endif() if (MSVC) Index: include/CMakeLists.txt =================================================================== --- include/CMakeLists.txt +++ include/CMakeLists.txt @@ -10,11 +10,18 @@ sanitizer/msan_interface.h sanitizer/tsan_interface_atomic.h) +set(XRAY_HEADERS + xray/xray_interface.h) + +set(COMPILER_RT_HEADERS + ${SANITIZER_HEADERS} + ${XRAY_HEADERS}) + set(output_dir ${COMPILER_RT_OUTPUT_DIR}/include) # Copy compiler-rt headers to the build tree. set(out_files) -foreach( f ${SANITIZER_HEADERS} ) +foreach( f ${COMPILER_RT_HEADERS} ) set( src ${CMAKE_CURRENT_SOURCE_DIR}/${f} ) set( dst ${output_dir}/${f} ) add_custom_command(OUTPUT ${dst} @@ -32,3 +39,7 @@ install(FILES ${SANITIZER_HEADERS} PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ DESTINATION ${COMPILER_RT_INSTALL_PATH}/include/sanitizer) +# Install xray headers. +install(FILES ${XRAY_HEADERS} + PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ + DESTINATION ${COMPILER_RT_INSTALL_PATH}/include/xray) Index: include/xray/xray_interface.h =================================================================== --- /dev/null +++ include/xray/xray_interface.h @@ -0,0 +1,66 @@ +//===-- xray_interface.h ----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of XRay, a dynamic runtime instrumentation system. +// +// APIs for controlling XRay functionality explicitly. +//===----------------------------------------------------------------------===// +#ifndef XRAY_XRAY_INTERFACE_H +#define XRAY_XRAY_INTERFACE_H + +#include + +extern "C" { + +enum XRayEntryType { ENTRY = 0, EXIT = 1 }; + +// Provide a function to invoke for when instrumentation points are hit. This is +// a user-visible control surface that overrides the default implementation. The +// function provided should take the following arguments: +// +// - function id: an identifier that indicates the id of a function; this id +// is generated by xray; the mapping between the function id +// and the actual function pointer is available through +// __xray_table. +// - entry type: identifies what kind of instrumentation point was encountered +// (function entry, function exit, etc.). See the enum +// XRayEntryType for more details. +// +// Returns 1 on success, 0 on error. +extern int __xray_set_handler(void (*entry)(int32_t, XRayEntryType)); + +// This removes whatever the currently provided handler is. Returns 1 on +// success, 0 on error. +extern int __xray_remove_handler(); + +enum XRayPatchingStatus { + NOT_INITIALIZED = 0, + NOTIFIED = 1, + ONGOING = 2, + FAILED = 3 +}; + +// This tells XRay to patch the instrumentation points. This is an asynchronous +// process, and returns the following status in specific cases: +// +// - 0 : XRay is not initialized. +// - 1 : We've done the notification. +// - 2 : Patching / un-patching is on-going. +extern XRayPatchingStatus __xray_patch(); + +// Reverses the effect of __xray_patch(). This is an asynchronous process, and +// returns the following status in specific cases. +// +// - 0 : XRay is not initialized. +// - 1 : We've done the notification. +// - 2 : Patching / un-patching is on-going. +extern int __xray_unpatch(); +} + +#endif Index: lib/CMakeLists.txt =================================================================== --- lib/CMakeLists.txt +++ lib/CMakeLists.txt @@ -57,3 +57,7 @@ add_subdirectory(scudo) endif() endif() + +if(COMPILER_RT_BUILD_XRAY) + add_subdirectory(xray) +endif() Index: lib/xray/CMakeLists.txt =================================================================== --- /dev/null +++ lib/xray/CMakeLists.txt @@ -0,0 +1,36 @@ +# Build for the XRay runtime support library. + +set(XRAY_SOURCES + xray_init.cc + xray_interface.cc + xray_flags.cc +) + +include_directories(..) +include_directories(../../include) + +set(XRAY_CFLAGS ${SANITIZER_COMMON_CFLAGS}) + +set(XRAY_COMMON_DEFINITIONS XRAY_HAS_EXCEPTIONS=1) + +add_compiler_rt_object_libraries(RTXray + ARCHS ${XRAY_SUPPORTED_ARCH} + SOURCES ${XRAY_SOURCES} CFLAGS ${XRAY_CFLAGS} + DEFS ${XRAY_COMMON_DEFINITIONS}) + +add_custom_target(xray) +set(XRAY_COMMON_RUNTIME_OBJECT_LIBS + RTSanitizerCommon + RTSanitizerCommonLibc) +add_compiler_rt_runtime(clang_rt.xray + STATIC + ARCHS ${XRAY_SUPPORTED_ARCH} + OBJECT_LIBS RTXray ${XRAY_COMMON_RUNTIME_OBJECT_LIBS} + CFLAGS ${XRAY_CFLAGS} + DEFS ${XRAY_COMMON_DEFINITIONS} + PARENT_TARGET xray) +add_dependencies(compiler-rt xray) + +# if(COMPILER_RT_INCLUDE_TESTS) +# add_subdirectory(tests) +# endif() Index: lib/xray/xray_flags.h =================================================================== --- /dev/null +++ lib/xray/xray_flags.h @@ -0,0 +1,37 @@ +//===-- xray_flags.h -------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of XRay, a dynamic runtime instruementation system. +// +// XRay runtime flags. +//===----------------------------------------------------------------------===// + +#ifndef XRAY_FLAGS_H +#define XRAY_FLAGS_H + +#include "sanitizer_common/sanitizer_flag_parser.h" + +namespace __xray { + +struct Flags { +#define XRAY_FLAG(Type, Name, DefaultValue, Description) Type Name; +#include "xray_flags.inc" +#undef XRAY_FLAG + + void SetDefaults(); +}; + +extern Flags xray_flags_dont_use_directly; +inline Flags *flags() { return &xray_flags_dont_use_directly; } + +void InitializeFlags(); + +} // namespace __xray + +#endif // XRAY_FLAGS_H Index: lib/xray/xray_flags.cc =================================================================== --- /dev/null +++ lib/xray/xray_flags.cc @@ -0,0 +1,47 @@ +//===-- xray_flags.cc -------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of XRay, a dynamic runtime instrumentation system. +// +// XRay flag parsing logic. +//===----------------------------------------------------------------------===// + +#include "xray_flags.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_flag_parser.h" + +namespace __xray { + +Flags xray_flags_dont_use_directly; // use via flags(). + +void Flags::SetDefaults() { +#define XRAY_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; +#include "xray_flags.inc" +#undef XRAY_FLAG +} + +static void RegisterXRayFlags(FlagParser *P, Flags *F) { +#define XRAY_FLAG(Type, Name, DefaultValue, Description) \ + RegisterFlag(P, #Name, Description, &F->Name); +#include "xray_flags.inc" +#undef XRAY_FLAG +} + +void InitializeFlags() { + auto *F = flags(); + F->SetDefaults(); + + FlagParser XRayParser; + RegisterXRayFlags(&XRayParser, F); + + // Override from command line. + XRayParser.ParseString(GetEnv("XRAY_OPTIONS")); +} + +} // namespace __xray Index: lib/xray/xray_flags.inc =================================================================== --- /dev/null +++ lib/xray/xray_flags.inc @@ -0,0 +1,18 @@ +//===-- xray_flags.inc ------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// XRay runtime flags. +// +//===----------------------------------------------------------------------===// +#ifndef XRAY_FLAG +#error "Define XRAY_FLAG prior to including this file!" +#endif + +XRAY_FLAG(bool, patch_premain, true, + "Whether to patch instrumentation points before main.") Index: lib/xray/xray_init.cc =================================================================== --- /dev/null +++ lib/xray/xray_init.cc @@ -0,0 +1,65 @@ +//===-- xray_init.cc --------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of XRay, a dynamic runtime instrumentation system. +// +// XRay initialisation logic. +//===----------------------------------------------------------------------===// + +#include +#include +#include +#include + +#include "sanitizer_common/sanitizer_common.h" +#include "xray_flags.h" +#include "xray_interface_internal.h" +#include "llvm/Support/ELF.h" + +extern "C" { +extern void __xray_init(); +extern const XRaySledEntry __start_xray_instr_map[] __attribute__((weak)); +extern const XRaySledEntry __stop_xray_instr_map[] __attribute__((weak)); +} + +// We initialize some global variables that pertain to specific sections of XRay +// data structures in the binary. We do this for the current process using +// /proc/curproc/map and make sure that we're able to get it. We signal failure +// via a global atomic boolean to indicate whether we've initialized properly. +// +std::atomic XRayInitialized{false}; + +// This should always be updated before XRayInitialized is updated. +std::atomic<__xray::XRaySledMap> XRayInstrMap{}; + +// __xray_init() will do the actual loading of the current process' memory map +// and then proceed to look for the .xray_instr_map section/segment. +void __xray_init() { + if (__start_xray_instr_map == nullptr) { + Report("XRay instrumentation map missing. Not initializing XRay.\n"); + return; + } + + // Now initialize the XRayInstrMap global struct with the address of the + // entries, reinterpreted as an array of XRaySledEntry objects. We use the + // virtual pointer we have from the section to provide us the correct + // information. + __xray::XRaySledMap SledMap{}; + SledMap.Sleds = __start_xray_instr_map; + SledMap.Entries = __stop_xray_instr_map - __start_xray_instr_map; + XRayInstrMap.store(SledMap, std::memory_order_release); + XRayInitialized.store(true, std::memory_order_release); + + // FIXME: Check the flag/environment before patching. + if (__xray::flags()->patch_premain) + __xray_patch(); +} + +__attribute__((section(".preinit_array"), + used)) void (*__local_xray_preinit)(void) = __xray_init; Index: lib/xray/xray_interface.cc =================================================================== --- /dev/null +++ lib/xray/xray_interface.cc @@ -0,0 +1,226 @@ +//===-- xray_interface.cpp --------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of XRay, a dynamic runtime instrumentation system. +// +// Implementation of the API functions. +// +//===----------------------------------------------------------------------===// + +#include "xray_interface_internal.h" +#include +#include +#include +#include +#include +#include + +namespace __xray { + +// This is the function to call when we encounter the entry or exit sleds. +std::atomic XRayPatchedFunction{nullptr}; +} + +extern std::atomic XRayInitialized; +extern std::atomic<__xray::XRaySledMap> XRayInstrMap; + +void __xray_FunctionEntry() { + // First thing we do is save the caller provided registers before doing any + // actual work. + uint64_t Rrdi, Rrax, Rrdx, Rrsi, Rrcx, Rr8, Rr9, Rrbp; + __asm__ __volatile__("mov %%rbp, %0" : "=m"(Rrbp)); + __asm__ __volatile__("mov %%rdi, %0" : "=m"(Rrdi)); + __asm__ __volatile__("mov %%rax, %0" : "=m"(Rrax)); + __asm__ __volatile__("mov %%rdx, %0" : "=m"(Rrdx)); + __asm__ __volatile__("mov %%rsi, %0" : "=m"(Rrsi)); + __asm__ __volatile__("mov %%rcx, %0" : "=m"(Rrcx)); + __asm__ __volatile__("mov %%r8, %0" : "=m"(Rr8)); + __asm__ __volatile__("mov %%r9, %0" : "=m"(Rr9)); + + // FIXME: Handle async signal safety, and prevent recursive calls. + auto Fn = __xray::XRayPatchedFunction.load(std::memory_order_acquire); + if (Fn != nullptr) { + int32_t FunctionID; + __asm__("mov %%r10d, %0" : "=g"(FunctionID) : : "%r10"); + (*Fn)(FunctionID, XRayEntryType::ENTRY); + } + + // Then restore the registers before returning. + __asm__ __volatile__("mov %0,%%r9" : : "m"(Rr9) : "%r9"); + __asm__ __volatile__("mov %0,%%r8" : : "m"(Rr8) : "%r8"); + __asm__ __volatile__("mov %0,%%rcx" : : "m"(Rrcx) : "%rcx"); + __asm__ __volatile__("mov %0,%%rsi" : : "m"(Rrsi) : "%rsi"); + __asm__ __volatile__("mov %0,%%rdx" : : "m"(Rrdx) : "%rdx"); + __asm__ __volatile__("mov %0,%%rax" : : "m"(Rrax) : "%rax"); + __asm__ __volatile__("mov %0,%%rdi" : : "m"(Rrdi) : "%rdi"); + __asm__ __volatile__("mov %0,%%rbp" : : "m"(Rrbp) : "%rbp"); +} + +void __xray_FunctionExit() { + // First thing we do is save the caller provided registers before doing any + // actual work. + uint64_t Rrax, Rrdx, Rrbp; + __asm__ __volatile__("mov %%rax, %0" : "=m"(Rrax)); + __asm__ __volatile__("mov %%rdx, %0" : "=m"(Rrdx)); + __asm__ __volatile__("mov %%rbp, %0" : "=m"(Rrbp)); + + // Then it's safe to call the provided function. + // FIXME: Handle async signal safety, and prevent recursive calls. + auto Fn = __xray::XRayPatchedFunction.load(std::memory_order_acquire); + if (Fn != nullptr) { + int32_t FunctionID; + __asm__("mov %%r10d, %0" : "=g"(FunctionID) : : "r10"); + (*Fn)(FunctionID, XRayEntryType::EXIT); + } + + // Then restore the registers before returning. + __asm__ __volatile__("mov %0,%%rbp" : : "m"(Rrbp) : "%rbp"); + __asm__ __volatile__("mov %0,%%rdx" : : "m"(Rrdx) : "%rdx"); + __asm__ __volatile__("mov %0,%%rax" : : "m"(Rrax) : "%rax"); +} + +int __xray_set_handler(void (*entry)(int32_t, XRayEntryType)) { + if (XRayInitialized.load(std::memory_order_acquire)) { + __xray::XRayPatchedFunction.store(entry, std::memory_order_release); + return 1; + } + return 0; +} + +std::atomic XRayPatching{false}; + +XRayPatchingStatus __xray_patch() { + // FIXME: Make this happen asynchronously. For now just do this sequentially. + if (!XRayInitialized.load(std::memory_order_acquire)) + return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized. + + static bool NotPatching = false; + if (!XRayPatching.compare_exchange_strong(NotPatching, true, + std::memory_order_acq_rel, + std::memory_order_acquire)) { + return XRayPatchingStatus::ONGOING; // Already patching. + } + + // Step 1: Compute the function id, as a unique identifier per function in the + // instrumentation map. + __xray::XRaySledMap InstrMap = XRayInstrMap.load(std::memory_order_acquire); + if (InstrMap.Entries == 0) + return XRayPatchingStatus::NOT_INITIALIZED; + + int32_t FuncId = 1; + static constexpr uint8_t CallOpCode = 0xe8; + static constexpr uint16_t MovR10Seq = 0xba41; + static constexpr uint8_t JmpOpCode = 0xe9; + uint64_t CurFun = 0; + for (std::size_t I = 0; I < InstrMap.Entries; I++) { + auto Sled = InstrMap.Sleds[I]; + auto F = Sled.Function; + if (CurFun == 0) + CurFun = F; + if (F != CurFun) { + ++FuncId; + CurFun = F; + } + + // While we're here, we should patch the nop sled. To do that we mprotect + // the page containing the function to be writeable. + void *PageAlignedAddr = + reinterpret_cast(Sled.Address & ~((2 << 16) - 1)); + std::size_t MProtectLen = + (Sled.Address + 12) - reinterpret_cast(PageAlignedAddr); + if (mprotect(PageAlignedAddr, MProtectLen, + PROT_READ | PROT_WRITE | PROT_EXEC) == -1) { + printf("Failed mprotect: %d\n", errno); + return XRayPatchingStatus::FAILED; + } + + static constexpr int64_t MinOffset{std::numeric_limits::min()}; + static constexpr int64_t MaxOffset{std::numeric_limits::max()}; + if (Sled.Kind == XRayEntryType::ENTRY) { + // Here we do the dance of replacing the following sled: + // + // xray_sled_n: + // jmp +9 + // <9 byte nop> + // + // With the following: + // + // mov r10d, + // call + // + // We need to do this in the following order: + // + // 1. Put the function id first, 2 bytes from the start of the sled (just + // after the 2-byte jmp instruction). + // 2. Put the call opcode 6 bytes from the start of the sled. + // 3. Put the relative offset 7 bytes from the start of the sled. + // 4. Do an atomic write over the jmp instruction for the "mov r10d" + // opcode and first operand. + // + // Prerequisite is to compute the relative offset to the + // __xray_FunctionEntry function's address. + int64_t TrampolineOffset = + reinterpret_cast(__xray_FunctionEntry) - + (static_cast(Sled.Address) + 11); + if (TrampolineOffset < MinOffset || TrampolineOffset > MaxOffset) { + // FIXME: Print out an error here. + continue; + } + *reinterpret_cast(Sled.Address + 2) = FuncId; + *reinterpret_cast(Sled.Address + 6) = CallOpCode; + *reinterpret_cast(Sled.Address + 7) = TrampolineOffset; + std::atomic_store_explicit( + reinterpret_cast *>(Sled.Address), MovR10Seq, + std::memory_order_release); + } + + if (Sled.Kind == XRayEntryType::EXIT) { + // Here we do the dance of replacing the following sled: + // + // xray_sled_n: + // ret + // <10 byte nop> + // + // With the following: + // + // mov r10d, + // jmp + // + // 1. Put the function id first, 2 bytes from the start of the sled (just + // after the 1-byte ret instruction). + // 2. Put the jmp opcode 6 bytes from the start of the sled. + // 3. Put the relative offset 7 bytes from the start of the sled. + // 4. Do an atomic write over the jmp instruction for the "mov r10d" + // opcode and first operand. + // + // Prerequisite is to compute the relative offset fo the + // __xray_FunctionExit function's address. + int64_t TrampolineOffset = + reinterpret_cast(__xray_FunctionExit) - + (static_cast(Sled.Address) + 11); + if (TrampolineOffset < MinOffset || TrampolineOffset > MaxOffset) { + // FIXME: Print out an error here. + continue; + } + *reinterpret_cast(Sled.Address + 2) = FuncId; + *reinterpret_cast(Sled.Address + 6) = JmpOpCode; + *reinterpret_cast(Sled.Address + 7) = TrampolineOffset; + std::atomic_store_explicit( + reinterpret_cast *>(Sled.Address), MovR10Seq, + std::memory_order_release); + } + + if (mprotect(PageAlignedAddr, MProtectLen, PROT_READ | PROT_EXEC) == -1) { + printf("Failed mprotect: %d\n", errno); + return XRayPatchingStatus::FAILED; + } + } + XRayPatching.store(false, std::memory_order_release); + return XRayPatchingStatus::NOTIFIED; +} Index: lib/xray/xray_interface_internal.h =================================================================== --- /dev/null +++ lib/xray/xray_interface_internal.h @@ -0,0 +1,40 @@ +//===-- xray_interface_internal.h -------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of XRay, a dynamic runtime instrumentation system. +// +// Implementation of the API functions. See also include/xray/xray_interface.h. +// +//===----------------------------------------------------------------------===// +#ifndef XRAY_INTERFACE_INTERNAL_H +#define XRAY_INTERFACE_INTERNAL_H + +#include "xray/xray_interface.h" +#include + +extern "C" { + +struct XRaySledEntry { + uint64_t Address; + uint64_t Function; + unsigned char Kind; + unsigned char AlwaysInstrument; + unsigned char Padding[14]; // Need 32 bytes +}; +} + +namespace __xray { + +struct XRaySledMap { + const XRaySledEntry *Sleds; + std::size_t Entries; +}; +} + +#endif