Index: lib/tsan/rtl/tsan_platform.h =================================================================== --- lib/tsan/rtl/tsan_platform.h +++ lib/tsan/rtl/tsan_platform.h @@ -96,9 +96,13 @@ // runtime for AArch64 uses an external memory read (vmaSize) to select which // mapping to use. Although slower, it make a same instrumented binary run on // multiple kernels. +// According to https://lkml.org/lkml/2015/12/21/340, the mmap random gap +// between heap and stack is determined at runtime, and can be too huge +// to create shadow memory for it. So we need to find heap space at runtime. /* C/C++ on linux/aarch64 (39-bit VMA) + 0000 0010 00 - 0100 0000 00: main binary 0100 0000 00 - 0800 0000 00: - 0800 0000 00 - 2000 0000 00: shadow memory @@ -108,9 +112,10 @@ 5500 0000 00 - 5600 0000 00: main binary (PIE) 5600 0000 00 - 6000 0000 00: - 6000 0000 00 - 6200 0000 00: traces -6200 0000 00 - 7d00 0000 00: - -7c00 0000 00 - 7d00 0000 00: heap -7d00 0000 00 - 7fff ffff ff: modules and main thread stack +6200 0000 00 - xx00 0000 00: - +xx00 0000 00 - yy00 0000 00: heap +yy00 0000 00 - 7f00 0000 00: - memory random gap +7f00 0000 00 - 7fff ffff ff: main thread stack */ struct Mapping39 { static const uptr kLoAppMemBeg = 0x0000001000ull; @@ -124,9 +129,12 @@ static const uptr kMidShadowOff = 0x5000000000ull; static const uptr kTraceMemBeg = 0x6000000000ull; static const uptr kTraceMemEnd = 0x6200000000ull; - static const uptr kHeapMemBeg = 0x7c00000000ull; - static const uptr kHeapMemEnd = 0x7d00000000ull; - static const uptr kHiAppMemBeg = 0x7e00000000ull; + static uptr kHeapMemBeg; + static uptr kHeapMemEnd; + static const uptr kHeapMemSize = 0x0100000000ull; + static const uptr kHeapShadowBeg = 0x1800000000ull; + static const uptr kHeapMetaBeg = 0x3200000000ull; + static const uptr kHiAppMemBeg = 0x7f00000000ull; static const uptr kHiAppMemEnd = 0x7fffffffffull; static const uptr kAppMemMsk = 0x7800000000ull; static const uptr kAppMemXor = 0x0200000000ull; @@ -144,8 +152,9 @@ 2aa00 0000 00 - 2ab00 0000 00: main binary (PIE) 2ab00 0000 00 - 36200 0000 00: - 36200 0000 00 - 36240 0000 00: traces -36240 0000 00 - 3e000 0000 00: - -3e000 0000 00 - 3f000 0000 00: heap +36240 0000 00 - xxxx0 0000 00: - +xxxx0 0000 00 - yyyy0 0000 00: heap +yyyy0 0000 00 - 3f000 0000 00: - memory random gap 3f000 0000 00 - 3ffff ffff ff: modules and main thread stack */ struct Mapping42 { @@ -160,13 +169,16 @@ static const uptr kMidShadowOff = 0x28000000000ull; static const uptr kTraceMemBeg = 0x36200000000ull; static const uptr kTraceMemEnd = 0x36400000000ull; - static const uptr kHeapMemBeg = 0x3e000000000ull; - static const uptr kHeapMemEnd = 0x3f000000000ull; + static uptr kHeapMemBeg; + static uptr kHeapMemEnd; + static const uptr kHeapMemSize = 0x01000000000ull; + static const uptr kHeapShadowBeg = 0x14000000000ull; + static const uptr kHeapMetaBeg = 0x26800000000ull; static const uptr kHiAppMemBeg = 0x3f000000000ull; static const uptr kHiAppMemEnd = 0x3ffffffffffull; static const uptr kAppMemMsk = 0x3c000000000ull; static const uptr kAppMemXor = 0x04000000000ull; - static const uptr kVdsoBeg = 0x37f00000000ull; + static const uptr kVdsoBeg = 0x3f000000000ull; }; // Indicates the runtime will define the memory regions at runtime. @@ -591,13 +603,25 @@ #endif } +template +uptr MemToShadowImplWithDynamicHeap(uptr x) { + DCHECK(IsAppMem(x)); +#if !defined(SANITIZER_GO) + if (x >= Mapping::kHeapMemBeg && + x < Mapping::kHeapMemBeg + Mapping::kHeapMemSize) + return ((x - Mapping::kHeapMemBeg) & ~(kShadowCell - 1)) * kShadowCnt + + Mapping::kHeapShadowBeg; +#endif + return MemToShadowImpl(x); +} + ALWAYS_INLINE uptr MemToShadow(uptr x) { #ifdef __aarch64__ if (vmaSize == 39) - return MemToShadowImpl(x); + return MemToShadowImplWithDynamicHeap(x); else - return MemToShadowImpl(x); + return MemToShadowImplWithDynamicHeap(x); DCHECK(0); #elif defined(__powerpc64__) if (vmaSize == 44) @@ -624,13 +648,25 @@ #endif } +template +u32 *MemToMetaImplWithDynamicHeap(uptr x) { + DCHECK(IsAppMem(x)); +#if !defined(SANITIZER_GO) + if (x >= Mapping::kHeapMemBeg && + x < Mapping::kHeapMemBeg + Mapping::kHeapMemSize) + return (u32*)(((x - Mapping::kHeapMemBeg) & ~(kMetaShadowCell - 1)) + / kMetaShadowCell * kMetaShadowSize + Mapping::kHeapMetaBeg); +#endif + return MemToMetaImpl(x); +} + ALWAYS_INLINE u32 *MemToMeta(uptr x) { #ifdef __aarch64__ if (vmaSize == 39) - return MemToMetaImpl(x); + return MemToMetaImplWithDynamicHeap(x); else - return MemToMetaImpl(x); + return MemToMetaImplWithDynamicHeap(x); DCHECK(0); #elif defined(__powerpc64__) if (vmaSize == 44) @@ -668,13 +704,24 @@ #endif } +template +uptr ShadowToMemImplWithDynamicHeap(uptr s) { + DCHECK(IsShadowMem(s)); +#if !defined(SANITIZER_GO) + if (s >= MemToShadow(Mapping::kHeapMemBeg) + && s <= MemToShadow(Mapping::kHeapMemEnd - 1)) + return (s - Mapping::kHeapShadowBeg) / kShadowCnt + Mapping::kHeapMemBeg; +#endif + return ShadowToMemImpl(s); +} + ALWAYS_INLINE uptr ShadowToMem(uptr s) { #ifdef __aarch64__ if (vmaSize == 39) - return ShadowToMemImpl(s); + return ShadowToMemImplWithDynamicHeap(s); else - return ShadowToMemImpl(s); + return ShadowToMemImplWithDynamicHeap(s); DCHECK(0); #elif defined(__powerpc64__) if (vmaSize == 44) Index: lib/tsan/rtl/tsan_platform_linux.cc =================================================================== --- lib/tsan/rtl/tsan_platform_linux.cc +++ lib/tsan/rtl/tsan_platform_linux.cc @@ -241,6 +241,13 @@ CHECK_LT((uptr)&g_data_start, g_data_end); } +#ifdef __aarch64__ +uptr Mapping39::kHeapMemBeg = 0; +uptr Mapping39::kHeapMemEnd = 0; +uptr Mapping42::kHeapMemBeg = 0; +uptr Mapping42::kHeapMemEnd = 0; +#endif + #endif // #ifndef SANITIZER_GO void InitializePlatformEarly() { @@ -253,6 +260,31 @@ Printf("FATAL: Found %d - Supported 39 and 42\n", vmaSize); Die(); } + MemoryMappingLayout proc_maps(true); + uptr heap_end = 0; + uptr start, end, prot; + while (proc_maps.Next(&start, &end, 0, 0, 0, &prot)) { + // Omit stack + if (start <= (uptr)&heap_end && (uptr)&heap_end < end) + continue; + if (end > heap_end) + heap_end = end; + } + if (vmaSize == 39) { + if (heap_end > Mapping39::kHiAppMemBeg) + heap_end = Mapping39::kHiAppMemBeg; + Mapping39::kHeapMemEnd = heap_end; + Mapping39::kHeapMemBeg = heap_end - Mapping39::kHeapMemSize; + DPrintf("Mapping39::kHeapMem = 0x%llx - 0x%llx\n", Mapping39::kHeapMemBeg, + Mapping39::kHeapMemEnd); + } else { + if (heap_end > Mapping42::kHiAppMemBeg) + heap_end = Mapping42::kHiAppMemBeg; + Mapping42::kHeapMemEnd = heap_end; + Mapping42::kHeapMemBeg = heap_end - Mapping42::kHeapMemSize; + DPrintf("Mapping42::kHeapMem = 0x%llx - 0x%llx\n", Mapping42::kHeapMemBeg, + Mapping42::kHeapMemEnd); + } #elif defined(__powerpc64__) if (vmaSize != 44 && vmaSize != 46) { Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");