Index: lib/tsan/rtl/tsan_platform.h =================================================================== --- lib/tsan/rtl/tsan_platform.h +++ lib/tsan/rtl/tsan_platform.h @@ -45,6 +45,22 @@ 0560 0000 0000 - 0760 0000 0000: traces 0760 0000 0000 - 07d0 0000 0000: metainfo (memory blocks and sync objects) 07d0 0000 0000 - 07ff ffff ffff: - + +C++ FreeBSD memory layout: +0000 0000 0000 - 0000 0100 0000: - +0000 0100 0000 - 0010 0000 0000: executable, modules and service heap +0010 0000 0000 - 1000 0000 0000: - +1000 0000 0000 - 1000 0400 0000: protected +1000 0400 0000 - 1040 0000 0000: low mem shadow +1040 0000 0000 - 2400 0000 0000: protected +2400 0000 0000 - 3000 0000 0000: high mem shadow +3000 0000 0000 - 4000 0000 0000: low metainfo (memory blocks and sync objects) +4000 0000 0000 - 5000 0000 0000: high metainfo (memory blocks and sync objects) +5000 0000 0000 - 6000 0000 0000: - +6000 0000 0000 - 6000 0010 0000: trace +6000 0010 0000 - 7d00 0000 0000: - +7d00 0000 0000 - 7e00 0000 0000: heap +7e00 0000 0000 - 7fff ffff ffff: stacks & system */ #ifndef TSAN_PLATFORM_H @@ -69,13 +85,31 @@ static const uptr kMetaSize = 0x100000000000ULL; # endif // if SANITIZER_WINDOWS #else // defined(TSAN_GO) +# if SANITIZER_FREEBSD +static const uptr kLowMetaShadow = 0x300000000000ULL; +static const uptr kLowMetaSize = 0x100000000000ULL; +static const uptr kHighMetaShadow = 0x400000000000ULL; +static const uptr kHighMetaSize = 0x100000000000ULL; +static const uptr kLinuxLowAppMemBeg = 0x000001000000ULL; +static const uptr kLinuxLowAppMemEnd = 0x000fffffffffULL; +static const uptr kLinuxHighAppMemBeg = 0x7d0000000000ULL; +static const uptr kLinuxHighAppMemEnd = 0x7fffffffffffULL; +static const uptr kLinuxLowShadowMsk = 0x100000000000ULL; +static const uptr kLinuxHighShadowMsk = 0x240000000000ULL; +# else static const uptr kMetaShadow = 0x300000000000ULL; static const uptr kMetaSize = 0x100000000000ULL; static const uptr kLinuxAppMemBeg = 0x7cf000000000ULL; static const uptr kLinuxAppMemEnd = 0x7fffffffffffULL; +# endif #endif +#if !defined(TSAN_GO) && SANITIZER_FREEBSD +static const uptr kLinuxLowAppMemMsk = 0x000000000000ULL; +static const uptr kLinuxHighAppMemMsk = 0x7c0000000000ULL; +#else static const uptr kLinuxAppMemMsk = 0x7c0000000000ULL; +#endif #if SANITIZER_WINDOWS const uptr kTraceMemBegin = 0x056000000000ULL; @@ -86,19 +120,86 @@ // This has to be a macro to allow constant initialization of constants below. #ifndef TSAN_GO -#define MemToShadow(addr) \ - ((((uptr)addr) & ~(kLinuxAppMemMsk | (kShadowCell - 1))) * kShadowCnt) -#define MemToMeta(addr) \ - (u32*)(((((uptr)addr) & ~(kLinuxAppMemMsk | (kMetaShadowCell - 1))) \ - / kMetaShadowCell * kMetaShadowSize) | kMetaShadow) -#else -#define MemToShadow(addr) \ +# if SANITIZER_FREEBSD +# define MemToLowShadow(addr) \ + (((((uptr)(addr)) & ~(kShadowCell - 1)) * kShadowCnt) \ + | kLinuxLowShadowMsk) +# define MemToHighShadow(addr) \ + (((((uptr)(addr)) & ~(kLinuxHighAppMemMsk | (kShadowCell - 1))) \ + * kShadowCnt) | kLinuxHighShadowMsk) + +# define MemToLowMeta(addr) \ + (u32*)(((((uptr)(addr)) & ~(kLinuxLowAppMemMsk | (kMetaShadowCell - 1))) \ + / kMetaShadowCell * kMetaShadowSize) | kLowMetaShadow) +# define MemToHighMeta(addr) \ + (u32*)(((((uptr)(addr)) & ~(kLinuxHighAppMemMsk | (kMetaShadowCell - 1))) \ + / kMetaShadowCell * kMetaShadowSize) | kHighMetaShadow) + +# define IsLowAppMem(mem) \ + ((mem) >= kLinuxLowAppMemBeg && (mem) <= kLinuxLowAppMemEnd) +# define IsHighAppMem(mem) \ + ((mem) >= kLinuxHighAppMemBeg && (mem) <= kLinuxHighAppMemEnd) + +# define MemToShadow(mem) \ + (IsLowAppMem((mem)) ? MemToLowShadow((mem)) : MemToHighShadow((mem))) +# define MemToMeta(mem) \ + (IsLowAppMem((mem)) ? MemToLowMeta((mem)) : MemToHighMeta((mem))) +# else // SANITIZER_FREEBSD +# define MemToShadow(addr) \ + ((((uptr)(addr)) & ~(kLinuxAppMemMsk | (kShadowCell - 1))) * kShadowCnt) +# define MemToMeta(addr) \ + (u32*)(((((uptr)(addr)) & ~(kLinuxAppMemMsk | (kMetaShadowCell - 1))) \ + / kMetaShadowCell * kMetaShadowSize) | kMetaShadow) +# endif // !SANITIZER_FREEBSD +#else // TSAN_GO +# define MemToShadow(addr) \ (((((uptr)addr) & ~(kShadowCell - 1)) * kShadowCnt) | kLinuxShadowMsk) -#define MemToMeta(addr) \ +# define MemToMeta(addr) \ (u32*)(((((uptr)addr) & ~(kMetaShadowCell - 1)) \ - / kMetaShadowCell * kMetaShadowSize) | kMetaShadow) -#endif + / kMetaShadowCell * kMetaShadowSize) | kMetaShadow) +#endif // !TSAN_GO + +#if !defined(TSAN_GO) && SANITIZER_FREEBSD +static const uptr kLinuxLowShadowBeg = MemToShadow(kLinuxLowAppMemBeg); +static const uptr kLinuxLowShadowEnd = + MemToShadow(kLinuxLowAppMemEnd) | 0xff; +static const uptr kLinuxHighShadowBeg = MemToShadow(kLinuxHighAppMemBeg); +static const uptr kLinuxHighShadowEnd = + MemToShadow(kLinuxHighAppMemEnd) | 0xff; + +static inline bool IsAppMem(uptr mem) { + return IsLowAppMem(mem) || IsHighAppMem(mem); +} + +static inline bool IsLowShadowMem(uptr mem) { + return mem >= kLinuxLowShadowBeg && mem <= kLinuxLowShadowEnd; +} + +static inline bool IsHighShadowMem(uptr mem) { + return mem >= kLinuxHighShadowBeg && mem <= kLinuxHighShadowEnd; +} + +static inline bool IsShadowMem(uptr mem) { + return IsLowShadowMem(mem) || IsHighShadowMem(mem); +} +static inline uptr LowShadowToMem(uptr shadow) { + CHECK(IsLowShadowMem(shadow)); + return ((shadow & ~kLinuxLowShadowMsk) / kShadowCnt) \ + | kLinuxLowAppMemMsk; +} + +static inline uptr HighShadowToMem(uptr shadow) { + CHECK(IsHighShadowMem(shadow)); + return (shadow / kShadowCnt) | kLinuxHighAppMemMsk; +} + +static inline uptr ShadowToMem(uptr shadow) { + CHECK(IsShadowMem(shadow)); + return IsLowShadowMem(shadow) ? + LowShadowToMem(shadow) : HighShadowToMem(shadow); +} +#else // !TSAN_GO && SANITIZER_FREEBSD static const uptr kLinuxShadowBeg = MemToShadow(kLinuxAppMemBeg); static const uptr kLinuxShadowEnd = MemToShadow(kLinuxAppMemEnd) | 0xff; @@ -123,6 +224,15 @@ return (shadow / kShadowCnt) | kLinuxAppMemMsk; #endif } +#endif // TSAN_GO || !SANITIZER_FREEBSD + +#ifndef TSAN_GO +# if SANITIZER_FREEBSD +const uptr kModulesBeg = 0x000001000000; +# else +const uptr kModulesBeg = kLinuxShadowBeg; +# endif +#endif // !TSAN_GO void FlushShadowMemory(); void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive); Index: lib/tsan/rtl/tsan_platform_linux.cc =================================================================== --- lib/tsan/rtl/tsan_platform_linux.cc +++ lib/tsan/rtl/tsan_platform_linux.cc @@ -9,7 +9,7 @@ // // This file is a part of ThreadSanitizer (TSan), a race detector. // -// Linux-specific code. +// Linux- and FreeBSD-specific code. //===----------------------------------------------------------------------===// @@ -221,70 +221,95 @@ internal_close(fd); } -void InitializeShadowMemory() { +static void MmapShadow(uptr shadowBeg, uptr shadowEnd, + uptr appMemBeg, uptr appMemEnd, + uptr metaShadow, uptr metaSize) { + CHECK_LT(shadowBeg, shadowEnd); + CHECK_LT(appMemBeg, appMemEnd); + // Map memory shadow. - uptr shadow = (uptr)MmapFixedNoReserve(kLinuxShadowBeg, - kLinuxShadowEnd - kLinuxShadowBeg); - if (shadow != kLinuxShadowBeg) { + uptr shadow = (uptr)MmapFixedNoReserve(shadowBeg, shadowEnd - shadowBeg); + if (shadow != shadowBeg) { Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n"); Printf("FATAL: Make sure to compile with -fPIE and " - "to link with -pie (%p, %p).\n", shadow, kLinuxShadowBeg); + "to link with -pie (%p, %p).\n", shadow, shadowBeg); Die(); } - // This memory range is used for thread stacks and large user mmaps. - // Frequently a thread uses only a small part of stack and similarly - // a program uses a small part of large mmap. On some programs - // we see 20% memory usage reduction without huge pages for this range. -#ifdef MADV_NOHUGEPAGE - madvise((void*)MemToShadow(0x7f0000000000ULL), - 0x10000000000ULL * kShadowMultiplier, MADV_NOHUGEPAGE); -#endif DPrintf("memory shadow: %zx-%zx (%zuGB)\n", - kLinuxShadowBeg, kLinuxShadowEnd, - (kLinuxShadowEnd - kLinuxShadowBeg) >> 30); + shadowBeg, shadowEnd, (shadowEnd - shadowBeg) >> 30); // Map meta shadow. - if (MemToMeta(kLinuxAppMemBeg) < (u32*)kMetaShadow) { + if (MemToMeta(appMemBeg) < (u32*)metaShadow) { Printf("ThreadSanitizer: bad meta shadow (%p -> %p < %p)\n", - kLinuxAppMemBeg, MemToMeta(kLinuxAppMemBeg), kMetaShadow); + appMemBeg, MemToMeta(appMemBeg), metaShadow); Die(); } - if (MemToMeta(kLinuxAppMemEnd) >= (u32*)(kMetaShadow + kMetaSize)) { + if (MemToMeta(appMemEnd) >= (u32*)(metaShadow + metaSize)) { Printf("ThreadSanitizer: bad meta shadow (%p -> %p >= %p)\n", - kLinuxAppMemEnd, MemToMeta(kLinuxAppMemEnd), kMetaShadow + kMetaSize); + appMemEnd, MemToMeta(appMemEnd), metaShadow + metaSize); Die(); } - uptr meta = (uptr)MmapFixedNoReserve(kMetaShadow, kMetaSize); - if (meta != kMetaShadow) { + uptr meta = (uptr)MmapFixedNoReserve(metaShadow, metaSize); + if (meta != metaShadow) { Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n"); Printf("FATAL: Make sure to compile with -fPIE and " - "to link with -pie (%p, %p).\n", meta, kMetaShadow); + "to link with -pie (%p, %p).\n", meta, metaShadow); Die(); } DPrintf("meta shadow: %zx-%zx (%zuGB)\n", - kMetaShadow, kMetaShadow + kMetaSize, kMetaSize >> 30); + metaShadow, metaShadow + metaSize, metaSize >> 30); + VPrintf(2, "app mem: %zx-%zx (%zuGB)\n", + appMemBeg, appMemEnd, (appMemEnd - appMemBeg) >> 30); +} + +void InitializeShadowMemory() { +#if SANITIZER_FREEBSD + MmapShadow(kLinuxLowShadowBeg, kLinuxLowShadowEnd, + kLinuxLowAppMemBeg, kLinuxLowAppMemEnd, + kLowMetaShadow, kLowMetaSize); + MmapShadow(kLinuxHighShadowBeg, kLinuxHighShadowEnd, + kLinuxHighAppMemBeg, kLinuxHighAppMemEnd, + kHighMetaShadow, kHighMetaSize); +#else + MmapShadow(kLinuxShadowBeg, kLinuxShadowEnd, + kLinuxAppMemBeg, kLinuxAppMemEnd, + kMetaShadow, kMetaSize); +#endif + // This memory range is used for thread stacks and large user mmaps. + // Frequently a thread uses only a small part of stack and similarly + // a program uses a small part of large mmap. On some programs + // we see 20% memory usage reduction without huge pages for this range. +#ifdef MADV_NOHUGEPAGE + madvise((void*)MemToShadow(0x7f0000000000ULL), + 0x10000000000ULL * kShadowMultiplier, MADV_NOHUGEPAGE); +#endif // Protect gaps. +#if SANITIZER_FREEBSD + const uptr kClosedLowBeg = 0x100000000000; + const uptr kClosedLowEnd = kLinuxLowShadowBeg - 1; + const uptr kClosedMidBeg = kLinuxLowShadowEnd + 1; + const uptr kClosedMidEnd = kLinuxHighShadowBeg - 1; +#else const uptr kClosedLowBeg = 0x200000; const uptr kClosedLowEnd = kLinuxShadowBeg - 1; const uptr kClosedMidBeg = kLinuxShadowEnd + 1; const uptr kClosedMidEnd = min(min(kLinuxAppMemBeg, kTraceMemBegin), kMetaShadow); - +#endif ProtectRange(kClosedLowBeg, kClosedLowEnd); ProtectRange(kClosedMidBeg, kClosedMidEnd); VPrintf(2, "kClosedLow %zx-%zx (%zuGB)\n", kClosedLowBeg, kClosedLowEnd, (kClosedLowEnd - kClosedLowBeg) >> 30); VPrintf(2, "kClosedMid %zx-%zx (%zuGB)\n", kClosedMidBeg, kClosedMidEnd, (kClosedMidEnd - kClosedMidBeg) >> 30); - VPrintf(2, "app mem: %zx-%zx (%zuGB)\n", - kLinuxAppMemBeg, kLinuxAppMemEnd, - (kLinuxAppMemEnd - kLinuxAppMemBeg) >> 30); - VPrintf(2, "stack: %zx\n", (uptr)&shadow); + + uptr stack = (uptr) &stack; + VPrintf(2, "stack: %zx\n", stack); MapRodata(); } -#endif +#endif // !TSAN_GO static uptr g_data_start; static uptr g_data_end; @@ -297,10 +322,10 @@ if (proc_maps.Next(&start, &end, /*offset*/0, /*filename*/0, /*filename_size*/0, /*protection*/0)) { - if ((u64)start < kLinuxAppMemBeg) { + if ((u64)start < kModulesBeg) { Printf("FATAL: ThreadSanitizer can not mmap the shadow memory (" "something is mapped at 0x%zx < 0x%zx)\n", - start, kLinuxAppMemBeg); + start, kModulesBeg); Printf("FATAL: Make sure to compile with -fPIE" " and to link with -pie.\n"); Die(); @@ -312,6 +337,20 @@ MemoryMappingLayout proc_maps(true); uptr start, end, offset; char name[128]; +#if SANITIZER_FREEBSD + // On FreeBSD BSS is usually the last block allocated within the + // low range and heap is the last block allocated within the range + // 0x800000000-0x8ffffffff. + while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name), + /*protection*/ 0)) { + DPrintf("%p-%p %p %s\n", start, end, offset, name); + if ((start & 0xffff00000000ULL) == 0 && (end & 0xffff00000000ULL) == 0 && + name[0] == '\0') { + g_data_start = start; + g_data_end = end; + } + } +#else bool prev_is_data = false; while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name), /*protection*/ 0)) { @@ -327,12 +366,12 @@ g_data_end = end; prev_is_data = is_data; } +#endif DPrintf("guessed data_start=%p data_end=%p\n", g_data_start, g_data_end); CHECK_LT(g_data_start, g_data_end); CHECK_GE((uptr)&g_data_start, g_data_start); CHECK_LT((uptr)&g_data_start, g_data_end); } - #endif // #ifndef TSAN_GO void InitializePlatform() { @@ -427,8 +466,8 @@ pthread_cleanup_pop(0); return res; } -#endif +#endif // #ifndef TSAN_GO } // namespace __tsan -#endif // SANITIZER_LINUX +#endif // SANITIZER_LINUX || SANITIZER_FREEBSD