Index: lib/tsan/rtl/tsan_platform.h =================================================================== --- lib/tsan/rtl/tsan_platform.h +++ lib/tsan/rtl/tsan_platform.h @@ -87,37 +87,46 @@ const uptr kAppMemXor = 0x0400000000ull; const uptr kVdsoBeg = 0xfffff00000ull; #elif defined(__aarch64__) -# if SANITIZER_AARCH64_VMA == 39 +// AArch64 supports multiple VMA which leads to multiple address transformation +// functions. To support these multiple VMAS transformations and mappings TSAN +// runtime for AArch64 uses an external memory read (vmaSize) to select which +// mapping to use. Although slower, it make a same instrumented binary run on +// multiple kernels. + /* C/C++ on linux/aarch64 (39-bit VMA) -0000 4000 00 - 0200 0000 00: main binary -2000 0000 00 - 4000 0000 00: shadow memory -4000 0000 00 - 5000 0000 00: metainfo -5000 0000 00 - 6000 0000 00: - +0000 0010 00 - 0100 0000 00: main binary +0100 0000 00 - 0800 0000 00: - +0800 0000 00 - 1F00 0000 00: shadow memory +1C00 0000 00 - 3100 0000 00: - +3100 0000 00 - 3400 0000 00: metainfo +3400 0000 00 - 6000 0000 00: - 6000 0000 00 - 6200 0000 00: traces 6200 0000 00 - 7d00 0000 00: - -7d00 0000 00 - 7e00 0000 00: heap -7e00 0000 00 - 7fff ffff ff: modules and main thread stack +7c00 0000 00 - 7d00 0000 00: heap +7d00 0000 00 - 7fff ffff ff: modules and main thread stack */ -const uptr kLoAppMemBeg = 0x0000400000ull; -const uptr kLoAppMemEnd = 0x0200000000ull; -const uptr kShadowBeg = 0x2000000000ull; -const uptr kShadowEnd = 0x4000000000ull; -const uptr kMetaShadowBeg = 0x4000000000ull; -const uptr kMetaShadowEnd = 0x5000000000ull; -const uptr kTraceMemBeg = 0x6000000000ull; -const uptr kTraceMemEnd = 0x6200000000ull; -const uptr kHeapMemBeg = 0x7d00000000ull; -const uptr kHeapMemEnd = 0x7e00000000ull; -const uptr kHiAppMemBeg = 0x7e00000000ull; -const uptr kHiAppMemEnd = 0x7fffffffffull; -const uptr kAppMemMsk = 0x7800000000ull; -const uptr kAppMemXor = 0x0800000000ull; -const uptr kVdsoBeg = 0x7f00000000ull; -# elif SANITIZER_AARCH64_VMA == 42 +struct Mapping39 { + static const uptr kLoAppMemBeg = 0x0000001000ull; + static const uptr kLoAppMemEnd = 0x0100000000ull; + static const uptr kShadowBeg = 0x0800000000ull; + static const uptr kShadowEnd = 0x1F00000000ull; + static const uptr kMetaShadowBeg = 0x3100000000ull; + static const uptr kMetaShadowEnd = 0x3400000000ull; + static const uptr kTraceMemBeg = 0x6000000000ull; + static const uptr kTraceMemEnd = 0x6200000000ull; + static const uptr kHeapMemBeg = 0x7c00000000ull; + static const uptr kHeapMemEnd = 0x7d00000000ull; + static const uptr kHiAppMemBeg = 0x7d00000000ull; + static const uptr kHiAppMemEnd = 0x7fffffffffull; + static const uptr kAppMemMsk = 0x7800000000ull; + static const uptr kAppMemXor = 0x0200000000ull; + static const uptr kVdsoBeg = 0x7f00000000ull; +}; + /* C/C++ on linux/aarch64 (42-bit VMA) -00000 4000 00 - 01000 0000 00: main binary +00000 0010 00 - 01000 0000 00: main binary 01000 0000 00 - 10000 0000 00: - 10000 0000 00 - 20000 0000 00: shadow memory 20000 0000 00 - 26000 0000 00: - @@ -126,27 +135,153 @@ 36200 0000 00 - 36240 0000 00: traces 36240 0000 00 - 3e000 0000 00: - 3e000 0000 00 - 3f000 0000 00: heap -3c000 0000 00 - 3ff00 0000 00: - -3ff00 0000 00 - 3ffff f000 00: modules and main thread stack +3f000 0000 00 - 3ffff ffff ff: modules and main thread stack */ -const uptr kLoAppMemBeg = 0x00000400000ull; -const uptr kLoAppMemEnd = 0x01000000000ull; -const uptr kShadowBeg = 0x10000000000ull; -const uptr kShadowEnd = 0x20000000000ull; -const uptr kMetaShadowBeg = 0x26000000000ull; -const uptr kMetaShadowEnd = 0x28000000000ull; -const uptr kTraceMemBeg = 0x36200000000ull; -const uptr kTraceMemEnd = 0x36400000000ull; -const uptr kHeapMemBeg = 0x3e000000000ull; -const uptr kHeapMemEnd = 0x3f000000000ull; -const uptr kHiAppMemBeg = 0x3ff00000000ull; -const uptr kHiAppMemEnd = 0x3fffff00000ull; -const uptr kAppMemMsk = 0x3c000000000ull; -const uptr kAppMemXor = 0x04000000000ull; -const uptr kVdsoBeg = 0x37f00000000ull; -# endif +struct Mapping42 { + static const uptr kLoAppMemBeg = 0x00000001000ull; + static const uptr kLoAppMemEnd = 0x01000000000ull; + static const uptr kShadowBeg = 0x10000000000ull; + static const uptr kShadowEnd = 0x20000000000ull; + static const uptr kMetaShadowBeg = 0x26000000000ull; + static const uptr kMetaShadowEnd = 0x28000000000ull; + static const uptr kTraceMemBeg = 0x36200000000ull; + static const uptr kTraceMemEnd = 0x36400000000ull; + static const uptr kHeapMemBeg = 0x3e000000000ull; + static const uptr kHeapMemEnd = 0x3f000000000ull; + static const uptr kHiAppMemBeg = 0x3f000000000ull; + static const uptr kHiAppMemEnd = 0x3ffffffffffull; + static const uptr kAppMemMsk = 0x3c000000000ull; + static const uptr kAppMemXor = 0x04000000000ull; + static const uptr kVdsoBeg = 0x37f00000000ull; +}; + +// Hold the detect VMA size at runtime. +extern uptr vmaSize; + +// Hold the detect Mapping based on runtime VMA. +extern uptr kLoAppMemBeg; +extern uptr kLoAppMemEnd; +extern uptr kShadowBeg; +extern uptr kShadowEnd; +extern uptr kMetaShadowBeg; +extern uptr kMetaShadowEnd; +extern uptr kTraceMemBeg; +extern uptr kTraceMemEnd; +extern uptr kHeapMemBeg; +extern uptr kHeapMemEnd; +extern uptr kHiAppMemBeg; +extern uptr kHiAppMemEnd; +extern uptr kAppMemMsk; +extern uptr kAppMemXor; +extern uptr kVdsoBeg; + + +template +bool IsAppMemImpl(uptr mem) { + return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) || + (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) || + (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd); +} + +ALWAYS_INLINE +bool IsAppMem(uptr mem) { + if (vmaSize == 39) + return IsAppMemImpl(mem); + else + return IsAppMemImpl(mem); + DCHECK(0); +} + + +template +bool IsShadowMemImpl(uptr mem) { + return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd; +} + +ALWAYS_INLINE +bool IsShadowMem(uptr mem) { + if (vmaSize == 39) + return IsShadowMemImpl(mem); + else + return IsShadowMemImpl(mem); + DCHECK(0); +} + + +template +bool IsMetaMemImpl(uptr mem) { + return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd; +} + +ALWAYS_INLINE +bool IsMetaMem(uptr mem) { + if (vmaSize == 39) + return IsMetaMemImpl(mem); + else + return IsMetaMemImpl(mem); + DCHECK(0); +} + + +template +uptr MemToShadowImpl(uptr x) { + return (((x) & ~(Mapping::kAppMemMsk | (kShadowCell - 1))) + ^ Mapping::kAppMemXor) * kShadowCnt; +} + +ALWAYS_INLINE +uptr MemToShadow(uptr x) { + if (vmaSize == 39) + return MemToShadowImpl(x); + else + return MemToShadowImpl(x); + DCHECK(0); +} + + +template +u32 *MemToMetaImpl(uptr x) { + DCHECK(IsAppMem(x)); + return (u32*)(((((x) & ~(Mapping::kAppMemMsk | (kMetaShadowCell - 1))) + ^ Mapping::kAppMemXor) / kMetaShadowCell * kMetaShadowSize) + | Mapping::kMetaShadowBeg); +} + +ALWAYS_INLINE +u32 *MemToMeta(uptr x) { + if (vmaSize == 39) + return MemToMetaImpl(x); + else + return MemToMetaImpl(x); + DCHECK(0); +} + + +template +uptr ShadowToMemImpl(uptr s) { + CHECK(IsShadowMem(s)); + if (s >= MemToShadow(Mapping::kLoAppMemBeg) + && s <= MemToShadow(Mapping::kLoAppMemEnd - 1)) + return (s / kShadowCnt) ^ Mapping::kAppMemXor; + else + return ((s / kShadowCnt) ^ Mapping::kAppMemXor) | Mapping::kAppMemMsk; +} + +ALWAYS_INLINE +uptr ShadowToMem(uptr s) { + if (vmaSize == 39) + return ShadowToMemImpl(s); + else + return ShadowToMemImpl(s); + DCHECK(0); +} + +// It will initialized at runtime. +extern uptr UserRegions[6]; +#define TSAN_NOINLINE_FUNCTIONS 1 #endif +#ifndef TSAN_NOINLINE_FUNCTIONS ALWAYS_INLINE bool IsAppMem(uptr mem) { return (mem >= kHeapMemBeg && mem < kHeapMemEnd) || @@ -192,6 +327,7 @@ kHiAppMemBeg, kHiAppMemEnd, kHeapMemBeg, kHeapMemEnd, }; +#endif #elif defined(SANITIZER_GO) && !SANITIZER_WINDOWS @@ -326,6 +462,41 @@ const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace) + (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1); +#ifdef TSAN_NOINLINE_FUNCTIONS +template +uptr GetThreadTraceImpl(int tid) { + uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize; + DCHECK_LT(p, Mapping::kTraceMemEnd); + return p; +} + +ALWAYS_INLINE +uptr GetThreadTrace(int tid) { + if (vmaSize == 39) + return GetThreadTraceImpl(tid); + else + return GetThreadTraceImpl(tid); + DCHECK(0); +} + + +template +uptr GetThreadTraceHeaderImpl(int tid) { + uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize + + kTraceSize * sizeof(Event); + DCHECK_LT(p, Mapping::kTraceMemEnd); + return p; +} + +ALWAYS_INLINE +uptr GetThreadTraceHeader(int tid) { + if (vmaSize == 39) + return GetThreadTraceHeaderImpl(tid); + else + return GetThreadTraceHeaderImpl(tid); + DCHECK(0); +} +#else uptr ALWAYS_INLINE GetThreadTrace(int tid) { uptr p = kTraceMemBeg + (uptr)tid * kTotalTraceSize; DCHECK_LT(p, kTraceMemEnd); @@ -338,8 +509,10 @@ DCHECK_LT(p, kTraceMemEnd); return p; } +#endif void InitializePlatform(); +void InitializePlatformEarly(); void CheckAndProtect(); void InitializeShadowMemoryPlatform(); void FlushShadowMemory(); Index: lib/tsan/rtl/tsan_platform_linux.cc =================================================================== --- lib/tsan/rtl/tsan_platform_linux.cc +++ lib/tsan/rtl/tsan_platform_linux.cc @@ -67,6 +67,26 @@ static uptr g_data_start; static uptr g_data_end; +#if !defined(SANITIZER_GO) && defined(TSAN_NOINLINE_FUNCTIONS) +uptr kLoAppMemBeg; +uptr kLoAppMemEnd; +uptr kShadowBeg; +uptr kShadowEnd; +uptr kMetaShadowBeg; +uptr kMetaShadowEnd; +uptr kTraceMemBeg; +uptr kTraceMemEnd; +uptr kHeapMemBeg; +uptr kHeapMemEnd; +uptr kHiAppMemBeg; +uptr kHiAppMemEnd; +uptr kAppMemMsk; +uptr kAppMemXor; +uptr kVdsoBeg; + +uptr vmaSize; +#endif + enum { MemTotal = 0, MemShadow = 1, @@ -235,6 +255,61 @@ #endif // #ifndef SANITIZER_GO +#if !defined(SANITIZER_GO) && defined(__aarch64__) +template +void InitMappingsValue(void) { + kLoAppMemBeg = Mapping::kLoAppMemBeg; + kLoAppMemEnd = Mapping::kLoAppMemEnd; + kShadowBeg = Mapping::kShadowBeg; + kShadowEnd = Mapping::kShadowEnd; + kMetaShadowBeg = Mapping::kMetaShadowBeg; + kMetaShadowEnd = Mapping::kMetaShadowEnd; + kTraceMemBeg = Mapping::kTraceMemBeg; + kTraceMemEnd = Mapping::kTraceMemEnd; + kHeapMemBeg = Mapping::kHeapMemEnd; + kHeapMemEnd = Mapping::kHeapMemEnd; + kHiAppMemBeg = Mapping::kHiAppMemBeg; + kHiAppMemEnd = Mapping::kHiAppMemEnd; + kAppMemMsk = Mapping::kAppMemMsk; + kAppMemXor = Mapping::kAppMemXor; + kVdsoBeg = Mapping::kVdsoBeg; +} + +uptr UserRegions[] = { + 0, 0, // kLoAppMemBeg, kLoAppMemEnd + 0, 0, // kHiAppMemBeg, kHiAppMemEnd, + 0, 0, // kHeapMemBeg, kHeapMemEnd, +}; + +template +void InitUserRegions(void) { + UserRegions[0] = Mapping::kLoAppMemBeg; + UserRegions[1] = Mapping::kLoAppMemEnd; + UserRegions[2] = Mapping::kHiAppMemBeg; + UserRegions[3] = Mapping::kHiAppMemEnd; + UserRegions[4] = Mapping::kHeapMemBeg; + UserRegions[5] = Mapping::kHeapMemEnd; +} +#endif + +void InitializePlatformEarly() { +#if !defined(SANITIZER_GO) && defined(__aarch64__) + vmaSize = + (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); + if (vmaSize == 39) { + InitMappingsValue(); + InitUserRegions(); + } else if (vmaSize == 42) { + InitMappingsValue(); + InitUserRegions(); + } else { + Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); + Printf("FATAL: Found %d - Supported 39 and 42\n", vmaSize); + Die(); + } +#endif +} + void InitializePlatform() { DisableCoreDumperIfNecessary(); Index: lib/tsan/rtl/tsan_platform_mac.cc =================================================================== --- lib/tsan/rtl/tsan_platform_mac.cc +++ lib/tsan/rtl/tsan_platform_mac.cc @@ -91,6 +91,9 @@ prev_pthread_introspection_hook(event, thread, addr, size); } +void InitializePlatformEarly() { +} + void InitializePlatform() { DisableCoreDumperIfNecessary(); #ifndef SANITIZER_GO Index: lib/tsan/rtl/tsan_platform_posix.cc =================================================================== --- lib/tsan/rtl/tsan_platform_posix.cc +++ lib/tsan/rtl/tsan_platform_posix.cc @@ -46,8 +46,17 @@ const uptr kMadviseRangeBeg = 0xff00000000ull; const uptr kMadviseRangeSize = 0x0100000000ull; #elif defined(__aarch64__) - const uptr kMadviseRangeBeg = 0x7e00000000ull; - const uptr kMadviseRangeSize = 0x0100000000ull; + unsigned runtimeVMA = + (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); + uptr kMadviseRangeBeg = 0; + uptr kMadviseRangeSize = 0; + if (runtimeVMA == 39) { + kMadviseRangeBeg = 0x7d00000000ull; + kMadviseRangeSize = 0x0300000000ull; + } else if (runtimeVMA == 42) { + kMadviseRangeBeg = 0x3f000000000ull; + kMadviseRangeSize = 0x01000000000ull; + } #endif NoHugePagesInRegion(MemToShadow(kMadviseRangeBeg), kMadviseRangeSize * kShadowMultiplier); Index: lib/tsan/rtl/tsan_platform_windows.cc =================================================================== --- lib/tsan/rtl/tsan_platform_windows.cc +++ lib/tsan/rtl/tsan_platform_windows.cc @@ -31,6 +31,9 @@ void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) { } +void InitializePlatformEarly() { +} + void InitializePlatform() { } Index: lib/tsan/rtl/tsan_rtl.cc =================================================================== --- lib/tsan/rtl/tsan_rtl.cc +++ lib/tsan/rtl/tsan_rtl.cc @@ -322,7 +322,7 @@ const char *options = GetEnv(kTsanOptionsEnv); CacheBinaryName(); InitializeFlags(&ctx->flags, options); - CheckVMASize(); + InitializePlatformEarly(); #ifndef SANITIZER_GO InitializeAllocator(); ReplaceSystemMalloc(); Index: test/tsan/test.h =================================================================== --- test/tsan/test.h +++ test/tsan/test.h @@ -53,7 +53,7 @@ else if (vma == 42) format = "0x%011lx"; else { - fprintf(stderr, "unsupported vma: %ul\n", vma); + fprintf(stderr, "unsupported vma: %lu\n", vma); exit(1); }