[tsan] Add support for linux/powerpc64 in buildgo.sh

[cseo]

Summary

The current implementation of the Go sanitizer only works on x86_64. Added
some modifications to the buildgo.sh script and the Tsan code to make it
work on powerpc64/linux.

[dvyukov]

Is there an upstream Go bug for race on powerpc? Please CC me on it.

lib/tsan/go/buildgo.sh

141

Please dedup the common part "-DSANITIZER_DEBUG=0 -O3 -fomit-frame-pointer" and then add arch-specific flags separately.

lib/tsan/rtl/tsan_platform.h

420

I see that Go heap always starts at 0x00c0 on powerpc:

			switch {
			case GOARCH == "arm64" && GOOS == "darwin":
				p = uintptr(i)<<40 | uintptrMask&(0x0013<<28)
			case GOARCH == "arm64":
				p = uintptr(i)<<40 | uintptrMask&(0x0040<<32)
			default:
				p = uintptr(i)<<40 | uintptrMask&(0x00c0<<32)
			}

And you don't cover this region, instead it's occupied by shadow.
How does this work?

[cseo]

Version 2: update memory mappings, changed buildgo.sh FLAGS.

• Note: decided to drop Mapping44 because I could not find a system old

enough for me to test it.

• Go issue related to this: 23731

[dvyukov]

lib/tsan/go/buildgo.sh

60

List of sources is completely identical to amd64 version, please don't duplicate it. Just keep it outside of the arch if.

lib/tsan/rtl/tsan_platform.h

445

kLoAppMemEnd is 0x010000000000ull, please adjust this line.

448

remove this line

464

I don't see where/how this is used.

468

What lives in kHiAppMemBeg-kHiAppMemEnd range?

472

I don't see where/how this is used.

819

This is full copy of the C++ mapping, right?
If so, I think it will be better to just say #if !SANITIZER_GO || defined(__powerpc64__) rather than duplicate it.
This also applies to some other functions that do the same.

876–877

Why?

lib/tsan/rtl/tsan_platform_linux.cc

239

No need to duplicate this call.
Let's do:

vmaSize = (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
...
#   elif defined(__powerpc64__)
#     if !SANITIZER_GO
if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) { ... }
#     else
if (vmaSize != 44 && vmaSize != 46) { ... }
#     endif

lib/tsan/rtl/tsan_rtl.cc

309 ↗

(On Diff #141886)

These VPrintf's look redundant (and mis-aligned). We already print p, s and MemToMeta(p) few lines above.
ShadowToMem(s) will be printed if CHECK fails (it prints argument values).

[cseo]

lib/tsan/rtl/tsan_platform.h

468

ld.so and shared libraries.

Actually, let me ask you about this, since it seems I'm understanding wrongly how this should work. Is it necessary to map these memory areas for shared libraries? Or just executable/heap/shadow/meta are enough for the Go sanitizer?

[dvyukov]

lib/tsan/rtl/tsan_platform.h

468

For Go we care about and only need shadow for Go's heap and data/bss segment. All other addresses never passed to tsan runtime, they are filtered here:
https://github.com/golang/go/blob/master/src/runtime/race_amd64.s#L147

So I think we don't need vdso, HiAppMem and HeapMem, we probably need something closer to x86_64 Go mapping.

[cseo]

lib/tsan/rtl/tsan_platform.h

468

Excellent, thanks!

I'll make the changes and resubmit.

[cseo]

Version 3: new mappings, dropped unneeded memory areas.

[dvyukov]

Much nicer now!
I think this is ready for commit after fixing these few nits.

lib/tsan/go/buildgo.sh

59

If I am reading this correctly, amd64 will not have -fPIC because this overrides whole OSCFLAGS rather than append to them. Do we need something like:

OSCFLAGS="$OSCFLAGS -ffreestanding -Wno-unused-const-variable -Werror -Wno-unknown-warning-option"

?

lib/tsan/rtl/tsan_platform.h

414

This should be:

0000 0000 1000 - 0000 1000 0000

438

This should be:

0000 0000 1000 - 0000 1000 0000

[cseo]

Version 3: fixed OSCFLAGS for amd64 in buildgo.sh and some comments in tsan_platform.h.

[dvyukov]

Looks good to me.
Do you have commit access, or you want me to commit it?

[cseo]

In D43025#1067279, @dvyukov wrote:

Looks good to me.
Do you have commit access, or you want me to commit it?

Could you please do it? I don't have commit access. Thanks!

lib/tsan/go/buildgo.sh

59

Oops... sorry about that.

[dvyukov]

Submitted in:
http://llvm.org/viewvc/llvm-project?view=revision&revision=330122
Thanks!

lib/tsan/CMakeLists.txt

if(arch STREQUAL "x86_64")
         DEPENDS clang_rt.tsan-${arch} ${BUILDGO_SCRIPT}
         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/go
         COMMENT "Checking TSan Go runtime..."
         VERBATIM)
     elseif(arch STREQUAL "aarch64")
       add_asm_sources(TSAN_ASM_SOURCES rtl/tsan_rtl_aarch64.S)
     elseif(arch MATCHES "powerpc64|powerpc64le")
       add_asm_sources(TSAN_ASM_SOURCES rtl/tsan_rtl_ppc64.S)
+      # Sanity check for Go runtime.
+      set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/go/buildgo.sh)
+      add_custom_target(GotsanRuntimeCheck
+  COMMAND env "CC=${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}"
+    IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT}
+  DEPENDS clang_rt.tsan-${arch} ${BUILDGO_SCRIPT}
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/go
+  COMMENT "Checking TSan Go runtime..."
+  VERBATIM)
     elseif(arch MATCHES "mips64|mips64le")
       add_asm_sources(TSAN_ASM_SOURCES rtl/tsan_rtl_mips64.S)
     else()
       set(TSAN_ASM_SOURCES)
     endif()
     add_compiler_rt_runtime(clang_rt.tsan
       STATIC
       ARCHS ${arch}

lib/tsan/go/buildgo.sh

SRCS="
   ../../sanitizer_common/sanitizer_thread_registry.cc
   ../../sanitizer_common/sanitizer_stackdepot.cc
   ../../sanitizer_common/sanitizer_stacktrace.cc
   ../../sanitizer_common/sanitizer_symbolizer.cc
   ../../sanitizer_common/sanitizer_termination.cc
 "
 
 if [ "`uname -a | grep Linux`" != "" ]; then
-  SUFFIX="linux_amd64"
-  OSCFLAGS="-fPIC -ffreestanding -Wno-maybe-uninitialized -Wno-unused-const-variable -Werror -Wno-unknown-warning-option"
+  OSCFLAGS="-fPIC -Wno-maybe-uninitialized"
   OSLDFLAGS="-lpthread -fPIC -fpie"
   SRCS="
     $SRCS
     ../rtl/tsan_platform_linux.cc
     ../../sanitizer_common/sanitizer_posix.cc
     ../../sanitizer_common/sanitizer_posix_libcdep.cc
     ../../sanitizer_common/sanitizer_procmaps_common.cc
     ../../sanitizer_common/sanitizer_procmaps_linux.cc
     ../../sanitizer_common/sanitizer_linux.cc
     ../../sanitizer_common/sanitizer_linux_libcdep.cc
     ../../sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc
     "
+  if [ "`uname -a | grep ppc64le`" != "" ]; then
+    SUFFIX="linux_ppc64le"
+  elif [ "`uname -a | grep x86_64`" != "" ]; then
+    SUFFIX="linux_amd64"
+    OSCFLAGS="$OSCFLAGS -ffreestanding -Wno-unused-const-variable -Werror -Wno-unknown-warning-option"
+  fi
 elif [ "`uname -a | grep FreeBSD`" != "" ]; then
   SUFFIX="freebsd_amd64"
   OSCFLAGS="-fno-strict-aliasing -fPIC -Werror"
   OSLDFLAGS="-lpthread -fPIC -fpie"
   SRCS="
     $SRCS
     ../rtl/tsan_platform_linux.cc
     ../../sanitizer_common/sanitizer_posix.cc

 
 rm -f $DIR/gotsan.cc
 for F in $SRCS; do
   cat $F >> $DIR/gotsan.cc
 done
 
 FLAGS=" -I../rtl -I../.. -I../../sanitizer_common -I../../../include -std=c++11 -m64 -Wall -fno-exceptions -fno-rtti -DSANITIZER_GO=1 -DSANITIZER_DEADLOCK_DETECTOR_VERSION=2 $OSCFLAGS"
 if [ "$DEBUG" = "" ]; then
-  FLAGS="$FLAGS -DSANITIZER_DEBUG=0 -O3 -msse3 -fomit-frame-pointer"
+  FLAGS="$FLAGS -DSANITIZER_DEBUG=0 -O3 -fomit-frame-pointer"
+  if [ "$SUFFIX" = "linux_ppc64le" ]; then
+    FLAGS="$FLAGS -mcpu=power8 -fno-function-sections"
+  elif [ "$SUFFIX" = "linux_amd64" ]; then
+    FLAGS="$FLAGS -msse3"
+  fi
 else
   FLAGS="$FLAGS -DSANITIZER_DEBUG=1 -g"
 fi
 
 if [ "$SILENT" != "1" ]; then
   echo $CC gotsan.cc -c -o $DIR/race_$SUFFIX.syso $FLAGS $CFLAGS
 fi
 $CC $DIR/gotsan.cc -c -o $DIR/race_$SUFFIX.syso $FLAGS $CFLAGS

lib/tsan/rtl/tsan_platform.h

struct Mapping47 {
   static const uptr kAppMemXor     = 0x020000000000ull;
   static const uptr kVdsoBeg       = 0x7800000000000000ull;
 };
 
 // Indicates the runtime will define the memory regions at runtime.
 #define TSAN_RUNTIME_VMA 1
 #endif
 
-#elif SANITIZER_GO && !SANITIZER_WINDOWS
+#elif SANITIZER_GO && !SANITIZER_WINDOWS && defined(__x86_64__)
 
-/* Go on linux, darwin and freebsd
+/* Go on linux, darwin and freebsd on x86_64
 0000 0000 1000 - 0000 1000 0000: executable
 0000 1000 0000 - 00c0 0000 0000: -
 00c0 0000 0000 - 00e0 0000 0000: heap
 00e0 0000 0000 - 2000 0000 0000: -
 2000 0000 0000 - 2380 0000 0000: shadow
 2380 0000 0000 - 3000 0000 0000: -
 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
 4000 0000 0000 - 6000 0000 0000: -
struct Mapping {
   static const uptr kTraceMemBeg   = 0x056000000000ull;
   static const uptr kTraceMemEnd   = 0x076000000000ull;
   static const uptr kShadowBeg     = 0x010000000000ull;
   static const uptr kShadowEnd     = 0x050000000000ull;
   static const uptr kAppMemBeg     = 0x000000001000ull;
   static const uptr kAppMemEnd     = 0x00e000000000ull;
 };
 
+#elif SANITIZER_GO && defined(__powerpc64__)
+
+/* Only Mapping46 and Mapping47 are currently supported for powercp64 on Go. */
+
+/* Go on linux/powerpc64 (46-bit VMA)
+0000 0000 1000 - 0000 1000 0000: executable
+0000 1000 0000 - 00c0 0000 0000: -
+00c0 0000 0000 - 00e0 0000 0000: heap
+00e0 0000 0000 - 2000 0000 0000: -
[phone] - 2380 0000 0000: shadow
[phone] - 2400 0000 0000: -
[phone] - 3400 0000 0000: metainfo (memory blocks and sync objects)
[phone] - 3600 0000 0000: -
[phone] - 3800 0000 0000: traces
[phone] - 4000 0000 0000: -
+*/
+
+struct Mapping46 {
+  static const uptr kMetaShadowBeg = 0x240000000000ull;
+  static const uptr kMetaShadowEnd = 0x340000000000ull;
+  static const uptr kTraceMemBeg   = 0x360000000000ull;
+  static const uptr kTraceMemEnd   = 0x380000000000ull;
+  static const uptr kShadowBeg     = 0x200000000000ull;
+  static const uptr kShadowEnd     = 0x238000000000ull;
+  static const uptr kAppMemBeg     = 0x000000001000ull;
+  static const uptr kAppMemEnd     = 0x00e000000000ull;
+};
+
+/* Go on linux/powerpc64 (47-bit VMA)
+0000 0000 1000 - 0000 1000 0000: executable
+0000 1000 0000 - 00c0 0000 0000: -
+00c0 0000 0000 - 00e0 0000 0000: heap
+00e0 0000 0000 - 2000 0000 0000: -
[phone] - 3000 0000 0000: shadow
[phone] - 3000 0000 0000: -
[phone] - 4000 0000 0000: metainfo (memory blocks and sync objects)
[phone] - 6000 0000 0000: -
[phone] - 6200 0000 0000: traces
[phone] - 8000 0000 0000: -
+*/
+
+struct Mapping47 {
+  static const uptr kMetaShadowBeg = 0x300000000000ull;
+  static const uptr kMetaShadowEnd = 0x400000000000ull;
+  static const uptr kTraceMemBeg   = 0x600000000000ull;
+  static const uptr kTraceMemEnd   = 0x620000000000ull;
+  static const uptr kShadowBeg     = 0x200000000000ull;
+  static const uptr kShadowEnd     = 0x300000000000ull;
+  static const uptr kAppMemBeg     = 0x000000001000ull;
+  static const uptr kAppMemEnd     = 0x00e000000000ull;
+};
+
+// Indicates the runtime will define the memory regions at runtime.
+#define TSAN_RUNTIME_VMA 1
+
 #else
 # error "Unknown platform"
 #endif
 
 
 #ifdef TSAN_RUNTIME_VMA
 extern uptr vmaSize;
 #endif
 
 
 enum MappingType {
   MAPPING_LO_APP_BEG,
   MAPPING_LO_APP_END,
   MAPPING_HI_APP_BEG,
   MAPPING_HI_APP_END,
 #ifdef TSAN_MID_APP_RANGE
   MAPPING_MID_APP_BEG,
switch (vmaSize) {
     case 39: return MappingImpl<Mapping39, Type>();
     case 42: return MappingImpl<Mapping42, Type>();
     case 48: return MappingImpl<Mapping48, Type>();
   }
   DCHECK(0);
   return 0;
 #elif defined(__powerpc64__)
   switch (vmaSize) {
+#if !SANITIZER_GO
     case 44: return MappingImpl<Mapping44, Type>();
+#endif
     case 46: return MappingImpl<Mapping46, Type>();
     case 47: return MappingImpl<Mapping47, Type>();
   }
   DCHECK(0);
   return 0;
 #else
   return MappingImpl<Mapping, Type>();
 #endif
switch (vmaSize) {
     case 39: return IsAppMemImpl<Mapping39>(mem);
     case 42: return IsAppMemImpl<Mapping42>(mem);
     case 48: return IsAppMemImpl<Mapping48>(mem);
   }
   DCHECK(0);
   return false;
 #elif defined(__powerpc64__)
   switch (vmaSize) {
+#if !SANITIZER_GO
     case 44: return IsAppMemImpl<Mapping44>(mem);
+#endif
     case 46: return IsAppMemImpl<Mapping46>(mem);
     case 47: return IsAppMemImpl<Mapping47>(mem);
   }
   DCHECK(0);
   return false;
 #else
   return IsAppMemImpl<Mapping>(mem);
 #endif
switch (vmaSize) {
     case 39: return IsShadowMemImpl<Mapping39>(mem);
     case 42: return IsShadowMemImpl<Mapping42>(mem);
     case 48: return IsShadowMemImpl<Mapping48>(mem);
   }
   DCHECK(0);
   return false;
 #elif defined(__powerpc64__)
   switch (vmaSize) {
+#if !SANITIZER_GO
     case 44: return IsShadowMemImpl<Mapping44>(mem);
+#endif
     case 46: return IsShadowMemImpl<Mapping46>(mem);
     case 47: return IsShadowMemImpl<Mapping47>(mem);
   }
   DCHECK(0);
   return false;
 #else
   return IsShadowMemImpl<Mapping>(mem);
 #endif
switch (vmaSize) {
     case 39: return IsMetaMemImpl<Mapping39>(mem);
     case 42: return IsMetaMemImpl<Mapping42>(mem);
     case 48: return IsMetaMemImpl<Mapping48>(mem);
   }
   DCHECK(0);
   return false;
 #elif defined(__powerpc64__)
   switch (vmaSize) {
+#if !SANITIZER_GO
     case 44: return IsMetaMemImpl<Mapping44>(mem);
+#endif
     case 46: return IsMetaMemImpl<Mapping46>(mem);
     case 47: return IsMetaMemImpl<Mapping47>(mem);
   }
   DCHECK(0);
   return false;
 #else
   return IsMetaMemImpl<Mapping>(mem);
 #endif
switch (vmaSize) {
     case 39: return MemToShadowImpl<Mapping39>(x);
     case 42: return MemToShadowImpl<Mapping42>(x);
     case 48: return MemToShadowImpl<Mapping48>(x);
   }
   DCHECK(0);
   return 0;
 #elif defined(__powerpc64__)
   switch (vmaSize) {
+#if !SANITIZER_GO
     case 44: return MemToShadowImpl<Mapping44>(x);
+#endif
     case 46: return MemToShadowImpl<Mapping46>(x);
     case 47: return MemToShadowImpl<Mapping47>(x);
   }
   DCHECK(0);
   return 0;
 #else
   return MemToShadowImpl<Mapping>(x);
 #endif
 }
 
 
 template<typename Mapping>
 u32 *MemToMetaImpl(uptr x) {
   DCHECK(IsAppMem(x));
 #if !SANITIZER_GO
   return (u32*)(((((x) & ~(Mapping::kAppMemMsk | (kMetaShadowCell - 1)))) /
       kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg);
 #else
 # ifndef SANITIZER_WINDOWS
   return (u32*)(((x & ~(kMetaShadowCell - 1)) / \
       kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg);
 # else
   return (u32*)(((x & ~(kMetaShadowCell - 1)) / \
       kMetaShadowCell * kMetaShadowSize) + Mapping::kMetaShadowBeg);
 # endif
 #endif
 }
 
 ALWAYS_INLINE
 u32 *MemToMeta(uptr x) {
 #if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return MemToMetaImpl<Mapping39>(x);
     case 42: return MemToMetaImpl<Mapping42>(x);
     case 48: return MemToMetaImpl<Mapping48>(x);
   }
   DCHECK(0);
   return 0;
 #elif defined(__powerpc64__)
   switch (vmaSize) {
+#if !SANITIZER_GO
     case 44: return MemToMetaImpl<Mapping44>(x);
+#endif
     case 46: return MemToMetaImpl<Mapping46>(x);
     case 47: return MemToMetaImpl<Mapping47>(x);
   }
   DCHECK(0);
   return 0;
 #else
   return MemToMetaImpl<Mapping>(x);
 #endif
p = ((s / kShadowCnt) ^ Mapping::kAppMemXor) +
       (Mapping::kMidAppMemBeg & Mapping::kAppMemMsk);
   if (p >= Mapping::kMidAppMemBeg && p < Mapping::kMidAppMemEnd &&
       MemToShadow(p) == s)
     return p;
 # endif
   return ((s / kShadowCnt) ^ Mapping::kAppMemXor) | Mapping::kAppMemMsk;
 #else  // #if !SANITIZER_GO
 # ifndef SANITIZER_WINDOWS
   return (s & ~Mapping::kShadowBeg) / kShadowCnt;
 # else
   return (s - Mapping::kShadowBeg) / kShadowCnt;
 # endif // SANITIZER_WINDOWS
 #endif
 }
 
 ALWAYS_INLINE
 uptr ShadowToMem(uptr s) {
 #if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return ShadowToMemImpl<Mapping39>(s);
     case 42: return ShadowToMemImpl<Mapping42>(s);
     case 48: return ShadowToMemImpl<Mapping48>(s);
   }
   DCHECK(0);
   return 0;
 #elif defined(__powerpc64__)
   switch (vmaSize) {
+#if !SANITIZER_GO
     case 44: return ShadowToMemImpl<Mapping44>(s);
+#endif
     case 46: return ShadowToMemImpl<Mapping46>(s);
     case 47: return ShadowToMemImpl<Mapping47>(s);
   }
   DCHECK(0);
   return 0;
 #else
   return ShadowToMemImpl<Mapping>(s);
 #endif
switch (vmaSize) {
     case 39: return GetThreadTraceImpl<Mapping39>(tid);
     case 42: return GetThreadTraceImpl<Mapping42>(tid);
     case 48: return GetThreadTraceImpl<Mapping48>(tid);
   }
   DCHECK(0);
   return 0;
 #elif defined(__powerpc64__)
   switch (vmaSize) {
+#if !SANITIZER_GO
     case 44: return GetThreadTraceImpl<Mapping44>(tid);
+#endif
     case 46: return GetThreadTraceImpl<Mapping46>(tid);
     case 47: return GetThreadTraceImpl<Mapping47>(tid);
   }
   DCHECK(0);
   return 0;
 #else
   return GetThreadTraceImpl<Mapping>(tid);
 #endif
switch (vmaSize) {
     case 39: return GetThreadTraceHeaderImpl<Mapping39>(tid);
     case 42: return GetThreadTraceHeaderImpl<Mapping42>(tid);
     case 48: return GetThreadTraceHeaderImpl<Mapping48>(tid);
   }
   DCHECK(0);
   return 0;
 #elif defined(__powerpc64__)
   switch (vmaSize) {
+#if !SANITIZER_GO
     case 44: return GetThreadTraceHeaderImpl<Mapping44>(tid);
+#endif
     case 46: return GetThreadTraceHeaderImpl<Mapping46>(tid);
     case 47: return GetThreadTraceHeaderImpl<Mapping47>(tid);
   }
   DCHECK(0);
   return 0;
 #else
   return GetThreadTraceHeaderImpl<Mapping>(tid);
 #endif

lib/tsan/rtl/tsan_platform_linux.cc

vmaSize =
     (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
 #if defined(__aarch64__)
   if (vmaSize != 39 && vmaSize != 42 && vmaSize != 48) {
     Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
     Printf("FATAL: Found %d - Supported 39, 42 and 48\n", vmaSize);
     Die();
   }
 #elif defined(__powerpc64__)
+# if !SANITIZER_GO
   if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) {
     Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
     Printf("FATAL: Found %d - Supported 44, 46, and 47\n", vmaSize);
     Die();
   }
+# else
+  if (vmaSize != 46 && vmaSize != 47) {
+    Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
+    Printf("FATAL: Found %d - Supported 46, and 47\n", vmaSize);
+    Die();
+  }
+# endif
 #endif
 #endif
 }
 
 void InitializePlatform() {
   DisableCoreDumperIfNecessary();
 
   // Go maps shadow memory lazily and works fine with limited address space.
   // Unlimited stack is not a problem as well, because the executable
   // is not compiled with -pie.
   if (!SANITIZER_GO) {
     bool reexec = false;
     // TSan doesn't play well with unlimited stack size (as stack
     // overlaps with shadow memory). If we detect unlimited stack size,
     // we re-exec the program with limited stack size as a best effort.