hwasan: Instrument globals.

[pcc]

Summary

Globals are instrumented by adding a pointer tag to their symbol values
and emitting metadata into a special section that allows the runtime to tag
their memory when the library is loaded.

Due to order of initialization issues explained in more detail in the comments,
shadow initialization cannot happen during regular global initialization.
Instead, the location of the global section is marked using an ELF note,
and we require libc support for calling a function provided by the HWASAN
runtime when libraries are loaded and unloaded.

Based on ideas discussed with @evgeny777 in D56672.

Depends on D65686

Depends on D65768

Depends on D65769

[vitalybuka]

compiler-rt/lib/hwasan/hwasan.cpp

221 ↗

(On Diff #213434)

Can this be just following?

u32 size:24;
u8 tag;

236 ↗

(On Diff #213434)

could this be a normal constant?

[vitalybuka]

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

1203 ↗

(On Diff #213434)

what is going to happen with size > 2^24?

[pcc]

compiler-rt/lib/hwasan/hwasan.cpp

221 ↗

(On Diff #213434)

It could, but since these structs are created by the pass I didn't want to make any assumptions about bitfield layout (even though I think we should always end up with the layout we expect on the platforms we care about), and the way I've written it the code is more "obviously" correct when comparing it against the pass.

236 ↗

(On Diff #213434)

You mean like an enum? Sure.

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

1203 ↗

(On Diff #213434)

We create multiple descriptors, see the loop on lines 1242-1262.

[pcc]

• Switch to an enum

[pcc]

• Increment num_descriptions_printed in the right place

[pcc]

The AOSP side of this is https://android-review.googlesource.com/c/platform/bionic/+/1096613

cfe/trunk/lib/Driver/SanitizerArgs.cpp

if (AsanInvalidPointerSub) {
     CmdArgs.push_back("-asan-detect-invalid-pointer-sub");
   }
 
   if (!HwasanAbi.empty()) {
     CmdArgs.push_back("-default-function-attr");
     CmdArgs.push_back(Args.MakeArgString("hwasan-abi=" + HwasanAbi));
   }
 
+  if (Sanitizers.has(SanitizerKind::HWAddress)) {
+    CmdArgs.push_back("-target-feature");
+    CmdArgs.push_back("+tagged-globals");
+  }
+
   // MSan: Workaround for PR16386.
   // ASan: This is mainly to help LSan with cases such as
   // https://github.com/google/sanitizers/issues/373
   // We can't make this conditional on -fsanitize=leak, as that flag shouldn't
   // affect compilation.
   if (Sanitizers.has(SanitizerKind::Memory) ||
       Sanitizers.has(SanitizerKind::Address))
     CmdArgs.push_back("-fno-assume-sane-operator-new");

cfe/trunk/test/Driver/fsanitize.c

 // RUN: %clang -target x86_64-linux-gnu -fsanitize=scudo,kernel-memory  %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SCUDO-KMSAN
 // CHECK-SCUDO-KMSAN: error: invalid argument '-fsanitize=kernel-memory' not allowed with '-fsanitize=scudo'
 
 // RUN: %clang -target x86_64-linux-gnu -fsanitize=hwaddress %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-HWASAN-INTERCEPTOR-ABI
 // RUN: %clang -target x86_64-linux-gnu -fsanitize=hwaddress -fsanitize-hwaddress-abi=interceptor %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-HWASAN-INTERCEPTOR-ABI
 // RUN: %clang -target x86_64-linux-gnu -fsanitize=hwaddress -fsanitize-hwaddress-abi=platform %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-HWASAN-PLATFORM-ABI
 // RUN: %clang -target x86_64-linux-gnu -fsanitize=hwaddress -fsanitize-hwaddress-abi=foo %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-HWASAN-FOO-ABI
 // CHECK-HWASAN-INTERCEPTOR-ABI: "-default-function-attr" "hwasan-abi=interceptor"
+// CHECK-HWASAN-INTERCEPTOR-ABI: "-target-feature" "+tagged-globals"
 // CHECK-HWASAN-PLATFORM-ABI: "-default-function-attr" "hwasan-abi=platform"
+// CHECK-HWASAN-PLATFORM-ABI: "-target-feature" "+tagged-globals"
 // CHECK-HWASAN-FOO-ABI: error: invalid value 'foo' in '-fsanitize-hwaddress-abi=foo'
 
 // RUN: %clang -target x86_64-linux-gnu -fsanitize=address,pointer-compare,pointer-subtract %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-POINTER-ALL
 // RUN: %clang -target x86_64-linux-gnu -fsanitize=pointer-compare %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-POINTER-CMP-NEEDS-ADDRESS
 // RUN: %clang -target x86_64-linux-gnu -fsanitize=pointer-subtract %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-POINTER-SUB-NEEDS-ADDRESS
 // RUN: %clang -target x86_64-linux-gnu -fsanitize=pointer-subtract -fno-sanitize=pointer-subtract %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-POINTER-SUB
 // RUN: %clang -target x86_64-linux-gnu -fsanitize=pointer-compare -fno-sanitize=pointer-compare %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-POINTER-CMP
 // CHECK-POINTER-ALL: -cc1{{.*}}-fsanitize={{[^"]*}}pointer-compare,pointer-subtract{{.*}}" {{.*}} "-mllvm" "-asan-detect-invalid-pointer-cmp" {{.*}}"-mllvm" "-asan-detect-invalid-pointer-sub"

compiler-rt/trunk/lib/hwasan/hwasan.h

 
 extern int hwasan_inited;
 extern bool hwasan_init_is_running;
 extern int hwasan_report_count;
 
 bool ProtectRange(uptr beg, uptr end);
 bool InitShadow();
 void InitThreads();
-void InitInstrumentation();
 void MadviseShadow();
 char *GetProcSelfMaps();
 void InitializeInterceptors();
 
 void HwasanAllocatorInit();
 void HwasanAllocatorThreadFinish();
 
 void *hwasan_malloc(uptr size, StackTrace *stack);

compiler-rt/trunk/lib/hwasan/hwasan.cpp

void UpdateMemoryUsage() {
   HwasanFormatMemoryUsage(s);
   internal_strncpy(memory_usage_buffer, s.data(), kMemoryUsageBufferSize - 1);
   memory_usage_buffer[kMemoryUsageBufferSize - 1] = '\0';
 }
 #else
 void UpdateMemoryUsage() {}
 #endif
 
-// Prepare to run instrumented code on the main thread.
-void InitInstrumentation() {
-  if (hwasan_instrumentation_inited) return;
-
-  if (!InitShadow()) {
-    Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n");
-    DumpProcessMap();
-    Die();
-  }
-
-  InitThreads();
-  hwasanThreadList().CreateCurrentThread();
-
-  hwasan_instrumentation_inited = 1;
-}
-
 } // namespace __hwasan
 
+using namespace __hwasan;
+
 void __sanitizer::BufferedStackTrace::UnwindImpl(
     uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) {
-  using namespace __hwasan;
   Thread *t = GetCurrentThread();
   if (!t) {
     // the thread is still being created.
     size = 0;
     return;
   }
   if (!StackTrace::WillUseFastUnwind(request_fast)) {
     // Block reports from our interceptors during _Unwind_Backtrace.
     SymbolizerScope sym_scope;
     return Unwind(max_depth, pc, bp, context, 0, 0, request_fast);
   }
   if (StackTrace::WillUseFastUnwind(request_fast))
     Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), true);
   else
     Unwind(max_depth, pc, 0, context, 0, 0, false);
 }
 
-// Interface.
+struct hwasan_global {
+  s32 gv_relptr;
+  u32 info;
+};
+
+static void InitGlobals(const hwasan_global *begin, const hwasan_global *end) {
+  for (auto *desc = begin; desc != end; ++desc) {
+    uptr gv = reinterpret_cast<uptr>(desc) + desc->gv_relptr;
+    uptr size = desc->info & 0xffffff;
+    uptr full_granule_size = RoundDownTo(size, 16);
+    u8 tag = desc->info >> 24;
+    TagMemoryAligned(gv, full_granule_size, tag);
+    if (size % 16)
+      TagMemoryAligned(gv + full_granule_size, 16, size % 16);
+  }
+}
 
-using namespace __hwasan;
+enum { NT_LLVM_HWASAN_GLOBALS = 3 };
+
+struct hwasan_global_note {
+  s32 begin_relptr;
+  s32 end_relptr;
+};
+
+static void InitGlobalsFromPhdrs(ElfW(Addr) base, const ElfW(Phdr) * phdr,
+                                 ElfW(Half) phnum) {
+  for (; phnum != 0; ++phdr, --phnum) {
+    if (phdr->p_type != PT_NOTE)
+      continue;
+    const char *note = reinterpret_cast<const char *>(base + phdr->p_vaddr);
+    const char *nend = note + phdr->p_memsz;
+    while (note < nend) {
+      auto *nhdr = reinterpret_cast<const ElfW(Nhdr) *>(note);
+      const char *name = note + sizeof(ElfW(Nhdr));
+      const char *desc = name + nhdr->n_namesz;
+      if (nhdr->n_type != NT_LLVM_HWASAN_GLOBALS ||
+          internal_strcmp(name, "LLVM") != 0) {
+        note = desc + nhdr->n_descsz;
+        continue;
+      }
+
+      auto *global_note = reinterpret_cast<const hwasan_global_note *>(desc);
+      auto *global_begin = reinterpret_cast<const hwasan_global *>(
+          note + global_note->begin_relptr);
+      auto *global_end = reinterpret_cast<const hwasan_global *>(
+          note + global_note->end_relptr);
+      InitGlobals(global_begin, global_end);
+      return;
+    }
+  }
+}
+
+static void InitLoadedGlobals() {
+  dl_iterate_phdr(
+      [](dl_phdr_info *info, size_t size, void *data) {
+        InitGlobalsFromPhdrs(info->dlpi_addr, info->dlpi_phdr,
+                             info->dlpi_phnum);
+        return 0;
+      },
+      nullptr);
+}
+
+// Prepare to run instrumented code on the main thread.
+static void InitInstrumentation() {
+  if (hwasan_instrumentation_inited) return;
+
+  if (!InitShadow()) {
+    Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n");
+    DumpProcessMap();
+    Die();
+  }
+
+  InitThreads();
+  hwasanThreadList().CreateCurrentThread();
+
+  hwasan_instrumentation_inited = 1;
+}
+
+// Interface.
 
 uptr __hwasan_shadow_memory_dynamic_address;  // Global interface symbol.
 
 // This function was used by the old frame descriptor mechanism. We keep it
 // around to avoid breaking ABI.
 void __hwasan_init_frames(uptr beg, uptr end) {}
 
 void __hwasan_init_static() {
   InitShadowGOT();
   InitInstrumentation();
+
+  // In the non-static code path we call dl_iterate_phdr here. But at this point
+  // libc might not have been initialized enough for dl_iterate_phdr to work.
+  // Fortunately, since this is a statically linked executable we can use the
+  // linker-defined symbol __ehdr_start to find the only relevant set of phdrs.
+  extern ElfW(Ehdr) __ehdr_start;
+  InitGlobalsFromPhdrs(
+      0,
+      reinterpret_cast<const ElfW(Phdr) *>(
+          reinterpret_cast<const char *>(&__ehdr_start) + __ehdr_start.e_phoff),
+      __ehdr_start.e_phnum);
 }
 
 void __hwasan_init() {
   CHECK(!hwasan_init_is_running);
   if (hwasan_inited) return;
   hwasan_init_is_running = 1;
   SanitizerToolName = "HWAddressSanitizer";
 
   InitTlsSize();
 
   CacheBinaryName();
   InitializeFlags();
 
   // Install tool-specific callbacks in sanitizer_common.
   SetCheckFailedCallback(HWAsanCheckFailed);
 
   __sanitizer_set_report_path(common_flags()->log_path);
 
   AndroidTestTlsSlot();
 
   DisableCoreDumperIfNecessary();
 
   InitInstrumentation();
+  InitLoadedGlobals();
 
   // Needs to be called here because flags()->random_tags might not have been
   // initialized when InitInstrumentation() was called.
   GetCurrentThread()->InitRandomState();
 
   MadviseShadow();
 
   SetPrintfAndReportCallback(AppendToErrorMessageBuffer);

 #endif
 
   VPrintf(1, "HWAddressSanitizer init done\n");
 
   hwasan_init_is_running = 0;
   hwasan_inited = 1;
 }
 
+void __hwasan_library_loaded(ElfW(Addr) base, const ElfW(Phdr) * phdr,
+                             ElfW(Half) phnum) {
+  InitGlobalsFromPhdrs(base, phdr, phnum);
+}
+
+void __hwasan_library_unloaded(ElfW(Addr) base, const ElfW(Phdr) * phdr,
+                               ElfW(Half) phnum) {
+  for (; phnum != 0; ++phdr, --phnum)
+    if (phdr->p_type == PT_LOAD)
+      TagMemory(base + phdr->p_vaddr, phdr->p_memsz, 0);
+}
+
 void __hwasan_print_shadow(const void *p, uptr sz) {
   uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p));
   uptr shadow_first = MemToShadow(ptr_raw);
   uptr shadow_last = MemToShadow(ptr_raw + sz - 1);
   Printf("HWASan shadow map for %zx .. %zx (pointer tag %x)\n", ptr_raw,
          ptr_raw + sz, GetTagFromPointer((uptr)p));
   for (uptr s = shadow_first; s <= shadow_last; ++s)
     Printf("  %zx: %x\n", ShadowToMem(s), *(tag_t *)s);

compiler-rt/trunk/lib/hwasan/hwasan_interface_internal.h

 // Private Hwasan interface header.
 //===----------------------------------------------------------------------===//
 
 #ifndef HWASAN_INTERFACE_INTERNAL_H
 #define HWASAN_INTERFACE_INTERNAL_H
 
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
+#include <link.h>
 
 extern "C" {
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_init_static();
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_init();
 
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_library_loaded(ElfW(Addr) base, const ElfW(Phdr) * phdr,
+                             ElfW(Half) phnum);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_library_unloaded(ElfW(Addr) base, const ElfW(Phdr) * phdr,
+                               ElfW(Half) phnum);
+
 using __sanitizer::uptr;
 using __sanitizer::sptr;
 using __sanitizer::uu64;
 using __sanitizer::uu32;
 using __sanitizer::uu16;
 using __sanitizer::u64;
 using __sanitizer::u32;
 using __sanitizer::u16;

compiler-rt/trunk/lib/hwasan/hwasan_report.cpp

if (chunk.IsAllocated()) {
                                : chunk.Beg() - untagged_addr,
              candidate == left ? "right" : "left", chunk.UsedSize(),
              chunk.Beg(), chunk.End());
       Printf("%s", d.Allocation());
       Printf("allocated here:\n");
       Printf("%s", d.Default());
       GetStackTraceFromId(chunk.GetAllocStackId()).Print();
       num_descriptions_printed++;
+    } else {
+      // Check whether the address points into a loaded library. If so, this is
+      // most likely a global variable.
+      const char *module_name;
+      uptr module_address;
+      Symbolizer *sym = Symbolizer::GetOrInit();
+      if (sym->GetModuleNameAndOffsetForPC(mem, &module_name,
+                                           &module_address)) {
+        DataInfo info;
+        if (sym->SymbolizeData(mem, &info) && info.start) {
+          Printf(
+              "%p is located %zd bytes to the %s of %zd-byte global variable "
+              "%s [%p,%p) in %s\n",
+              untagged_addr,
+              candidate == left ? untagged_addr - (info.start + info.size)
+                                : info.start - untagged_addr,
+              candidate == left ? "right" : "left", info.size, info.name,
+              info.start, info.start + info.size, module_name);
+        } else {
+          Printf("%p is located to the %s of a global variable in (%s+0x%x)\n",
+                 untagged_addr, candidate == left ? "right" : "left",
+                 module_name, module_address);
+        }
+        num_descriptions_printed++;
+      }
     }
   }
 
   hwasanThreadList().VisitAllLiveThreads([&](Thread *t) {
     // Scan all threads' ring buffers to find if it's a heap-use-after-free.
     HeapAllocationRecord har;
     if (uptr D = FindHeapAllocation(t->heap_allocations(), tagged_addr, &har)) {
       Printf("%s", d.Location());

compiler-rt/trunk/test/hwasan/TestCases/global.c

+// RUN: %clang_hwasan %s -o %t
+// RUN: %run %t 0
+// RUN: not %run %t 1 2>&1 | FileCheck --check-prefixes=CHECK,RSYM %s
+// RUN: not %env_hwasan_opts=symbolize=0 %run %t 1 2>&1 | FileCheck --check-prefixes=CHECK,RNOSYM %s
+// RUN: not %run %t -1 2>&1 | FileCheck --check-prefixes=CHECK,LSYM %s
+// RUN: not %env_hwasan_opts=symbolize=0 %run %t -1 2>&1 | FileCheck --check-prefixes=CHECK,LNOSYM %s
+
+int x = 1;
+
+int main(int argc, char **argv) {
+  // RSYM: is located 0 bytes to the right of 4-byte global variable x {{.*}} in {{.*}}global.c.tmp
+  // RNOSYM: is located to the right of a global variable in ({{.*}}global.c.tmp+{{.*}})
+  // LSYM: is located 4 bytes to the left of 4-byte global variable x {{.*}} in {{.*}}global.c.tmp
+  // LNOSYM: is located to the left of a global variable in ({{.*}}global.c.tmp+{{.*}})
+  // CHECK-NOT: can not describe
+  (&x)[atoi(argv[1])] = 1;
+}

compiler-rt/trunk/test/hwasan/lit.cfg.py

 # -*- Python -*-
 
 import os
 
 # Setup config name.
 config.name = 'HWAddressSanitizer' + getattr(config, 'name_suffix', 'default')
 
 # Setup source root.
 config.test_source_root = os.path.dirname(__file__)
 
 # Setup default compiler flags used with -fsanitize=memory option.
 clang_cflags = [config.target_cflags] + config.debug_info_flags
 clang_cxxflags = config.cxx_mode_flags + clang_cflags
-clang_hwasan_cflags = ["-fsanitize=hwaddress"] + clang_cflags
+clang_hwasan_cflags = ["-fsanitize=hwaddress", "-mllvm", "-hwasan-globals"] + clang_cflags
+if config.target_arch == 'x86_64':
+  # This does basically the same thing as tagged-globals on aarch64. Because
+  # the x86_64 implementation is for testing purposes only there is no
+  # equivalent target feature implemented on x86_64.
+  clang_hwasan_cflags += ["-mcmodel=large"]
 clang_hwasan_cxxflags = config.cxx_mode_flags + clang_hwasan_cflags
 
 def build_invocation(compile_flags):
   return " " + " ".join([config.clang] + compile_flags) + " "
 
 config.substitutions.append( ("%clangxx ", build_invocation(clang_cxxflags)) )
 config.substitutions.append( ("%clang_hwasan ", build_invocation(clang_hwasan_cflags)) )
 config.substitutions.append( ("%clangxx_hwasan ", build_invocation(clang_hwasan_cxxflags)) )

llvm/trunk/include/llvm/BinaryFormat/ELF.h

enum : unsigned {
   NT_ARM_SVE = 0x405,
   NT_ARM_PAC_MASK = 0x406,
 
   NT_FILE = 0x46494c45,
   NT_PRXFPREG = 0x46e62b7f,
   NT_SIGINFO = 0x53494749,
 };
 
+// LLVM-specific notes.
+enum {
+  NT_LLVM_HWASAN_GLOBALS = 3,
+};
+
 // GNU note types
 enum {
   NT_GNU_ABI_TAG = 1,
   NT_GNU_HWCAP = 2,
   NT_GNU_BUILD_ID = 3,
   NT_GNU_GOLD_VERSION = 4,
   NT_GNU_PROPERTY_TYPE_0 = 5,
 };

llvm/trunk/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

 /// based on tagged addressing.
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
+#include "llvm/BinaryFormat/ELF.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Function.h"

 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
 #include <sstream>
 
 using namespace llvm;
 
 #define DEBUG_TYPE "hwasan"
 
 static const char *const kHwasanModuleCtorName = "hwasan.module_ctor";
+static const char *const kHwasanNoteName = "hwasan.note";
 static const char *const kHwasanInitName = "__hwasan_init";
 
 static const char *const kHwasanShadowMemoryDynamicAddress =
     "__hwasan_shadow_memory_dynamic_address";
 
 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
 static const size_t kNumberOfAccessSizes = 5;
 
cl::desc("Clear alloca tags before returning from the function to allow "
              "function to detect use after return."),
     cl::Hidden, cl::init(true));
 
 static cl::opt<bool> ClGenerateTagsWithCalls(
     "hwasan-generate-tags-with-calls",
     cl::desc("generate new tags with runtime library calls"), cl::Hidden,
     cl::init(false));
 
+static cl::opt<bool> ClGlobals("hwasan-globals", cl::desc("Instrument globals"),
+                               cl::Hidden, cl::init(false));
+
 static cl::opt<int> ClMatchAllTag(
     "hwasan-match-all-tag",
     cl::desc("don't report bad accesses via pointers with this tag"),
     cl::Hidden, cl::init(-1));
 
 static cl::opt<bool> ClEnableKhwasan(
     "hwasan-kernel",
     cl::desc("Enable KernelHWAddressSanitizer instrumentation"),

 
 namespace {
 
 /// An instrumentation pass implementing detection of addressability bugs
 /// using tagged pointers.
 class HWAddressSanitizer {
 public:
   explicit HWAddressSanitizer(Module &M, bool CompileKernel = false,
-                              bool Recover = false) {
+                              bool Recover = false) : M(M) {
     this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover;
     this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 ?
         ClEnableKhwasan : CompileKernel;
 
-    initializeModule(M);
+    initializeModule();
   }
 
   bool sanitizeFunction(Function &F);
-  void initializeModule(Module &M);
+  void initializeModule();
 
   void initializeCallbacks(Module &M);
 
   Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
   Value *getDynamicShadowNonTls(IRBuilder<> &IRB);
 
   void untagPointerOperand(Instruction *I, Value *Addr);
   Value *shadowBase();
public:
   Value *getStackBaseTag(IRBuilder<> &IRB);
   Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, AllocaInst *AI,
                      unsigned AllocaNo);
   Value *getUARTag(IRBuilder<> &IRB, Value *StackTag);
 
   Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty);
   void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord);
 
+  void instrumentGlobal(GlobalVariable *GV, uint8_t Tag);
+  void instrumentGlobals();
+
 private:
   LLVMContext *C;
+  Module &M;
   Triple TargetTriple;
   FunctionCallee HWAsanMemmove, HWAsanMemcpy, HWAsanMemset;
   FunctionCallee HWAsanHandleVfork;
 
   /// This struct defines the shadow mapping using the rule:
   ///   shadow = (mem >> Scale) + Offset.
   /// If InGlobal is true, then
   ///   extern char __hwasan_shadow[];
   ///   shadow = (mem >> Scale) + &__hwasan_shadow
   /// If InTls is true, then
   ///   extern char *__hwasan_tls;
   ///   shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment)
   struct ShadowMapping {
     int Scale;
     uint64_t Offset;
     bool InGlobal;
     bool InTls;
 
     void init(Triple &TargetTriple);
-    unsigned getAllocaAlignment() const { return 1U << Scale; }
+    unsigned getObjectAlignment() const { return 1U << Scale; }
   };
   ShadowMapping Mapping;
 
   Type *IntptrTy;
   Type *Int8PtrTy;
   Type *Int8Ty;
   Type *Int32Ty;
+  Type *Int64Ty = Type::getInt64Ty(M.getContext());
 
   bool CompileKernel;
   bool Recover;
 
   Function *HwasanCtorFunction;
 
   FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
   FunctionCallee HwasanMemoryAccessCallbackSized[2];
PreservedAnalyses HWAddressSanitizerPass::run(Module &M,
   if (Modified)
     return PreservedAnalyses::none();
   return PreservedAnalyses::all();
 }
 
 /// Module-level initialization.
 ///
 /// inserts a call to __hwasan_init to the module's constructor list.
-void HWAddressSanitizer::initializeModule(Module &M) {
+void HWAddressSanitizer::initializeModule() {
   LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n");
   auto &DL = M.getDataLayout();
 
   TargetTriple = Triple(M.getTargetTriple());
 
   Mapping.init(TargetTriple);
 
   C = &(M.getContext());
std::tie(HwasanCtorFunction, std::ignore) =
             /*InitArgs=*/{},
             // This callback is invoked when the functions are created the first
             // time. Hook them into the global ctors list in that case:
             [&](Function *Ctor, FunctionCallee) {
               Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
               Ctor->setComdat(CtorComdat);
               appendToGlobalCtors(M, Ctor, 0, Ctor);
             });
+
+    // Older versions of Android do not have the required runtime support for
+    // global instrumentation. On other platforms we currently require using the
+    // latest version of the runtime.
+    bool InstrumentGlobals =
+        !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(30);
+    if (ClGlobals.getNumOccurrences())
+      InstrumentGlobals = ClGlobals;
+    if (InstrumentGlobals)
+      instrumentGlobals();
   }
 
   if (!TargetTriple.isAndroid()) {
     Constant *C = M.getOrInsertGlobal("__hwasan_tls", IntptrTy, [&] {
       auto *GV = new GlobalVariable(M, IntptrTy, /*isConstant=*/false,
                                     GlobalValue::ExternalLinkage, nullptr,
                                     "__hwasan_tls", nullptr,
                                     GlobalVariable::InitialExecTLSModel);
static uint64_t getAllocaSizeInBytes(const AllocaInst &AI) {
   }
   Type *Ty = AI.getAllocatedType();
   uint64_t SizeInBytes = AI.getModule()->getDataLayout().getTypeAllocSize(Ty);
   return SizeInBytes * ArraySize;
 }
 
 bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI,
                                    Value *Tag, size_t Size) {
-  size_t AlignedSize = alignTo(Size, Mapping.getAllocaAlignment());
+  size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
 
   Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty());
   if (ClInstrumentWithCalls) {
     IRB.CreateCall(HwasanTagMemoryFunc,
                    {IRB.CreatePointerCast(AI, Int8PtrTy), JustTag,
                     ConstantInt::get(IntptrTy, AlignedSize)});
   } else {
     size_t ShadowSize = Size >> Mapping.Scale;
     Value *ShadowPtr = memToShadow(IRB.CreatePointerCast(AI, IntptrTy), IRB);
     // If this memset is not inlined, it will be intercepted in the hwasan
     // runtime library. That's OK, because the interceptor skips the checks if
     // the address is in the shadow region.
     // FIXME: the interceptor is not as fast as real memset. Consider lowering
     // llvm.memset right here into either a sequence of stores, or a call to
     // hwasan_tag_memory.
     if (ShadowSize)
       IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, /*Align=*/1);
     if (Size != AlignedSize) {
       IRB.CreateStore(
-          ConstantInt::get(Int8Ty, Size % Mapping.getAllocaAlignment()),
+          ConstantInt::get(Int8Ty, Size % Mapping.getObjectAlignment()),
           IRB.CreateConstGEP1_32(Int8Ty, ShadowPtr, ShadowSize));
       IRB.CreateStore(JustTag, IRB.CreateConstGEP1_32(
                                    Int8Ty, IRB.CreateBitCast(AI, Int8PtrTy),
                                    AlignedSize - 1));
     }
   }
   return true;
 }
for (unsigned N = 0; N < Allocas.size(); ++N) {
     size_t Size = getAllocaSizeInBytes(*AI);
     tagAlloca(IRB, AI, Tag, Size);
 
     for (auto RI : RetVec) {
       IRB.SetInsertPoint(RI);
 
       // Re-tag alloca memory with the special UAR tag.
       Value *Tag = getUARTag(IRB, StackTag);
-      tagAlloca(IRB, AI, Tag, alignTo(Size, Mapping.getAllocaAlignment()));
+      tagAlloca(IRB, AI, Tag, alignTo(Size, Mapping.getObjectAlignment()));
     }
   }
 
   return true;
 }
 
 bool HWAddressSanitizer::isInterestingAlloca(const AllocaInst &AI) {
   return (AI.getAllocatedType()->isSized() &&
bool HWAddressSanitizer::sanitizeFunction(Function &F) {
   }
 
   // Pad and align each of the allocas that we instrumented to stop small
   // uninteresting allocas from hiding in instrumented alloca's padding and so
   // that we have enough space to store real tags for short granules.
   DenseMap<AllocaInst *, AllocaInst *> AllocaToPaddedAllocaMap;
   for (AllocaInst *AI : AllocasToInstrument) {
     uint64_t Size = getAllocaSizeInBytes(*AI);
-    uint64_t AlignedSize = alignTo(Size, Mapping.getAllocaAlignment());
-    AI->setAlignment(std::max(AI->getAlignment(), 16u));
+    uint64_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
+    AI->setAlignment(
+        std::max(AI->getAlignment(), Mapping.getObjectAlignment()));
     if (Size != AlignedSize) {
       Type *AllocatedType = AI->getAllocatedType();
       if (AI->isArrayAllocation()) {
         uint64_t ArraySize =
             cast<ConstantInt>(AI->getArraySize())->getZExtValue();
         AllocatedType = ArrayType::get(AllocatedType, ArraySize);
       }
       Type *TypeWithPadding = StructType::get(
for (auto Inst : ToInstrument)
     Changed |= instrumentMemAccess(Inst);
 
   LocalDynamicShadow = nullptr;
   StackBaseTag = nullptr;
 
   return Changed;
 }
 
+void HWAddressSanitizer::instrumentGlobal(GlobalVariable *GV, uint8_t Tag) {
+  Constant *Initializer = GV->getInitializer();
+  uint64_t SizeInBytes =
+      M.getDataLayout().getTypeAllocSize(Initializer->getType());
+  uint64_t NewSize = alignTo(SizeInBytes, Mapping.getObjectAlignment());
+  if (SizeInBytes != NewSize) {
+    // Pad the initializer out to the next multiple of 16 bytes and add the
+    // required short granule tag.
+    std::vector<uint8_t> Init(NewSize - SizeInBytes, 0);
+    Init.back() = Tag;
+    Constant *Padding = ConstantDataArray::get(*C, Init);
+    Initializer = ConstantStruct::getAnon({Initializer, Padding});
+  }
+
+  auto *NewGV = new GlobalVariable(M, Initializer->getType(), GV->isConstant(),
+                                   GlobalValue::ExternalLinkage, Initializer,
+                                   GV->getName() + ".hwasan");
+  NewGV->copyAttributesFrom(GV);
+  NewGV->setLinkage(GlobalValue::PrivateLinkage);
+  NewGV->copyMetadata(GV, 0);
+  NewGV->setAlignment(
+      std::max(GV->getAlignment(), Mapping.getObjectAlignment()));
+
+  // It is invalid to ICF two globals that have different tags. In the case
+  // where the size of the global is a multiple of the tag granularity the
+  // contents of the globals may be the same but the tags (i.e. symbol values)
+  // may be different, and the symbols are not considered during ICF. In the
+  // case where the size is not a multiple of the granularity, the short granule
+  // tags would discriminate two globals with different tags, but there would
+  // otherwise be nothing stopping such a global from being incorrectly ICF'd
+  // with an uninstrumented (i.e. tag 0) global that happened to have the short
+  // granule tag in the last byte.
+  NewGV->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
+
+  // Descriptor format (assuming little-endian):
+  // bytes 0-3: relative address of global
+  // bytes 4-6: size of global (16MB ought to be enough for anyone, but in case
+  // it isn't, we create multiple descriptors)
+  // byte 7: tag
+  auto *DescriptorTy = StructType::get(Int32Ty, Int32Ty);
+  const uint64_t MaxDescriptorSize = 0xfffff0;
+  for (uint64_t DescriptorPos = 0; DescriptorPos < SizeInBytes;
+       DescriptorPos += MaxDescriptorSize) {
+    auto *Descriptor =
+        new GlobalVariable(M, DescriptorTy, true, GlobalValue::PrivateLinkage,
+                           nullptr, GV->getName() + ".hwasan.descriptor");
+    auto *GVRelPtr = ConstantExpr::getTrunc(
+        ConstantExpr::getAdd(
+            ConstantExpr::getSub(
+                ConstantExpr::getPtrToInt(NewGV, Int64Ty),
+                ConstantExpr::getPtrToInt(Descriptor, Int64Ty)),
+            ConstantInt::get(Int64Ty, DescriptorPos)),
+        Int32Ty);
+    uint32_t Size = std::min(SizeInBytes - DescriptorPos, MaxDescriptorSize);
+    auto *SizeAndTag = ConstantInt::get(Int32Ty, Size | (uint32_t(Tag) << 24));
+    Descriptor->setComdat(NewGV->getComdat());
+    Descriptor->setInitializer(ConstantStruct::getAnon({GVRelPtr, SizeAndTag}));
+    Descriptor->setSection("hwasan_globals");
+    Descriptor->setMetadata(LLVMContext::MD_associated,
+                            MDNode::get(*C, ValueAsMetadata::get(NewGV)));
+    appendToCompilerUsed(M, Descriptor);
+  }
+
+  Constant *Aliasee = ConstantExpr::getIntToPtr(
+      ConstantExpr::getAdd(
+          ConstantExpr::getPtrToInt(NewGV, Int64Ty),
+          ConstantInt::get(Int64Ty, uint64_t(Tag) << kPointerTagShift)),
+      GV->getType());
+  auto *Alias = GlobalAlias::create(GV->getValueType(), GV->getAddressSpace(),
+                                    GV->getLinkage(), "", Aliasee, &M);
+  Alias->setVisibility(GV->getVisibility());
+  Alias->takeName(GV);
+  GV->replaceAllUsesWith(Alias);
+  GV->eraseFromParent();
+}
+
+void HWAddressSanitizer::instrumentGlobals() {
+  // Start by creating a note that contains pointers to the list of global
+  // descriptors. Adding a note to the output file will cause the linker to
+  // create a PT_NOTE program header pointing to the note that we can use to
+  // find the descriptor list starting from the program headers. A function
+  // provided by the runtime initializes the shadow memory for the globals by
+  // accessing the descriptor list via the note. The dynamic loader needs to
+  // call this function whenever a library is loaded.
+  //
+  // The reason why we use a note for this instead of a more conventional
+  // approach of having a global constructor pass a descriptor list pointer to
+  // the runtime is because of an order of initialization problem. With
+  // constructors we can encounter the following problematic scenario:
+  //
+  // 1) library A depends on library B and also interposes one of B's symbols
+  // 2) B's constructors are called before A's (as required for correctness)
+  // 3) during construction, B accesses one of its "own" globals (actually
+  //    interposed by A) and triggers a HWASAN failure due to the initialization
+  //    for A not having happened yet
+  //
+  // Even without interposition it is possible to run into similar situations in
+  // cases where two libraries mutually depend on each other.
+  //
+  // We only need one note per binary, so put everything for the note in a
+  // comdat.
+  Comdat *NoteComdat = M.getOrInsertComdat(kHwasanNoteName);
+
+  Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0);
+  auto Start =
+      new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
+                         nullptr, "__start_hwasan_globals");
+  Start->setVisibility(GlobalValue::HiddenVisibility);
+  Start->setDSOLocal(true);
+  auto Stop =
+      new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
+                         nullptr, "__stop_hwasan_globals");
+  Stop->setVisibility(GlobalValue::HiddenVisibility);
+  Stop->setDSOLocal(true);
+
+  // Null-terminated so actually 8 bytes, which are required in order to align
+  // the note properly.
+  auto *Name = ConstantDataArray::get(*C, "LLVM\0\0\0");
+
+  auto *NoteTy = StructType::get(Int32Ty, Int32Ty, Int32Ty, Name->getType(),
+                                 Int32Ty, Int32Ty);
+  auto *Note =
+      new GlobalVariable(M, NoteTy, /*isConstantGlobal=*/true,
+                         GlobalValue::PrivateLinkage, nullptr, kHwasanNoteName);
+  Note->setSection(".note.hwasan.globals");
+  Note->setComdat(NoteComdat);
+  Note->setAlignment(4);
+  Note->setDSOLocal(true);
+
+  // The pointers in the note need to be relative so that the note ends up being
+  // placed in rodata, which is the standard location for notes.
+  auto CreateRelPtr = [&](Constant *Ptr) {
+    return ConstantExpr::getTrunc(
+        ConstantExpr::getSub(ConstantExpr::getPtrToInt(Ptr, Int64Ty),
+                             ConstantExpr::getPtrToInt(Note, Int64Ty)),
+        Int32Ty);
+  };
+  Note->setInitializer(ConstantStruct::getAnon(
+      {ConstantInt::get(Int32Ty, 8),                           // n_namesz
+       ConstantInt::get(Int32Ty, 8),                           // n_descsz
+       ConstantInt::get(Int32Ty, ELF::NT_LLVM_HWASAN_GLOBALS), // n_type
+       Name, CreateRelPtr(Start), CreateRelPtr(Stop)}));
+  appendToCompilerUsed(M, Note);
+
+  // Create a zero-length global in hwasan_globals so that the linker will
+  // always create start and stop symbols.
+  auto Dummy = new GlobalVariable(
+      M, Int8Arr0Ty, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
+      Constant::getNullValue(Int8Arr0Ty), "hwasan.dummy.global");
+  Dummy->setSection("hwasan_globals");
+  Dummy->setComdat(NoteComdat);
+  Dummy->setMetadata(LLVMContext::MD_associated,
+                     MDNode::get(*C, ValueAsMetadata::get(Note)));
+  appendToCompilerUsed(M, Dummy);
+
+  std::vector<GlobalVariable *> Globals;
+  for (GlobalVariable &GV : M.globals()) {
+    if (GV.isDeclarationForLinker() || GV.getName().startswith("llvm.") ||
+        GV.isThreadLocal())
+      continue;
+
+    // Common symbols can't have aliases point to them, so they can't be tagged.
+    if (GV.hasCommonLinkage())
+      continue;
+
+    // Globals with custom sections may be used in __start_/__stop_ enumeration,
+    // which would be broken both by adding tags and potentially by the extra
+    // padding/alignment that we insert.
+    if (GV.hasSection())
+      continue;
+
+    Globals.push_back(&GV);
+  }
+
+  MD5 Hasher;
+  Hasher.update(M.getSourceFileName());
+  MD5::MD5Result Hash;
+  Hasher.final(Hash);
+  uint8_t Tag = Hash[0];
+
+  for (GlobalVariable *GV : Globals) {
+    // Skip tag 0 in order to avoid collisions with untagged memory.
+    if (Tag == 0)
+      Tag = 1;
+    instrumentGlobal(GV, Tag++);
+  }
+}
+
 void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple) {
   Scale = kDefaultShadowScale;
   if (ClMappingOffset.getNumOccurrences() > 0) {
     InGlobal = false;
     InTls = false;
     Offset = ClMappingOffset;
   } else if (ClEnableKhwasan || ClInstrumentWithCalls) {
     InGlobal = false;

llvm/trunk/test/Instrumentation/HWAddressSanitizer/globals.ll

+; RUN: opt < %s -S -hwasan -mtriple=aarch64--linux-android29 | FileCheck --check-prefix=CHECK29 %s
+; RUN: opt < %s -S -hwasan -mtriple=aarch64--linux-android30 | FileCheck --check-prefix=CHECK30 %s
+
+; CHECK29-NOT: @hwasan.note
+; CHECK29: @four = global
+
+; CHECK30: @__start_hwasan_globals = external hidden constant [0 x i8]
+; CHECK30: @__stop_hwasan_globals = external hidden constant [0 x i8]
+
+; CHECK30: @hwasan.note = private constant { i32, i32, i32, [8 x i8], i32, i32 } { i32 8, i32 8, i32 3, [8 x i8] c"LLVM\00\00\00\00", i32 trunc (i64 sub (i64 ptrtoint ([0 x i8]* @__start_hwasan_globals to i64), i64 ptrtoint ({ i32, i32, i32, [8 x i8], i32, i32 }* @hwasan.note to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint ([0 x i8]* @__stop_hwasan_globals to i64), i64 ptrtoint ({ i32, i32, i32, [8 x i8], i32, i32 }* @hwasan.note to i64)) to i32) }, section ".note.hwasan.globals", comdat, align 4
+
+; CHECK30: @hwasan.dummy.global = private constant [0 x i8] zeroinitializer, section "hwasan_globals", comdat($hwasan.note), !associated [[NOTE:![0-9]+]]
+
+; CHECK30: @four.hwasan = private global { i32, [12 x i8] } { i32 1, [12 x i8] c"\00\00\00\00\00\00\00\00\00\00\00\AC" }, align 16
+; CHECK30: @four.hwasan.descriptor = private constant { i32, i32 } { i32 trunc (i64 sub (i64 ptrtoint ({ i32, [12 x i8] }* @four.hwasan to i64), i64 ptrtoint ({ i32, i32 }* @four.hwasan.descriptor to i64)) to i32), i32 -1409286140 }, section "hwasan_globals", !associated [[FOUR:![0-9]+]]
+
+; CHECK30: @sixteen.hwasan = private global [16 x i8] zeroinitializer, align 16
+; CHECK30: @sixteen.hwasan.descriptor = private constant { i32, i32 } { i32 trunc (i64 sub (i64 ptrtoint ([16 x i8]* @sixteen.hwasan to i64), i64 ptrtoint ({ i32, i32 }* @sixteen.hwasan.descriptor to i64)) to i32), i32 -1392508912 }, section "hwasan_globals", !associated [[SIXTEEN:![0-9]+]]
+
+; CHECK30: @huge.hwasan = private global [16777232 x i8] zeroinitializer, align 16
+; CHECK30: @huge.hwasan.descriptor = private constant { i32, i32 } { i32 trunc (i64 sub (i64 ptrtoint ([16777232 x i8]* @huge.hwasan to i64), i64 ptrtoint ({ i32, i32 }* @huge.hwasan.descriptor to i64)) to i32), i32 -1358954512 }, section "hwasan_globals", !associated [[HUGE:![0-9]+]]
+; CHECK30: @huge.hwasan.descriptor.1 = private constant { i32, i32 } { i32 trunc (i64 add (i64 sub (i64 ptrtoint ([16777232 x i8]* @huge.hwasan to i64), i64 ptrtoint ({ i32, i32 }* @huge.hwasan.descriptor.1 to i64)), i64 16777200) to i32), i32 -1375731680 }, section "hwasan_globals", !associated [[HUGE]]
+
+; CHECK30: @four = alias i32, inttoptr (i64 add (i64 ptrtoint ({ i32, [12 x i8] }* @four.hwasan to i64), i64 -6052837899185946624) to i32*)
+; CHECK30: @sixteen = alias [16 x i8], inttoptr (i64 add (i64 ptrtoint ([16 x i8]* @sixteen.hwasan to i64), i64 -5980780305148018688) to [16 x i8]*)
+; CHECK30: @huge = alias [16777232 x i8], inttoptr (i64 add (i64 ptrtoint ([16777232 x i8]* @huge.hwasan to i64), i64 -5908722711110090752) to [16777232 x i8]*)
+
+; CHECK30: [[NOTE]] = !{{{{}} i32, i32, i32, [8 x i8], i32, i32 }* @hwasan.note}
+; CHECK30: [[FOUR]] = !{{{{}} i32, [12 x i8] }* @four.hwasan}
+; CHECK30: [[SIXTEEN]] = !{[16 x i8]* @sixteen.hwasan}
+; CHECK30: [[HUGE]] = !{[16777232 x i8]* @huge.hwasan}
+
+source_filename = "foo"
+
+@four = global i32 1
+@sixteen = global [16 x i8] zeroinitializer
+@huge = global [16777232 x i8] zeroinitializer