[asan] Make ASan compatible with linker dead stripping on Windows

[rnk]

Summary

This is similar to what was done for Darwin in rL264645 /
http://reviews.llvm.org/D16737, but it uses COFF COMDATs to achive the
same result instead of relying on new custom linker features.

As on MachO, this creates one metadata global per instrumented global.
The metadata global is placed in the custom .ASAN$GL section, which the
ASan runtime will iterate over during initialization. There are no other
references to the metadata, so normal linker dead stripping would
discard it. However, the metadata is put in a COMDAT group with the
instrumented global, so that it will be discarded if and only if the
instrumented global is discarded.

I didn't update the ASan ABI version check since this doesn't affect
non-Windows platforms, and the WinASan ABI isn't really stable yet.

Implementing this for ELF will require extending LLVM IR and MC a bit so
that we can use non-COMDAT section groups.

[rnk]

This passed check-asan, but it goes without saying that this needs an LLVM-only test too.

[rnk]

• Add instrumentation test

[inglorion]

lib/Transforms/Instrumentation/AddressSanitizer.cpp

1566

This would make it not use comdat metadata on ELF, right? Would it work to instead do the equivalent of

bool UseMachOGlobalsSection = ShouldUseMachOGlobalsSection();
bool UseComdatMetadata = !UseMachOGlobalsSection && TargetTriple.supportsCOMDAT();
bool UseMetadataArray = !(UseComdatMetadata || UseMachOGlobalsSection);

so that we get the MachO special implementation where applicable, else try to use comdats, and only fall back to the metadata array if neither of the other options will work?

[rnk]

lib/Transforms/Instrumentation/AddressSanitizer.cpp

1566

There's a bit more work to do for ELF:

1. COMDAT groups around internal symbols don't work the way I expected, this is what my comment about "Implementing this for ELF will require extending LLVM IR and MC a bit so that we can use non-COMDAT section groups." is about. If you enable this code on ELF, you end up with "relocation against discarded section" errors for strings and other static globals.
2. Once that's all prototyped, I need to do a bit more work to verify that --gc-sections actually discards unused globals and retains asan global metadata for used globals
3. We need to add an initializer to every DSO containing asan-instrumented code to register these globals. On Windows, users are required to statically link a small bit of ASan RTL code into every DLL that uses ASan, so I put the registration code there.

[rnk]

Ping for LLVM side

[vitalybuka]

LGTM

lib/Transforms/Instrumentation/AddressSanitizer.cpp

public:
   StringRef getPassName() const override { return "AddressSanitizerModule"; }
 
 private:
   void initializeCallbacks(Module &M);
 
   bool InstrumentGlobals(IRBuilder<> &IRB, Module &M);
   bool ShouldInstrumentGlobal(GlobalVariable *G);
   bool ShouldUseMachOGlobalsSection() const;
+  StringRef getGlobalMetadataSection() const;
   void poisonOneInitializer(Function &GlobalInit, GlobalValue *ModuleName);
   void createInitializerPoisonCalls(Module &M, GlobalValue *ModuleName);
   size_t MinRedzoneSizeForGlobal() const {
     return RedzoneSizeForScale(Mapping.Scale);
   }
 
   GlobalsMetadata GlobalsMD;
   bool CompileKernel;
bool AddressSanitizerModule::ShouldUseMachOGlobalsSection() const {
   if (TargetTriple.isiOS() /* or tvOS */ && !TargetTriple.isOSVersionLT(9))
     return true;
   if (TargetTriple.isWatchOS() && !TargetTriple.isOSVersionLT(2))
     return true;
 
   return false;
 }
 
+StringRef AddressSanitizerModule::getGlobalMetadataSection() const {
+  switch (TargetTriple.getObjectFormat()) {
+  case Triple::COFF:  return ".ASAN$GL";
+  case Triple::ELF:   return "asan_globals";
+  case Triple::MachO: return "__DATA,__asan_globals,regular";
+  default: break;
+  }
+  llvm_unreachable("unsupported object format");
+}
+
 void AddressSanitizerModule::initializeCallbacks(Module &M) {
   IRBuilder<> IRB(*C);
 
   // Declare our poisoning and unpoisoning functions.
   AsanPoisonGlobals = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
       kAsanPoisonGlobalsName, IRB.getVoidTy(), IntptrTy, nullptr));
   AsanPoisonGlobals->setLinkage(Function::ExternalLinkage);
   AsanUnpoisonGlobals = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M) {
 
   for (auto &G : M.globals()) {
     if (ShouldInstrumentGlobal(&G)) GlobalsToChange.push_back(&G);
   }
 
   size_t n = GlobalsToChange.size();
   if (n == 0) return false;
 
+  bool UseComdatMetadata = TargetTriple.isOSBinFormatCOFF();
+  bool UseMachOGlobalsSection = ShouldUseMachOGlobalsSection();
+  bool UseMetadataArray = !(UseComdatMetadata || UseMachOGlobalsSection);
+
   // A global is described by a structure
   //   size_t beg;
   //   size_t size;
   //   size_t size_with_redzone;
   //   const char *name;
   //   const char *module_name;
   //   size_t has_dynamic_init;
   //   void *source_location;
   //   size_t odr_indicator;
   // We initialize an array of such structures and pass it to a run-time call.
   StructType *GlobalStructTy =
       StructType::get(IntptrTy, IntptrTy, IntptrTy, IntptrTy, IntptrTy,
                       IntptrTy, IntptrTy, IntptrTy, nullptr);
-  SmallVector<Constant *, 16> Initializers(n);
+  SmallVector<Constant *, 16> Initializers(UseMetadataArray ? n : 0);
+
+  // On recent Mach-O platforms, use a structure which binds the liveness of
+  // the global variable to the metadata struct. Keep the list of "Liveness" GV
+  // created to be added to llvm.compiler.used
+  StructType *LivenessTy  = nullptr;
+  if (UseMachOGlobalsSection)
+    LivenessTy = StructType::get(IntptrTy, IntptrTy, nullptr);
+  SmallVector<GlobalValue *, 16> LivenessGlobals(
+      UseMachOGlobalsSection ? n : 0);
 
   bool HasDynamicallyInitializedGlobals = false;
 
   // We shouldn't merge same module names, as this string serves as unique
   // module ID in runtime.
   GlobalVariable *ModuleName = createPrivateGlobalForString(
       M, M.getModuleIdentifier(), /*AllowMerging*/ false);
 
for (size_t i = 0; i < n; i++) {
     Value *Indices2[2];
     Indices2[0] = IRB.getInt32(0);
     Indices2[1] = IRB.getInt32(0);
 
     G->replaceAllUsesWith(
         ConstantExpr::getGetElementPtr(NewTy, NewGlobal, Indices2, true));
     NewGlobal->takeName(G);
     G->eraseFromParent();
+    G = NewGlobal;
+
+    if (UseComdatMetadata) {
+      // Get or create a COMDAT for G so that we can use it with our metadata.
+      Comdat *C = G->getComdat();
+      if (!C) {
+        if (!G->hasName()) {
+          // If G is unnamed, it must be internal. Give it an artificial name
+          // so we can put it in a comdat.
+          assert(G->hasLocalLinkage());
+          G->setName(Twine(kAsanGenPrefix) + "_anon_global");
+        }
+        C = M.getOrInsertComdat(G->getName());
+        // Make this IMAGE_COMDAT_SELECT_NODUPLICATES on COFF.
+        if (TargetTriple.isOSBinFormatCOFF())
+          C->setSelectionKind(Comdat::NoDuplicates);
+        G->setComdat(C);
+      }
+    }
 
     Constant *SourceLoc;
     if (!MD.SourceLoc.empty()) {
       auto SourceLocGlobal = createPrivateGlobalForSourceLoc(M, MD.SourceLoc);
       SourceLoc = ConstantExpr::getPointerCast(SourceLocGlobal, IntptrTy);
     } else {
       SourceLoc = ConstantInt::get(IntptrTy, 0);
     }
if (CanUsePrivateAliases && ClUsePrivateAliasForGlobals) {
       // Set meaningful attributes for indicator symbol.
       ODRIndicatorSym->setVisibility(NewGlobal->getVisibility());
       ODRIndicatorSym->setDLLStorageClass(NewGlobal->getDLLStorageClass());
       ODRIndicatorSym->setAlignment(1);
       ODRIndicator = ODRIndicatorSym;
       InstrumentedGlobal = GA;
     }
 
-    Initializers[i] = ConstantStruct::get(
+    Constant *Initializer = ConstantStruct::get(
         GlobalStructTy,
         ConstantExpr::getPointerCast(InstrumentedGlobal, IntptrTy),
         ConstantInt::get(IntptrTy, SizeInBytes),
         ConstantInt::get(IntptrTy, SizeInBytes + RightRedzoneSize),
         ConstantExpr::getPointerCast(Name, IntptrTy),
         ConstantExpr::getPointerCast(ModuleName, IntptrTy),
         ConstantInt::get(IntptrTy, MD.IsDynInit), SourceLoc,
         ConstantExpr::getPointerCast(ODRIndicator, IntptrTy), nullptr);
 
     if (ClInitializers && MD.IsDynInit) HasDynamicallyInitializedGlobals = true;
 
     DEBUG(dbgs() << "NEW GLOBAL: " << *NewGlobal << "\n");
+
+    // If we aren't using separate metadata globals, add it to the initializer
+    // list and continue.
+    if (UseMetadataArray) {
+      Initializers[i] = Initializer;
+      continue;
+    }
+
+    // Create a separate metadata global and put it in the appropriate ASan
+    // global registration section.
+    GlobalVariable *Metadata = new GlobalVariable(
+        M, GlobalStructTy, false, GlobalVariable::InternalLinkage,
+        Initializer, Twine("__asan_global_") +
+                             GlobalValue::getRealLinkageName(G->getName()));
+    Metadata->setSection(getGlobalMetadataSection());
+    Metadata->setAlignment(1); // Don't leave padding in between.
+
+    // On platforms that support comdats, put the metadata and the
+    // instrumented global in the same group. This ensures that the metadata
+    // is discarded if the instrumented global is discarded.
+    if (UseComdatMetadata) {
+      assert(G->hasComdat());
+      Metadata->setComdat(G->getComdat());
+      continue;
+    }
+    assert(UseMachOGlobalsSection);
+
+    // On recent Mach-O platforms, we emit the global metadata in a way that
+    // allows the linker to properly strip dead globals.
+    auto LivenessBinder = ConstantStruct::get(
+        LivenessTy, Initializer->getAggregateElement(0u),
+        ConstantExpr::getPointerCast(Metadata, IntptrTy), nullptr);
+    GlobalVariable *Liveness = new GlobalVariable(
+        M, LivenessTy, false, GlobalVariable::InternalLinkage, LivenessBinder,
+        Twine("__asan_binder_") + G->getName());
+    Liveness->setSection("__DATA,__asan_liveness,regular,live_support");
+    LivenessGlobals[i] = Liveness;
   }
 
+  // Create calls for poisoning before initializers run and unpoisoning after.
+  if (HasDynamicallyInitializedGlobals)
+    createInitializerPoisonCalls(M, ModuleName);
+
+  // Platforms with a dedicated metadata section don't need to emit any more
+  // code.
+  if (UseComdatMetadata)
+    return true;
 
   GlobalVariable *AllGlobals = nullptr;
   GlobalVariable *RegisteredFlag = nullptr;
 
-  // On recent Mach-O platforms, we emit the global metadata in a way that
-  // allows the linker to properly strip dead globals.
-  if (ShouldUseMachOGlobalsSection()) {
+  if (UseMachOGlobalsSection) {
     // RegisteredFlag serves two purposes. First, we can pass it to dladdr()
     // to look up the loaded image that contains it. Second, we can store in it
     // whether registration has already occurred, to prevent duplicate
     // registration.
     //
-    // Common linkage allows us to coalesce needles defined in each object
-    // file so that there's only one per shared library.
+    // common linkage ensures that there is only one global per shared library.
     RegisteredFlag = new GlobalVariable(
         M, IntptrTy, false, GlobalVariable::CommonLinkage,
         ConstantInt::get(IntptrTy, 0), kAsanGlobalsRegisteredFlagName);
 
-    // We also emit a structure which binds the liveness of the global
-    // variable to the metadata struct.
-    StructType *LivenessTy = StructType::get(IntptrTy, IntptrTy, nullptr);
-
-    // Keep the list of "Liveness" GV created to be added to llvm.compiler.used
-    SmallVector<GlobalValue *, 16> LivenessGlobals;
-    LivenessGlobals.reserve(n);
-
-    for (size_t i = 0; i < n; i++) {
-      GlobalVariable *Metadata = new GlobalVariable(
-          M, GlobalStructTy, false, GlobalVariable::InternalLinkage,
-          Initializers[i], "");
-      Metadata->setSection("__DATA,__asan_globals,regular");
-      Metadata->setAlignment(1); // don't leave padding in between
-
-      auto LivenessBinder = ConstantStruct::get(LivenessTy,
-          Initializers[i]->getAggregateElement(0u),
-          ConstantExpr::getPointerCast(Metadata, IntptrTy),
-          nullptr);
-
-      // Recover the name of the variable this global is pointing to
-      StringRef GVName =
-          Initializers[i]->getAggregateElement(0u)->getOperand(0)->getName();
-
-      GlobalVariable *Liveness = new GlobalVariable(
-          M, LivenessTy, false, GlobalVariable::InternalLinkage, LivenessBinder,
-          Twine("__asan_binder_") + GVName);
-      Liveness->setSection("__DATA,__asan_liveness,regular,live_support");
-      LivenessGlobals.push_back(Liveness);
-    }
-
     // Update llvm.compiler.used, adding the new liveness globals. This is
     // needed so that during LTO these variables stay alive. The alternative
     // would be to have the linker handling the LTO symbols, but libLTO
     // current API does not expose access to the section for each symbol.
     if (!LivenessGlobals.empty())
       appendToCompilerUsed(M, LivenessGlobals);
-  } else {
-    // On all other platfoms, we just emit an array of global metadata
-    // structures.
+  } else if (UseMetadataArray) {
+    // On platforms that don't have a custom metadata section, we emit an array
+    // of global metadata structures.
     ArrayType *ArrayOfGlobalStructTy = ArrayType::get(GlobalStructTy, n);
     AllGlobals = new GlobalVariable(
         M, ArrayOfGlobalStructTy, false, GlobalVariable::InternalLinkage,
         ConstantArray::get(ArrayOfGlobalStructTy, Initializers), "");
   }
 
-  // Create calls for poisoning before initializers run and unpoisoning after.
-  if (HasDynamicallyInitializedGlobals)
-    createInitializerPoisonCalls(M, ModuleName);
-
   // Create a call to register the globals with the runtime.
-  if (ShouldUseMachOGlobalsSection()) {
+  if (UseMachOGlobalsSection) {
     IRB.CreateCall(AsanRegisterImageGlobals,
                    {IRB.CreatePointerCast(RegisteredFlag, IntptrTy)});
   } else {
     IRB.CreateCall(AsanRegisterGlobals,
                    {IRB.CreatePointerCast(AllGlobals, IntptrTy),
                     ConstantInt::get(IntptrTy, n)});
   }
 
   // We also need to unregister globals at the end, e.g., when a shared library
   // gets closed.
   Function *AsanDtorFunction =
       Function::Create(FunctionType::get(Type::getVoidTy(*C), false),
                        GlobalValue::InternalLinkage, kAsanModuleDtorName, &M);
   BasicBlock *AsanDtorBB = BasicBlock::Create(*C, "", AsanDtorFunction);
   IRBuilder<> IRB_Dtor(ReturnInst::Create(*C, AsanDtorBB));
 
-  if (ShouldUseMachOGlobalsSection()) {
+  if (UseMachOGlobalsSection) {
     IRB_Dtor.CreateCall(AsanUnregisterImageGlobals,
                         {IRB.CreatePointerCast(RegisteredFlag, IntptrTy)});
   } else {
     IRB_Dtor.CreateCall(AsanUnregisterGlobals,
                         {IRB.CreatePointerCast(AllGlobals, IntptrTy),
                          ConstantInt::get(IntptrTy, n)});
   }
 

test/Instrumentation/AddressSanitizer/global_metadata_darwin.ll

 @global = global [1 x i32] zeroinitializer, align 4
 
 !llvm.asan.globals = !{!0}
 
 !0 = !{[1 x i32]* @global, !1, !"global", i1 false, i1 false}
 !1 = !{!"test-globals.c", i32 1, i32 5}
 
 
-; Test that there is the flag global variable:
-; CHECK: @__asan_globals_registered = common global i64 0
-
 ; Find the metadata for @global:
-; CHECK: [[METADATA:@[0-9]+]] = internal global {{.*}} @global {{.*}} section "__DATA,__asan_globals,regular", align 1
+; CHECK: [[METADATA:@.+]] = internal global {{.*}} @global {{.*}} section "__DATA,__asan_globals,regular", align 1
 
 ; Find the liveness binder for @global and its metadata:
 ; CHECK: @__asan_binder_global = internal global {{.*}} @global {{.*}} [[METADATA]] {{.*}} section "__DATA,__asan_liveness,regular,live_support"
 
+; Test that there is the flag global variable:
+; CHECK: @__asan_globals_registered = common global i64 0
+
 ; The binder has to be inserted to llvm.compiler.used to avoid being stripped
 ; during LTO.
 ; CHECK: @llvm.compiler.used {{.*}} @__asan_binder_global {{.*}} section "llvm.metadata"
 
 
 ; Test that __asan_register_image_globals is invoked from the constructor:
 ; CHECK-LABEL: define internal void @asan.module_ctor
 ; CHECK-NOT: ret
 ; CHECK: call void @__asan_register_image_globals(i64 ptrtoint (i64* @__asan_globals_registered to i64))
 ; CHECK: ret
 
 ; Test that __asan_unregister_image_globals is invoked from the destructor:
 ; CHECK-LABEL: define internal void @asan.module_dtor
 ; CHECK-NOT: ret
 ; CHECK: call void @__asan_unregister_image_globals(i64 ptrtoint (i64* @__asan_globals_registered to i64))
 ; CHECK: ret

test/Instrumentation/AddressSanitizer/global_metadata_windows.ll

+; Test that global metadata is placed in a separate section on Windows, and that
+; it is in the same comdat group as the instrumented global. This ensures that
+; linker dead stripping (/OPT:REF) works as intended.
+
+; FIXME: Later we can use this to instrument linkonce odr string literals.
+
+; RUN: opt < %s -asan -asan-module -S | FileCheck %s
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+$mystr = comdat any
+
+; CHECK: $dead_global = comdat noduplicates
+; CHECK: @dead_global = local_unnamed_addr global { i32, [60 x i8] } { i32 42, [60 x i8] zeroinitializer }, comdat, align 32
+; CHECK: @__asan_global_dead_global = internal global { {{.*}} }, section ".ASAN$GL", comdat($dead_global), align 1
+
+@dead_global = local_unnamed_addr global i32 42, align 4
+@mystr = linkonce_odr unnamed_addr constant [5 x i8] c"main\00", comdat, align 1
+
+; Function Attrs: nounwind uwtable
+define i32 @main() local_unnamed_addr #0 {
+entry:
+  %call = tail call i32 @puts(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @mystr, i64 0, i64 0))
+  ret i32 0
+}
+
+; Function Attrs: nounwind
+declare i32 @puts(i8* nocapture readonly) local_unnamed_addr #1
+
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"PIC Level", i32 2}
+!1 = !{!"clang version 4.0.0 "}