[Windows] Symbolize with llvm-symbolizer instead of dbghelp in a self-host

[rnk]

Summary

llvm-symbolizer understands both PDBs and DWARF, so it is more likely to
succeed at symbolization. If llvm-symbolizer is unavailable, we will
fall back to dbghelp. This also makes our crash traces more similar
between Windows and Linux.

[zturner]

lib/Support/Signals.cpp

99–100

Does createTemporaryFile not open the file in such a way that it has FILE_FLAG_DELETE_ON_CLOSE or the equivalent on unix platforms?

138

os << format_hex() already behaves consistently. It's not a big deal, but it seems better to use type-safe formatting whenever possible, and I think format_hex would work just fine in that regard here.

lib/Support/Windows/Signals.inc

282–283

Seems like this code could move after the #ifdef, no need to initialize anything if we end up not using Dbghelp anyway

289–292

Why only when self-hosting? Wouldn't it be worth it to try llvm-symbolizer no matter what?

[rnk]

lib/Support/Signals.cpp

99–100

Apparently not. =/

138

Cool, I switched to that.

lib/Support/Windows/Signals.inc

282–283

StackWalk64 is part of dbghelp, and we need to initialize it first.

289–292

Yeah, I'll get rid of that. My idea was to avoid changing behavior for a standard Windows build, but we might as well do it.

[rnk]

• Use format_hex instead of format()

lib/Support/Signals.cpp

 //===- Signals.cpp - Signal Handling support --------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file defines some helpful functions for dealing with the possibility of
 // Unix signals occurring while your program is running.
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Config/config.h"
+#include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
+#include "llvm/Support/Format.h"
 #include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Mutex.h"
+#include "llvm/Support/Program.h"
 #include "llvm/Support/Signals.h"
-
+#include "llvm/Support/StringSaver.h"
+#include "llvm/Support/raw_ostream.h"
 #include <vector>
 
 namespace llvm {
 using namespace sys;
 
 //===----------------------------------------------------------------------===//
 //=== WARNING: Implementation here must contain only TRULY operating system
 //===          independent code.
 //===----------------------------------------------------------------------===//
 
 static ManagedStatic<std::vector<std::pair<void (*)(void *), void *>>>
     CallBacksToRun;
 void sys::RunSignalHandlers() {
   if (!CallBacksToRun.isConstructed())
     return;
   for (auto &I : *CallBacksToRun)
     I.first(I.second);
   CallBacksToRun->clear();
 }
 }
 
+using namespace llvm;
+
+static bool findModulesAndOffsets(void **StackTrace, int Depth,
+                                  const char **Modules, intptr_t *Offsets,
+                                  const char *MainExecutableName,
+                                  StringSaver &StrPool);
+
+/// Format a pointer value as hexadecimal. Zero pad it out so its always the
+/// same width.
+static FormattedNumber format_ptr(void *PC) {
+  // Each byte is two hex digits plus 2 for the 0x prefix.
+  unsigned PtrWidth = 2 + 2 * sizeof(void *);
+  return format_hex((uint64_t)PC, PtrWidth);
+}
+
+/// Helper that launches llvm-symbolizer and symbolizes a backtrace.
+static bool printSymbolizedStackTrace(void **StackTrace, int Depth,
+                                      llvm::raw_ostream &OS) {
+  // FIXME: Subtract necessary number from StackTrace entries to turn return addresses
+  // into actual instruction addresses.
+  // Use llvm-symbolizer tool to symbolize the stack traces.
+  ErrorOr<std::string> LLVMSymbolizerPathOrErr =
+      sys::findProgramByName("llvm-symbolizer");
+  if (!LLVMSymbolizerPathOrErr)
+    return false;
+  const std::string &LLVMSymbolizerPath = *LLVMSymbolizerPathOrErr;
+  // We don't know argv0 or the address of main() at this point, but try
+  // to guess it anyway (it's possible on some platforms).
+  std::string MainExecutableName = sys::fs::getMainExecutable(nullptr, nullptr);
+  if (MainExecutableName.empty() ||
+      MainExecutableName.find("llvm-symbolizer") != std::string::npos)
+    return false;
+
+  BumpPtrAllocator Allocator;
+  StringSaver StrPool(Allocator);
+  std::vector<const char *> Modules(Depth, nullptr);
+  std::vector<intptr_t> Offsets(Depth, 0);
+  if (!findModulesAndOffsets(StackTrace, Depth, Modules.data(), Offsets.data(),
+                             MainExecutableName.c_str(), StrPool))
+    return false;
+  int InputFD;
+  SmallString<32> InputFile, OutputFile;
+  sys::fs::createTemporaryFile("symbolizer-input", "", InputFD, InputFile);
+  sys::fs::createTemporaryFile("symbolizer-output", "", OutputFile);
+  FileRemover InputRemover(InputFile.c_str());
+  FileRemover OutputRemover(OutputFile.c_str());
+
+  {
+    raw_fd_ostream Input(InputFD, true);
+    for (int i = 0; i < Depth; i++) {
+      if (Modules[i])
+        Input << Modules[i] << " " << (void*)Offsets[i] << "\n";
+    }
+  }
+
+  StringRef InputFileStr(InputFile);
+  StringRef OutputFileStr(OutputFile);
+  StringRef StderrFileStr;
+  const StringRef *Redirects[] = {&InputFileStr, &OutputFileStr,
+                                  &StderrFileStr};
+  const char *Args[] = {"llvm-symbolizer", "--functions=linkage", "--inlining",
+#ifdef LLVM_ON_WIN32
+                        // Pass --relative-address on Windows so that we don't
+                        // have to add ImageBase from PE file.
+                        // FIXME: Make this the default for llvm-symbolizer.
+                        "--relative-address",
+#endif
+                        "--demangle", nullptr};
+  int RunResult =
+      sys::ExecuteAndWait(LLVMSymbolizerPath, Args, nullptr, Redirects);
+  if (RunResult != 0)
+    return false;
+
+  // This report format is based on the sanitizer stack trace printer.  See
+  // sanitizer_stacktrace_printer.cc in compiler-rt.
+  auto OutputBuf = MemoryBuffer::getFile(OutputFile.c_str());
+  if (!OutputBuf)
+    return false;
+  StringRef Output = OutputBuf.get()->getBuffer();
+  SmallVector<StringRef, 32> Lines;
+  Output.split(Lines, "\n");
+  auto CurLine = Lines.begin();
+  int frame_no = 0;
+  for (int i = 0; i < Depth; i++) {
+    if (!Modules[i]) {
+      OS << '#' << frame_no++ << ' ' << format_ptr(StackTrace[i]) << '\n';
+      continue;
+    }
+    // Read pairs of lines (function name and file/line info) until we
+    // encounter empty line.
+    for (;;) {
+      if (CurLine == Lines.end())
+        return false;
+      StringRef FunctionName = *CurLine++;
+      if (FunctionName.empty())
+        break;
+      OS << '#' << frame_no++ << ' ' << format_ptr(StackTrace[i]) << ' ';
+      if (!FunctionName.startswith("??"))
+        OS << FunctionName << ' ';
+      if (CurLine == Lines.end())
+        return false;
+      StringRef FileLineInfo = *CurLine++;
+      if (!FileLineInfo.startswith("??"))
+        OS << FileLineInfo;
+      else
+        OS << "(" << Modules[i] << '+' << format_hex(Offsets[i], 0) << ")";
+      OS << "\n";
+    }
+  }
+  return true;
+}
+
 // Include the platform-specific parts of this class.
 #ifdef LLVM_ON_UNIX
 #include "Unix/Signals.inc"
 #endif
 #ifdef LLVM_ON_WIN32
 #include "Windows/Signals.inc"
 #endif

lib/Support/Unix/Signals.inc

for (int j = 0; j < data->depth; j++) {
       }
     }
   }
   return 0;
 }
 
 static bool findModulesAndOffsets(void **StackTrace, int Depth,
                                   const char **Modules, intptr_t *Offsets,
-                                  const char *MainExecutableName) {
+                                  const char *MainExecutableName,
+                                  StringSaver &StrPool) {
   DlIteratePhdrData data = {StackTrace, Depth,   true,
                             Modules,    Offsets, MainExecutableName};
   dl_iterate_phdr(dl_iterate_phdr_cb, &data);
   return true;
 }
 #else
 static bool findModulesAndOffsets(void **StackTrace, int Depth,
                                   const char **Modules, intptr_t *Offsets,
                                   const char *MainExecutableName) {
   return false;
 }
 #endif
-
-static bool printSymbolizedStackTrace(void **StackTrace, int Depth,
-                                      llvm::raw_ostream &OS) {
-  // FIXME: Subtract necessary number from StackTrace entries to turn return addresses
-  // into actual instruction addresses.
-  // Use llvm-symbolizer tool to symbolize the stack traces.
-  ErrorOr<std::string> LLVMSymbolizerPathOrErr =
-      sys::findProgramByName("llvm-symbolizer");
-  if (!LLVMSymbolizerPathOrErr)
-    return false;
-  const std::string &LLVMSymbolizerPath = *LLVMSymbolizerPathOrErr;
-  // We don't know argv0 or the address of main() at this point, but try
-  // to guess it anyway (it's possible on some platforms).
-  std::string MainExecutableName = sys::fs::getMainExecutable(nullptr, nullptr);
-  if (MainExecutableName.empty() ||
-      MainExecutableName.find("llvm-symbolizer") != std::string::npos)
-    return false;
-
-  std::vector<const char *> Modules(Depth, nullptr);
-  std::vector<intptr_t> Offsets(Depth, 0);
-  if (!findModulesAndOffsets(StackTrace, Depth, Modules.data(), Offsets.data(),
-                             MainExecutableName.c_str()))
-    return false;
-  int InputFD;
-  SmallString<32> InputFile, OutputFile;
-  sys::fs::createTemporaryFile("symbolizer-input", "", InputFD, InputFile);
-  sys::fs::createTemporaryFile("symbolizer-output", "", OutputFile);
-  FileRemover InputRemover(InputFile.c_str());
-  FileRemover OutputRemover(OutputFile.c_str());
-
-  {
-    raw_fd_ostream Input(InputFD, true);
-    for (int i = 0; i < Depth; i++) {
-      if (Modules[i])
-        Input << Modules[i] << " " << (void*)Offsets[i] << "\n";
-    }
-  }
-
-  StringRef InputFileStr(InputFile);
-  StringRef OutputFileStr(OutputFile);
-  StringRef StderrFileStr;
-  const StringRef *Redirects[] = {&InputFileStr, &OutputFileStr,
-                                  &StderrFileStr};
-  const char *Args[] = {"llvm-symbolizer", "--functions=linkage", "--inlining",
-                        "--demangle", nullptr};
-  int RunResult =
-      sys::ExecuteAndWait(LLVMSymbolizerPath, Args, nullptr, Redirects);
-  if (RunResult != 0)
-    return false;
-
-  auto OutputBuf = MemoryBuffer::getFile(OutputFile.c_str());
-  if (!OutputBuf)
-    return false;
-  StringRef Output = OutputBuf.get()->getBuffer();
-  SmallVector<StringRef, 32> Lines;
-  Output.split(Lines, "\n");
-  auto CurLine = Lines.begin();
-  int frame_no = 0;
-  for (int i = 0; i < Depth; i++) {
-    if (!Modules[i]) {
-      OS << format("#%d %p\n", frame_no++, StackTrace[i]);
-      continue;
-    }
-    // Read pairs of lines (function name and file/line info) until we
-    // encounter empty line.
-    for (;;) {
-      if (CurLine == Lines.end())
-        return false;
-      StringRef FunctionName = *CurLine++;
-      if (FunctionName.empty())
-        break;
-      OS << format("#%d %p ", frame_no++, StackTrace[i]);
-      if (!FunctionName.startswith("??"))
-        OS << format("%s ", FunctionName.str().c_str());
-      if (CurLine == Lines.end())
-        return false;
-      StringRef FileLineInfo = *CurLine++;
-      if (!FileLineInfo.startswith("??"))
-        OS << format("%s", FileLineInfo.str().c_str());
-      else
-        OS << format("(%s+%p)", Modules[i], (void *)Offsets[i]);
-      OS << "\n";
-    }
-  }
-  return true;
-}
 #endif // defined(HAVE_BACKTRACE) && defined(ENABLE_BACKTRACES)
 
 // PrintStackTrace - In the case of a program crash or fault, print out a stack
 // trace so that the user has an indication of why and where we died.
 //
 // On glibc systems we have the 'backtrace' function, which works nicely, but
 // doesn't demangle symbols.
 void llvm::sys::PrintStackTrace(raw_ostream &OS) {

lib/Support/Windows/Signals.inc

 typedef BOOL (WINAPI *fpSymGetSymFromAddr64)(HANDLE, DWORD64,
                       PDWORD64, PIMAGEHLP_SYMBOL64);
 static fpSymGetSymFromAddr64 fSymGetSymFromAddr64;
 
 typedef BOOL (WINAPI *fpSymGetLineFromAddr64)(HANDLE, DWORD64,
                       PDWORD, PIMAGEHLP_LINE64);
 static fpSymGetLineFromAddr64 fSymGetLineFromAddr64;
 
+typedef BOOL(WINAPI *fpSymGetModuleInfo64)(HANDLE hProcess, DWORD64 dwAddr,
+                                           PIMAGEHLP_MODULE64 ModuleInfo);
+static fpSymGetModuleInfo64 fSymGetModuleInfo64;
+
 typedef PVOID (WINAPI *fpSymFunctionTableAccess64)(HANDLE, DWORD64);
 static fpSymFunctionTableAccess64 fSymFunctionTableAccess64;
 
 typedef DWORD (WINAPI *fpSymSetOptions)(DWORD);
 static fpSymSetOptions fSymSetOptions;
 
 typedef BOOL (WINAPI *fpSymInitialize)(HANDLE, PCSTR, BOOL);
 static fpSymInitialize fSymInitialize;
 
+typedef BOOL (WINAPI *fpEnumerateLoadedModules)(HANDLE,PENUMLOADED_MODULES_CALLBACK64,PVOID);
+static fpEnumerateLoadedModules fEnumerateLoadedModules;
+
 static bool load64BitDebugHelp(void) {
   HMODULE hLib = ::LoadLibraryW(L"Dbghelp.dll");
   if (hLib) {
     fStackWalk64 = (fpStackWalk64)
                       ::GetProcAddress(hLib, "StackWalk64");
     fSymGetModuleBase64 = (fpSymGetModuleBase64)
                       ::GetProcAddress(hLib, "SymGetModuleBase64");
     fSymGetSymFromAddr64 = (fpSymGetSymFromAddr64)
                       ::GetProcAddress(hLib, "SymGetSymFromAddr64");
     fSymGetLineFromAddr64 = (fpSymGetLineFromAddr64)
                       ::GetProcAddress(hLib, "SymGetLineFromAddr64");
+    fSymGetModuleInfo64 = (fpSymGetModuleInfo64)
+                      ::GetProcAddress(hLib, "SymGetModuleInfo64");
     fSymFunctionTableAccess64 = (fpSymFunctionTableAccess64)
                      ::GetProcAddress(hLib, "SymFunctionTableAccess64");
     fSymSetOptions = (fpSymSetOptions)::GetProcAddress(hLib, "SymSetOptions");
     fSymInitialize = (fpSymInitialize)::GetProcAddress(hLib, "SymInitialize");
+    fEnumerateLoadedModules = (fpEnumerateLoadedModules)
+      ::GetProcAddress(hLib, "EnumerateLoadedModules64");
   }
   return fStackWalk64 && fSymInitialize && fSymSetOptions;
 }
 
 using namespace llvm;
 
 // Forward declare.
 static LONG WINAPI LLVMUnhandledExceptionFilter(LPEXCEPTION_POINTERS ep);
 static BOOL WINAPI LLVMConsoleCtrlHandler(DWORD dwCtrlType);
 
 // InterruptFunction - The function to call if ctrl-c is pressed.
 static void (*InterruptFunction)() = 0;
 
 static std::vector<std::string> *FilesToRemove = NULL;
 static bool RegisteredUnhandledExceptionFilter = false;
 static bool CleanupExecuted = false;
 static PTOP_LEVEL_EXCEPTION_FILTER OldFilter = NULL;
 
 // Windows creates a new thread to execute the console handler when an event
 // (such as CTRL/C) occurs.  This causes concurrency issues with the above
 // globals which this critical section addresses.
 static CRITICAL_SECTION CriticalSection;
 static bool CriticalSectionInitialized = false;
 
-static void PrintStackTraceForThread(llvm::raw_ostream &OS, HANDLE hProcess,
-                                     HANDLE hThread, STACKFRAME64 &StackFrame,
-                                     CONTEXT *Context) {
-  DWORD machineType;
+enum {
 #if defined(_M_X64)
-  machineType = IMAGE_FILE_MACHINE_AMD64;
+  NativeMachineType = IMAGE_FILE_MACHINE_AMD64
 #else
-  machineType = IMAGE_FILE_MACHINE_I386;
+  NativeMachineType = IMAGE_FILE_MACHINE_I386
 #endif
+};
 
+static bool printStackTraceWithLLVMSymbolizer(llvm::raw_ostream &OS,
+                                              HANDLE hProcess, HANDLE hThread,
+                                              STACKFRAME64 &StackFrameOrig,
+                                              CONTEXT *ContextOrig) {
+  // StackWalk64 modifies the incoming stack frame and context, so copy them.
+  STACKFRAME64 StackFrame = StackFrameOrig;
+
+  // Copy the register context so that we don't modify it while we unwind. We
+  // could use InitializeContext + CopyContext, but that's only required to get
+  // at AVX registers, which typically aren't needed by StackWalk64. Reduce the
+  // flag set to indicate that there's less data.
+  CONTEXT Context = *ContextOrig;
+  Context.ContextFlags = CONTEXT_CONTROL | CONTEXT_INTEGER;
+
+  static void *StackTrace[256];
+  int Depth = 0;
+  while (fStackWalk64(NativeMachineType, hProcess, hThread, &StackFrame,
+                      &Context, 0, fSymFunctionTableAccess64,
+                      fSymGetModuleBase64, 0)) {
+    if (StackFrame.AddrFrame.Offset == 0)
+      break;
+    StackTrace[Depth++] = (void *)(uintptr_t)StackFrame.AddrPC.Offset;
+    if (Depth >= array_lengthof(StackTrace))
+      break;
+  }
+
+  return printSymbolizedStackTrace(&StackTrace[0], Depth, OS);
+}
+
+namespace {
+struct FindModuleData {
+  void **StackTrace;
+  int Depth;
+  const char **Modules;
+  intptr_t *Offsets;
+  StringSaver *StrPool;
+};
+}
+
+static BOOL CALLBACK findModuleCallback(WIN32_ELMCB_PCSTR ModuleName,
+                                        DWORD64 ModuleBase, ULONG ModuleSize,
+                                        void *VoidData) {
+  FindModuleData *Data = (FindModuleData*)VoidData;
+  intptr_t Beg = ModuleBase;
+  intptr_t End = Beg + ModuleSize;
+  for (int I = 0; I < Data->Depth; I++) {
+    if (Data->Modules[I])
+      continue;
+    intptr_t Addr = (intptr_t)Data->StackTrace[I];
+    if (Beg <= Addr && Addr < End) {
+      Data->Modules[I] = Data->StrPool->save(ModuleName);
+      Data->Offsets[I] = Addr - Beg;
+    }
+  }
+  return TRUE;
+}
+
+static bool findModulesAndOffsets(void **StackTrace, int Depth,
+                                  const char **Modules, intptr_t *Offsets,
+                                  const char *MainExecutableName,
+                                  StringSaver &StrPool) {
+  if (!fEnumerateLoadedModules)
+    return false;
+  FindModuleData Data;
+  Data.StackTrace = StackTrace;
+  Data.Depth = Depth;
+  Data.Modules = Modules;
+  Data.Offsets = Offsets;
+  Data.StrPool = &StrPool;
+  fEnumerateLoadedModules(GetCurrentProcess(), findModuleCallback, &Data);
+  return true;
+}
+
+static void PrintStackTraceForThread(llvm::raw_ostream &OS, HANDLE hProcess,
+                                     HANDLE hThread, STACKFRAME64 &StackFrame,
+                                     CONTEXT *Context) {
   // Initialize the symbol handler.
   fSymSetOptions(SYMOPT_DEFERRED_LOADS | SYMOPT_LOAD_LINES);
   fSymInitialize(hProcess, NULL, TRUE);
 
+  // Try llvm-symbolizer first. llvm-symbolizer knows how to deal with both PDBs
+  // and DWARF, so it should do a good job regardless of what debug info or
+  // linker is in use.
+  if (printStackTraceWithLLVMSymbolizer(OS, hProcess, hThread, StackFrame,
+                                        Context)) {
+    return;
+  }
+
   while (true) {
-    if (!fStackWalk64(machineType, hProcess, hThread, &StackFrame, Context, 0,
-                      fSymFunctionTableAccess64, fSymGetModuleBase64, 0)) {
+    if (!fStackWalk64(NativeMachineType, hProcess, hThread, &StackFrame,
+                      Context, 0, fSymFunctionTableAccess64,
+                      fSymGetModuleBase64, 0)) {
       break;
     }
 
     if (StackFrame.AddrFrame.Offset == 0)
       break;
 
     using namespace llvm;
     // Print the PC in hexadecimal.

 
 #if defined(__MINGW32__) && !defined(__MINGW64_VERSION_MAJOR)
 // Provide a prototype for RtlCaptureContext, mingw32 from mingw.org is
 // missing it but mingw-w64 has it.
 extern "C" VOID WINAPI RtlCaptureContext(PCONTEXT ContextRecord);
 #endif
 
 void llvm::sys::PrintStackTrace(raw_ostream &OS) {
-
   STACKFRAME64 StackFrame = {};
   CONTEXT Context = {};
   ::RtlCaptureContext(&Context);
 #if defined(_M_X64)
   StackFrame.AddrPC.Offset = Context.Rip;
   StackFrame.AddrStack.Offset = Context.Rsp;
   StackFrame.AddrFrame.Offset = Context.Rbp;
 #else