[lldb/linux] Handle main thread exits

[labath]

Summary

This patch handles the situation where the main thread exits (through
the SYS_exit syscall). In this case, the process as a whole continues
running until all of the other threads exit, or one of them issues an
exit_group syscall.

The patch consists of two changes:

• a moderate redesign of the handling of thread exit (WIFEXITED) events. Previously, we were removing (forgetting) a thread once we received the WIFEXITED (or WIFSIGNALED) event. This was problematic for the main thread, since the main thread WIFEXITED event (which is better thought of as a process-wide event) gets reported only after the entire process exits. This resulted in deadlocks, where we were waiting for the process to stop (because we still considered the main thread "live").
  
  This patch changes the logic such that the main thread is removed as soon as its PTRACE_EVENT_EXIT (the pre-exit) event is received. At this point we can consider the thread gone (for most purposes). As a corrolary, I needed to add special logic to catch process-wide exit events in the cases where we don't have the main thread around.

• The second part of the patch is the removal of the assumptions that the main thread is always available. This generally meant replacing the uses of GetThreadByID(process_id) with GetCurrentThread() in various process-wide operations (such as memory reads).

[mgorny]

Makes sense.

lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp

LLDB_LOG(log,
              "got exit status({0}) , tid = {1} ({2} main thread), process "
              "state = {3}",
              status, thread.GetID(), is_main_thread ? "is" : "is not",
              GetState());
 
     // This is a thread that exited.  Ensure we're not tracking it anymore.
     StopTrackingThread(thread);
 
-    if (is_main_thread) {
-      // The main thread exited.  We're done monitoring.  Report to delegate.
-      SetExitStatus(status, true);
-
-      // Notify delegate that our process has exited.
-      SetState(StateType::eStateExited, true);
-    }
+    assert(!is_main_thread && "Main thread exits handled elsewhere");
     return;
   }
 
   siginfo_t info;
   const auto info_err = GetSignalInfo(thread.GetID(), &info);
 
   // Get details on the signal raised.
   if (info_err.Success()) {
if (!StateIsRunningState(state)) {
       // gets a SIGKILL. This confuses our state tracking logic in
       // ResumeThread(), since normally, we should not be receiving any ptrace
       // events while the inferior is stopped. This makes sure that the
       // inferior is resumed and exits normally.
       state = eStateRunning;
     }
     ResumeThread(thread, state, LLDB_INVALID_SIGNAL_NUMBER);
 
+    if (is_main_thread) {
+      // Main thread report the read (WIFEXITED) event only after all threads in
+      // the process exit, so we need to stop tracking it here instead of in
+      // MonitorCallback
+      StopTrackingThread(thread);
+    }
+
     break;
   }
 
   case (SIGTRAP | (PTRACE_EVENT_VFORK_DONE << 8)): {
     if (bool(m_enabled_extensions & Extension::vfork)) {
       thread.SetStoppedByVForkDone();
       StopRunningThreads(thread.GetID());
     }
LinuxMapCallback callback = [&](llvm::Expected<MemoryRegionInfo> Info) {
     Result = Info.takeError();
     m_supports_mem_region = LazyBool::eLazyBoolNo;
     LLDB_LOG(log, "failed to parse proc maps: {0}", Result);
     return false;
   };
 
   // Linux kernel since 2.6.14 has /proc/{pid}/smaps
   // if CONFIG_PROC_PAGE_MONITOR is enabled
-  auto BufferOrError = getProcFile(GetID(), "smaps");
+  auto BufferOrError = getProcFile(GetID(), GetCurrentThreadID(), "smaps");
   if (BufferOrError)
     ParseLinuxSMapRegions(BufferOrError.get()->getBuffer(), callback);
   else {
-    BufferOrError = getProcFile(GetID(), "maps");
+    BufferOrError = getProcFile(GetID(), GetCurrentThreadID(), "maps");
     if (!BufferOrError) {
       m_supports_mem_region = LazyBool::eLazyBoolNo;
       return BufferOrError.getError();
     }
 
     ParseLinuxMapRegions(BufferOrError.get()->getBuffer(), callback);
   }
 
auto region_it = llvm::find_if(m_mem_region_cache, [](const auto &pair) {
     return pair.first.GetExecutable() == MemoryRegionInfo::eYes;
   });
   if (region_it == m_mem_region_cache.end())
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "No executable memory region found!");
 
   addr_t exe_addr = region_it->first.GetRange().GetRangeBase();
 
-  NativeThreadLinux &thread = *GetThreadByID(GetID());
+  NativeThreadLinux &thread = *GetCurrentThread();
   assert(thread.GetState() == eStateStopped);
   NativeRegisterContextLinux &reg_ctx = thread.GetRegisterContext();
 
   NativeRegisterContextLinux::SyscallData syscall_data =
       *reg_ctx.GetSyscallData();
 
   DataBufferSP registers_sp;
   if (llvm::Error Err = reg_ctx.ReadAllRegisterValues(registers_sp).ToError())
Status NativeProcessLinux::ReadMemoryTags(int32_t type, lldb::addr_t addr,
 
   // This call can return partial data so loop until we error or
   // get all tags back.
   while (num_tags) {
     tags_iovec.iov_base = dest;
     tags_iovec.iov_len = num_tags;
 
     Status error = NativeProcessLinux::PtraceWrapper(
-        details->ptrace_read_req, GetID(), reinterpret_cast<void *>(read_addr),
-        static_cast<void *>(&tags_iovec), 0, nullptr);
+        details->ptrace_read_req, GetCurrentThreadID(),
+        reinterpret_cast<void *>(read_addr), static_cast<void *>(&tags_iovec),
+        0, nullptr);
 
     if (error.Fail()) {
       // Discard partial reads
       tags.resize(0);
       return error;
     }
 
     size_t tags_read = tags_iovec.iov_len;
Status NativeProcessLinux::WriteMemoryTags(int32_t type, lldb::addr_t addr,
 
   // This call can partially write tags, so we loop until we
   // error or all tags have been written.
   while (num_tags > 0) {
     tags_vec.iov_base = src;
     tags_vec.iov_len = num_tags;
 
     Status error = NativeProcessLinux::PtraceWrapper(
-        details->ptrace_write_req, GetID(),
+        details->ptrace_write_req, GetCurrentThreadID(),
         reinterpret_cast<void *>(write_addr), static_cast<void *>(&tags_vec), 0,
         nullptr);
 
     if (error.Fail()) {
       // Don't attempt to restore the original values in the case of a partial
       // write
       return error;
     }

 }
 
 Status NativeProcessLinux::ReadMemory(lldb::addr_t addr, void *buf, size_t size,
                                       size_t &bytes_read) {
   if (ProcessVmReadvSupported()) {
     // The process_vm_readv path is about 50 times faster than ptrace api. We
     // want to use this syscall if it is supported.
 
-    const ::pid_t pid = GetID();
-
     struct iovec local_iov, remote_iov;
     local_iov.iov_base = buf;
     local_iov.iov_len = size;
     remote_iov.iov_base = reinterpret_cast<void *>(addr);
     remote_iov.iov_len = size;
 
-    bytes_read = process_vm_readv(pid, &local_iov, 1, &remote_iov, 1, 0);
+    bytes_read = process_vm_readv(GetCurrentThreadID(), &local_iov, 1,
+                                  &remote_iov, 1, 0);
     const bool success = bytes_read == size;
 
     Log *log = GetLog(POSIXLog::Process);
     LLDB_LOG(log,
              "using process_vm_readv to read {0} bytes from inferior "
              "address {1:x}: {2}",
              size, addr, success ? "Success" : llvm::sys::StrError(errno));
 
     if (success)
       return Status();
     // else the call failed for some reason, let's retry the read using ptrace
     // api.
   }
 
   unsigned char *dst = static_cast<unsigned char *>(buf);
   size_t remainder;
   long data;
 
   Log *log = GetLog(POSIXLog::Memory);
   LLDB_LOG(log, "addr = {0}, buf = {1}, size = {2}", addr, buf, size);
 
   for (bytes_read = 0; bytes_read < size; bytes_read += remainder) {
     Status error = NativeProcessLinux::PtraceWrapper(
-        PTRACE_PEEKDATA, GetID(), (void *)addr, nullptr, 0, &data);
+        PTRACE_PEEKDATA, GetCurrentThreadID(), (void *)addr, nullptr, 0, &data);
     if (error.Fail())
       return error;
 
     remainder = size - bytes_read;
     remainder = remainder > k_ptrace_word_size ? k_ptrace_word_size : remainder;
 
     // Copy the data into our buffer
     memcpy(dst, &data, remainder);
for (bytes_written = 0; bytes_written < size; bytes_written += remainder) {
     remainder = size - bytes_written;
     remainder = remainder > k_ptrace_word_size ? k_ptrace_word_size : remainder;
 
     if (remainder == k_ptrace_word_size) {
       unsigned long data = 0;
       memcpy(&data, src, k_ptrace_word_size);
 
       LLDB_LOG(log, "[{0:x}]:{1:x}", addr, data);
-      error = NativeProcessLinux::PtraceWrapper(PTRACE_POKEDATA, GetID(),
-                                                (void *)addr, (void *)data);
+      error = NativeProcessLinux::PtraceWrapper(
+          PTRACE_POKEDATA, GetCurrentThreadID(), (void *)addr, (void *)data);
       if (error.Fail())
         return error;
     } else {
       unsigned char buff[8];
       size_t bytes_read;
       error = ReadMemory(addr, buff, k_ptrace_word_size, bytes_read);
       if (error.Fail())
         return error;
void NativeProcessLinux::ThreadWasCreated(NativeThreadLinux &thread) {
   if (m_pending_notification_tid != LLDB_INVALID_THREAD_ID &&
       StateIsRunningState(thread.GetState())) {
     // We will need to wait for this new thread to stop as well before firing
     // the notification.
     thread.RequestStop();
   }
 }
 
-void NativeProcessLinux::SigchldHandler() {
+static llvm::Optional<WaitStatus> HandlePid(::pid_t pid) {
   Log *log = GetLog(POSIXLog::Process);
 
-  // Threads can appear or disappear as a result of event processing, so gather
-  // the events upfront.
-  llvm::DenseMap<lldb::tid_t, WaitStatus> tid_events;
-  for (const auto &thread_up : m_threads) {
-    int status = -1;
-    ::pid_t wait_pid =
-        llvm::sys::RetryAfterSignal(-1, ::waitpid, thread_up->GetID(), &status,
-                                    __WALL | __WNOTHREAD | WNOHANG);
+  int status;
+  ::pid_t wait_pid = llvm::sys::RetryAfterSignal(
+      -1, ::waitpid, pid, &status, __WALL | __WNOTHREAD | WNOHANG);
 
-    if (wait_pid == 0)
-      continue; // Nothing to do for this thread.
+    return llvm::None;
 
-    if (wait_pid == -1) {
-      Status error(errno, eErrorTypePOSIX);
-      LLDB_LOG(log, "waitpid({0}, &status, _) failed: {1}", thread_up->GetID(),
-               error);
-      continue;
-    }
+    LLDB_LOG(log, "waitpid({0}, &status, _) failed: {1}", pid,
+    return llvm::None;
 
-    assert(wait_pid == static_cast<::pid_t>(thread_up->GetID()));
+  assert(wait_pid == pid);
 
-    WaitStatus wait_status = WaitStatus::Decode(status);
 
-    LLDB_LOG(log, "waitpid({0})  got status = {1}", thread_up->GetID(),
-             wait_status);
-    tid_events.try_emplace(thread_up->GetID(), wait_status);
+  LLDB_LOG(log, "waitpid({0})  got status = {1}", pid, wait_status);
+  return wait_status;
+}
+
+void NativeProcessLinux::SigchldHandler() {
+  Log *log = GetLog(POSIXLog::Process);
+
+  // Threads can appear or disappear as a result of event processing, so gather
+  // the events upfront.
+  llvm::DenseMap<lldb::tid_t, WaitStatus> tid_events;
+  bool checked_main_thread = false;
+  for (const auto &thread_up : m_threads) {
+    if (thread_up->GetID() == GetID())
+      checked_main_thread = true;
+
+    if (llvm::Optional<WaitStatus> status = HandlePid(thread_up->GetID()))
+      tid_events.try_emplace(thread_up->GetID(), *status);
+  }
+  // Check the main thread even when we're not tracking it as process exit
+  // events are reported that way.
+  if (!checked_main_thread) {
+    if (llvm::Optional<WaitStatus> status = HandlePid(GetID()))
+      tid_events.try_emplace(GetID(), *status);
   }
 
   for (auto &KV : tid_events) {
     LLDB_LOG(log, "processing {0}({1}) ...", KV.first, KV.second);
+    if (KV.first == GetID() && (KV.second.type == WaitStatus::Exit ||
+                                KV.second.type == WaitStatus::Signal)) {
+
+      // The process exited.  We're done monitoring.  Report to delegate.
+      SetExitStatus(KV.second, true);
+      return;
+    }
     NativeThreadLinux *thread = GetThreadByID(KV.first);
-    if (thread) {
-      MonitorCallback(*thread, KV.second);
-    } else {
-      // This can happen if one of the events is an main thread exit.
-      LLDB_LOG(log, "... but the thread has disappeared");
-    }
+    assert(thread && "Why did this thread disappear?");
   }
 }
 
 // Wrapper for ptrace to catch errors and log calls. Note that ptrace sets
 // errno on error because -1 can be a valid result (i.e. for PTRACE_PEEK*)
 Status NativeProcessLinux::PtraceWrapper(int req, lldb::pid_t pid, void *addr,
                                          void *data, size_t data_size,
                                          long *result) {

lldb/test/API/functionalities/thread/main_thread_exit/Makefile

+ENABLE_THREADS := YES
+CXX_SOURCES := main.cpp
+include Makefile.rules

lldb/test/API/functionalities/thread/main_thread_exit/TestMainThreadExit.py

+"""
+Test handling of the situation where the main thread exits but the other threads
+in the process keep running.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+import lldbsuite.test.lldbutil as lldbutil
+
+
+class ThreadExitTestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+    NO_DEBUG_INFO_TESTCASE = True
+
+    # Needs os-specific implementation in the inferior
+    @skipIf(oslist=no_match(["linux"]))
+    def test(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(self, "// break here",
+                lldb.SBFileSpec("main.cpp"))
+
+        # There should be one (non-main) thread left
+        self.assertEquals(self.process().GetNumThreads(), 1)
+
+        # Ensure we can evaluate_expressions in this state
+        self.expect_expr("call_me()", result_value="12345")
+
+        self.runCmd("continue")
+        self.assertEquals(self.process().GetExitStatus(), 47)

lldb/test/API/functionalities/thread/main_thread_exit/main.cpp

+#include <thread>
+
+#ifdef __linux__
+#include <sys/syscall.h>
+#include <unistd.h>
+
+void exit_thread(int result) { syscall(SYS_exit, result); }
+#else
+#error Needs OS-specific implementation
+#endif
+
+int call_me() { return 12345; }
+
+void thread() {
+  std::this_thread::sleep_for(
+      std::chrono::seconds(10)); // Let the main thread exit.
+  exit_thread(42);               // break here
+}
+
+int main() {
+  std::thread(thread).detach();
+  exit_thread(47);
+}