Remote Non-Stop Support

[EwanCrawford]

Summary

This patch is the beginnings of support for Non-stop mode in the remote protocol. Letting a user examine stopped threads, while other threads execute freely.

Non-stop mode is enabled using the setting target.non-stop-mode, which sends a QNonStop packet when establishing the remote connection.
Changes are also made to treat the '?' stop reply packet differently in non-stop mode, according to spec https://sourceware.org/gdb/current/onlinedocs/gdb/Remote-Non_002dStop.html#Remote-Non_002dStop.
A setting for querying the remote for default thread on setup is also included.

Handling of '%' async notification packets will be added next.

[clayborg]

Everything looks good. Just one quests on why we need the "default-thread" setting. Can you explain why we need a user settable setting here? Shouldn't we just do this if non stop mode is enabled?

source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp

111 ↗

(On Diff #25458)

Why do we need a setting for this? Why would anyone want to set this manually? It would be great to not require this and just "do the right thing". Please explain why this is needed.

[EwanCrawford]

Thanks for having a look Greg. I removed the setting so that we only query for a default thread when non stop mode is enabled.

[clayborg]

As long as default mode still works, this looks good.

[jingham]

This patch seems okay for what it does, but there are a bunch of places where the stepping machinery assumes that it can stop all the threads to do things like stepping over breakpoints, etc. You are going to get lots of subtle funny behavior - like missing breakpoint hits on live threads when stepping over the breakpoint hit on another thread, or getting confused if a live thread hits a breakpoint when we think we are only running one thread for some other purpose. That part is going to take a bit more thought to get it to work correctly. This needs to be designed...

lldb/trunk/include/lldb/Target/Target.h

//    SetDisassemblyFlavor(const char *flavor);
     MemoryModuleLoadLevel
     GetMemoryModuleLoadLevel() const;
 
     bool
     GetUserSpecifiedTrapHandlerNames (Args &args) const;
 
     void
     SetUserSpecifiedTrapHandlerNames (const Args &args);
-    
     bool
+    GetNonStopModeEnabled () const;
+    
+    bool
     GetDisplayRuntimeSupportValues () const;
     
     void
     SetDisplayRuntimeSupportValues (bool b);
 
     const ProcessLaunchInfo &
     GetProcessLaunchInfo();
 

lldb/trunk/source/Commands/CommandObjectThread.cpp

public:
         }
 
         void
         OptionParsingStarting ()
         {
             m_step_in_avoid_no_debug = eLazyBoolCalculate;
             m_step_out_avoid_no_debug = eLazyBoolCalculate;
             m_run_mode = eOnlyDuringStepping;
+
+            // Check if we are in Non-Stop mode
+            lldb::TargetSP target_sp = m_interpreter.GetDebugger().GetSelectedTarget();
+            if (target_sp.get() != nullptr && target_sp->GetNonStopModeEnabled())
+                m_run_mode = eOnlyThisThread;
+
             m_avoid_regexp.clear();
             m_step_in_target.clear();
             m_class_name.clear();
             m_step_count = 1;
         }
 
         const OptionDefinition*
         GetDefinitions ()

lldb/trunk/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp

if (!m_bytes.empty())
         switch (m_bytes[0])
         {
             case '+':       // Look for ack
             case '-':       // Look for cancel
             case '\x03':    // ^C to halt target
                 content_length = total_length = 1;  // The command is one byte long...
                 break;
 
+            case '%': // Async notify packet
             case '$':
                 // Look for a standard gdb packet?
                 {
                     size_t hash_pos = m_bytes.find('#');
                     if (hash_pos != std::string::npos)
                     {
                         if (hash_pos + 2 < m_bytes.size())
                         {
if (!m_bytes.empty())
                     uint32_t idx;
                     for (idx = 1; !done && idx < bytes_len; ++idx)
                     {
                         switch (m_bytes[idx])
                         {
                         case '+':
                         case '-':
                         case '\x03':
+                        case '%':
                         case '$':
                             done = true;
                             break;
                                 
                         default:
                             break;
                         }
                     }
if (!m_bytes.empty())
                     packet_str.push_back(escapee);
                 }
                 else
                 {
                     packet_str.push_back(*c);
                 }
             }
 
-            if (m_bytes[0] == '$')
+            if (m_bytes[0] == '$' || m_bytes[0] == '%')
             {
                 assert (checksum_idx < m_bytes.size());
                 if (::isxdigit (m_bytes[checksum_idx+0]) || 
                     ::isxdigit (m_bytes[checksum_idx+1]))
                 {
                     if (GetSendAcks ())
                     {
                         const char *packet_checksum_cstr = &m_bytes[checksum_idx];

lldb/trunk/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h

public:
     bool
     GetSyncThreadStateSupported();
     
     void
     ResetDiscoverableSettings();
 
     bool
     GetHostInfo (bool force = false);
-    
     bool
+    GetDefaultThreadId (lldb::tid_t &tid);
+    
+    bool
     GetOSVersion (uint32_t &major, 
                   uint32_t &minor, 
                   uint32_t &update);
 
     bool
     GetOSBuildString (std::string &s);
     
     bool
SupportsGDBStoppointPacket (GDBStoppointType type)
         }
     }
     uint8_t
     SendGDBStoppointTypePacket (GDBStoppointType type,   // Type of breakpoint or watchpoint
                                 bool insert,              // Insert or remove?
                                 lldb::addr_t addr,        // Address of breakpoint or watchpoint
                                 uint32_t length);         // Byte Size of breakpoint or watchpoint
 
+    bool
+    SetNonStopMode (const bool enable);
+
     void
     TestPacketSpeed (const uint32_t num_packets);
 
     // This packet is for testing the speed of the interface only. Both
     // the client and server need to support it, but this allows us to
     // measure the packet speed without any other work being done on the
     // other end and avoids any of that work affecting the packet send
     // and response times.

lldb/trunk/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp

 GDBRemoteCommunicationClient::GetGDBServerProgramVersion()
 {
     if (GetGDBServerVersion())
         return m_gdb_server_version;
     return 0;
 }
 
 bool
+GDBRemoteCommunicationClient::GetDefaultThreadId (lldb::tid_t &tid)
+{
+    StringExtractorGDBRemote response;
+    if (SendPacketAndWaitForResponse("qC",response,false) !=  PacketResult::Success)
+        return false;
+
+    if (!response.IsNormalResponse())
+        return false;
+
+    if (response.GetChar() == 'Q' && response.GetChar() == 'C')
+        tid = response.GetHexMaxU32(true, -1);
+
+    return true;
+}
+
+bool
 GDBRemoteCommunicationClient::GetHostInfo (bool force)
 {
     Log *log (ProcessGDBRemoteLog::GetLogIfAnyCategoryIsSet (GDBR_LOG_PROCESS));
 
     if (force || m_qHostInfo_is_valid == eLazyBoolCalculate)
     {
         m_qHostInfo_is_valid = eLazyBoolNo;
         StringExtractorGDBRemote response;
if (m_supports_qGroupName)
         {
             m_supports_qGroupName = false;
             return false;
         }
     }
     return false;
 }
 
+bool
+GDBRemoteCommunicationClient::SetNonStopMode (const bool enable)
+{
+    // Form non-stop packet request
+    char packet[32];
+    const int packet_len = ::snprintf(packet, sizeof(packet), "QNonStop:%1d", (int)enable);
+    assert(packet_len < (int)sizeof(packet));
+
+    StringExtractorGDBRemote response;
+    // Send to target
+    if (SendPacketAndWaitForResponse(packet, packet_len, response, false) == PacketResult::Success)
+        if (response.IsOKResponse())
+            return true;
+
+    // Failed or not supported
+    return false;
+
+}
+
 void
 GDBRemoteCommunicationClient::TestPacketSpeed (const uint32_t num_packets)
 {
     uint32_t i;
     TimeValue start_time, end_time;
     uint64_t total_time_nsec;
     if (SendSpeedTestPacket (0, 0))
     {

lldb/trunk/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h

protected:
     uint64_t m_max_memory_size;       // The maximum number of bytes to read/write when reading and writing memory
     uint64_t m_remote_stub_max_memory_size;    // The maximum memory size the remote gdb stub can handle
     MMapMap m_addr_to_mmap_size;
     lldb::BreakpointSP m_thread_create_bp_sp;
     bool m_waiting_for_attach;
     bool m_destroy_tried_resuming;
     lldb::CommandObjectSP m_command_sp;
     int64_t m_breakpoint_pc_offset;
+    lldb::tid_t m_initial_tid; // The inital thread ID, given by stub on attach
 
     bool
     StartAsyncThread ();
 
     void
     StopAsyncThread ();
 
     static lldb::thread_result_t
     AsyncThread (void *arg);
 
     static bool
     MonitorDebugserverProcess (void *callback_baton,
                                lldb::pid_t pid,
                                bool exited,
                                int signo,
                                int exit_status);
 
     lldb::StateType
     SetThreadStopInfo (StringExtractor& stop_packet);
 
     void
+    HandleStopReplySequence ();
+
+    void
     ClearThreadIDList ();
 
     bool
     UpdateThreadIDList ();
 
     void
     DidLaunchOrAttach (ArchSpec& process_arch);
 

lldb/trunk/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp

ProcessGDBRemote::ProcessGDBRemote(Target& target, Listener &listener) :
     m_continue_S_tids (),
     m_max_memory_size (0),
     m_remote_stub_max_memory_size (0),
     m_addr_to_mmap_size (),
     m_thread_create_bp_sp (),
     m_waiting_for_attach (false),
     m_destroy_tried_resuming (false),
     m_command_sp (),
-    m_breakpoint_pc_offset (0)
+    m_breakpoint_pc_offset (0),
+    m_initial_tid (LLDB_INVALID_THREAD_ID)
 {
     m_async_broadcaster.SetEventName (eBroadcastBitAsyncThreadShouldExit,   "async thread should exit");
     m_async_broadcaster.SetEventName (eBroadcastBitAsyncContinue,           "async thread continue");
     m_async_broadcaster.SetEventName (eBroadcastBitAsyncThreadDidExit,      "async thread did exit");
     const uint64_t timeout_seconds = GetGlobalPluginProperties()->GetPacketTimeout();
     if (timeout_seconds > 0)
         m_gdb_comm.SetPacketTimeout(timeout_seconds);
 }
if (pid == LLDB_INVALID_PROCESS_ID)
         // listings...
         SetPrivateState (eStateConnected);
     }
     else
     {
         // We have a valid process
         SetID (pid);
         GetThreadList();
-        if (m_gdb_comm.SendPacketAndWaitForResponse("?", 1, m_last_stop_packet, false) == GDBRemoteCommunication::PacketResult::Success)
+        if (m_gdb_comm.GetStopReply(m_last_stop_packet))
         {
+
+            // '?' Packets must be handled differently in non-stop mode
+            if (GetTarget().GetNonStopModeEnabled())
+                HandleStopReplySequence();
+
             if (!m_target.GetArchitecture().IsValid()) 
             {
                 if (m_gdb_comm.GetProcessArchitecture().IsValid())
                 {
                     m_target.SetArchitecture(m_gdb_comm.GetProcessArchitecture());
                 }
                 else
                 {
if (object_file)
             if (GetID() == LLDB_INVALID_PROCESS_ID)
             {
                 if (log)
                     log->Printf("failed to connect to debugserver: %s", error.AsCString());
                 KillDebugserverProcess ();
                 return error;
             }
 
-            if (m_gdb_comm.SendPacketAndWaitForResponse("?", 1, m_last_stop_packet, false) == GDBRemoteCommunication::PacketResult::Success)
+            if (m_gdb_comm.GetStopReply(m_last_stop_packet))
             {
+
+                // '?' Packets must be handled differently in non-stop mode
+                if (GetTarget().GetNonStopModeEnabled())
+                    HandleStopReplySequence();
+
                 const ArchSpec &process_arch = m_gdb_comm.GetProcessArchitecture();
 
                 if (process_arch.IsValid())
                 {
                     m_target.MergeArchitecture(process_arch);
                 }
                 else
                 {
ProcessGDBRemote::ConnectToDebugserver (const char *connect_url)
     // alright.
     if (!m_gdb_comm.HandshakeWithServer (&error))
     {
         m_gdb_comm.Disconnect();
         if (error.Success())
             error.SetErrorString("not connected to remote gdb server");
         return error;
     }
+
+    // Send $QNonStop:1 packet on startup if required
+    if (GetTarget().GetNonStopModeEnabled())
+        m_gdb_comm.SetNonStopMode(true);
+
     m_gdb_comm.GetThreadSuffixSupported ();
     m_gdb_comm.GetListThreadsInStopReplySupported ();
     m_gdb_comm.GetHostInfo ();
     m_gdb_comm.GetVContSupported ('c');
     m_gdb_comm.GetVAttachOrWaitSupported();
-    
+    // Ask the remote server for the default thread id
+    if (GetTarget().GetNonStopModeEnabled())
+        m_gdb_comm.GetDefaultThreadId(m_initial_tid);
+
+
     size_t num_cmds = GetExtraStartupCommands().GetArgumentCount();
     for (size_t idx = 0; idx < num_cmds; idx++)
     {
         StringExtractorGDBRemote response;
         m_gdb_comm.SendPacketAndWaitForResponse (GetExtraStartupCommands().GetArgumentAtIndex(idx), response, false);
     }
     return error;
 }
if (listener.StartListeningForEvents (&m_gdb_comm, GDBRemoteCommunication::eBroadcastBitRunPacketSent))
         listener.StartListeningForEvents (&m_async_broadcaster, ProcessGDBRemote::eBroadcastBitAsyncThreadDidExit);
         
         const size_t num_threads = GetThreadList().GetSize();
 
         StreamString continue_packet;
         bool continue_packet_error = false;
         if (m_gdb_comm.HasAnyVContSupport ())
         {
-            if (m_continue_c_tids.size() == num_threads ||
+            if (!GetTarget().GetNonStopModeEnabled() &&
+                (m_continue_c_tids.size() == num_threads ||
                 (m_continue_c_tids.empty() &&
                  m_continue_C_tids.empty() &&
                  m_continue_s_tids.empty() &&
-                 m_continue_S_tids.empty()))
+                 m_continue_S_tids.empty())))
             {
                 // All threads are continuing, just send a "c" packet
                 continue_packet.PutCString ("c");
             }
             else
             {
                 continue_packet.PutCString ("vCont");
             
if (listener.StartListeningForEvents (&m_gdb_comm, GDBRemoteCommunication::eBroadcastBitRunPacketSent))
             }
         }
     }
 
     return error;
 }
 
 void
+ProcessGDBRemote::HandleStopReplySequence ()
+{
+    while(true)
+    {
+        // Send vStopped
+        StringExtractorGDBRemote response;
+        m_gdb_comm.SendPacketAndWaitForResponse("vStopped", response, false);
+
+        // OK represents end of signal list
+        if (response.IsOKResponse())
+            break;
+
+        // If not OK or a normal packet we have a problem
+        if (!response.IsNormalResponse())
+            break;
+
+        SetLastStopPacket(response);
+    }
+}
+
+void
 ProcessGDBRemote::ClearThreadIDList ()
 {
     Mutex::Locker locker(m_thread_list_real.GetMutex());
     m_thread_ids.clear();
 }
 
 bool
 ProcessGDBRemote::UpdateThreadIDList ()
ProcessGDBRemote::RefreshStateAfterStop ()
     SetThreadStopInfo (m_last_stop_packet);
     // Check to see if SetThreadStopInfo() filled in m_thread_ids?
     if (m_thread_ids.empty())
     {
         // No, we need to fetch the thread list manually
         UpdateThreadIDList();
     }
 
+    // If we have queried for a default thread id
+    if (m_initial_tid != LLDB_INVALID_THREAD_ID)
+    {
+        m_thread_list.SetSelectedThreadByID(m_initial_tid);
+        m_initial_tid = LLDB_INVALID_THREAD_ID;
+    }
+
     // Let all threads recover from stopping and do any clean up based
     // on the previous thread state (if any).
     m_thread_list_real.RefreshStateAfterStop();
     
 }
 
 Error
 ProcessGDBRemote::DoHalt (bool &caused_stop)

lldb/trunk/source/Target/Target.cpp

// FIXME: This is the wrong way to do per-architecture settings, but we don't have a general per architecture settings system in place yet.
         "Loading modules from memory can be slow as reading the symbol tables and other data can take a long time depending on your connection to the debug target. "
         "This setting helps users control how much information gets loaded when loading modules from memory."
         "'complete' is the default value for this setting which will load all sections and symbols by reading them from memory (slowest, most accurate). "
         "'partial' will load sections and attempt to find function bounds without downloading the symbol table (faster, still accurate, missing symbol names). "
         "'minimal' is the fastest setting and will load section data with no symbols, but should rarely be used as stack frames in these memory regions will be inaccurate and not provide any context (fastest). " },
     { "display-expression-in-crashlogs"    , OptionValue::eTypeBoolean   , false, false,                      NULL, NULL, "Expressions that crash will show up in crash logs if the host system supports executable specific crash log strings and this setting is set to true." },
     { "trap-handler-names"                 , OptionValue::eTypeArray     , true,  OptionValue::eTypeString,   NULL, NULL, "A list of trap handler function names, e.g. a common Unix user process one is _sigtramp." },
     { "display-runtime-support-values"     , OptionValue::eTypeBoolean   , false, false,                      NULL, NULL, "If true, LLDB will show variables that are meant to support the operation of a language's runtime support." },
+    { "non-stop-mode"                      , OptionValue::eTypeBoolean   , false, 0,                          NULL, NULL, "Disable lock-step debugging, instead control threads independently." },
     { NULL                                 , OptionValue::eTypeInvalid   , false, 0                         , NULL, NULL, NULL }
 };
 
 enum
 {
     ePropertyDefaultArch,
     ePropertyExprPrefix,
     ePropertyPreferDynamic,
enum
     ePropertyDisassemblyFlavor,
     ePropertyUseHexImmediates,
     ePropertyHexImmediateStyle,
     ePropertyUseFastStepping,
     ePropertyLoadScriptFromSymbolFile,
     ePropertyMemoryModuleLoadLevel,
     ePropertyDisplayExpressionsInCrashlogs,
     ePropertyTrapHandlerNames,
-    ePropertyDisplayRuntimeSupportValues
+    ePropertyDisplayRuntimeSupportValues,
+    ePropertyNonStopModeEnabled
 };
 
 
 class TargetOptionValueProperties : public OptionValueProperties
 {
 public:
     TargetOptionValueProperties (const ConstString &name) :
         OptionValueProperties (name),

 
 void
 TargetProperties::SetDisplayRuntimeSupportValues (bool b)
 {
     const uint32_t idx = ePropertyDisplayRuntimeSupportValues;
     m_collection_sp->SetPropertyAtIndexAsBoolean (NULL, idx, b);
 }
 
+bool
+TargetProperties::GetNonStopModeEnabled () const
+{
+    const uint32_t idx = ePropertyNonStopModeEnabled;
+    return m_collection_sp->GetPropertyAtIndexAsBoolean (NULL, idx, false);
+}
+
 const ProcessLaunchInfo &
 TargetProperties::GetProcessLaunchInfo ()
 {
     m_launch_info.SetArg0(GetArg0()); // FIXME: Arg0 callback doesn't work
     return m_launch_info;
 }
 
 void

lldb/trunk/source/Utility/StringExtractorGDBRemote.h

enum ServerPacketType
         eServerPacketType_s,
         eServerPacketType_S,
         eServerPacketType_T,
         eServerPacketType_Z,
         eServerPacketType_z,
 
         eServerPacketType__M,
         eServerPacketType__m,
+        eServerPacketType_notify, // '%' notification
     };
     
     ServerPacketType
     GetServerPacketType () const;
 
     enum ResponseType
     {
         eUnsupported = 0,

lldb/trunk/source/Utility/StringExtractorGDBRemote.cpp

#define PACKET_STARTS_WITH(s) ((packet_size >= (sizeof(s)-1)) && ::strncmp(packet_cstr, s, (sizeof(s)-1))==0)
     // Empty is not a supported packet...
     if (m_packet.empty())
         return eServerPacketType_invalid;
 
     const size_t packet_size = m_packet.size();
     const char *packet_cstr = m_packet.c_str();
     switch (m_packet[0])
     {
+
+    case '%':
+        return eServerPacketType_notify;
+
     case '\x03':
         if (packet_size == 1) return eServerPacketType_interrupt;
         break;
 
     case '-':
         if (packet_size == 1) return eServerPacketType_nack;
         break;