gdb-remote: Fix checksum verification for messages with escape chars

[labath]

Summary

We've had a mismatch in the checksum computation between the sender and
receiver. The sender computed the payload checksum using the wire
encoding of the packet, while the receiver did this after expanding
un-escaping and expanding run-length-encoded sequences. This resulted in
communication breakdown if packets using these feature were sent in the
ack mode.

Normally, this did not cause any issues since the only packet we send in
the ack-mode is the QStartNoAckMode packet, but I ran into this when
debugging the lldb-server tests which (for better or worse) don't use
this mode.

According to the gdb-remote documentation "The two-digit checksum is computed as
the modulo 256 sum of all characters between the leading ‘$’ and the
trailing ‘#’", it seems that our sender is doing the right thing here.
Therefore, I fix the receiver the match the sender behavior and add a
test.

With this bug fixed, we can see that lldb-server is sending a stop-reply
after receiving the "k" in the same way as debugserver does (but we
weren't detecting this because at that point the connection was dead
already). I fix that expectation as well.

lldb/trunk/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp

if (content_length == std::string::npos) {
 
       if (m_bytes[0] == '$' || m_bytes[0] == '%') {
         assert(checksum_idx < m_bytes.size());
         if (::isxdigit(m_bytes[checksum_idx + 0]) ||
             ::isxdigit(m_bytes[checksum_idx + 1])) {
           if (GetSendAcks()) {
             const char *packet_checksum_cstr = &m_bytes[checksum_idx];
             char packet_checksum = strtol(packet_checksum_cstr, NULL, 16);
-            char actual_checksum = CalculcateChecksum(packet_str);
+            char actual_checksum = CalculcateChecksum(
+                llvm::StringRef(m_bytes).slice(content_start, content_end));
             success = packet_checksum == actual_checksum;
             if (!success) {
               if (log)
                 log->Printf("error: checksum mismatch: %.*s expected 0x%2.2x, "
                             "got 0x%2.2x",
                             (int)(total_length), m_bytes.c_str(),
                             (uint8_t)packet_checksum, (uint8_t)actual_checksum);
             }

lldb/trunk/unittests/Process/gdb-remote/CMakeLists.txt

 add_lldb_unittest(ProcessGdbRemoteTests
   GDBRemoteClientBaseTest.cpp
   GDBRemoteCommunicationClientTest.cpp
+  GDBRemoteCommunicationTest.cpp
   GDBRemoteTestUtils.cpp
 
   LINK_LIBS
     lldbCore
     lldbHost
     lldbPluginPlatformMacOSX
     lldbPluginProcessUtility
     lldbPluginProcessGDBRemote
 
     LLVMTestingSupport
 
   LINK_COMPONENTS
     Support
   )

lldb/trunk/unittests/Process/gdb-remote/GDBRemoteCommunicationTest.cpp

+//===-- GDBRemoteCommunicationTest.cpp --------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+#include "GDBRemoteTestUtils.h"
+#include "llvm/Testing/Support/Error.h"
+
+using namespace lldb_private::process_gdb_remote;
+using namespace lldb_private;
+using namespace lldb;
+typedef GDBRemoteCommunication::PacketResult PacketResult;
+
+namespace {
+
+class TestClient : public GDBRemoteCommunication {
+public:
+  TestClient()
+      : GDBRemoteCommunication("test.client", "test.client.listener") {}
+
+  PacketResult ReadPacket(StringExtractorGDBRemote &response) {
+    return GDBRemoteCommunication::ReadPacket(response, std::chrono::seconds(1),
+                                              /*sync_on_timeout*/ false);
+  }
+};
+
+class GDBRemoteCommunicationTest : public GDBRemoteTest {
+public:
+  void SetUp() override {
+    ASSERT_THAT_ERROR(Connect(client, server), llvm::Succeeded());
+  }
+
+protected:
+  TestClient client;
+  MockServer server;
+
+  bool Write(llvm::StringRef packet) {
+    ConnectionStatus status;
+    return server.Write(packet.data(), packet.size(), status, nullptr) ==
+           packet.size();
+  }
+};
+} // end anonymous namespace
+
+TEST_F(GDBRemoteCommunicationTest, ReadPacket_checksum) {
+  struct TestCase {
+    llvm::StringLiteral Packet;
+    llvm::StringLiteral Payload;
+  };
+  static constexpr TestCase Tests[] = {
+      {{"$#00"}, {""}},
+      {{"$foobar#79"}, {"foobar"}},
+      {{"$}}#fa"}, {"]"}},
+      {{"$x*%#c7"}, {"xxxxxxxxx"}},
+  };
+  for (const auto &Test : Tests) {
+    SCOPED_TRACE(Test.Packet + " -> " + Test.Payload);
+    StringExtractorGDBRemote response;
+    ASSERT_TRUE(Write(Test.Packet));
+    ASSERT_EQ(PacketResult::Success, client.ReadPacket(response));
+    ASSERT_EQ(Test.Payload, response.GetStringRef());
+    ASSERT_EQ(PacketResult::Success, server.GetAck());
+  }
+}

lldb/trunk/unittests/tools/lldb-server/tests/TestClient.cpp

TestClient::TestClient(std::unique_ptr<Connection> Conn) {
 
   SendAck(); // Send this as a handshake.
 }
 
 TestClient::~TestClient() {
   if (!IsConnected())
     return;
 
-  std::string response;
-  // Debugserver (non-conformingly?) sends a reply to the k packet instead of
-  // simply closing the connection.
-  PacketResult result =
-      IsDebugServer() ? PacketResult::Success : PacketResult::ErrorDisconnected;
-  EXPECT_THAT_ERROR(SendMessage("k", response, result), Succeeded());
+  EXPECT_THAT_ERROR(SendMessage("k"), Succeeded());
 }
 
 Expected<std::unique_ptr<TestClient>> TestClient::launch(StringRef Log) {
   return launch(Log, {});
 }
 
 Expected<std::unique_ptr<TestClient>> TestClient::launch(StringRef Log, ArrayRef<StringRef> InferiorArgs) {
   return launchCustom(Log, {}, InferiorArgs);