[lldb] [gdb-remote] Fix displaying i387_ext & vec regs with gdbserver

[mgorny]

Summary

Adjust the encoding and format applied to i387_ext and vec* type
registers from gdbserver to match lldb-server. Both types are now
displayed as vector of uint8 instead of float and integer formats used
before. Additionally, this fixes display of STi registers when they do
not carry floating-point data (they are also used to hold MMX vectors).

Add client tests using GDB-style target.xml.

[mgorny]

Correct the test case to actually include data for all XMM registers (NFC).

[mgorny]

Actually, the test should include 16 XMM registers and MXCSR.

[labath]

Is printing STi as a vector actually desired? It seems to me that printing them as floating point would be more useful. Have you looked at what it would take to make lldb-server print them as FP as well?

[mgorny]

At a quick glance, we'd have to enhance Scalar to fully support x87 extended type. FWICS it's currently modeled to use convert APFloat from/to float/double/long double types. Maybe we could get away with some lossy rounding but it doesn't like a good way forward. It would probably be a better idea to drop some of the abstraction and/or use APFloat more.

I don't think an effort to getting proper x87 float support should be blocking this. If anyone is sufficiently motivated to implement it, it will be easy to switch the client to it, both for lldb-server and gdbserver usage.

lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp

 //===-- ProcessGDBRemote.cpp ----------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Host/Config.h"
 
 #include <cerrno>
 #include <cstdlib>
 #if LLDB_ENABLE_POSIX
 #include <netinet/in.h>
 #include <sys/mman.h>
 #include <sys/socket.h>
 #include <unistd.h>
feature_node.ForEachChildElementWithName(
 
         if (!gdb_type.empty() && !(encoding_set || format_set)) {
           if (llvm::StringRef(gdb_type).startswith("int")) {
             reg_info.format = eFormatHex;
             reg_info.encoding = eEncodingUint;
           } else if (gdb_type == "data_ptr" || gdb_type == "code_ptr") {
             reg_info.format = eFormatAddressInfo;
             reg_info.encoding = eEncodingUint;
-          } else if (gdb_type == "i387_ext" || gdb_type == "float") {
+          } else if (gdb_type == "float") {
             reg_info.format = eFormatFloat;
             reg_info.encoding = eEncodingIEEE754;
+          } else if (llvm::StringRef(gdb_type).startswith("vec") ||
+                     gdb_type == "i387_ext") {
+            reg_info.format = eFormatVectorOfUInt8;
+            reg_info.encoding = eEncodingVector;
           }
         }
 
         // Only update the register set name if we didn't get a "reg_set"
         // attribute. "set_name" will be empty if we didn't have a "reg_set"
         // attribute.
         if (!set_name) {
           if (!gdb_group.empty()) {

lldb/test/API/functionalities/gdb_remote_client/TestGDBServerTargetXML.py

+from __future__ import print_function
+import lldb
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test.decorators import *
+from gdbclientutils import *
+
+
+class TestGDBServerTargetXML(GDBRemoteTestBase):
+
+    @skipIfXmlSupportMissing
+    @skipIfRemote
+    def test_x86_64_vec_regs(self):
+        """Test rendering of x86_64 vector registers from gdbserver."""
+        class MyResponder(MockGDBServerResponder):
+            def qXferRead(self, obj, annex, offset, length):
+                if annex == "target.xml":
+                    return """<?xml version="1.0"?>
+                        <!DOCTYPE feature SYSTEM "gdb-target.dtd">
+                        <target>
+                          <architecture>i386:x86-64</architecture>
+                          <osabi>GNU/Linux</osabi>
+                          <feature name="org.gnu.gdb.i386.core">
+                            <reg name="rsp" bitsize="64" type="data_ptr" regnum="7"/>
+                            <reg name="rip" bitsize="64" type="code_ptr" regnum="16"/>
+                            <reg name="st0" bitsize="80" type="i387_ext" regnum="24"/>
+                            <reg name="st1" bitsize="80" type="i387_ext" regnum="25"/>
+                            <reg name="st2" bitsize="80" type="i387_ext" regnum="26"/>
+                            <reg name="st3" bitsize="80" type="i387_ext" regnum="27"/>
+                            <reg name="st4" bitsize="80" type="i387_ext" regnum="28"/>
+                            <reg name="st5" bitsize="80" type="i387_ext" regnum="29"/>
+                            <reg name="st6" bitsize="80" type="i387_ext" regnum="30"/>
+                            <reg name="st7" bitsize="80" type="i387_ext" regnum="31"/>
+                          </feature>
+                          <feature name="org.gnu.gdb.i386.sse">
+                            <reg name="xmm0" bitsize="128" type="vec128" regnum="40"/>
+                            <reg name="xmm1" bitsize="128" type="vec128" regnum="41"/>
+                            <reg name="xmm2" bitsize="128" type="vec128" regnum="42"/>
+                            <reg name="xmm3" bitsize="128" type="vec128" regnum="43"/>
+                            <reg name="xmm4" bitsize="128" type="vec128" regnum="44"/>
+                            <reg name="xmm5" bitsize="128" type="vec128" regnum="45"/>
+                            <reg name="xmm6" bitsize="128" type="vec128" regnum="46"/>
+                            <reg name="xmm7" bitsize="128" type="vec128" regnum="47"/>
+                            <reg name="xmm8" bitsize="128" type="vec128" regnum="48"/>
+                            <reg name="xmm9" bitsize="128" type="vec128" regnum="49"/>
+                            <reg name="xmm10" bitsize="128" type="vec128" regnum="50"/>
+                            <reg name="xmm11" bitsize="128" type="vec128" regnum="51"/>
+                            <reg name="xmm12" bitsize="128" type="vec128" regnum="52"/>
+                            <reg name="xmm13" bitsize="128" type="vec128" regnum="53"/>
+                            <reg name="xmm14" bitsize="128" type="vec128" regnum="54"/>
+                            <reg name="xmm15" bitsize="128" type="vec128" regnum="55"/>
+                            <reg name="mxcsr" bitsize="32" type="i386_mxcsr" regnum="56" group="vector"/>
+                          </feature>
+                        </target>""", False
+                else:
+                    return None, False
+
+            def readRegister(self, regnum):
+                return ""
+
+            def readRegisters(self):
+                return (
+                    "0102030405060708"  # rsp
+                    "1112131415161718"  # rip
+                    "0102030405060708090a"  # st0
+                    "1112131415161718191a" +  # st1
+                    "2122232425262728292a" * 6 +  # st2..st7
+                    "8182838485868788898a8b8c8d8e8f90"  # xmm0
+                    "9192939495969798999a9b9c9d9e9fa0"  # xmm1
+                    "a1a2a3a4a5a6a7a8a9aaabacadaeafb0" * 6  # xmm2..xmm7
+                )
+
+            def haltReason(self):
+                return "T02thread:1ff0d;threads:1ff0d;thread-pcs:000000010001bc00;07:0102030405060708;10:1112131415161718;"
+
+        self.server.responder = MyResponder()
+
+        target = self.createTarget("basic_eh_frame.yaml")
+        process = self.connect(target)
+        lldbutil.expect_state_changes(self, self.dbg.GetListener(), process,
+                                      [lldb.eStateStopped])
+
+        # rsp and rip should be displayed as uints
+        self.match("register read rsp",
+                   ["rsp = 0x0807060504030201"])
+        self.match("register read rip",
+                   ["rip = 0x1817161514131211"])
+
+        # both stX and xmmX should be displayed as vectors
+        self.match("register read st0",
+                   ["st0 = {0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 0x09 0x0a}"])
+        self.match("register read st1",
+                   ["st1 = {0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1a}"])
+        self.match("register read xmm0",
+                   ["xmm0 = {0x81 0x82 0x83 0x84 0x85 0x86 0x87 0x88 "
+                    "0x89 0x8a 0x8b 0x8c 0x8d 0x8e 0x8f 0x90}"])
+        self.match("register read xmm1",
+                   ["xmm1 = {0x91 0x92 0x93 0x94 0x95 0x96 0x97 0x98 "
+                    "0x99 0x9a 0x9b 0x9c 0x9d 0x9e 0x9f 0xa0}"])