[lldb] [gdb-remote] Fix displaying i387_ext & vec regs with gdbserver

[mgorny]

Summary

Adjust the encoding and format applied to i387_ext and vec* type
registers from gdbserver to match lldb-server. Both types are now
displayed as vector of uint8 instead of float and integer formats used
before. Additionally, this fixes display of STi registers when they do
not carry floating-point data (they are also used to hold MMX vectors).

Add client tests using GDB-style target.xml.

[mgorny]

Correct the test case to actually include data for all XMM registers (NFC).

[mgorny]

Actually, the test should include 16 XMM registers and MXCSR.

[labath]

Is printing STi as a vector actually desired? It seems to me that printing them as floating point would be more useful. Have you looked at what it would take to make lldb-server print them as FP as well?

[mgorny]

At a quick glance, we'd have to enhance Scalar to fully support x87 extended type. FWICS it's currently modeled to use convert APFloat from/to float/double/long double types. Maybe we could get away with some lossy rounding but it doesn't like a good way forward. It would probably be a better idea to drop some of the abstraction and/or use APFloat more.

I don't think an effort to getting proper x87 float support should be blocking this. If anyone is sufficiently motivated to implement it, it will be easy to switch the client to it, both for lldb-server and gdbserver usage.

lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp

feature_node.ForEachChildElementWithName(
 
         if (!gdb_type.empty() && !(encoding_set || format_set)) {
           if (llvm::StringRef(gdb_type).startswith("int")) {
             reg_info.format = eFormatHex;
             reg_info.encoding = eEncodingUint;
           } else if (gdb_type == "data_ptr" || gdb_type == "code_ptr") {
             reg_info.format = eFormatAddressInfo;
             reg_info.encoding = eEncodingUint;
-          } else if (gdb_type == "i387_ext" || gdb_type == "float") {
+          } else if (gdb_type == "float") {
             reg_info.format = eFormatFloat;
             reg_info.encoding = eEncodingIEEE754;
-          } else if (gdb_type == "aarch64v") {
+          } else if (gdb_type == "aarch64v" ||
+                     llvm::StringRef(gdb_type).startswith("vec") ||
+                     gdb_type == "i387_ext") {
             reg_info.format = eFormatVectorOfUInt8;
             reg_info.encoding = eEncodingVector;
           }
         }
 
         // Only update the register set name if we didn't get a "reg_set"
         // attribute. "set_name" will be empty if we didn't have a "reg_set"
         // attribute.

lldb/test/API/functionalities/gdb_remote_client/TestGDBServerTargetXML.py

def test_x86_64_regs(self):
                    ["r8 = 0x6867666564636261"])
         self.match("register read arg6",
                    ["r9 = 0x7877767574737271"])
         self.match("register read pc",
                    ["rip = 0x8887868584838281"])
         self.match("register read flags",
                    ["eflags = 0x94939291"])
 
+        # both stX and xmmX should be displayed as vectors
+        self.match("register read st0",
+                   ["st0 = {0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 0x09 0x0a}"])
+        self.match("register read st1",
+                   ["st1 = {0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1a}"])
+        self.match("register read xmm0",
+                   ["xmm0 = {0x81 0x82 0x83 0x84 0x85 0x86 0x87 0x88 "
+                    "0x89 0x8a 0x8b 0x8c 0x8d 0x8e 0x8f 0x90}"])
+        self.match("register read xmm1",
+                   ["xmm1 = {0x91 0x92 0x93 0x94 0x95 0x96 0x97 0x98 "
+                    "0x99 0x9a 0x9b 0x9c 0x9d 0x9e 0x9f 0xa0}"])
+
     @skipIfXmlSupportMissing
     @skipIfRemote
     @skipIfLLVMTargetMissing("X86")
     def test_i386_regs(self):
         """Test grabbing various i386 registers from gdbserver."""
         reg_data = [
             "01020304",  # eax
             "11121314",  # ecx
def test_i386_regs(self):
         # test generic aliases
         self.match("register read fp",
                    ["ebp = 0x54535251"])
         self.match("register read pc",
                    ["eip = 0x84838281"])
         self.match("register read flags",
                    ["eflags = 0x94939291"])
 
+        # both stX and xmmX should be displayed as vectors
+        self.match("register read st0",
+                   ["st0 = {0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 0x09 0x0a}"])
+        self.match("register read st1",
+                   ["st1 = {0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1a}"])
+        self.match("register read xmm0",
+                   ["xmm0 = {0x81 0x82 0x83 0x84 0x85 0x86 0x87 0x88 "
+                    "0x89 0x8a 0x8b 0x8c 0x8d 0x8e 0x8f 0x90}"])
+        self.match("register read xmm1",
+                   ["xmm1 = {0x91 0x92 0x93 0x94 0x95 0x96 0x97 0x98 "
+                    "0x99 0x9a 0x9b 0x9c 0x9d 0x9e 0x9f 0xa0}"])
+
     @skipIfXmlSupportMissing
     @skipIfRemote
     @skipIfLLVMTargetMissing("AArch64")
     def test_aarch64_regs(self):
         """Test grabbing various aarch64 registers from gdbserver."""
         class MyResponder(MockGDBServerResponder):
             reg_data = (
                 "0102030405060708"  # x0