[intel-pt] Fix python support and add a full python test

[wallace]

Summary

The initial implementation of the Intel PT support for LLDB came with
some Python support. However, when LLDB was upgraded to more modern versions of
SWIG and Python3, the Intel Pt Python API broke and has not been updated since
then.
This makes me believe that no one has used that API in a looong time.

As a fix, I replicated some fixes done in the core LLDB Python support.
Additionally, I added a test that uses entirely the Intel PT Python API and made
some additional fixes to the existing test.

I'm not fond of the existing API design, so I'll be improving it in following
diffs.

[wallace]

lldb/source/API/SBDebugger.cpp

66–69

I'm adding this small change, which helped a lot to figure out some issues when loading the intel plugin shared object. I hope you don't mind.

[wallace]

fix incorrect comment

[wallace]

After syncing up with Greg, we decided to redo this in a different way

lldb/source/API/SBDebugger.cpp

 #include "llvm/Support/ManagedStatic.h"
 
 using namespace lldb;
 using namespace lldb_private;
 
 static llvm::sys::DynamicLibrary LoadPlugin(const lldb::DebuggerSP &debugger_sp,
                                             const FileSpec &spec,
                                             Status &error) {
+  std::string error_message;
   llvm::sys::DynamicLibrary dynlib =
-      llvm::sys::DynamicLibrary::getPermanentLibrary(spec.GetPath().c_str());
+      llvm::sys::DynamicLibrary::getPermanentLibrary(spec.GetPath().c_str(),
+                                                     &error_message);
   if (dynlib.isValid()) {
     typedef bool (*LLDBCommandPluginInit)(lldb::SBDebugger & debugger);
 
     lldb::SBDebugger debugger_sb(debugger_sp);
     // This calls the bool lldb::PluginInitialize(lldb::SBDebugger debugger)
     // function.
     // TODO: mangle this differently for your system - on OSX, the first
     // underscore needs to be removed and the second one stays
     LLDBCommandPluginInit init_func =
         (LLDBCommandPluginInit)(uintptr_t)dynlib.getAddressOfSymbol(
             "_ZN4lldb16PluginInitializeENS_10SBDebuggerE");
     if (init_func) {
       if (init_func(debugger_sb))
         return dynlib;
       else
         error.SetErrorString("plug-in refused to load "
                              "(lldb::PluginInitialize(lldb::SBDebugger) "
                              "returned false)");
     } else {
       error.SetErrorString("plug-in is missing the required initialization: "
                            "lldb::PluginInitialize(lldb::SBDebugger)");
     }
   } else {
     if (FileSystem::Instance().Exists(spec))
-      error.SetErrorString("this file does not represent a loadable dylib");
+      error.SetErrorStringWithFormat(
+          "this file does not represent a loadable dylib. %s",
+          error_message.c_str());
     else
       error.SetErrorString("no such file");
   }
   return llvm::sys::DynamicLibrary();
 }
 
 static llvm::ManagedStatic<SystemLifetimeManager> g_debugger_lifetime;
 

lldb/test/API/tools/intel-features/intel-pt/test/TestIntelPTSimpleBinary.py

 from __future__ import print_function
 
 import os
 import lldb
 import time
+import lldbIntelFeatures
 
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 from lldbsuite.test import configuration
 
 
 class TestIntelPTSimpleBinary(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
     NO_DEBUG_INFO_TESTCASE = True
 
     def setUp(self):
         TestBase.setUp(self)
 
         if 'intel-pt' not in configuration.enabled_plugins:
             self.skipTest("The intel-pt test plugin is not enabled")
 
         plugin_path = os.path.join(os.environ["LLDB_IMPLIB_DIR"], "liblldbIntelFeatures.so")
         self.runCmd("plugin load " + plugin_path)
 
+    def waitUntilNotNone(self, callback):
+        obj = None
+        for _ in range(20):
+            obj = callback()
+            if obj:
+                break
+            time.sleep(0.5)
+        self.assertIsNotNone(obj)
+        return obj
+
+    def waitUntilTrue(self, callback):
+        res = False
+        for _ in range(20):
+            res = callback()
+            if res:
+                break
+            time.sleep(0.5)
+        self.assertTrue(res)
+
+    def find_start_address_of_function(self, target, function_name):
+        return target.FindFunctions(function_name)[0].GetSymbol() \
+            .GetStartAddress().GetLoadAddress(target)
+
     @skipIf(oslist=no_match(['linux']))
     @skipIf(archs=no_match(['i386', 'x86_64']))
     @skipIfRemote
     def test_basic_flow(self):
         """Test collection, decoding, and dumping instructions"""
 
         self.build()
         exe = self.getBuildArtifact("a.out")
         lldbutil.run_to_name_breakpoint(self, "main", exe_name=exe)
-        # We start tracing from main
+        # We will start tracing from main
         self.runCmd("processor-trace start all")
 
         # We check the trace after the for loop
         self.runCmd("b " + str(line_number('main.cpp', '// Break 1')))
         self.runCmd("c")
 
-        # We wait a little bit to ensure the processor has send the PT packets to
-        # the memory
-        time.sleep(.1)
+        command = "processor-trace show-instr-log -c 100"
 
-        # We find the start address of the 'fun' function for a later check
-        target = self.dbg.GetSelectedTarget()
-        fun_start_adddress = target.FindFunctions("fun")[0].GetSymbol() \
-            .GetStartAddress().GetLoadAddress(target)
+        def isTraceAvailable():
+            res = lldb.SBCommandReturnObject()
+            self.dbg.GetCommandInterpreter().HandleCommand(command, res)
+            return "Instruction Log not available" not in res.GetOutput()
+
+        self.waitUntilTrue(isTraceAvailable)
 
-        # We print the last instructions
-        self.expect("processor-trace show-instr-log -c 100",
+        self.expect(command,
             patterns=[
                 # We expect to have seen the first instruction of 'fun'
-                hex(fun_start_adddress),
+                hex(self.find_start_address_of_function(self.dbg.GetSelectedTarget(), "fun")),
                 # We expect to see the exit condition of the for loop
                 "at main.cpp:" + str(line_number('main.cpp', '// Break for loop'))
             ])
 
         self.runCmd("processor-trace stop")
+
+    @skipIf(oslist=no_match(['linux']))
+    @skipIf(archs=no_match(['i386', 'x86_64']))
+    @skipIfRemote
+    def test_basic_flow_with_python_api(self):
+        """Test collection, decoding, and dumping instructions using the Python API"""
+
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+
+        # We will start tracing from main
+        lldbutil.run_to_name_breakpoint(self, "main", exe_name=exe)
+
+        decoder = lldbIntelFeatures.PTDecoder(self.dbg)
+
+        target = self.dbg.GetSelectedTarget()
+        process = target.GetProcess()
+        tid = process.GetProcessID()
+
+        # We configure the tracing options
+        trace_opts = lldb.SBTraceOptions()
+        trace_opts.setThreadID(tid)
+        trace_opts.setType(lldb.eTraceTypeProcessorTrace)
+        trace_opts.setMetaDataBufferSize(0)
+        trace_opts.setTraceBufferSize(4096)
+
+        stream = lldb.SBStream()
+        stream.Print('{"trace-tech":"intel-pt"}')
+        custom_params = lldb.SBStructuredData()
+        self.assertSuccess(custom_params.SetFromJSON(stream))
+        trace_opts.setTraceParams(custom_params)
+
+        # We start tracing
+        error = lldb.SBError()
+        decoder.StartProcessorTrace(process, trace_opts, error)
+        self.assertSuccess(error)
+
+        # We check the trace after the for loop
+        self.runCmd("b " + str(line_number('main.cpp', '// Break 1')))
+        self.runCmd("c")
+
+        # We'll check repeteadly until the trace is available
+        def getInstructions():
+            instruction_list = lldbIntelFeatures.PTInstructionList()
+            decoder.GetInstructionLogAtOffset(
+                process, tid, offset=99, count=100, result_list=instruction_list, sberror=error)
+            if error.Fail() or instruction_list.GetSize() == 0:
+                return None
+            return instruction_list
+
+        instruction_list = self.waitUntilNotNone(getInstructions)
+
+        # We assert we executed the "fun" function in the trace
+        addresses = set()
+        for i in range(0, instruction_list.GetSize()):
+            insn = instruction_list.GetInstructionAtIndex(i)
+            addresses.add(insn.GetInsnAddress())
+        self.assertIn(self.find_start_address_of_function(target, "fun"), addresses)
+
+        # We assert stopping the trace works
+        decoder.StopProcessorTrace(process, error, tid)
+        self.assertSuccess(error)

lldb/tools/intel-features/CMakeLists.txt

add_lldb_library(lldbIntelFeatures SHARED
   LINK_LIBS
     ${FEATURE_LIBS}
     ${PYTHON_LIBRARIES}
   )
 
 # Add link dependencies for python wrapper
 if (LLDB_ENABLE_PYTHON AND LLDB_BUILD_INTEL_PT)
   add_dependencies(lldbIntelFeatures intel-features-swig_wrapper)
+  add_dependencies(lldbIntelFeatures finish-intel-features-swig)
 endif()
 
 install(TARGETS lldbIntelFeatures
   LIBRARY DESTINATION lib${LLVM_LIBDIR_SUFFIX})

lldb/tools/intel-features/scripts/CMakeLists.txt

add_custom_command(
   COMMENT "Generating python wrapper for features library")
 
 set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/IntelFeaturesPythonWrap.cpp PROPERTIES GENERATED 1)
 set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/lldbIntelFeatures.py PROPERTIES GENERATED 1)
 
 add_custom_target(intel-features-swig_wrapper ALL
   DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/IntelFeaturesPythonWrap.cpp
   )
+
+# Set up the python module in a similar way as in lldb/CmakeLists.txt
+
+if(LLDB_BUILD_FRAMEWORK)
+  set(lldb_intel_features_python_build_path
+    "${LLDB_FRAMEWORK_ABSOLUTE_BUILD_DIR}/LLDB.framework/Resources/Python/lldbIntelFeatures")
+else()
+  set(lldb_intel_features_python_build_path
+    "${CMAKE_BINARY_DIR}/${CMAKE_CFG_INTDIR}/${LLDB_PYTHON_RELATIVE_PATH}/lldbIntelFeatures")
+endif()
+
+# Add a Post-Build Event to copy over Python files and create the symlink
+# to libIntelFeatures.so for the Python API.
+add_custom_target(finish-intel-features-swig ALL VERBATIM
+  COMMAND ${CMAKE_COMMAND} -E make_directory ${lldb_intel_features_python_build_path}
+  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/lldbIntelFeatures.py
+  COMMENT "Python script sym-linking LLDB Intel Features Python API")
+
+add_custom_command(TARGET finish-intel-features-swig POST_BUILD VERBATIM
+  COMMAND ${CMAKE_COMMAND} -E copy
+    "${CMAKE_CURRENT_BINARY_DIR}/lldbIntelFeatures.py"
+    "${lldb_intel_features_python_build_path}/__init__.py")
+
+set(libIntelFeatures_symlink_dest
+  "${LLVM_SHLIB_OUTPUT_INTDIR}/liblldbIntelFeatures${CMAKE_SHARED_LIBRARY_SUFFIX}")
+
+function(create_relative_symlink target dest_file output_dir output_name)
+  get_filename_component(dest_file ${dest_file} ABSOLUTE)
+  get_filename_component(output_dir ${output_dir} ABSOLUTE)
+  file(RELATIVE_PATH rel_dest_file ${output_dir} ${dest_file})
+  if(CMAKE_HOST_UNIX)
+    set(LLVM_LINK_OR_COPY create_symlink)
+  else()
+    set(LLVM_LINK_OR_COPY copy)
+  endif()
+  add_custom_command(TARGET ${target} POST_BUILD VERBATIM
+    COMMAND ${CMAKE_COMMAND} -E ${LLVM_LINK_OR_COPY} ${rel_dest_file} ${output_name}
+    WORKING_DIRECTORY ${output_dir})
+endfunction()
+
+create_relative_symlink(finish-intel-features-swig ${libIntelFeatures_symlink_dest}
+  ${lldb_intel_features_python_build_path} _lldbIntelFeatures.so)

lldb/tools/intel-features/scripts/lldb-intel-features.swig

-%module lldbIntelFeatures
+/*
+Swig import fix copied from lldb/bindings/python.swig
+*/
+
+%define MODULEIMPORT
+"try:
+    # Try an absolute import first.
+    import $module
+except ImportError:
+    # Relative import should work if we are being loaded by Python.
+    from . import $module"
+%enddef
+// These versions will not generate working python modules, so error out early.
+#if SWIG_VERSION >= 0x030009 && SWIG_VERSION < 0x030011
+#error Swig versions 3.0.9 and 3.0.10 are incompatible with lldb.
+#endif
+
+%module(moduleimport=MODULEIMPORT) lldbIntelFeatures
 
 %{
 #include "lldb/lldb-public.h"
 #include "intel-pt/PTDecoder.h"
 using namespace ptdecoder;
 %}
 
 /* Undefine GCC keyword to make Swig happy when processing glibc's stdint.h */
 #define __extension__
 
 /* Combined python typemap for all features */
 %include "python-typemaps.txt"
 
 /* Feature specific python interface files*/
 %include "../intel-pt/interface/PTDecoder.i"