[lldb] Unify python site-packages path

[hhb]

Summary

Before this change, the procedure of installing and loading python
modules are as follow:

1. finishSwigPythonLLDB.py writes necessary files to

${CMAKE_BINARY_DIR}/${CMAKE_CFG_INTDIR}/distutils.sysconfig.get_python_lib()

2. lldb/scripts/CMakeLists.txt installs them to lib${LLVM_LIBDIR_SUFFIX}

3. ScriptInterpreterPython.cpp adds the right path into PYTHONPATH.

(XCode is using a different path but that's not changed here.)

The issue is that get_python_lib() is platform dependent. It is
difficult to guess the right path in cmake. And it is impossible to be
right when cross compiling.

This change updates finishSwigPythonLLDB.py to output to ${liblldb_build_dir}.
Since cmake will always install them into lib${LLVM_LIBDIR_SUFFIX} , this change
Also updates ScriptInterpreterPython.cpp to load from
${liblldb_path}/../lib${LLVM_LIBDIR_SUFFIX}/site-packages.

[hhb]

Rebase

[hhb]

Fix symbol linker

[mgorny]

I do not believe this to be the right approach. Unconditionally installing Python modules to the wrong directory is a bug, not a feature. It should be fixed (as I proposed in D67890), not made more prominent by making everything hinge on it.

[labath]

Like I said in the other patch, I think we're stuck with having to match how the python layouts differ between operating systems (though there is definitely a lot of room for improvement in the way that it is done now).

lldb/CMakeLists.txt

if (WIN32 AND NOT "${PYTHON_DLL}" STREQUAL "")
         file(TO_NATIVE_PATH "${CMAKE_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin" LLDB_BIN_DIR)
         file(TO_NATIVE_PATH "${PYTHON_DLL}" PYTHON_DLL_NATIVE_PATH)
         add_custom_command(
             TARGET finish_swig
             POST_BUILD
             COMMAND ${CMAKE_COMMAND} -E copy ${PYTHON_DLL_NATIVE_PATH} ${LLDB_BIN_DIR} VERBATIM
             COMMENT "Copying Python DLL to LLDB binaries directory.")
     endif ()
+
+    if(NOT LLDB_BUILD_FRAMEWORK)
+        set(SWIG_PYTHON_DIR ${liblldb_build_dir}/site-packages)
+        set(SWIG_INSTALL_DIR lib${LLVM_LIBDIR_SUFFIX})
+
+        # Install the LLDB python module
+        install(DIRECTORY ${SWIG_PYTHON_DIR} DESTINATION ${SWIG_INSTALL_DIR})
+    endif()
 endif ()
 
 if(LLDB_BUILT_STANDALONE AND NOT LLVM_ENABLE_IDE)
   llvm_distribution_add_targets()
 endif()

lldb/scripts/CMakeLists.txt

COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/prepare_bindings.py
       --swigExecutable=${SWIG_EXECUTABLE}
   VERBATIM
   COMMENT "Python script building LLDB Python wrapper")
 
 add_custom_target(swig_wrapper ALL DEPENDS
   ${CMAKE_CURRENT_BINARY_DIR}/LLDBWrapPython.cpp
   ${CMAKE_CURRENT_BINARY_DIR}/lldb.py
 )
-
-if(NOT LLDB_BUILD_FRAMEWORK)
-  if(CMAKE_SYSTEM_NAME MATCHES "Windows")
-    set(swig_python_subdir site-packages)
-  else()
-    set(swig_python_subdir python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR})
-  endif()
-
-  set(SWIG_PYTHON_DIR ${LLVM_LIBRARY_OUTPUT_INTDIR}/${swig_python_subdir})
-  set(SWIG_INSTALL_DIR lib${LLVM_LIBDIR_SUFFIX})
-
-  # Install the LLDB python module
-  install(DIRECTORY ${SWIG_PYTHON_DIR} DESTINATION ${SWIG_INSTALL_DIR})
-endif()

lldb/scripts/Python/finishSwigPythonLLDB.py

 """ Python SWIG post process script for each language
 
     --------------------------------------------------------------------------
     File:           finishSwigPythonLLDB.py
 
     Overview:       Python script(s) to post process SWIG Python C++ Script
                     Bridge wrapper code on the Windows/LINUX/OSX platform.
-                    The Python scripts are equivalent to the shell script (.sh)
-                    files.
                     For the Python script interpreter (external to liblldb) to
                     be able to import and use the lldb module, there must be
                     two files, lldb.py and _lldb.so, that it can find. lldb.py
                     is generated by SWIG at the same time it generates the C++
                     file.  _lldb.so is actually a symlink file that points to
                     the LLDB shared library/framework.
                     The Python script interpreter needs to be able to
                     automatically find these two files. On Darwin systems it

 
 # User facing text:
 strMsgOsVersion = "The current OS is %s"
 strMsgPyVersion = "The Python version is %d.%d"
 strErrMsgProgFail = "Program failure: "
 strErrMsgLLDBPyFileNotNotFound = "Unable to locate lldb.py at path '%s'"
 strMsgCopyLLDBPy = "Copying lldb.py from '%s' to '%s'"
 strErrMsgFrameWkPyDirNotExist = "Unable to find the LLDB.framework directory '%s'"
+strErrMsgTargetDirNotExist = "Unable to find the --target directory '%s'"
 strMsgCreatePyPkgCopyPkgFile = "create_py_pkg: Copied file '%s' to folder '%s'"
 strMsgCreatePyPkgInitFile = "create_py_pkg: Creating pakage init file '%s'"
 strMsgCreatePyPkgMkDir = "create_py_pkg: Created folder '%s'"
 strMsgConfigBuildDir = "Configuration build directory located at '%s'"
 strMsgFoundLldbFrameWkDir = "Found '%s'"
 strMsgPyFileLocatedHere = "Python file will be put in '%s'"
 strMsgFrameWkPyExists = "Python output folder '%s' already exists"
 strMsgFrameWkPyMkDir = "Python output folder '%s' will be created"
def make_symlink(
 
     os.chdir(vstrFrameworkPythonDir)
     bMakeFileCalled = "-m" in vDictArgs
     eOSType = utilsOsType.determine_os_type()
     if not bMakeFileCalled:
         strBuildDir = os.path.join("..", "..", "..")
     else:
         # Resolve vstrSrcFile path relatively the build directory
-        if eOSType == utilsOsType.EnumOsType.Windows:
-            # On a Windows platform the vstrFrameworkPythonDir looks like:
-            # llvm\\build\\Lib\\site-packages\\lldb
-            strBuildDir = os.path.join("..", "..", "..")
-        else:
-            # On a UNIX style platform the vstrFrameworkPythonDir looks like:
-            # llvm/build/lib/python2.7/site-packages/lldb
-            strBuildDir = os.path.join("..", "..", "..", "..")
+        # vstrFrameworkPythonDir looks like:
+        # llvm/build/lib/site-packages/lldb
     strSrc = os.path.normcase(os.path.join(strBuildDir, vstrSrcFile))
 
     return make_symlink_native(vDictArgs, strSrc, strTarget)
 
 
 #++---------------------------------------------------------------------------
 # Details:  Make the symbolic that the script bridge for Python will need in
 #           the Python framework directory.
if bHaveConfigBldDir:
         strConfigBldDir = vDictArgs["--cfgBldDir"]
     if (bHaveConfigBldDir == False) or (strConfigBldDir.__len__() == 0):
         strConfigBldDir = vstrFrameworkPythonDir
 
     return (bOk, strConfigBldDir, strErrMsg)
 
 #++---------------------------------------------------------------------------
 # Details:  Determine where to put the files. Retrieve the directory path for
-#           Python's dist_packages/ site_package folder on a Windows platform.
+#           Python's dist_packages / site_package folder when build with llvm
+#           build system.
 # Args:     vDictArgs   - (R) Program input parameters.
 # Returns:  Bool - True = function success, False = failure.
 #           Str - Python Framework directory path.
 #           strErrMsg - Error description on task failure.
 # Throws:   None.
 #--
 
 
-def get_framework_python_dir_windows(vDictArgs):
+def get_framework_python_dir_llvm(vDictArgs):
     dbg = utilsDebug.CDebugFnVerbose(
-        "Python script get_framework_python_dir_windows()")
+        "Python script get_framework_python_dir_llvm()")
     bOk = True
     strWkDir = ""
     strErrMsg = ""
 
+    dbg.dump_text("Built by LLVM")
     # We are being built by LLVM, so use the PYTHON_INSTALL_DIR argument,
     # and append the python version directory to the end of it.  Depending
     # on the system other stuff may need to be put here as well.
+    strWkDir = vDictArgs["--targetDir"]
     from distutils.sysconfig import get_python_lib
-    strPythonInstallDir = ""
-    bHaveArgPrefix = "--prefix" in vDictArgs
-    if bHaveArgPrefix:
-        strPythonInstallDir = os.path.normpath(vDictArgs["--prefix"])
-
-    bHaveArgCmakeBuildConfiguration = "--cmakeBuildConfiguration" in vDictArgs
-    if bHaveArgCmakeBuildConfiguration:
-        strPythonInstallDir = os.path.join(
-            strPythonInstallDir,
-            vDictArgs["--cmakeBuildConfiguration"])
-
-    if strPythonInstallDir.__len__() != 0:
-        strWkDir = get_python_lib(True, False, strPythonInstallDir)
+    if os.path.exists(strWkDir):
+        strWkDir = os.path.join(strWkDir, 'site-packages')
     else:
-        strWkDir = get_python_lib(True, False)
+        bOk = False
+        strErrMsg = strErrMsgTargetDirNotExist % strWkDir
     strWkDir = os.path.normcase(os.path.join(strWkDir, "lldb"))
 
     return (bOk, strWkDir, strErrMsg)
 
 #++---------------------------------------------------------------------------
-# Details:  Retrieve the directory path for Python's dist_packages/
-#           site_package folder on a UNIX style platform.
+# Details:  Retrieve the directory path for Python's dist_packages /
+#           site_package folder for xcode.
 # Args:     vDictArgs   - (R) Program input parameters.
 # Returns:  Bool - True = function success, False = failure.
 #           Str - Python Framework directory path.
 #           strErrMsg - Error description on task failure.
 # Throws:   None.
 #--
 
 
-def get_framework_python_dir_other_platforms(vDictArgs):
+def get_framework_python_dir_xcode(vDictArgs):
     dbg = utilsDebug.CDebugFnVerbose(
-        "Python script get_framework_python_dir_other_platform()")
+        "Python script get_framework_python_dir_xcode()")
     bOk = True
     strWkDir = ""
     strErrMsg = ""
-    bDbg = "-d" in vDictArgs
 
-    bMakeFileCalled = "-m" in vDictArgs
-    if bMakeFileCalled:
-        dbg.dump_text("Built by LLVM")
-        return get_framework_python_dir_windows(vDictArgs)
-    else:
-        dbg.dump_text("Built by XCode")
-        # We are being built by XCode, so all the lldb Python files can go
-        # into the LLDB.framework/Resources/Python subdirectory.
-        strWkDir = vDictArgs["--targetDir"]
-        strWkDir = os.path.join(strWkDir, "LLDB.framework")
-        if os.path.exists(strWkDir):
-            if bDbg:
-                print((strMsgFoundLldbFrameWkDir % strWkDir))
-            strWkDir = os.path.join(strWkDir, "Resources", "Python", "lldb")
-            strWkDir = os.path.normcase(strWkDir)
-        else:
-            bOk = False
-            strErrMsg = strErrMsgFrameWkPyDirNotExist % strWkDir
+    bDbg = "-d" in vDictArgs
 
     return (bOk, strWkDir, strErrMsg)
 
 #++---------------------------------------------------------------------------
 # Details:  Retrieve the directory path for Python's dist_packages/
 #           site_package folder depending on the type of OS platform being
 #           used.
 # Args:     vDictArgs   - (R) Program input parameters.
 # Returns:  Bool - True = function success, False = failure.
 #           Str - Python Framework directory path.
 #           strErrMsg - Error description on task failure.
 # Throws:   None.
 #--
 
 
 def get_framework_python_dir(vDictArgs):
     dbg = utilsDebug.CDebugFnVerbose(
         "Python script get_framework_python_dir()")
-    bOk = True
-    strWkDir = ""
-    strErrMsg = ""
 
-    eOSType = utilsOsType.determine_os_type()
-    if eOSType == utilsOsType.EnumOsType.Unknown:
-        bOk = False
-        strErrMsg = strErrMsgOsTypeUnknown
-    elif eOSType == utilsOsType.EnumOsType.Windows:
-        bOk, strWkDir, strErrMsg = get_framework_python_dir_windows(vDictArgs)
+    bMakeFileCalled = "-m" in vDictArgs
+    if bMakeFileCalled:
+        return get_framework_python_dir_llvm(vDictArgs)
     else:
-        bOk, strWkDir, strErrMsg = get_framework_python_dir_other_platforms(
-            vDictArgs)
-
-    return (bOk, strWkDir, strErrMsg)
+        return get_framework_python_dir_xcode(vDictArgs)
 
 #++---------------------------------------------------------------------------
 # Details:  Retrieve the liblldb directory path, if it exists and is valid.
 # Args:     vDictArgs               - (R) Program input parameters.
 # Returns:  Bool - True = function success, False = failure.
 #           Str - liblldb directory path.
 #           strErrMsg - Error description on task failure.
 # Throws:   None.

lldb/scripts/finishSwigWrapperClasses.py

 """ Post process SWIG Bridge wrapper code Python script for Windows/LINUX/OSX platform
 
     --------------------------------------------------------------------------
     File:           finishSwigWrapperClasses.py
 
     Overview:       Python script(s) to finish off the SWIG Python C++ Script
                     Bridge wrapper code on the Windows/LINUX/OSX platform.
-                    The Python scripts are equivalent to the shell script (.sh)
-                    files.
                     We use SWIG to create a C++ file containing the appropriate
                     wrapper classes and functions for each scripting language,
                     before liblldb is built (thus the C++ file can be compiled
                     into liblldb.  In some cases, additional work may need to be
                     done after liblldb has been compiled, to make the scripting
                     language stuff fully functional.  Any such post-processing
                     is handled through the Python scripts called here.
 
def run_post_process_for_each_script_supported(vDictArgs):
     if not bOk:
         return (-8, strScriptDirNotFound)
 
     # Look for any script language directories to build for
     listDirs = ["Python"]
 
     # Iterate script directory find any script language directories
     for scriptLang in listDirs:
-        # __pycache__ is a magic directory in Python 3 that holds .pyc files
-        if scriptLang != "__pycache__" and scriptLang != "swig_bot_lib":
-            dbg.dump_text("Executing language script for \'%s\'" % scriptLang)
-            nResult, strStatusMsg = run_post_process(
-                scriptLang, strFinishFileName, vDictArgs)
         if nResult < 0:
             break
 
     if nResult < 0:
         strTmp = strPostProcessError % scriptLang
         strTmp += strStatusMsg
         strStatusMsg = strTmp
 

lldb/scripts/get_relative_lib_dir.py

-import distutils.sysconfig
-import os
-import platform
-import re
-import sys
-
-
-def get_python_relative_libdir():
-    """Returns the appropropriate python libdir relative to the build directory.
-
-    @param exe_path the path to the lldb executable
-
-    @return the python path that needs to be added to sys.path (PYTHONPATH)
-    in order to find the lldb python module.
-    """
-    if platform.system() != 'Linux':
-        return None
-
-    # We currently have a bug in lldb -P that does not account for
-    # architecture variants in python paths for
-    # architecture-specific modules.  Handle the lookup here.
-    # When that bug is fixed, we should just ask lldb for the
-    # right answer always.
-    arch_specific_libdir = distutils.sysconfig.get_python_lib(True, False)
-    split_libdir = arch_specific_libdir.split(os.sep)
-    lib_re = re.compile(r"^lib.+$")
-
-    for i in range(len(split_libdir)):
-        match = lib_re.match(split_libdir[i])
-        if match is not None:
-            # We'll call this the relative root of the lib dir.
-            # Things like RHEL will have an arch-specific python
-            # lib dir, which isn't 'lib' on x86_64.
-            return os.sep.join(split_libdir[i:])
-    # Didn't resolve it.
-    return None
-
-if __name__ == '__main__':
-    lib_dir = get_python_relative_libdir()
-    if lib_dir is not None:
-        sys.stdout.write(lib_dir)
-        sys.exit(0)
-    else:
-        sys.exit(1)

lldb/source/Plugins/ScriptInterpreter/Python/CMakeLists.txt

-if (NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
-  # Call a python script to gather the arch-specific libdir for
-  # modules like the lldb module.
-  execute_process(
-    COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/../../../../scripts/get_relative_lib_dir.py
-    RESULT_VARIABLE get_libdir_status
-    OUTPUT_VARIABLE relative_libdir
-    )
-  if (get_libdir_status EQUAL 0)
-    add_definitions(-DLLDB_PYTHON_RELATIVE_LIBDIR="${relative_libdir}")
-  endif()
-endif()
-
 add_lldb_library(lldbPluginScriptInterpreterPython PLUGIN
   PythonDataObjects.cpp
   PythonExceptionState.cpp
   ScriptInterpreterPython.cpp
 
   LINK_LIBS
     lldbBreakpoint
     lldbCore

lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.h

public:
   static void Initialize();
   static void Terminate();
   static lldb_private::ConstString GetPluginNameStatic();
   static const char *GetPluginDescriptionStatic();
   static FileSpec GetPythonDir();
 
 protected:
   static void ComputePythonDirForApple(llvm::SmallVectorImpl<char> &path);
-  static void ComputePythonDirForPosix(llvm::SmallVectorImpl<char> &path);
-  static void ComputePythonDirForWindows(llvm::SmallVectorImpl<char> &path);
+  static void ComputePythonDir(llvm::SmallVectorImpl<char> &path);
 };
 } // namespace lldb_private
 
 #endif // LLDB_DISABLE_PYTHON
 #endif // LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_SCRIPTINTERPRETERPYTHON_H

lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp

 #include "lldb/API/SBValue.h"
 #include "lldb/Breakpoint/StoppointCallbackContext.h"
 #include "lldb/Breakpoint/WatchpointOptions.h"
 #include "lldb/Core/Communication.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/ValueObject.h"
 #include "lldb/DataFormatters/TypeSummary.h"
+#include "lldb/Host/Config.h"
 #include "lldb/Host/ConnectionFileDescriptor.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Host/HostInfo.h"
 #include "lldb/Host/Pipe.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
 #include "lldb/Target/Thread.h"
 #include "lldb/Target/ThreadPlan.h"
void ScriptInterpreterPython::ComputePythonDirForApple(
     llvm::SmallVectorImpl<char> &path) {
   auto style = llvm::sys::path::Style::posix;
 
   llvm::StringRef path_ref(path.begin(), path.size());
   auto rbegin = llvm::sys::path::rbegin(path_ref, style);
   auto rend = llvm::sys::path::rend(path_ref);
   auto framework = std::find(rbegin, rend, "LLDB.framework");
   if (framework == rend) {
-    ComputePythonDirForPosix(path);
+    ComputePythonDir(path);
     return;
   }
   path.resize(framework - rend);
   llvm::sys::path::append(path, style, "LLDB.framework", "Resources", "Python");
 }
 
-void ScriptInterpreterPython::ComputePythonDirForPosix(
+void ScriptInterpreterPython::ComputePythonDir(
     llvm::SmallVectorImpl<char> &path) {
-  auto style = llvm::sys::path::Style::posix;
-#if defined(LLDB_PYTHON_RELATIVE_LIBDIR)
-  // Build the path by backing out of the lib dir, then building with whatever
-  // the real python interpreter uses.  (e.g. lib for most, lib64 on RHEL
-  // x86_64).
-  llvm::sys::path::remove_filename(path, style);
-  llvm::sys::path::append(path, style, LLDB_PYTHON_RELATIVE_LIBDIR);
+#if defined(_WIN32)
+    auto style = llvm::sys::path::Style::windows;
 #else
-  llvm::sys::path::append(path, style,
-                          "python" + llvm::Twine(PY_MAJOR_VERSION) + "." +
-                              llvm::Twine(PY_MINOR_VERSION),
-                          "site-packages");
+    auto style = llvm::sys::path::Style::posix;
 #endif
-}
-
-void ScriptInterpreterPython::ComputePythonDirForWindows(
-    llvm::SmallVectorImpl<char> &path) {
-  auto style = llvm::sys::path::Style::windows;
   llvm::sys::path::remove_filename(path, style);
-  llvm::sys::path::append(path, style, "lib", "site-packages");
+  llvm::sys::path::append(path, style, "lib" LLDB_LIBDIR_SUFFIX, "site-packages");
 
   // This will be injected directly through FileSpec.GetDirectory().SetString(),
   // so we need to normalize manually.
   std::replace(path.begin(), path.end(), '\\', '/');
 }
 
 FileSpec ScriptInterpreterPython::GetPythonDir() {
   static FileSpec g_spec = []() {
     FileSpec spec = HostInfo::GetShlibDir();
     if (!spec)
       return FileSpec();
     llvm::SmallString<64> path;
     spec.GetPath(path);
 
 #if defined(__APPLE__)
     ComputePythonDirForApple(path);
-#elif defined(_WIN32)
-    ComputePythonDirForWindows(path);
 #else
-    ComputePythonDirForPosix(path);
+    ComputePythonDir(path);
 #endif
     spec.GetDirectory().SetString(path);
     return spec;
   }();
   return g_spec;
 }
 
 lldb_private::ConstString ScriptInterpreterPython::GetPluginNameStatic() {