Don't try to link against a release python during a debug build.

[zturner]

Summary

When doing a debug build of LLDB, python headers automatically force a link against python27_d.lib. Since obtaining a copy of python27_d.lib requires building python manually, we attempted to avoid requiring LLDB developers to build python themselves as a matter of convenience. This leads to undefined behavior.

To reproduce the failures, you can do the following:

1. Build a debug build of LLDB on Windows with python enabled.
2. Run LLDB
3. Type "script" with no arguments.

When the python interpreter launches, you get a CRT error from the Visual Studio runtime. This is most likely due to the fact that in order to make this work, we #undef'ed _DEBUG, then included stdlib.h, then included python.h, then #define'd _DEBUG again. This leads to stdlib.h being included with a different defines in different translation units. For example, a TU which indirectly includes lldb-python.h will see a different pre-processed stdlib.h than a translation unit which does not include lldb-python.h

[zturner]

Note that this means that if someone on Windows wants to enable python support and/or run tests, they will have to build Python themselves. This is unfortunate, but it seems like the way it has to be.

We can try to streamline this in the future by checking in a hermetic copy of Python with necessary build files, but until that day comes, building manually seems like the best option. Note that this only applies if you have LLDB_DISABLE_PYTHON=0, which is NOT the default.

[zturner]

Stop normalizing the case of .py files when they're copied over. Python module names are case-sensitive, so this can lead to errors.

[tfiala]

We can try to streamline this in the future by checking in a hermetic

copy of Python with necessary build files, but until that day comes,
building manually seems like the best option. Note that this only applies
if you have LLDB_DISABLE_PYTHON=0, which is NOT the default.

FWIW this is highly unlikely to happen due to the purity of license that
LLVM wants to adhere to - we have way too many organizations using
LLVM-based code to want to pollute the license in any way.

I'll have a look at the patch as soon as I get a few cycles. Should be
today.

[tfiala]

Looks fine, I'm giving it a test run now...

scripts/Python/finishSwigPythonLLDB.py

154 ↗

(On Diff #11227)

Hey what was the normcase() trying to take care of?

[zturner]

Good question, I don't know. But I don't see how it could be correct,
because python module names are case-sensitive, and existing python code is
written against the un-normalized names.

[tfiala]

LGTM. No issues on Linux and MacOSX. No new test failures on either. Assuming this is working on Windows, all good here.

[zturner]

Closed by commit rL212785 (authored by @zturner).

lldb/trunk/CMakeLists.txt

macro(add_lldb_definitions)
   # We don't want no semicolons on LLDB_DEFINITIONS:
   foreach(arg ${ARGN})
     set(LLDB_DEFINITIONS "${LLVM_DEFINITIONS} ${arg}")
   endforeach(arg)
   add_definitions( ${ARGN} )
 endmacro(add_lldb_definitions)
 
 if (NOT LLDB_DISABLE_PYTHON)
-    if (MSVC)
-        # this definition will stop python from auto linking python27_d.lib when Python.h is included
-        add_definitions( -DSWIG_PYTHON_INTERPRETER_NO_DEBUG )
-    endif()
     find_package(PythonLibs REQUIRED)
     include_directories(${PYTHON_INCLUDE_DIRS})
 endif()
 
 include_directories(../clang/include)
 include_directories("${CMAKE_CURRENT_BINARY_DIR}/../clang/include")
 
 # lldb requires c++11 to build. Make sure that we have a compiler and standard

lldb/trunk/include/lldb/lldb-python.h

 #ifndef LLDB_lldb_python_h_
 #define LLDB_lldb_python_h_
 
 // Python.h needs to be included before any system headers in order to avoid redefinition of macros
 
 #ifdef LLDB_DISABLE_PYTHON
 // Python is disabled in this build
 #else
-  // If this is a visual studio build
-  #if defined( _MSC_VER )
-    // Special case for debug build since python unfortunately
-    // adds library to the linker path through a #pragma directive
-    #if defined( _DEBUG )
-      // Python forces a header link to python27_d.lib when building debug.
-      // To get around this (because most python packages for Windows
-      // don't come with debug python libraries), we undefine _DEBUG, include
-      // python.h and then restore _DEBUG.
-
-      // The problem with this is that any system level headers included from
-      // python.h were also effectively included in 'release' mode when we undefined
-      // _DEBUG. To work around this we include headers that python includes 
-      // before undefining _DEBUG.
-      #           include <stdlib.h>
-      // Undefine to force python to link against the release distro
-      #           undef _DEBUG
-      #           include <Python.h>
-      #           define _DEBUG
-      
-    #else
-      #include <Python.h>
-    #endif
-
-  #else
-    #if defined(__linux__)
-      // features.h will define _POSIX_C_SOURCE if _GNU_SOURCE is defined.  This value
-      // may be different from the value that Python defines it to be which results
-      // in a warning.  Undefine _POSIX_C_SOURCE before including Python.h  The same
-      // holds for _XOPEN_SOURCE.
-      #undef _POSIX_C_SOURCE
-      #undef _XOPEN_SOURCE
-    #endif
 
-    // Include python for non windows machines
-    #include <Python.h>
-
-  #endif
 #endif // LLDB_DISABLE_PYTHON
 
 #endif  // LLDB_lldb_python_h_

lldb/trunk/scripts/Python/finishSwigPythonLLDB.py

 strMsgCreatePyPkgMkDir = "create_py_pkg: Created folder '%s'";
 strMsgConfigBuildDir = "Configuration build directory located at '%s'";
 strMsgFoundLldbFrameWkDir = "Found '%s'";
 strMsgPyFileLocatedHere = "Python file will be put in '%s'";
 strMsgFrameWkPyExists = "Python output folder '%s' already exists";
 strMsgFrameWkPyMkDir = "Python output folder '%s' will be created";
 strErrMsgCreateFrmWkPyDirFailed = "Unable to create directory '%s' error: %s";
 strMsglldbsoExists = "Symlink '%s' already exists";
-strMsglldbsoMk = "Creating symlink for _lldb.so";
+strMsglldbsoMk = "Creating symlink for _lldb.so  (%s -> %s)";
 strErrMsgCpLldbpy = "copying lldb to lldb package directory";
 strErrMsgCreatePyPkgMissingSlash = "Parameter 3 fn create_py_pkg() missing slash"; 
 strErrMsgMkLinkExecute = "Command mklink failed: %s";
 strErrMsgMakeSymlink = "creating symbolic link";
 strErrMsgUnexpected = "Unexpected error: %s";
   
 #++---------------------------------------------------------------------------
 # Details:  Copy files needed by lldb/macosx/heap.py to build libheap.dylib.
def create_py_pkg( vDictArgs, vstrFrameworkPythonDir, vstrPkgDir, vListPkgFiles ):
   
   if vstrPkgDir.__len__() != 0 and vstrPkgDir[ 0 ] != "/":
     bOk = False;
     strMsg = strErrMsgCreatePyPkgMissingSlash;
     return (bOk, strMsg);
 
   strPkgName = vstrPkgDir;
   strPkgName = "lldb" + strPkgName.replace( "/", "." );
-  strPkgName = os.path.normcase( strPkgName );
   
   strPkgDir = vstrFrameworkPythonDir;
   strPkgDir += vstrPkgDir;
   strPkgDir = os.path.normcase( strPkgDir );
   
   if not(os.path.exists( strPkgDir ) and os.path.isdir( strPkgDir )):
     if bDbg:
       print(strMsgCreatePyPkgMkDir % strPkgDir);
     os.makedirs( strPkgDir );
     
   for strPkgFile in vListPkgFiles:
     if os.path.exists( strPkgFile ) and os.path.isfile( strPkgFile ):
-      strPyFile = os.path.normcase( strPkgFile );
       if bDbg:
         print(strMsgCreatePyPkgCopyPkgFile % (strPkgFile, strPkgDir));
-      shutil.copy( strPyFile, strPkgDir );
+      shutil.copy( strPkgFile, strPkgDir );
   
   # Create a packet init files if there wasn't one
   strPkgIniFile = strPkgDir + "/__init__.py";
   strPkgIniFile = os.path.normcase( strPkgIniFile );
   if os.path.exists( strPkgIniFile ) and os.path.isfile( strPkgIniFile ):
     return (bOk, strMsg);
   
   strPyScript = "__all__ = [";
   strDelimiter = "";
   for strPkgFile in vListPkgFiles:
-    strPyFile = os.path.normcase( strPkgFile );
-    if os.path.exists( strPyFile ) and os.path.isfile( strPyFile ):
+    if os.path.exists( strPkgFile ) and os.path.isfile( strPkgFile ):
       strBaseName = os.path.basename( strPkgFile );
       nPos = strBaseName.find( "." );
       if nPos != -1:
         strBaseName = strBaseName[ 0 : nPos ];
       strPyScript += "%s\"%s\"" % (strDelimiter, strBaseName);
       strDelimiter = ",";
   strPyScript += "]\n";   
   strPyScript += "for x in __all__:\n";
else:
     strSrc = os.path.normcase( "../../../bin/liblldb%s" % strLibFileExtn );
 
   if os.path.isfile( strTarget ):
     if bDbg:
       print strMsglldbsoExists % strTarget;
     return (bOk, strMsg);
 
   if bDbg:
-    print strMsglldbsoMk;
+    print strMsglldbsoMk % (os.path.abspath(strSrc), os.path.abspath(strTarget));
     
   try:
     csl = ctypes.windll.kernel32.CreateHardLinkW
     csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
     csl.restype = ctypes.c_ubyte
     if csl(strTarget, strSrc, 0) == 0:
       raise ctypes.WinError()
   except Exception as e:

lldb/trunk/source/CMakeLists.txt

if ( NOT CMAKE_SYSTEM_NAME MATCHES "Windows" )
 
   set_property(
     SOURCE lldb.cpp
     PROPERTY COMPILE_DEFINITIONS "LLDB_REVISION=\"${LLDB_REVISION}\"" "LLDB_REPOSITORY=\"${LLDB_REPOSITORY}\"")
 endif ()
 # FIXME: implement svn/git revision and repository parsing solution on Windows. There is an SVN-only
 #        revision parsing solution in tools/clang/lib/Basic/CMakelists.txt.
 
+if ( LLDB_ENABLE_PYTHON_SCRIPTS_SWIG_API_GENERATION )
+  # Add a Post-Build Event to copy over Python files and create the symlink to liblldb.so for the Python API(hardlink on Windows)
+  if ( NOT LLDB_DISABLE_PYTHON )
+        message("Adding post build step to run finishSwigWrapperClasses.py")
+      add_custom_command( TARGET liblldb
+          POST_BUILD
+            DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/finishSwigWrapperClasses.py
+          COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/finishSwigWrapperClasses.py "--srcRoot=${LLDB_SOURCE_DIR}" "--targetDir=${CMAKE_CURRENT_BINARY_DIR}/../scripts" "--cfgBldDir=${CMAKE_CURRENT_BINARY_DIR}/../scripts" "--prefix=${CMAKE_BINARY_DIR}" "--cmakeBuildConfiguration=${CMAKE_CFG_INTDIR}" -m
+          COMMENT "Python script sym-linking LLDB Python API")
+  endif ()
+endif ()
+
 install(TARGETS liblldb
   RUNTIME DESTINATION bin
   LIBRARY DESTINATION lib
   ARCHIVE DESTINATION lib)

lldb/trunk/source/Interpreter/ScriptInterpreterPython.cpp

if (Host::GetLLDBPath (ePathTypePythonDir, file_spec))
                 python_path.append (python_dir_path);
                 python_path.append ("\")");
                 PyRun_SimpleString (python_path.c_str());
                 python_path.resize (orig_len);
             }
         }
     }
 
-    PyRun_SimpleString ("sys.dont_write_bytecode = 1; import lldb.embedded_interpreter; from lldb.embedded_interpreter import run_python_interpreter; from lldb.embedded_interpreter import run_one_line; from termios import *");
+    PyRun_SimpleString ("sys.dont_write_bytecode = 1; import lldb.embedded_interpreter; from lldb.embedded_interpreter import run_python_interpreter; from lldb.embedded_interpreter import run_one_line");
 
     if (threads_already_initialized) {
         if (log)
             log->Printf("Releasing PyGILState. Returning to state = %slocked\n", gstate == PyGILState_UNLOCKED ? "un" : "");
         PyGILState_Release (gstate);
     } else {
         // We initialized the threads in this function, just unlock the GIL.
         PyEval_SaveThread();