Update swig typemaps and interfaces to not call PyString_FromStringAndSize

[zturner]

Summary

Python 3's native API has a number of incompatibilities with Python 2's. Most of these incompatibilities are hidden inside of the various PythonObject classes, but our swig typemaps and interfaces use Python native APIs directly.

This patch addresses the first of these incompatibilities, the use of PyString_FromStringAndSize. It is fixed by changing these to use `PythonString. It's possible more work may need to be done here in the future because there are still numerous calls to native Python APIs from the swig typemaps, but for now this is the minimum amount of work necessary to get a successful compilation under Python 3.

The second of these incompatibilities, the use of various PyFile_xxx functions, will be addressed in a followup patch.

scripts/Python/python-extensions.swig

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBBlock {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBBreakpoint {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
         
     %pythoncode %{ 

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
         
         /* the write() and flush() calls are not part of the SB API proper, and are solely for Python usage

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
     %pythoncode %{ 
         def __eq__(self, rhs):

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBDebugger {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBDeclaration {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
         
     %pythoncode %{ 

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBFileSpec {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBFrame {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBFunction {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
         
     %pythoncode %{ 

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBInstructionList {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBLineEntry {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
         
     %pythoncode %{ 

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
         
     %pythoncode %{ 

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBSection {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
         
     %pythoncode %{ 

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
     %pythoncode %{
         def __eq__(self, rhs):

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBSymbolContextList {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 
     %pythoncode %{ 

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBTypeCategory {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBTypeFilter {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
     %pythoncode %{ 
         def __eq__(self, rhs):

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBTypeMember {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBTypeEnumMember {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBTypeNameSpecifier {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
     %pythoncode %{ 
         def __eq__(self, rhs):

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
     %pythoncode %{ 
         def __eq__(self, rhs):

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
     %pythoncode %{ 
         def __eq__(self, rhs):

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
     %pythoncode %{ 
         def __eq__(self, rhs):

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBValueList {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 %extend lldb::SBWatchpoint {

                 if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r'))
                     --desc_len;
                 if (desc_len > 0)
-                    return PyString_FromStringAndSize (desc, desc_len);
+                    return lldb_private::PythonString(llvm::StringRef(desc, desc_len)).release();
                 else
-                    return PyString_FromString("");
+                    return lldb_private::PythonString("").release();
         }
 }
 

scripts/Python/python-typemaps.swig

   int i;
   len = 0;
   while ($1[len]) len++;
-  $result = PyList_New(len);
-  for (i = 0; i < len; i++) {
-    PyList_SetItem($result, i, PyString_FromString($1[i]));
-  }
+  lldb_private::PythonList list(len);
+  for (i = 0; i < len; i++)
+    list.SetItemAtIndex(i, lldb_private::PythonString($1[i]));
+  $result = list.release();
 }
 
 %typemap(in) char const ** {

 // See also SBThread::GetStopDescription.
 %typemap(argout) (char *dst, size_t dst_len) {
    Py_XDECREF($result);   /* Blow away any previous result */
-   $result = PyString_FromStringAndSize(($1),result);
+   lldb_private::PythonString str($1);
+   $result = str.release();
    free($1);
 }
 

 // See also SBProcess::ReadMemory.
 %typemap(argout) (void *buf, size_t size) {
    Py_XDECREF($result);   /* Blow away any previous result */
-   $result = PyString_FromStringAndSize(static_cast<const char*>($1),result);
+   llvm::StringRef ref(static_cast<const char*>($1), result);
+   lldb_private::PythonString string(ref);
+   $result = string.release();
    free($1);
 }
 

scripts/lldb.swig

 #include "lldb/API/SBWatchpoint.h"
 #include "lldb/API/SBUnixSignals.h"
 
-#include "../scripts/Python/python-swigsafecast.swig"
+#include "../source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h"
 
+#include "../scripts/Python/python-swigsafecast.swig"
 %}
 
 /* Various liblldb typedefs that SWIG needs to know about.  */

    as INT32_MAX should only be defined if __STDC_LIMIT_MACROS is. */
 #define __STDC_LIMIT_MACROS
 %include "stdint.i"
+
 %include "lldb/lldb-defines.h"
 %include "lldb/lldb-enumerations.h"
 %include "lldb/lldb-forward.h"

source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h

     {
         // Avoid calling the virtual method if it's not necessary
         // to actually validate the type of the PyObject.
-        if (!rhs.get())
+        if (!rhs.IsValid())
             Reset();
         else
             Reset(PyRefType::Borrowed, rhs.m_py_obj);

     Dump (Stream &strm) const;
 
     PyObject*
-    get () const
+    get() const
     {
         return m_py_obj;
     }
 
-    PyObjectType GetObjectType() const;
+    PyObject*
+    release()
+    {
+        PyObject *result = m_py_obj;
+        m_py_obj = nullptr;
+        return result;
+    }
+
+    PyObjectType
+    GetObjectType() const;
 
     PythonString
     Repr ();

source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp

     int fd = fileno(fp);
 #endif
 
-    return PyRef(PyRefType::Owned,
+    return PythonObject(PyRefType::Owned,
         PyFile_FromFd(fd, nullptr, cmode, -1, nullptr, "ignore", nullptr, 0));
 #else
     // Read through the Python source, doesn't seem to modify these strings