Add Python 3 support to clang.cindex

[jbcoe]

Summary

Introduce an interop string to convert from unicode to c-strings where needed.
Add missing conversions from _CXString to strings in function registrations.
Explicitly evaluate lists where Python 3's lazy iterators would not otherwise do so.

This is an improvement upon the reverted change proposed in https://reviews.llvm.org/D26082

[jbcoe]

Remove a couple of unnecessary type conversions.

[compnerd]

I think it would've been nice to split this up into the changes for map/filter rather than group it together. But sure, this looks reasonable.

clang/bindings/python/clang/__init__.py

 
 The available modules are:
 
   cindex
 
     Bindings for the Clang indexing library.
 """
 
-
-# Python 3 uses unicode for strings. The bindings, in particular the interaction
-# with ctypes, need modifying to handle conversions between unicode and
-# c-strings.
-import sys 
-if sys.version_info[0] != 2: 
-    raise Exception("Only Python 2 is supported.")
-
 __all__ = ['cindex']
 

clang/bindings/python/clang/cindex.py

 #
 # o implement additional SourceLocation, SourceRange, and File methods.
 
 from ctypes import *
 import collections
 
 import clang.enumerations
 
+import sys
+if sys.version_info[0] == 3:
+    # Python 3 strings are unicode, translate them to/from utf8 for C-interop.
+    class c_interop_string(c_char_p):
+
+        def __init__(self, p=None):
+            if p is None:
+                p = ""
+            if isinstance(p, str):
+                p = p.encode("utf8")
+            super(c_char_p, self).__init__(p)
+
+        def __str__(self):
+            return self.value
+
+        @property
+        def value(self):
+            if super(c_char_p, self).value is None:
+                return None
+            return super(c_char_p, self).value.decode("utf8")
+
+        @classmethod
+        def from_param(cls, param):
+            if isinstance(param, str):
+                return cls(param)
+            if isinstance(param, bytes):
+                return cls(param)
+            raise TypeError("Cannot convert '{}' to '{}'".format(type(param).__name__, cls.__name__))
+
+        @staticmethod
+        def to_python_string(x, *args):
+            return x.value
+
+    def b(x):
+        if isinstance(x, bytes):
+            return x
+        return x.encode('utf8')
+
+    xrange = range
+
+elif sys.version_info[0] == 2:
+    # Python 2 strings are utf8 byte strings, no translation is needed for
+    # C-interop.
+    c_interop_string = c_char_p
+
+    def _to_python_string(x, *args):
+        return x
+
+    c_interop_string.to_python_string = staticmethod(_to_python_string)
+
+    def b(x):
+        return x
+
+
 # ctypes doesn't implicitly convert c_void_p to the appropriate wrapper
 # object. This is a problem, because it means that from_parameter will see an
 # integer and pass the wrong value on platforms where int != void*. Work around
 # this by marshalling object arguments as void**.
 c_object_p = POINTER(c_void_p)
 
 callbacks = {}
 
class _CXString(Structure):
     def __del__(self):
         conf.lib.clang_disposeString(self)
 
     @staticmethod
     def from_result(res, fn, args):
         assert isinstance(res, _CXString)
         return conf.lib.clang_getCString(res)
 
+
 class SourceLocation(Structure):
     """
     A SourceLocation represents a particular location within a source file.
     """
     _fields_ = [("ptr_data", c_void_p * 2), ("int_data", c_uint)]
     _data = None
 
     def _get_instantiation(self):
class CursorKind(BaseEnumeration):
 
     # The required BaseEnumeration declarations.
     _kinds = []
     _name_map = None
 
     @staticmethod
     def get_all_kinds():
         """Return all CursorKind enumeration instances."""
-        return filter(None, CursorKind._kinds)
+        return [x for x in CursorKind._kinds if not x is None]
 
     def is_declaration(self):
         """Test if this is a declaration kind."""
         return conf.lib.clang_isDeclaration(self)
 
     def is_reference(self):
         """Test if this is a reference kind."""
         return conf.lib.clang_isReference(self)
def get_size(self):
         Retrieve the size of the record.
         """
         return conf.lib.clang_Type_getSizeOf(self)
 
     def get_offset(self, fieldname):
         """
         Retrieve the offset of a field in the record.
         """
-        return conf.lib.clang_Type_getOffsetOf(self, c_char_p(fieldname))
+        return conf.lib.clang_Type_getOffsetOf(self, fieldname)
 
     def get_ref_qualifier(self):
         """
         Retrieve the ref-qualifier of the type.
         """
         return RefQualifierKind.from_id(
                 conf.lib.clang_Type_getCXXRefQualifier(self))
 
class Kind:
             return "<ChunkKind: %s>" % self
 
     def __init__(self, completionString, key):
         self.cs = completionString
         self.key = key
         self.__kindNumberCache = -1
 
     def __repr__(self):
-        return "{'" + self.spelling + "', " + str(self.kind) + "}"
+        return "{'" + str(self.spelling) + "', " + str(self.kind) + "}"
 
     @CachedProperty
     def spelling(self):
         if self.__kindNumber in SpellingCache:
                 return SpellingCache[self.__kindNumber]
-        return conf.lib.clang_getCompletionChunkText(self.cs, self.key).spelling
+        return conf.lib.clang_getCompletionChunkText(self.cs, self.key)
 
     # We do not use @CachedProperty here, as the manual implementation is
     # apparently still significantly faster. Please profile carefully if you
     # would like to add CachedProperty back.
     @property
     def __kindNumber(self):
         if self.__kindNumberCache == -1:
             self.__kindNumberCache = \
def briefComment(self):
         if conf.function_exists("clang_getCompletionBriefComment"):
             return conf.lib.clang_getCompletionBriefComment(self.obj)
         return _CXString()
 
     def __repr__(self):
         return " | ".join([str(a) for a in self]) \
                + " || Priority: " + str(self.priority) \
                + " || Availability: " + str(self.availability) \
-               + " || Brief comment: " + str(self.briefComment.spelling)
+               + " || Brief comment: " + str(self.briefComment)
 
 availabilityKinds = {
             0: CompletionChunk.Kind("Available"),
             1: CompletionChunk.Kind("Deprecated"),
             2: CompletionChunk.Kind("NotAvailable"),
             3: CompletionChunk.Kind("NotAccessible")}
 
 class CodeCompletionResult(Structure):
def from_source(cls, filename, args=None, unsaved_files=None, options=0,
         if unsaved_files is None:
             unsaved_files = []
 
         if index is None:
             index = Index.create()
 
         args_array = None
         if len(args) > 0:
-            args_array = (c_char_p * len(args))(* args)
+            args_array = (c_char_p * len(args))(*[b(x) for x in args])
 
         unsaved_array = None
         if len(unsaved_files) > 0:
             unsaved_array = (_CXUnsavedFile * len(unsaved_files))()
             for i, (name, contents) in enumerate(unsaved_files):
                 if hasattr(contents, "read"):
                     contents = contents.read()
 
-                unsaved_array[i].name = name
-                unsaved_array[i].contents = contents
+                unsaved_array[i].name = b(name)
+                unsaved_array[i].contents = b(contents)
                 unsaved_array[i].length = len(contents)
 
-        ptr = conf.lib.clang_parseTranslationUnit(index, filename, args_array,
+        ptr = conf.lib.clang_parseTranslationUnit(index, b(filename), args_array,
                                     len(args), unsaved_array,
                                     len(unsaved_files), options)
 
         if not ptr:
             raise TranslationUnitLoadError("Error parsing translation unit.")
 
         return cls(ptr, index=index)
 
def codeComplete(self, path, line, column, unsaved_files=None,
             for i,(name,value) in enumerate(unsaved_files):
                 if not isinstance(value, str):
                     # FIXME: It would be great to support an efficient version
                     # of this, one day.
                     value = value.read()
                     print(value)
                 if not isinstance(value, str):
                     raise TypeError('Unexpected unsaved file contents.')
-                unsaved_files_array[i].name = name
-                unsaved_files_array[i].contents = value
+                unsaved_files_array[i].name = b(name)
+                unsaved_files_array[i].contents = b(value)
                 unsaved_files_array[i].length = len(value)
         ptr = conf.lib.clang_codeCompleteAt(self, path, line, column,
                 unsaved_files_array, len(unsaved_files), options)
         if ptr:
             return CodeCompletionResults(ptr)
         return None
 
     def get_tokens(self, locations=None, extent=None):
class File(ClangObject):
     @staticmethod
     def from_name(translation_unit, file_name):
         """Retrieve a file handle within the given translation unit."""
         return File(conf.lib.clang_getFile(translation_unit, file_name))
 
     @property
     def name(self):
         """Return the complete file and path name of the file."""
-        return conf.lib.clang_getCString(conf.lib.clang_getFileName(self))
+        return conf.lib.clang_getFileName(self)
 
     @property
     def time(self):
         """Return the last modification time of the file."""
         return conf.lib.clang_getFileTime(self)
 
     def __str__(self):
         return self.name

 functionList = [
   ("clang_annotateTokens",
    [TranslationUnit, POINTER(Token), c_uint, POINTER(Cursor)]),
 
   ("clang_CompilationDatabase_dispose",
    [c_object_p]),
 
   ("clang_CompilationDatabase_fromDirectory",
-   [c_char_p, POINTER(c_uint)],
+   [c_interop_string, POINTER(c_uint)],
    c_object_p,
    CompilationDatabase.from_result),
 
   ("clang_CompilationDatabase_getAllCompileCommands",
    [c_object_p],
    c_object_p,
    CompileCommands.from_result),
 
   ("clang_CompilationDatabase_getCompileCommands",
-   [c_object_p, c_char_p],
+   [c_object_p, c_interop_string],
    c_object_p,
    CompileCommands.from_result),
 
   ("clang_CompileCommands_dispose",
    [c_object_p]),
 
   ("clang_CompileCommands_getCommand",
    [c_object_p, c_uint],
functionList = [
    _CXString,
    _CXString.from_result),
 
   ("clang_CompileCommand_getNumArgs",
    [c_object_p],
    c_uint),
 
   ("clang_codeCompleteAt",
-   [TranslationUnit, c_char_p, c_int, c_int, c_void_p, c_int, c_int],
+   [TranslationUnit, c_interop_string, c_int, c_int, c_void_p, c_int, c_int],
    POINTER(CCRStructure)),
 
   ("clang_codeCompleteGetDiagnostic",
    [CodeCompletionResults, c_int],
    Diagnostic),
 
   ("clang_codeCompleteGetNumDiagnostics",
    [CodeCompletionResults],
    c_int),
 
   ("clang_createIndex",
    [c_int, c_int],
    c_object_p),
 
   ("clang_createTranslationUnit",
-   [Index, c_char_p],
+   [Index, c_interop_string],
    c_object_p),
 
   ("clang_CXXConstructor_isConvertingConstructor",
    [Cursor],
    bool),
 
   ("clang_CXXConstructor_isCopyConstructor",
    [Cursor],
#  [CXTUResourceUsage]),
    bool),
 
   ("clang_equalTypes",
    [Type, Type],
    bool),
 
   ("clang_formatDiagnostic",
    [Diagnostic, c_uint],
-   _CXString),
+   _CXString,
+   _CXString.from_result),
 
   ("clang_getArgType",
    [Type, c_uint],
    Type,
    Type.from_result),
 
   ("clang_getArrayElementType",
    [Type],
#  [CXTUResourceUsage]),
    c_object_p),
 
   ("clang_getCompletionAvailability",
    [c_void_p],
    c_int),
 
   ("clang_getCompletionBriefComment",
    [c_void_p],
-   _CXString),
+   _CXString,
+   _CXString.from_result),
 
   ("clang_getCompletionChunkCompletionString",
    [c_void_p, c_int],
    c_object_p),
 
   ("clang_getCompletionChunkKind",
    [c_void_p, c_int],
    c_int),
 
   ("clang_getCompletionChunkText",
    [c_void_p, c_int],
-   _CXString),
+   _CXString,
+   _CXString.from_result),
 
   ("clang_getCompletionPriority",
    [c_void_p],
    c_int),
 
   ("clang_getCString",
    [_CXString],
-   c_char_p),
+   c_interop_string,
+   c_interop_string.to_python_string),
 
   ("clang_getCursor",
    [TranslationUnit, SourceLocation],
    Cursor),
 
   ("clang_getCursorDefinition",
    [Cursor],
    Cursor,
#  CXTUResourceUsage),
    c_longlong),
 
   ("clang_getEnumDeclIntegerType",
    [Cursor],
    Type,
    Type.from_result),
 
   ("clang_getFile",
-   [TranslationUnit, c_char_p],
+   [TranslationUnit, c_interop_string],
    c_object_p),
 
   ("clang_getFileName",
    [File],
-   _CXString), # TODO go through _CXString.from_result?
+   _CXString,
+   _CXString.from_result),
 
   ("clang_getFileTime",
    [File],
    c_uint),
 
   ("clang_getIBOutletCollectionType",
    [Cursor],
    Type,
#  CXTUResourceUsage),
 
   ("clang_getTranslationUnitSpelling",
    [TranslationUnit],
    _CXString,
    _CXString.from_result),
 
   ("clang_getTUResourceUsageName",
    [c_uint],
-   c_char_p),
+   c_interop_string,
+   c_interop_string.to_python_string),
 
   ("clang_getTypeDeclaration",
    [Type],
    Cursor,
    Cursor.from_result),
 
   ("clang_getTypedefDeclUnderlyingType",
    [Cursor],
#  CXTUResourceUsage),
    [Cursor],
    bool),
 
   ("clang_isVolatileQualifiedType",
    [Type],
    bool),
 
   ("clang_parseTranslationUnit",
-   [Index, c_char_p, c_void_p, c_int, c_void_p, c_int, c_int],
+   [Index, c_interop_string, c_void_p, c_int, c_void_p, c_int, c_int],
    c_object_p),
 
   ("clang_reparseTranslationUnit",
    [TranslationUnit, c_int, c_void_p, c_int],
    c_int),
 
   ("clang_saveTranslationUnit",
-   [TranslationUnit, c_char_p, c_uint],
+   [TranslationUnit, c_interop_string, c_uint],
    c_int),
 
   ("clang_tokenize",
    [TranslationUnit, SourceRange, POINTER(POINTER(Token)), POINTER(c_uint)]),
 
   ("clang_visitChildren",
    [Cursor, callbacks['cursor_visit'], py_object],
    c_uint),
#  CXTUResourceUsage),
    c_longlong),
 
   ("clang_Type_getClassType",
    [Type],
    Type,
    Type.from_result),
 
   ("clang_Type_getOffsetOf",
-   [Type, c_char_p],
+   [Type, c_interop_string],
    c_longlong),
 
   ("clang_Type_getSizeOf",
    [Type],
    c_longlong),
 
   ("clang_Type_getCXXRefQualifier",
    [Type],
def register_functions(lib, ignore_errors):
 
     This must be called as part of library instantiation so Python knows how
     to call out to the shared library.
     """
 
     def register(item):
         return register_function(lib, item, ignore_errors)
 
-    map(register, functionList)
+    for f in functionList:
+        register(f)
 
 class Config:
     library_path = None
     library_file = None
     compatibility_check = True
     loaded = False
 
     @staticmethod

clang/bindings/python/tests/cindex/test_translation_unit.py

 #define SOME_DEFINE y
 """)
             ])
     spellings = [c.spelling for c in tu.cursor.get_children()]
     assert spellings[-2] == 'x'
     assert spellings[-1] == 'y'
 
 def test_unsaved_files_2():
-    import StringIO
+    try:
+        from StringIO import StringIO
+    except:
+        from io import StringIO
     tu = TranslationUnit.from_source('fake.c', unsaved_files = [
-            ('fake.c', StringIO.StringIO('int x;'))])
+            ('fake.c', StringIO('int x;'))])
     spellings = [c.spelling for c in tu.cursor.get_children()]
     assert spellings[-1] == 'x'
 
 def normpaths_equal(path1, path2):
     """ Compares two paths for equality after normalizing them with
         os.path.normpath
     """
     return os.path.normpath(path1) == os.path.normpath(path2)