[libclang] Keep track of TranslationUnit instance when annotating tokens

[jklaehn]

Summary

Previously the _tu was not propagated to the returned cursor, leading to errors when calling any
method on that cursor (e.g. cursor.referenced).

[jbcoe]

Do you need someone to commit this change for you?

I'm happy to do so if you don't have commit access.

[jklaehn]

In D36953#877367, @jbcoe wrote:

Do you need someone to commit this change for you?

I'm happy to do so if you don't have commit access.

Yes that would be great, thanks!

bindings/python/clang/cindex.py

class Token(Structure):
     def extent(self):
         """The SourceRange this Token occupies."""
         return conf.lib.clang_getTokenExtent(self._tu, self)
 
     @property
     def cursor(self):
         """The Cursor this Token corresponds to."""
         cursor = Cursor()
+        cursor._tu = self._tu
 
         conf.lib.clang_annotateTokens(self._tu, byref(self), 1, byref(cursor))
 
         return cursor
 
 # Now comes the plumbing to hook up the C library.
 
 # Register callback types in common container.

bindings/python/tests/cindex/test_cursor.py

def test_get_tokens():
     tu = get_tu('int foo(int i);')
     foo = get_cursor(tu, 'foo')
 
     tokens = list(foo.get_tokens())
     assert len(tokens) == 6
     assert tokens[0].spelling == 'int'
     assert tokens[1].spelling == 'foo'
 
+def test_get_token_cursor():
+    """Ensure we can map tokens to cursors."""
+    tu = get_tu('class A {}; int foo(A var = A());', lang='cpp')
+    foo = get_cursor(tu, 'foo')
+
+    for cursor in foo.walk_preorder():
+        if cursor.kind.is_expression() and not cursor.kind.is_statement():
+            break
+    else:
+        assert False, "Could not find default value expression"
+
+    tokens = list(cursor.get_tokens())
+    assert len(tokens) == 4, [t.spelling for t in tokens]
+    assert tokens[0].spelling == '='
+    assert tokens[1].spelling == 'A'
+    assert tokens[2].spelling == '('
+    assert tokens[3].spelling == ')'
+    t_cursor = tokens[1].cursor
+    assert t_cursor.kind == CursorKind.TYPE_REF
+    r_cursor = t_cursor.referenced # should not raise an exception
+    assert r_cursor.kind == CursorKind.CLASS_DECL
+
 def test_get_arguments():
     tu = get_tu('void foo(int i, int j);')
     foo = get_cursor(tu, 'foo')
     arguments = list(foo.get_arguments())
 
     assert len(arguments) == 2
     assert arguments[0].spelling == "i"
     assert arguments[1].spelling == "j"