Create PythonCallable and PythonTuple wrappers with appropriate unit tests

[zturner]

Summary

This adds PythonTuple and PythonCallable classes to PythonDataObjects.

Additionally, unit tests are provided that exercise this functionality,
including invoking manipulating and checking for validity of tuples,
and invoking and checking for validity of callables using a variety
of different syntaxes.

The goal here is to eventually replace the code in python-wrapper.swig
that directly uses the Python C API to deal with callables and name
resolution with this code that can be more easily tested and debugged.

Note that this patch depends on D14524 (still pending review from Greg) before it can go in.

[Eugene.Zelenko]

Please also run Clang-tidy modernize checks.

source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp

895

Unnecessary line.

source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h

300

Should be PythonList() = default; Same for other similar default constructors.

337

Should be ~PythonTuple() override = default; Same for PythonCallable.

source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h

     List,
     String,
     Module,
+    Callable,
+    Tuple,
     File
 };
 

     Str() const;
 
     static PythonObject
-    ResolveNameGlobal(llvm::StringRef name);
+    ResolveNameWithDictionary(llvm::StringRef name, PythonDictionary dict);
 
     PythonObject
     ResolveName(llvm::StringRef name) const;

 class PythonList : public PythonObject
 {
 public:
+    PythonList() {}
     explicit PythonList(PyInitialValue value);
     explicit PythonList(int list_size);
     PythonList(PyRefType type, PyObject *o);

     StructuredData::ArraySP CreateStructuredArray() const;
 };
 
+class PythonTuple : public PythonObject
+{
+public:
+    PythonTuple() {}
+    explicit PythonTuple(PyInitialValue value);
+    explicit PythonTuple(int tuple_size);
+    PythonTuple(PyRefType type, PyObject *o);
+    PythonTuple(const PythonTuple &tuple);
+    PythonTuple(std::initializer_list<PythonObject> objects);
+    PythonTuple(std::initializer_list<PyObject*> objects);
+
+    ~PythonTuple() override;
+
+    static bool Check(PyObject *py_obj);
+
+    // Bring in the no-argument base class version
+    using PythonObject::Reset;
+
+    void Reset(PyRefType type, PyObject *py_obj) override;
+
+    uint32_t GetSize() const;
+
+    PythonObject GetItemAtIndex(uint32_t index) const;
+
+    void SetItemAtIndex(uint32_t index, const PythonObject &object);
+
+    StructuredData::ArraySP CreateStructuredArray() const;
+};
+
 class PythonDictionary : public PythonObject
 {
 public:
+    PythonDictionary() {}
     explicit PythonDictionary(PyInitialValue value);
     PythonDictionary(PyRefType type, PyObject *o);
     PythonDictionary(const PythonDictionary &dict);

 
     static bool Check(PyObject *py_obj);
 
-    static PythonModule MainModule();
+    static PythonModule
+    BuiltinsModule();
+
+    static PythonModule
+    MainModule();
+
+    static PythonModule
+    AddModule(llvm::StringRef module);
 
     // Bring in the no-argument base class version
     using PythonObject::Reset;

     PythonDictionary GetDictionary() const;
 };
 
+class PythonCallable : public PythonObject
+{
+public:
+    PythonCallable();
+    PythonCallable(PyRefType type, PyObject *o);
+    PythonCallable(const PythonCallable &dict);
+
+    ~PythonCallable() override;
+
+    static bool
+    Check(PyObject *py_obj);
+
+    // Bring in the no-argument base class version
+    using PythonObject::Reset;
+
+    void
+    Reset(PyRefType type, PyObject *py_obj) override;
+
+    void
+    GetNumArguments(size_t &num_args, bool &has_varargs, bool &has_kwargs) const;
+
+    PythonObject
+    operator ()(std::initializer_list<PyObject*> args);
+
+    PythonObject
+    operator ()(std::initializer_list<PythonObject> args);
+};
+
+
 class PythonFile : public PythonObject
 {
   public:

source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp

         return PyObjectType::Module;
     if (PythonList::Check(m_py_obj))
         return PyObjectType::List;
+    if (PythonTuple::Check(m_py_obj))
+        return PyObjectType::Tuple;
     if (PythonDictionary::Check(m_py_obj))
         return PyObjectType::Dictionary;
     if (PythonString::Check(m_py_obj))

         return PyObjectType::Integer;
     if (PythonFile::Check(m_py_obj))
         return PyObjectType::File;
+    if (PythonCallable::Check(m_py_obj))
+        return PyObjectType::Callable;
     return PyObjectType::Unknown;
 }
 

 }
 
 PythonObject
-PythonObject::ResolveNameGlobal(llvm::StringRef name)
+PythonObject::ResolveNameWithDictionary(llvm::StringRef name, PythonDictionary dict)
 {
-    return PythonModule::MainModule().ResolveName(name);
+    return dict.GetItemForKey(PythonString(name));
 }
 
 PythonObject

     if (dot_pos == llvm::StringRef::npos)
     {
         // No dots in the name, we should be able to find the value immediately
-        // as an attribute of `use_object`.
+        // as an attribute of `m_py_obj`.
         return GetAttributeValue(name);
     }
 

 }
 
 //----------------------------------------------------------------------
+// PythonTuple
+//----------------------------------------------------------------------
+
+PythonTuple::PythonTuple(PyInitialValue value)
+    : PythonObject()
+{
+    if (value == PyInitialValue::Empty)
+        Reset(PyRefType::Owned, PyTuple_New(0));
+}
+
+PythonTuple::PythonTuple(int tuple_size)
+    : PythonObject()
+{
+    Reset(PyRefType::Owned, PyTuple_New(tuple_size));
+}
+
+PythonTuple::PythonTuple(PyRefType type, PyObject *py_obj)
+    : PythonObject()
+{
+    Reset(type, py_obj); // Use "Reset()" to ensure that py_obj is a tuple
+}
+
+PythonTuple::PythonTuple(const PythonTuple &tuple)
+    : PythonObject(tuple)
+{
+}
+
+PythonTuple::PythonTuple(std::initializer_list<PythonObject> objects)
+{
+    m_py_obj = PyTuple_New(objects.size());
+
+    uint32_t idx = 0;
+    for (auto object : objects)
+    {
+        if (object.IsValid())
+            SetItemAtIndex(idx, object);
+        idx++;
+    }
+}
+
+PythonTuple::PythonTuple(std::initializer_list<PyObject*> objects)
+{
+    m_py_obj = PyTuple_New(objects.size());
+
+    uint32_t idx = 0;
+    for (auto py_object : objects)
+    {
+        PythonObject object(PyRefType::Borrowed, py_object);
+        if (object.IsValid())
+            SetItemAtIndex(idx, object);
+        idx++;
+    }
+}
+
+PythonTuple::~PythonTuple()
+{
+}
+
+bool
+PythonTuple::Check(PyObject *py_obj)
+{
+    if (!py_obj)
+        return false;
+    return PyTuple_Check(py_obj);
+}
+
+void
+PythonTuple::Reset(PyRefType type, PyObject *py_obj)
+{
+    // Grab the desired reference type so that if we end up rejecting
+    // `py_obj` it still gets decremented if necessary.
+    PythonObject result(type, py_obj);
+
+    if (!PythonTuple::Check(py_obj))
+    {
+        PythonObject::Reset();
+        return;
+    }
+
+    // Calling PythonObject::Reset(const PythonObject&) will lead to stack overflow since it calls
+    // back into the virtual implementation.
+    PythonObject::Reset(PyRefType::Borrowed, result.get());
+}
+
+uint32_t
+PythonTuple::GetSize() const
+{
+    if (IsValid())
+        return PyTuple_GET_SIZE(m_py_obj);
+    return 0;
+}
+
+PythonObject
+PythonTuple::GetItemAtIndex(uint32_t index) const
+{
+    if (IsValid())
+        return PythonObject(PyRefType::Borrowed, PyTuple_GetItem(m_py_obj, index));
+    return PythonObject();
+}
+
+void
+PythonTuple::SetItemAtIndex(uint32_t index, const PythonObject &object)
+{
+    if (IsAllocated() && object.IsValid())
+    {
+        // PyTuple_SetItem is documented to "steal" a reference, so we need to
+        // convert it to an owned reference by incrementing it.
+        Py_INCREF(object.get());
+        PyTuple_SetItem(m_py_obj, index, object.get());
+    }
+}
+
+StructuredData::ArraySP
+PythonTuple::CreateStructuredArray() const
+{
+    StructuredData::ArraySP result(new StructuredData::Array);
+    uint32_t count = GetSize();
+    for (uint32_t i = 0; i < count; ++i)
+    {
+        PythonObject obj = GetItemAtIndex(i);
+        result->AddItem(obj.CreateStructuredObject());
+    }
+    return result;
+}
+
+//----------------------------------------------------------------------
 // PythonDictionary
 //----------------------------------------------------------------------
 

 }
 
 PythonModule
+PythonModule::BuiltinsModule()
+{
+#if PY_MAJOR_VERSION >= 3
+    return AddModule("builtins");
+#else
+    return AddModule("__builtin__");
+#endif
+}
+
+PythonModule
 PythonModule::MainModule()
 {
-    return PythonModule(PyRefType::Borrowed, PyImport_AddModule("__main__"));
+    return AddModule("__main__");
+}
+
+PythonModule
+PythonModule::AddModule(llvm::StringRef module)
+{
+    std::string str = module.str();
+    return PythonModule(PyRefType::Borrowed, PyImport_AddModule(str.c_str()));
 }
 
 bool

     return PythonDictionary(PyRefType::Borrowed, PyModule_GetDict(m_py_obj));
 }
 
+PythonCallable::PythonCallable() : PythonObject()
+{
+}
+
+PythonCallable::PythonCallable(PyRefType type, PyObject *py_obj)
+{
+    Reset(type, py_obj); // Use "Reset()" to ensure that py_obj is a callable
+}
+
+PythonCallable::PythonCallable(const PythonCallable &callable)
+    : PythonObject(callable)
+{
+}
+
+PythonCallable::~PythonCallable()
+{
+}
+
+bool
+PythonCallable::Check(PyObject *py_obj)
+{
+    if (!py_obj)
+        return false;
+
+    return PyCallable_Check(py_obj);
+}
+
+void
+PythonCallable::Reset(PyRefType type, PyObject *py_obj)
+{
+    // Grab the desired reference type so that if we end up rejecting
+    // `py_obj` it still gets decremented if necessary.
+    PythonObject result(type, py_obj);
+
+    if (!PythonCallable::Check(py_obj))
+    {
+        PythonObject::Reset();
+        return;
+    }
+
+    // Calling PythonObject::Reset(const PythonObject&) will lead to stack overflow since it calls
+    // back into the virtual implementation.
+    PythonObject::Reset(PyRefType::Borrowed, result.get());
+}
+
+
+void
+PythonCallable::GetNumArguments(size_t &num_args, bool &has_varargs, bool &has_kwargs) const
+{
+    num_args = 0;
+    has_varargs = false;
+    has_kwargs = false;
+    if (!IsValid())
+        return;
+
+    PyObject *py_func_obj = m_py_obj;
+    if (PyMethod_Check(py_func_obj))
+        py_func_obj = PyMethod_GET_FUNCTION(py_func_obj);
+
+    if (!py_func_obj)
+        return;
+
+    PyCodeObject* code = (PyCodeObject*)PyFunction_GET_CODE(py_func_obj);
+    if (!code)
+        return;
+
+    num_args = code->co_argcount;
+    if (code->co_flags & CO_VARARGS)
+        has_varargs = true;
+    if (code->co_flags & CO_VARKEYWORDS)
+        has_kwargs = true;
+}
+
+PythonObject
+PythonCallable::operator ()(std::initializer_list<PyObject*> args)
+{
+    PythonTuple arg_tuple(args);
+    return PythonObject(PyRefType::Owned,
+        PyObject_CallObject(m_py_obj, arg_tuple.get()));
+}
+
+PythonObject
+PythonCallable::operator ()(std::initializer_list<PythonObject> args)
+{
+    PythonTuple arg_tuple(args);
+    return PythonObject(PyRefType::Owned,
+        PyObject_CallObject(m_py_obj, arg_tuple.get()));
+}
+
 PythonFile::PythonFile()
     : PythonObject()
 {

unittests/ScriptInterpreter/Python/PythonDataObjectsTests.cpp

         // Py_INCREF.  So acquire the GIL for the entire duration of this
         // test suite.
         m_gil_state = PyGILState_Ensure();
+
+        PythonString sys_module("sys");
+        m_sys_module.Reset(PyRefType::Owned, PyImport_Import(sys_module.get()));
+        m_main_module = PythonModule::MainModule();
+        m_builtins_module = PythonModule::BuiltinsModule();
     }
 
     void
     TearDown() override
     {
+        m_sys_module.Reset();
+        m_main_module.Reset();
+        m_builtins_module.Reset();
         PyGILState_Release(m_gil_state);
 
         ScriptInterpreterPython::Terminate();
     }
 
+  protected:
+    PythonModule m_sys_module;
+    PythonModule m_main_module;
+    PythonModule m_builtins_module;
+
   private:
     PyGILState_STATE m_gil_state;
 };

 
 TEST_F(PythonDataObjectsTest, TestGlobalNameResolutionNoDot)
 {
-    PythonObject sys_module = PythonObject::ResolveNameGlobal("sys");
+    PythonObject sys_module = m_main_module.ResolveName("sys");
+    EXPECT_EQ(m_sys_module.get(), sys_module.get());
     EXPECT_TRUE(sys_module.IsAllocated());
     EXPECT_TRUE(PythonModule::Check(sys_module.get()));
 }
 
 TEST_F(PythonDataObjectsTest, TestModuleNameResolutionNoDot)
 {
-    PythonObject sys_module = PythonObject::ResolveNameGlobal("sys");
-    PythonObject sys_path = sys_module.ResolveName("path");
-    PythonObject sys_version_info = sys_module.ResolveName("version_info");
+    PythonObject sys_path = m_sys_module.ResolveName("path");
+    PythonObject sys_version_info = m_sys_module.ResolveName("version_info");
     EXPECT_TRUE(sys_path.IsAllocated());
     EXPECT_TRUE(sys_version_info.IsAllocated());
 

 
 TEST_F(PythonDataObjectsTest, TestTypeNameResolutionNoDot)
 {
-    PythonObject sys_module = PythonObject::ResolveNameGlobal("sys");
-    PythonObject sys_version_info = sys_module.ResolveName("version_info");
+    PythonObject sys_version_info = m_sys_module.ResolveName("version_info");
 
     PythonObject version_info_type(PyRefType::Owned, PyObject_Type(sys_version_info.get()));
     EXPECT_TRUE(version_info_type.IsAllocated());

 
 TEST_F(PythonDataObjectsTest, TestInstanceNameResolutionNoDot)
 {
-    PythonObject sys_module = PythonObject::ResolveNameGlobal("sys");
-    PythonObject sys_version_info = sys_module.ResolveName("version_info");
+    PythonObject sys_version_info = m_sys_module.ResolveName("version_info");
     PythonObject major_version_field = sys_version_info.ResolveName("major");
     PythonObject minor_version_field = sys_version_info.ResolveName("minor");
 

 
 TEST_F(PythonDataObjectsTest, TestGlobalNameResolutionWithDot)
 {
-    PythonObject sys_path = PythonObject::ResolveNameGlobal("sys.path");
+    PythonObject sys_path = m_main_module.ResolveName("sys.path");
     EXPECT_TRUE(sys_path.IsAllocated());
     EXPECT_TRUE(PythonList::Check(sys_path.get()));
 
-    PythonInteger version_major = PythonObject::ResolveNameGlobal("sys.version_info.major").AsType<PythonInteger>();
-    PythonInteger version_minor = PythonObject::ResolveNameGlobal("sys.version_info.minor").AsType<PythonInteger>();
+    PythonInteger version_major = m_main_module.ResolveName("sys.version_info.major").AsType<PythonInteger>();
+    PythonInteger version_minor = m_main_module.ResolveName("sys.version_info.minor").AsType<PythonInteger>();
     EXPECT_TRUE(version_major.IsAllocated());
     EXPECT_TRUE(version_minor.IsAllocated());
     EXPECT_EQ(PY_MAJOR_VERSION, version_major.GetInteger());

     EXPECT_STREQ(string_value1, string_sp->GetValue().c_str());
 }
 
+TEST_F(PythonDataObjectsTest, TestPythonTupleSize)
+{
+    PythonTuple tuple(PyInitialValue::Empty);
+    EXPECT_EQ(0, tuple.GetSize());
+
+    tuple = PythonTuple(3);
+    EXPECT_EQ(3, tuple.GetSize());
+}
+
+TEST_F(PythonDataObjectsTest, TestPythonTupleValues)
+{
+    PythonTuple tuple(3);
+
+    PythonInteger int_value(1);
+    PythonString string_value("Test");
+    PythonObject none_value(PyRefType::Borrowed, Py_None);
+
+    tuple.SetItemAtIndex(0, int_value);
+    tuple.SetItemAtIndex(1, string_value);
+    tuple.SetItemAtIndex(2, none_value);
+
+    EXPECT_EQ(tuple.GetItemAtIndex(0).get(), int_value.get());
+    EXPECT_EQ(tuple.GetItemAtIndex(1).get(), string_value.get());
+    EXPECT_EQ(tuple.GetItemAtIndex(2).get(), none_value.get());
+}
+
+TEST_F(PythonDataObjectsTest, TestPythonTupleInitializerList)
+{
+    PythonInteger int_value(1);
+    PythonString string_value("Test");
+    PythonObject none_value(PyRefType::Borrowed, Py_None);
+    PythonTuple tuple{ int_value, string_value, none_value };
+    EXPECT_EQ(3, tuple.GetSize());
+
+    EXPECT_EQ(tuple.GetItemAtIndex(0).get(), int_value.get());
+    EXPECT_EQ(tuple.GetItemAtIndex(1).get(), string_value.get());
+    EXPECT_EQ(tuple.GetItemAtIndex(2).get(), none_value.get());
+}
+
+TEST_F(PythonDataObjectsTest, TestPythonTupleInitializerList2)
+{
+    PythonInteger int_value(1);
+    PythonString string_value("Test");
+    PythonObject none_value(PyRefType::Borrowed, Py_None);
+
+    PythonTuple tuple{ int_value.get(), string_value.get(), none_value.get() };
+    EXPECT_EQ(3, tuple.GetSize());
+
+    EXPECT_EQ(tuple.GetItemAtIndex(0).get(), int_value.get());
+    EXPECT_EQ(tuple.GetItemAtIndex(1).get(), string_value.get());
+    EXPECT_EQ(tuple.GetItemAtIndex(2).get(), none_value.get());
+}
+
+TEST_F(PythonDataObjectsTest, TestPythonTupleToStructuredList)
+{
+    PythonInteger int_value(1);
+    PythonString string_value("Test");
+
+    PythonTuple tuple{ int_value.get(), string_value.get() };
+
+    auto array_sp = tuple.CreateStructuredArray();
+    EXPECT_EQ(tuple.GetSize(), array_sp->GetSize());
+    EXPECT_EQ(StructuredData::Type::eTypeInteger, array_sp->GetItemAtIndex(0)->GetType());
+    EXPECT_EQ(StructuredData::Type::eTypeString, array_sp->GetItemAtIndex(1)->GetType());
+}
+
 TEST_F(PythonDataObjectsTest, TestPythonDictionaryValueEquality)
 {
     // Test that a dictionary which is built through the native

     EXPECT_EQ(int_value1, int_sp->GetValue());
 }
 
+TEST_F(PythonDataObjectsTest, TestPythonCallableCheck)
+{
+    PythonObject sys_exc_info = m_sys_module.ResolveName("exc_info");
+    PythonObject none(PyRefType::Borrowed, Py_None);
+
+    EXPECT_TRUE(PythonCallable::Check(sys_exc_info.get()));
+    EXPECT_FALSE(PythonCallable::Check(none.get()));
+}
+
+TEST_F(PythonDataObjectsTest, TestPythonCallableInvoke)
+{
+    auto list = m_builtins_module.ResolveName("list").AsType<PythonCallable>();
+    PythonInteger one(1);
+    PythonString two("two");
+    PythonTuple three = { one, two };
+
+    PythonTuple tuple_to_convert = { one, two, three };
+    PythonObject result = list({ tuple_to_convert });
+
+    EXPECT_TRUE(PythonList::Check(result.get()));
+    auto list_result = result.AsType<PythonList>();
+    EXPECT_EQ(3, list_result.GetSize());
+    EXPECT_EQ(one.get(), list_result.GetItemAtIndex(0).get());
+    EXPECT_EQ(two.get(), list_result.GetItemAtIndex(1).get());
+    EXPECT_EQ(three.get(), list_result.GetItemAtIndex(2).get());
+}
+
 TEST_F(PythonDataObjectsTest, TestPythonFile)
 {
     File file(FileSystem::DEV_NULL, File::eOpenOptionRead);