diff --git a/lldb/include/lldb/API/SBFile.h b/lldb/include/lldb/API/SBFile.h --- a/lldb/include/lldb/API/SBFile.h +++ b/lldb/include/lldb/API/SBFile.h @@ -18,6 +18,7 @@ public: SBFile(); + SBFile(FileSP file_sp); SBFile(FILE *file, bool transfer_ownership); SBFile(int fd, const char *mode, bool transfer_ownership); ~SBFile(); @@ -33,7 +34,6 @@ private: FileSP m_opaque_sp; - SBFile(FileSP file_sp); }; } // namespace lldb diff --git a/lldb/packages/Python/lldbsuite/test/python_api/file_handle/TestFileHandle.py b/lldb/packages/Python/lldbsuite/test/python_api/file_handle/TestFileHandle.py --- a/lldb/packages/Python/lldbsuite/test/python_api/file_handle/TestFileHandle.py +++ b/lldb/packages/Python/lldbsuite/test/python_api/file_handle/TestFileHandle.py @@ -13,8 +13,53 @@ import lldb from lldbsuite.test import lldbtest from lldbsuite.test.decorators import ( - add_test_categories, no_debug_info_test, skipIf) - + add_test_categories, skipIf, skipIfWindows) + +class OhNoe(Exception): + pass + +class BadIO(io.TextIOBase): + @property + def closed(self): + return False + def writable(self): + return True + def readable(self): + return True + def write(self, s): + raise OhNoe('OH NOE') + def read(self, n): + raise OhNoe("OH NOE") + def flush(self): + raise OhNoe('OH NOE') + +# This class will raise an exception while it's being +# converted into a C++ object by swig +class ReallyBadIO(io.TextIOBase): + def fileno(self): + return 999 + def writable(self): + raise OhNoe("OH NOE!!!") + +class MutableBool(): + def __init__(self, value): + self.value = value + def set(self, value): + self.value = bool(value) + def __bool__(self): + return self.value + +class FlushTestIO(io.StringIO): + def __init__(self, mutable_flushed, mutable_closed): + super(FlushTestIO, self).__init__() + self.mut_flushed = mutable_flushed + self.mut_closed = mutable_closed + def close(self): + self.mut_closed.set(True) + return super(FlushTestIO, self).close() + def flush(self): + self.mut_flushed.set(True) + return super(FlushTestIO, self).flush() @contextmanager def replace_stdout(new): @@ -36,6 +81,7 @@ class FileHandleTestCase(lldbtest.TestBase): + NO_DEBUG_INFO_TESTCASE = True mydir = lldbtest.Base.compute_mydir(__file__) # The way this class interacts with the debugger is different @@ -84,7 +130,8 @@ @add_test_categories(['pyapi']) - @no_debug_info_test + @skipIfWindows # FIXME pre-existing bug, should be fixed + # when we delete the FILE* typemaps. def test_legacy_file_out_script(self): with open(self.out_filename, 'w') as f: self.debugger.SetOutputFileHandle(f, False) @@ -100,7 +147,6 @@ @add_test_categories(['pyapi']) - @no_debug_info_test def test_legacy_file_out(self): with open(self.out_filename, 'w') as f: self.debugger.SetOutputFileHandle(f, False) @@ -110,7 +156,8 @@ self.assertIn('deadbeef', f.read()) @add_test_categories(['pyapi']) - @no_debug_info_test + @skipIfWindows # FIXME pre-existing bug, should be fixed + # when we delete the FILE* typemaps. def test_legacy_file_err_with_get(self): with open(self.out_filename, 'w') as f: self.debugger.SetErrorFileHandle(f, False) @@ -124,7 +171,6 @@ @add_test_categories(['pyapi']) - @no_debug_info_test def test_legacy_file_err(self): with open(self.out_filename, 'w') as f: self.debugger.SetErrorFileHandle(f, False) @@ -135,7 +181,6 @@ @add_test_categories(['pyapi']) - @no_debug_info_test def test_sbfile_type_errors(self): sbf = lldb.SBFile() self.assertRaises(TypeError, sbf.Write, None) @@ -146,8 +191,7 @@ @add_test_categories(['pyapi']) - @no_debug_info_test - def test_sbfile_write(self): + def test_sbfile_write_fileno(self): with open(self.out_filename, 'w') as f: sbf = lldb.SBFile(f.fileno(), "w", False) self.assertTrue(sbf.IsValid()) @@ -161,8 +205,20 @@ @add_test_categories(['pyapi']) - @no_debug_info_test - def test_sbfile_read(self): + def test_sbfile_write(self): + with open(self.out_filename, 'w') as f: + sbf = lldb.SBFile(f) + e, n = sbf.Write(b'FOO\n') + self.assertTrue(e.Success()) + self.assertEqual(n, 4) + sbf.Close() + self.assertTrue(f.closed) + with open(self.out_filename, 'r') as f: + self.assertEqual(f.read().strip(), 'FOO') + + + @add_test_categories(['pyapi']) + def test_sbfile_read_fileno(self): with open(self.out_filename, 'w') as f: f.write('FOO') with open(self.out_filename, 'r') as f: @@ -175,7 +231,21 @@ @add_test_categories(['pyapi']) - @no_debug_info_test + def test_sbfile_read(self): + with open(self.out_filename, 'w') as f: + f.write('foo') + with open(self.out_filename, 'r') as f: + sbf = lldb.SBFile(f) + buf = bytearray(100) + e, n = sbf.Read(buf) + self.assertTrue(e.Success()) + self.assertEqual(n, 3) + self.assertEqual(buf[:n], b'foo') + sbf.Close() + self.assertTrue(f.closed) + + + @add_test_categories(['pyapi']) def test_fileno_out(self): with open(self.out_filename, 'w') as f: sbf = lldb.SBFile(f.fileno(), "w", False) @@ -189,7 +259,6 @@ @add_test_categories(['pyapi']) - @no_debug_info_test def test_fileno_help(self): with open(self.out_filename, 'w') as f: sbf = lldb.SBFile(f.fileno(), "w", False) @@ -201,7 +270,6 @@ @add_test_categories(['pyapi']) - @no_debug_info_test def test_immediate(self): with open(self.out_filename, 'w') as f: ret = lldb.SBCommandReturnObject() @@ -220,7 +288,6 @@ @add_test_categories(['pyapi']) - @no_debug_info_test def test_fileno_inout(self): with open(self.in_filename, 'w') as f: f.write("help help\n") @@ -244,7 +311,6 @@ @add_test_categories(['pyapi']) - @no_debug_info_test def test_fileno_error(self): with open(self.out_filename, 'w') as f: @@ -263,7 +329,6 @@ #FIXME This shouldn't fail for python2 either. @add_test_categories(['pyapi']) - @no_debug_info_test @skipIf(py_version=['<', (3,)]) def test_replace_stdout(self): f = io.StringIO() @@ -272,3 +337,222 @@ self.handleCmd('script sys.stdout.write("lol")', collect_result=False, check=False) self.assertEqual(sys.stdout, f) + + + @add_test_categories(['pyapi']) + def test_sbfile_write_borrowed(self): + with open(self.out_filename, 'w') as f: + sbf = lldb.SBFile.Create(f, borrow=True) + e, n = sbf.Write(b'FOO') + self.assertTrue(e.Success()) + self.assertEqual(n, 3) + sbf.Close() + self.assertFalse(f.closed) + f.write('BAR\n') + with open(self.out_filename, 'r') as f: + self.assertEqual(f.read().strip(), 'FOOBAR') + + + + @add_test_categories(['pyapi']) + @skipIf(py_version=['<', (3,)]) + def test_sbfile_write_forced(self): + with open(self.out_filename, 'w') as f: + written = MutableBool(False) + orig_write = f.write + def mywrite(x): + written.set(True) + return orig_write(x) + f.write = mywrite + sbf = lldb.SBFile.Create(f, force_io_methods=True) + e, n = sbf.Write(b'FOO') + self.assertTrue(written) + self.assertTrue(e.Success()) + self.assertEqual(n, 3) + sbf.Close() + self.assertTrue(f.closed) + with open(self.out_filename, 'r') as f: + self.assertEqual(f.read().strip(), 'FOO') + + + @add_test_categories(['pyapi']) + @skipIf(py_version=['<', (3,)]) + def test_sbfile_write_forced_borrowed(self): + with open(self.out_filename, 'w') as f: + written = MutableBool(False) + orig_write = f.write + def mywrite(x): + written.set(True) + return orig_write(x) + f.write = mywrite + sbf = lldb.SBFile.Create(f, borrow=True, force_io_methods=True) + e, n = sbf.Write(b'FOO') + self.assertTrue(written) + self.assertTrue(e.Success()) + self.assertEqual(n, 3) + sbf.Close() + self.assertFalse(f.closed) + with open(self.out_filename, 'r') as f: + self.assertEqual(f.read().strip(), 'FOO') + + + @add_test_categories(['pyapi']) + @skipIf(py_version=['<', (3,)]) + def test_sbfile_write_string(self): + f = io.StringIO() + sbf = lldb.SBFile(f) + e, n = sbf.Write(b'FOO') + self.assertEqual(f.getvalue().strip(), "FOO") + self.assertTrue(e.Success()) + self.assertEqual(n, 3) + sbf.Close() + self.assertTrue(f.closed) + + @add_test_categories(['pyapi']) + @skipIf(py_version=['<', (3,)]) + def test_sbfile_write_bytes(self): + f = io.BytesIO() + sbf = lldb.SBFile(f) + e, n = sbf.Write(b'FOO') + self.assertEqual(f.getvalue().strip(), b"FOO") + self.assertTrue(e.Success()) + self.assertEqual(n, 3) + sbf.Close() + self.assertTrue(f.closed) + + @add_test_categories(['pyapi']) + @skipIf(py_version=['<', (3,)]) + def test_sbfile_read_string(self): + f = io.StringIO('zork') + sbf = lldb.SBFile(f) + buf = bytearray(100) + e, n = sbf.Read(buf) + self.assertTrue(e.Success()) + self.assertEqual(buf[:n], b'zork') + + + @add_test_categories(['pyapi']) + @skipIf(py_version=['<', (3,)]) + def test_sbfile_read_string_one_byte(self): + f = io.StringIO('z') + sbf = lldb.SBFile(f) + buf = bytearray(1) + e, n = sbf.Read(buf) + self.assertTrue(e.Fail()) + self.assertEqual(n, 0) + self.assertEqual(e.GetCString(), "can't read less than 6 bytes from a utf8 text stream") + + + @add_test_categories(['pyapi']) + @skipIf(py_version=['<', (3,)]) + def test_sbfile_read_bytes(self): + f = io.BytesIO(b'zork') + sbf = lldb.SBFile(f) + buf = bytearray(100) + e, n = sbf.Read(buf) + self.assertTrue(e.Success()) + self.assertEqual(buf[:n], b'zork') + + + @add_test_categories(['pyapi']) + @skipIf(py_version=['<', (3,)]) + def test_sbfile_out(self): + with open(self.out_filename, 'w') as f: + sbf = lldb.SBFile(f) + status = self.debugger.SetOutputFile(sbf) + self.assertTrue(status.Success()) + self.handleCmd('script 2+2') + with open(self.out_filename, 'r') as f: + self.assertEqual(f.read().strip(), '4') + + + @add_test_categories(['pyapi']) + def test_sbfile_error(self): + with open(self.out_filename, 'w') as f: + sbf = lldb.SBFile(f) + status = self.debugger.SetErrorFile(sbf) + self.assertTrue(status.Success()) + self.handleCmd('lolwut', check=False, collect_result=False) + with open(self.out_filename, 'r') as f: + errors = f.read() + self.assertTrue(re.search(r'error:.*lolwut', errors)) + + + @add_test_categories(['pyapi']) + def test_exceptions(self): + self.assertRaises(TypeError, lldb.SBFile, None) + self.assertRaises(TypeError, lldb.SBFile, "ham sandwich") + if sys.version_info[0] < 3: + self.assertRaises(TypeError, lldb.SBFile, ReallyBadIO()) + else: + self.assertRaises(OhNoe, lldb.SBFile, ReallyBadIO()) + error, n = lldb.SBFile(BadIO()).Write(b"FOO") + self.assertEqual(n, 0) + self.assertTrue(error.Fail()) + self.assertEqual(error.GetCString(), "OhNoe('OH NOE')") + error, n = lldb.SBFile(BadIO()).Read(bytearray(100)) + self.assertEqual(n, 0) + self.assertTrue(error.Fail()) + self.assertEqual(error.GetCString(), "OhNoe('OH NOE')") + + + @add_test_categories(['pyapi']) + @skipIf(py_version=['<', (3,)]) + def test_exceptions_logged(self): + messages = list() + self.debugger.SetLoggingCallback(messages.append) + self.handleCmd('log enable lldb script') + self.debugger.SetOutputFile(lldb.SBFile(BadIO())) + self.handleCmd('script 1+1') + self.assertTrue(any('OH NOE' in msg for msg in messages)) + + + @add_test_categories(['pyapi']) + @skipIf(py_version=['<', (3,)]) + def test_flush(self): + flushed = MutableBool(False) + closed = MutableBool(False) + f = FlushTestIO(flushed, closed) + self.assertFalse(flushed) + self.assertFalse(closed) + sbf = lldb.SBFile(f) + self.assertFalse(flushed) + self.assertFalse(closed) + sbf = None + self.assertFalse(flushed) + self.assertTrue(closed) + self.assertTrue(f.closed) + + flushed = MutableBool(False) + closed = MutableBool(False) + f = FlushTestIO(flushed, closed) + self.assertFalse(flushed) + self.assertFalse(closed) + sbf = lldb.SBFile.Create(f, borrow=True) + self.assertFalse(flushed) + self.assertFalse(closed) + sbf = None + self.assertTrue(flushed) + self.assertFalse(closed) + self.assertFalse(f.closed) + + + @add_test_categories(['pyapi']) + def test_fileno_flush(self): + with open(self.out_filename, 'w') as f: + f.write("foo") + sbf = lldb.SBFile(f) + sbf.Write(b'bar') + sbf = None + self.assertTrue(f.closed) + with open(self.out_filename, 'r') as f: + self.assertEqual(f.read(), 'foobar') + + with open(self.out_filename, 'w+') as f: + f.write("foo") + sbf = lldb.SBFile.Create(f, borrow=True) + sbf.Write(b'bar') + sbf = None + self.assertFalse(f.closed) + f.seek(0) + self.assertEqual(f.read(), 'foobar') diff --git a/lldb/scripts/Python/python-typemaps.swig b/lldb/scripts/Python/python-typemaps.swig --- a/lldb/scripts/Python/python-typemaps.swig +++ b/lldb/scripts/Python/python-typemaps.swig @@ -372,6 +372,69 @@ $1 = $1 || PyCallable_Check(reinterpret_cast($input)); } + +%typemap(in) lldb::FileSP { + using namespace lldb_private; + PythonFile py_file(PyRefType::Borrowed, $input); + if (!py_file) { + PyErr_SetString(PyExc_TypeError, "not a file"); + return nullptr; + } + auto sp = unwrapOrSetPythonException(py_file.ConvertToFile()); + if (!sp) + return nullptr; + $1 = sp; +} + +%typemap(in) lldb::FileSP FORCE_IO_METHODS { + using namespace lldb_private; + PythonFile py_file(PyRefType::Borrowed, $input); + if (!py_file) { + PyErr_SetString(PyExc_TypeError, "not a file"); + return nullptr; + } + auto sp = unwrapOrSetPythonException(py_file.ConvertToFileForcingUseOfScriptingIOMethods()); + if (!sp) + return nullptr; + $1 = sp; +} + +%typemap(in) lldb::FileSP BORROWED { + using namespace lldb_private; + PythonFile py_file(PyRefType::Borrowed, $input); + if (!py_file) { + PyErr_SetString(PyExc_TypeError, "not a file"); + return nullptr; + } + auto sp = unwrapOrSetPythonException(py_file.ConvertToFile(/*borrowed=*/true)); + if (!sp) + return nullptr; + $1 = sp; +} + +%typemap(in) lldb::FileSP BORROWED_FORCE_IO_METHODS { + using namespace lldb_private; + PythonFile py_file(PyRefType::Borrowed, $input); + if (!py_file) { + PyErr_SetString(PyExc_TypeError, "not a file"); + return nullptr; + } + auto sp = unwrapOrSetPythonException(py_file.ConvertToFileForcingUseOfScriptingIOMethods(/*borrowed=*/true)); + if (!sp) + return nullptr; + $1 = sp; +} + +%typecheck(SWIG_TYPECHECK_POINTER) lldb::FileSP { + if (lldb_private::PythonFile::Check($input)) { + $1 = 1; + } else { + PyErr_Clear(); + $1 = 0; + } +} + + // FIXME both of these paths wind up calling fdopen() with no provision for ever calling // fclose() on the result. SB interfaces that use FILE* should be deprecated for scripting // use and this typemap should eventually be removed. diff --git a/lldb/scripts/interface/SBFile.i b/lldb/scripts/interface/SBFile.i --- a/lldb/scripts/interface/SBFile.i +++ b/lldb/scripts/interface/SBFile.i @@ -15,9 +15,53 @@ class SBFile { public: + SBFile(); + + %feature("docstring", " + Initialize a SBFile from a file descriptor. mode is + 'r', 'r+', or 'w', like fdopen."); SBFile(int fd, const char *mode, bool transfer_ownership); + %feature("docstring", "initialize a SBFile from a python file object"); + SBFile(FileSP file); + + %extend { + static lldb::SBFile MakeBorrowed(lldb::FileSP BORROWED) { + return lldb::SBFile(BORROWED); + } + static lldb::SBFile MakeForcingIOMethods(lldb::FileSP FORCE_IO_METHODS) { + return lldb::SBFile(FORCE_IO_METHODS); + } + static lldb::SBFile MakeBorrowedForcingIOMethods(lldb::FileSP BORROWED_FORCE_IO_METHODS) { + return lldb::SBFile(BORROWED_FORCE_IO_METHODS); + } + } + + %pythoncode { + @classmethod + def Create(cls, file, borrow=False, force_io_methods=False): + """ + Create a SBFile from a python file object, with options. + + If borrow is set then the underlying file will + not be closed when the SBFile is closed or destroyed. + + If force_scripting_io is set then the python read/write + methods will be called even if a file descriptor is available. + """ + if borrow: + if force_io_methods: + return cls.MakeBorrowedForcingIOMethods(file) + else: + return cls.MakeBorrowed(file) + else: + if force_io_methods: + return cls.MakeForcingIOMethods(file) + else: + return cls(file) + } + ~SBFile (); %feature("autodoc", "Read(buffer) -> SBError, bytes_read") Read; diff --git a/lldb/source/API/SBFile.cpp b/lldb/source/API/SBFile.cpp --- a/lldb/source/API/SBFile.cpp +++ b/lldb/source/API/SBFile.cpp @@ -16,7 +16,9 @@ SBFile::~SBFile() {} -SBFile::SBFile(FileSP file_sp) : m_opaque_sp(file_sp) {} +SBFile::SBFile(FileSP file_sp) : m_opaque_sp(file_sp) { + LLDB_RECORD_DUMMY(void, SBfile, SBFile, (FileSP), file_sp); +} SBFile::SBFile() { LLDB_RECORD_CONSTRUCTOR_NO_ARGS(SBFile); } diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h --- a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h +++ b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h @@ -6,6 +6,26 @@ // //===----------------------------------------------------------------------===// +// +// !! FIXME FIXME FIXME !! +// +// Python APIs nearly all can return an exception. They do this +// by returning NULL, or -1, or some such value and setting +// the exception state with PyErr_Set*(). Exceptions must be +// handled before further python API functions are called. Failure +// to do so will result in asserts on debug builds of python. +// It will also sometimes, but not usually result in crashes of +// release builds. +// +// Nearly all the code in this header does not handle python exceptions +// correctly. It should all be converted to return Expected<> or +// Error types to capture the exception. +// +// Everything in this file except functions that return Error or +// Expected<> is considered deprecated and should not be +// used in new code. If you need to use it, fix it first. +// + #ifndef LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_PYTHONDATAOBJECTS_H #define LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_PYTHONDATAOBJECTS_H @@ -21,11 +41,16 @@ namespace lldb_private { +using llvm::Error; +using llvm::Expected; + +class PythonObject; class PythonBytes; class PythonString; class PythonList; class PythonDictionary; class PythonInteger; +class PythonException; class StructuredPythonObject : public StructuredData::Generic { public: @@ -72,10 +97,44 @@ // not call Py_INCREF. }; +namespace python { + +// Take a reference that you already own, and turn it into +// a PythonObject. +template static T Take(PyObject *obj) { + return T(PyRefType::Owned, obj); +} + +// Retain a reference you have borrowed, and turn it into +// a PythonObject. +template static T Retain(PyObject *obj) { + return T(PyRefType::Borrowed, obj); +} + +// Most python API methods will return NULL if and only if +// they set an exception. Use this to collect such return +// values. +// +// If you pass something ohter than PythonObject as T, +// you are NOT asserting that the thing is actually of +// type T. You'll get an invalid T back in that case, +// so check if you need to. +template static T AssertTake(PyObject *obj) { + assert(obj); + assert(!PyErr_Occurred()); + T thing(PyRefType::Owned, obj); + return thing; +} + +} // namespace python + enum class PyInitialValue { Invalid, Empty }; class PythonObject { public: + + operator PyObject *() const { return m_py_obj; }; + PythonObject() : m_py_obj(nullptr) {} PythonObject(PyRefType type, PyObject *py_obj) : m_py_obj(nullptr) { @@ -84,14 +143,19 @@ PythonObject(const PythonObject &rhs) : m_py_obj(nullptr) { Reset(rhs); } + PythonObject(PythonObject &&rhs) { + m_py_obj = rhs.m_py_obj; + rhs.m_py_obj = nullptr; + } + virtual ~PythonObject() { Reset(); } void Reset() { // Avoid calling the virtual method since it's not necessary // to actually validate the type of the PyObject if we're // just setting to null. - if (Py_IsInitialized()) - Py_XDECREF(m_py_obj); + if (m_py_obj && Py_IsInitialized()) + Py_DECREF(m_py_obj); m_py_obj = nullptr; } @@ -110,6 +174,8 @@ // PyRefType doesn't make sense, and the copy constructor should be used. void Reset(PyRefType type, const PythonObject &ref) = delete; + // FIXME We shouldn't have virtual anything. PythonObject should be a + // strictly pass-by-value type. virtual void Reset(PyRefType type, PyObject *py_obj) { if (py_obj == m_py_obj) return; @@ -123,7 +189,7 @@ // an owned reference by incrementing it. If it is an owned // reference (for example the caller allocated it with PyDict_New() // then we must *not* increment it. - if (Py_IsInitialized() && type == PyRefType::Borrowed) + if (m_py_obj && Py_IsInitialized() && type == PyRefType::Borrowed) Py_XINCREF(m_py_obj); } @@ -149,6 +215,17 @@ return *this; } + void Reset(PythonObject &&other) { + Reset(); + m_py_obj = other.m_py_obj; + other.m_py_obj = nullptr; + } + + PythonObject &operator=(PythonObject &&other) { + Reset(std::move(other)); + return *this; + } + PyObjectType GetObjectType() const; PythonString Repr() const; @@ -174,11 +251,13 @@ PythonObject GetAttributeValue(llvm::StringRef attribute) const; - bool IsValid() const; + bool IsNone() const { return m_py_obj == Py_None; } + + bool IsValid() const { return m_py_obj != nullptr; } - bool IsAllocated() const; + bool IsAllocated() const { return IsValid() && !IsNone(); } - bool IsNone() const; + operator bool() const { return IsValid() && !IsNone(); } template T AsType() const { if (!T::Check(m_py_obj)) @@ -188,10 +267,91 @@ StructuredData::ObjectSP CreateStructuredObject() const; +protected: + static Error nullDeref() { + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "A NULL PyObject* was dereferenced"); + } + static Error exception(const char *s = nullptr) { + return llvm::make_error(s); + } + +public: + template + Expected CallMethod(const char *name, const char *format, + Args... args) { + if (!m_py_obj) + return nullDeref(); +#if PY_MAJOR_VERSION < 3 + PyObject *obj = PyObject_CallMethod(m_py_obj, const_cast(name), + const_cast(format), args...); +#else + PyObject *obj = PyObject_CallMethod(m_py_obj, name, format, args...); +#endif + if (!obj) + return exception(); + return python::AssertTake(obj); + } + + Expected GetAttribute(const char *name) const { + if (!m_py_obj) + return nullDeref(); + PyObject *obj = PyObject_GetAttrString(m_py_obj, name); + if (!obj) + return exception(); + return python::AssertTake(obj); + } + + Expected IsTrue() { + if (!m_py_obj) + return nullDeref(); + int r = PyObject_IsTrue(m_py_obj); + if (r < 0) + return exception(); + return !!r; + } + + Expected AsLongLong() { + if (!m_py_obj) + return nullDeref(); + assert(!PyErr_Occurred()); + long long r = PyLong_AsLongLong(m_py_obj); + if (PyErr_Occurred()) + return exception(); + return r; + } + + Expected IsInstance(PyObject *cls) { + if (!m_py_obj || !cls) + return nullDeref(); + int r = PyObject_IsInstance(m_py_obj, cls); + if (r < 0) + return exception(); + return !!r; + } + protected: PyObject *m_py_obj; }; +namespace python { +// This is why C++ needs monads. +Expected AsBool(Expected &&obj); + +Expected AsLongLong(Expected &&obj); + +template Expected AsType(Expected &&obj) { + if (obj) { + if (!T::Check(obj.get())) + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "type error"); + return T(PyRefType::Borrowed, std::move(obj.get())); + } else { + return obj.takeError(); + } +} +} // namespace python + class PythonBytes : public PythonObject { public: PythonBytes(); @@ -245,9 +405,12 @@ class PythonString : public PythonObject { public: + static Expected FromUTF8(llvm::StringRef string); + static Expected FromUTF8(const char *string); + PythonString(); - explicit PythonString(llvm::StringRef string); - explicit PythonString(const char *string); + explicit PythonString(llvm::StringRef string); // safe, null on error + explicit PythonString(const char *string); // safe, null on error PythonString(PyRefType type, PyObject *o); ~PythonString() override; @@ -259,11 +422,13 @@ void Reset(PyRefType type, PyObject *py_obj) override; - llvm::StringRef GetString() const; + llvm::StringRef GetString() const; // safe, empty string on error + + Expected AsUTF8() const; size_t GetSize() const; - void SetString(llvm::StringRef string); + void SetString(llvm::StringRef string); // safe, null on error StructuredData::StringSP CreateStructuredString() const; }; @@ -467,8 +632,38 @@ void Reset(File &file, const char *mode); lldb::FileUP GetUnderlyingFile() const; + + llvm::Expected ConvertToFile(bool borrowed = false); + llvm::Expected + ConvertToFileForcingUseOfScriptingIOMethods(bool borrowed = false); }; +class PythonException : public llvm::ErrorInfo { +private: + PyObject *m_exception_type, *m_exception, *m_traceback; + PyObject *m_repr_bytes; + +public: + static char ID; + const char *toCString() const; + PythonException(const char *caller = nullptr); + void Restore(); + ~PythonException(); + void log(llvm::raw_ostream &OS) const override; + std::error_code convertToErrorCode() const override; +}; + +template T unwrapOrSetPythonException(llvm::Expected expected) { + if (expected) + return expected.get(); + llvm::handleAllErrors( + expected.takeError(), [](PythonException &E) { E.Restore(); }, + [](const llvm::ErrorInfoBase &E) { + PyErr_SetString(PyExc_Exception, E.message().c_str()); + }); + return T(); +} + } // namespace lldb_private #endif diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp --- a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp +++ b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp @@ -18,6 +18,7 @@ #include "lldb/Host/File.h" #include "lldb/Host/FileSystem.h" #include "lldb/Interpreter/ScriptInterpreter.h" +#include "lldb/Utility/Log.h" #include "lldb/Utility/Stream.h" #include "llvm/ADT/StringSwitch.h" @@ -28,6 +29,27 @@ using namespace lldb_private; using namespace lldb; +using namespace lldb_private::python; + +namespace lldb_private { +namespace python { +// This is why C++ needs monads. +Expected AsBool(Expected &&obj) { + if (obj) { + return obj.get().IsTrue(); + } else { + return obj.takeError(); + } +} +Expected AsLongLong(Expected &&obj) { + if (obj) { + return obj.get().AsLongLong(); + } else { + return obj.takeError(); + } +} +} // namespace python +} // namespace lldb_private void StructuredPythonObject::Serialize(llvm::json::OStream &s) const { s.value(llvm::formatv("Python Obj: {0:X}", GetValue()).str()); @@ -167,12 +189,6 @@ PyObject_GetAttr(m_py_obj, py_attr.get())); } -bool PythonObject::IsNone() const { return m_py_obj == Py_None; } - -bool PythonObject::IsValid() const { return m_py_obj != nullptr; } - -bool PythonObject::IsAllocated() const { return IsValid() && !IsNone(); } - StructuredData::ObjectSP PythonObject::CreateStructuredObject() const { switch (GetObjectType()) { case PyObjectType::Dictionary: @@ -334,6 +350,21 @@ // PythonString +Expected PythonString::FromUTF8(llvm::StringRef string) { +#if PY_MAJOR_VERSION >= 3 + PyObject *str = PyUnicode_FromStringAndSize(string.data(), string.size()); +#else + PyObject *str = PyString_FromStringAndSize(string.data(), string.size()); +#endif + if (!str) + return llvm::make_error(); + return AssertTake(str); +} + +Expected PythonString::FromUTF8(const char *string) { + return FromUTF8(llvm::StringRef(string)); +} + PythonString::PythonString(PyRefType type, PyObject *py_obj) : PythonObject() { Reset(type, py_obj); // Use "Reset()" to ensure that py_obj is a string } @@ -376,8 +407,12 @@ // In Python 2, Don't store PyUnicode objects directly, because we need // access to their underlying character buffers which Python 2 doesn't // provide. - if (PyUnicode_Check(py_obj)) - result.Reset(PyRefType::Owned, PyUnicode_AsUTF8String(result.get())); + if (PyUnicode_Check(py_obj)) { + PyObject *s = PyUnicode_AsUTF8String(result.get()); + if (s == NULL) + PyErr_Clear(); + result.Reset(PyRefType::Owned, s); + } #endif // Calling PythonObject::Reset(const PythonObject&) will lead to stack // overflow since it calls back into the virtual implementation. @@ -385,8 +420,17 @@ } llvm::StringRef PythonString::GetString() const { + auto s = AsUTF8(); + if (!s) { + llvm::consumeError(s.takeError()); + return llvm::StringRef(""); + } + return s.get(); +} + +Expected PythonString::AsUTF8() const { if (!IsValid()) - return llvm::StringRef(); + return nullDeref(); Py_ssize_t size; const char *data; @@ -394,10 +438,16 @@ #if PY_MAJOR_VERSION >= 3 data = PyUnicode_AsUTF8AndSize(m_py_obj, &size); #else - char *c; - PyString_AsStringAndSize(m_py_obj, &c, &size); + char *c = NULL; + int r = PyString_AsStringAndSize(m_py_obj, &c, &size); + if (r < 0) + c = NULL; data = c; #endif + + if (!data) + return exception(); + return llvm::StringRef(data, size); } @@ -413,13 +463,13 @@ } void PythonString::SetString(llvm::StringRef string) { -#if PY_MAJOR_VERSION >= 3 - PyObject *unicode = PyUnicode_FromStringAndSize(string.data(), string.size()); - PythonObject::Reset(PyRefType::Owned, unicode); -#else - PyObject *str = PyString_FromStringAndSize(string.data(), string.size()); - PythonObject::Reset(PyRefType::Owned, str); -#endif + auto s = FromUTF8(string); + if (!s) { + llvm::consumeError(s.takeError()); + Reset(); + } else { + PythonObject::Reset(std::move(s.get())); + } } StructuredData::StringSP PythonString::CreateStructuredString() const { @@ -963,20 +1013,19 @@ // first-class object type anymore. `PyFile_FromFd` is just a thin wrapper // over `io.open()`, which returns some object derived from `io.IOBase`. As a // result, the only way to detect a file in Python 3 is to check whether it - // inherits from `io.IOBase`. Since it is possible for non-files to also - // inherit from `io.IOBase`, we additionally verify that it has the `fileno` - // attribute, which should guarantee that it is backed by the file system. + // inherits from `io.IOBase`. PythonObject io_module(PyRefType::Owned, PyImport_ImportModule("io")); PythonDictionary io_dict(PyRefType::Borrowed, PyModule_GetDict(io_module.get())); PythonObject io_base_class = io_dict.GetItemForKey(PythonString("IOBase")); + assert(!PyErr_Occurred()); PythonObject object_type(PyRefType::Owned, PyObject_Type(py_obj)); - if (1 != PyObject_IsSubclass(object_type.get(), io_base_class.get())) - return false; - if (!object_type.HasAttribute("fileno")) + if (!PyObject_IsSubclass(object_type.get(), io_base_class.get())) { + PyErr_Clear(); return false; + } return true; #endif @@ -1031,4 +1080,432 @@ return file; } +namespace { +class GIL { +public: + GIL() { + m_state = PyGILState_Ensure(); + assert(!PyErr_Occurred()); + } + ~GIL() { PyGILState_Release(m_state); } + +protected: + PyGILState_STATE m_state; +}; +} + +const char *PythonException::toCString() const { + if (m_repr_bytes) { + return PyBytes_AS_STRING(m_repr_bytes); + } else { + return "unknown exception"; + } +} + +PythonException::PythonException(const char *caller) { + assert(PyErr_Occurred()); + m_exception_type = m_exception = m_traceback = m_repr_bytes = NULL; + PyErr_Fetch(&m_exception_type, &m_exception, &m_traceback); + PyErr_NormalizeException(&m_exception_type, &m_exception, &m_traceback); + if (m_exception) { + PyObject *repr = PyObject_Repr(m_exception); + if (repr) { + m_repr_bytes = PyUnicode_AsEncodedString(repr, "utf-8", nullptr); + Py_XDECREF(repr); + } + } + Log *log = GetLogIfAllCategoriesSet(LIBLLDB_LOG_SCRIPT); + if (caller) + LLDB_LOGF(log, "%s failed with exception: %s", caller, toCString()); + else + LLDB_LOGF(log, "python exception: %s", toCString()); + PyErr_Clear(); +} +void PythonException::Restore() { + if (m_exception_type && m_exception) { + PyErr_Restore(m_exception_type, m_exception, m_traceback); + } else { + PyErr_SetString(PyExc_Exception, toCString()); + } + m_exception_type = m_exception = m_traceback = NULL; +} + +PythonException::~PythonException() { + Py_XDECREF(m_exception_type); + Py_XDECREF(m_exception); + Py_XDECREF(m_traceback); + Py_XDECREF(m_repr_bytes); +} + +void PythonException::log(llvm::raw_ostream & OS) const { OS << toCString(); } + +std::error_code PythonException::convertToErrorCode() const { + return llvm::inconvertibleErrorCode(); +} + +char PythonException::ID = 0; + +llvm::Expected GetOptionsForPyObject(PythonObject & obj) { + uint32_t options = 0; +#if PY_MAJOR_VERSION >= 3 + auto readable = AsBool(obj.CallMethod("readable", "()")); + if (!readable) + return readable.takeError(); + auto writable = AsBool(obj.CallMethod("writable", "()")); + if (!writable) + return writable.takeError(); + if (readable.get()) + options |= File::eOpenOptionRead; + if (writable.get()) + options |= File::eOpenOptionWrite; +#else + PythonString py_mode = obj.GetAttributeValue("mode").AsType(); + options = File::GetOptionsFromMode(py_mode.GetString()); +#endif + return options; +} + +// Base class template for python files. All it knows how to do +// is hold a reference to the python object and close or flush it +// when the File is closed. +namespace{ +template class OwnedPythonFile : public Base { +public: + template + OwnedPythonFile(const PythonFile &file, bool borrowed, Args... args) + : Base(args...), m_py_obj(file), m_borrowed(borrowed) { + assert(m_py_obj); + } + + ~OwnedPythonFile() override { + assert(m_py_obj); + GIL takeGIL; + Close(); + m_py_obj.Reset(); + } + + bool IsPythonSideValid() const { + GIL takeGIL; + auto closed = AsBool(m_py_obj.GetAttribute("closed")); + if (!closed) { + llvm::consumeError(closed.takeError()); + return false; + } + return !closed.get(); + } + + bool IsValid() const override { + return IsPythonSideValid() && Base::IsValid(); + } + + Status Close() override { + assert(m_py_obj); + Status py_error, base_error; + GIL takeGIL; + if (!m_borrowed) { + auto r = m_py_obj.CallMethod("close", "()"); + if (!r) + py_error = Status(r.takeError()); + } + base_error = Base::Close(); + if (py_error.Fail()) + return py_error; + return base_error; + }; + +protected: + PythonFile m_py_obj; + bool m_borrowed; +}; +} + +// A SimplePythonFile is a OwnedPythonFile that just does all I/O as +// a NativeFile +namespace { +class SimplePythonFile : public OwnedPythonFile { +public: + SimplePythonFile(const PythonFile &file, bool borrowed, int fd, + uint32_t options) + : OwnedPythonFile(file, borrowed, fd, options, false){} +}; +} + + +#if PY_MAJOR_VERSION >= 3 + +namespace { +class PythonBuffer { +public: + // you must check PyErr_Occurred() after calling this constructor. + PythonBuffer(PythonObject &obj, int flags = PyBUF_SIMPLE) + : m_buffer({}), m_error(Error::success()) { + PyObject_GetBuffer(obj.get(), &m_buffer, flags); + if (!m_buffer.obj) { + m_error = llvm::make_error(); + } + } + operator bool() { return m_buffer.obj != nullptr; } + Error takeError() { return std::move(m_error); } + ~PythonBuffer() { + if (m_buffer.obj) { + PyBuffer_Release(&m_buffer); + } + if (!m_error) { + llvm::consumeError(std::move(m_error)); + } + } + Py_buffer &get() { return m_buffer; } + +protected: + Py_buffer m_buffer; + Error m_error; +}; +} + +// Shared methods between TextPythonFile and BinaryPythonFile +namespace { +class PythonIOFile : public OwnedPythonFile { +public: + PythonIOFile(const PythonFile &file, bool borrowed) + : OwnedPythonFile(file, borrowed) {} + + ~PythonIOFile() override { Close(); } + + bool IsValid() const override { + return IsPythonSideValid(); + } + + Status Close() override { + assert(m_py_obj); + GIL takeGIL; + if (m_borrowed) + return Flush(); + auto r = m_py_obj.CallMethod("close", "()"); + if (!r) + return Status(r.takeError()); + return Status(); + } + + Status Flush() override { + GIL takeGIL; + auto r = m_py_obj.CallMethod("flush", "()"); + if (!r) + return Status(r.takeError()); + return Status(); + } +}; +} + +namespace { +class BinaryPythonFile : public PythonIOFile { +protected: + int m_descriptor; + +public: + BinaryPythonFile(int fd, const PythonFile &file, bool borrowed) + : PythonIOFile(file, borrowed), + m_descriptor(File::DescriptorIsValid(fd) ? fd + : File::kInvalidDescriptor) {} + + int GetDescriptor() const override { return m_descriptor; } + + Status Write(const void *buf, size_t &num_bytes) override { + GIL takeGIL; + auto pybuffer = Take(PyMemoryView_FromMemory( + const_cast((const char *)buf), num_bytes, PyBUF_READ)); + num_bytes = 0; + + auto bytes_written = AsLongLong( + m_py_obj.CallMethod("write", "(O)", (PyObject *)pybuffer.get())); + if (!bytes_written) + return Status(bytes_written.takeError()); + if (bytes_written.get() < 0) + return Status(".write() method returned a negative number!"); + static_assert(sizeof(long long) >= sizeof(size_t), "overflow"); + num_bytes = bytes_written.get(); + return Status(); + } + + Status Read(void *buf, size_t &num_bytes) override { + GIL takeGIL; + static_assert(sizeof(long long) >= sizeof(size_t), "overflow"); + auto pybuffer_obj = + m_py_obj.CallMethod("read", "(L)", (unsigned long long)num_bytes); + if (!pybuffer_obj) + return Status(pybuffer_obj.takeError()); + num_bytes = 0; + if (pybuffer_obj.get().IsNone()) { + // EOF + num_bytes = 0; + return Status(); + } + PythonBuffer pybuffer(pybuffer_obj.get()); + if (!pybuffer) + return Status(pybuffer.takeError()); + memcpy(buf, pybuffer.get().buf, pybuffer.get().len); + num_bytes = pybuffer.get().len; + return Status(); + } +}; +} + +namespace { +class TextPythonFile : public PythonIOFile { +protected: + int m_descriptor; + +public: + TextPythonFile(int fd, const PythonFile &file, bool borrowed) + : PythonIOFile(file, borrowed), + m_descriptor(File::DescriptorIsValid(fd) ? fd + : File::kInvalidDescriptor) {} + + int GetDescriptor() const override { return m_descriptor; } + + Status Write(const void *buf, size_t &num_bytes) override { + GIL takeGIL; + auto pystring = + PythonString::FromUTF8(llvm::StringRef((const char *)buf, num_bytes)); + if (!pystring) + return Status(pystring.takeError()); + num_bytes = 0; + auto bytes_written = AsLongLong( + m_py_obj.CallMethod("write", "(O)", (PyObject *)pystring.get())); + if (!bytes_written) + return Status(bytes_written.takeError()); + if (bytes_written.get() < 0) + return Status(".write() method returned a negative number!"); + static_assert(sizeof(long long) >= sizeof(size_t), "overflow"); + num_bytes = bytes_written.get(); + return Status(); + } + + Status Read(void *buf, size_t &num_bytes) override { + GIL takeGIL; + size_t num_chars = num_bytes / 6; + size_t orig_num_bytes = num_bytes; + num_bytes = 0; + if (orig_num_bytes < 6) { + return Status("can't read less than 6 bytes from a utf8 text stream"); + } + auto pystring = AsType( + m_py_obj.CallMethod("read", "(L)", (unsigned long long)num_chars)); + if (!pystring) + return Status(pystring.takeError()); + if (pystring.get().IsNone()) { + // EOF + return Status(); + } + auto stringref = pystring.get().AsUTF8(); + if (!stringref) + return Status(stringref.takeError()); + num_bytes = stringref.get().size(); + memcpy(buf, stringref.get().begin(), num_bytes); + return Status(); + } +}; +} + +#endif + +llvm::Expected PythonFile::ConvertToFile(bool borrowed) { + if (!IsValid()) + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "invalid PythonFile"); + + int fd = PyObject_AsFileDescriptor(m_py_obj); + if (fd < 0) { + PyErr_Clear(); + return ConvertToFileForcingUseOfScriptingIOMethods(borrowed); + } + auto options = GetOptionsForPyObject(*this); + if (!options) + return options.takeError(); + + // LLDB and python will not share I/O buffers. We should probably + // flush the python buffers now. + auto r = CallMethod("flush", "()"); + if (!r) + return r.takeError(); + + FileSP file_sp; + if (borrowed) { + // In this case we we don't need to retain the python + // object at all. + file_sp = std::make_shared(fd, options.get(), false); + } else { + file_sp = std::static_pointer_cast( + std::make_shared(*this, borrowed, fd, options.get())); + } + if (!file_sp->IsValid()) + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "invalid File"); + + return file_sp; +} + +llvm::Expected PythonFile::ConvertToFileForcingUseOfScriptingIOMethods( + bool borrowed) { + + assert(!PyErr_Occurred()); + + if (!IsValid()) + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "invalid PythonFile"); + +#if PY_MAJOR_VERSION < 3 + + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "not supported on python 2"); + +#else + + int fd = PyObject_AsFileDescriptor(m_py_obj); + if (fd < 0) { + PyErr_Clear(); + fd = File::kInvalidDescriptor; + } + + auto io_module = Take(PyImport_ImportModule("io")); + auto io_dict = Retain(PyModule_GetDict(io_module.get())); + auto textIOBase = io_dict.GetItemForKey(PythonString("TextIOBase")); + auto rawIOBase = io_dict.GetItemForKey(PythonString("BufferedIOBase")); + auto bufferedIOBase = io_dict.GetItemForKey(PythonString("RawIOBase")); + // python interpreter badly janked if we can't get those. + assert(!PyErr_Occurred()); + + FileSP file_sp; + + auto isTextIO = IsInstance(textIOBase); + if (!isTextIO) + return isTextIO.takeError(); + if (isTextIO.get()) + file_sp = std::static_pointer_cast( + std::make_shared(fd, *this, borrowed)); + + auto isRawIO = IsInstance(rawIOBase); + if (!isRawIO) + return isRawIO.takeError(); + auto isBufferedIO = IsInstance(bufferedIOBase); + if (!isBufferedIO) + return isBufferedIO.takeError(); + + if (isRawIO.get() || isBufferedIO.get()) { + file_sp = std::static_pointer_cast( + std::make_shared(fd, *this, borrowed)); + } + + if (!file_sp) + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "python file is neither text nor binary"); + + if (!file_sp->IsValid()) + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "invalid File"); + + return file_sp; + +#endif +} + #endif