This patch fixes some refcounting problems with Python 3.8 .
One incompatible change was announced in the what's new
document, but actually there were two more problems which
were not explicitly mentioned but took much time to sort out.
The patch is compatible with the limited API changes
(tested with debug build and API error disabled).
It is also independent of the Python version which is
full Limited API support.
For more info, see the documentation mentioned below.
The flag error is circumvented now! We either find a better
solution or leave it as it is. For now this is ok.
Fixes: PYSIDE-939
Change-Id: Iff4a9816857a6ebe86efd4b654d8921e4e464939
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>
Gbp-Pq: Name 0002-Fix-Python-3.8-problems.patch
static PyType_Slot PropertyListType_slots[] = {
{Py_tp_init, (void *)propListTpInit},
{Py_tp_free, (void *)propListTpFree},
- {Py_tp_dealloc, (void *)object_dealloc},
+ {Py_tp_dealloc, (void *)Sbk_object_dealloc},
{0, 0}
};
static PyType_Spec PropertyListType_spec = {
{Py_tp_str, (void *)reinterpret_cast<reprfunc>(QtQml_VolatileBoolObject_str)},
{Py_tp_methods, (void *)QtQml_VolatileBoolObject_methods},
{Py_tp_new, (void *)QtQml_VolatileBoolObject_new},
- {Py_tp_dealloc, (void *)object_dealloc},
+ {Py_tp_dealloc, (void *)Sbk_object_dealloc},
{0, 0}
};
static PyType_Spec QtQml_VolatileBoolType_spec = {
{Py_tp_init, (void *)classInfoTpInit},
{Py_tp_new, (void *)classInfoTpNew},
{Py_tp_free, (void *)classInfoFree},
- {Py_tp_dealloc, (void *)object_dealloc},
+ {Py_tp_dealloc, (void *)Sbk_object_dealloc},
{0, 0}
};
static PyType_Spec PySideClassInfoType_spec = {
{Py_tp_call, (void *)functionCall},
{Py_tp_new, (void *)PyType_GenericNew},
{Py_tp_free, (void *)functionFree},
- {Py_tp_dealloc, (void *)object_dealloc},
+ {Py_tp_dealloc, (void *)Sbk_object_dealloc},
{0, 0}
};
static PyType_Spec PySideMetaFunctionType_spec = {
void qpropertyDeAlloc(PyObject *self)
{
qpropertyClear(self);
+ if (PepRuntime_38_flag) {
+ // PYSIDE-939: Handling references correctly.
+ // This was not needed before Python 3.8 (Python issue 35810)
+ Py_DECREF(Py_TYPE(self));
+ }
Py_TYPE(self)->tp_free(self);
}
#endif
{Py_tp_new, (void *)PySideQFlagsNew},
{Py_tp_richcompare, (void *)PySideQFlagsRichCompare},
- {Py_tp_dealloc, (void *)object_dealloc},
+ {Py_tp_dealloc, (void *)Sbk_object_dealloc},
{0, 0}
};
static PyType_Spec SbkNewQFlagsType_spec = {
{Py_tp_methods, (void *)MetaSignal_methods},
{Py_tp_base, (void *)&PyType_Type},
{Py_tp_free, (void *)PyObject_GC_Del},
- {Py_tp_dealloc, (void *)object_dealloc},
+ {Py_tp_dealloc, (void *)Sbk_object_dealloc},
{0, 0}
};
static PyType_Spec PySideMetaSignalType_spec = {
{Py_tp_init, (void *)signalTpInit},
{Py_tp_new, (void *)PyType_GenericNew},
{Py_tp_free, (void *)signalFree},
- {Py_tp_dealloc, (void *)object_dealloc},
+ {Py_tp_dealloc, (void *)Sbk_object_dealloc},
{0, 0}
};
static PyType_Spec PySideSignalType_spec = {
{Py_tp_methods, (void *)SignalInstance_methods},
{Py_tp_new, (void *)PyType_GenericNew},
{Py_tp_free, (void *)signalInstanceFree},
- {Py_tp_dealloc, (void *)object_dealloc},
+ {Py_tp_dealloc, (void *)Sbk_object_dealloc},
{0, 0}
};
static PyType_Spec PySideSignalInstanceType_spec = {
{
if (SbkSpecial_Type_Ready(module, PySideMetaSignalTypeF(), MetaSignal_SignatureStrings) < 0)
return;
- Py_INCREF(PySideSignalTypeF());
+ Py_INCREF(PySideMetaSignalTypeF());
PyModule_AddObject(module, "MetaSignal", reinterpret_cast<PyObject *>(PySideMetaSignalTypeF()));
if (SbkSpecial_Type_Ready(module, PySideSignalTypeF(), Signal_SignatureStrings) < 0)
{Py_tp_call, (void *)slotCall},
{Py_tp_init, (void *)slotTpInit},
{Py_tp_new, (void *)PyType_GenericNew},
- {Py_tp_dealloc, (void *)object_dealloc},
+ {Py_tp_dealloc, (void *)Sbk_object_dealloc},
{0, 0}
};
static PyType_Spec PySideSlotType_spec = {
static PyType_Slot PySideCallableObjectType_slots[] = {
{Py_tp_call, (void *)CallableObject_call},
- {Py_tp_dealloc, (void *)object_dealloc},
+ {Py_tp_dealloc, (void *)Sbk_object_dealloc},
{0, 0}
};
static PyType_Spec PySideCallableObjectType_spec = {
PyType_Ready(PySideCallableObjectTypeF());
}
- PySideCallableObject *callable = PyObject_New(PySideCallableObject, PySideCallableObjectTypeF());
+ PyTypeObject *type = PySideCallableObjectTypeF();
+ PySideCallableObject *callable = PyObject_New(PySideCallableObject, type);
if (!callable || PyErr_Occurred())
return 0;
+ if (!PepRuntime_38_flag) {
+ // PYSIDE-939: Handling references correctly.
+ // Workaround for Python issue 35810; no longer necessary in Python 3.8
+ Py_INCREF(type);
+ }
PyObject *weak = PyWeakref_NewRef(obj, reinterpret_cast<PyObject *>(callable));
if (!weak || PyErr_Occurred())
# if this is no debug build, then we check at least that
# we do not crash any longer.
if not skiptest:
- total = sys.gettotalrefcount()
+ total = gettotalrefcount()
for idx in range(1000):
o.selectionModel().destroyed.connect(self.callback)
o.selectionModel().destroyed.disconnect(self.callback)
gc.collect()
if not skiptest:
- self.assertTrue(abs(gettotalrefcount() - total) < 10)
+ delta = gettotalrefcount() - total
+ print("delta total refcount =", delta)
+ self.assertTrue(abs(delta) < 10)
if __name__ == '__main__':
if (metaClass->isNamespace() || metaClass->hasPrivateDestructor()) {
tp_dealloc = metaClass->hasPrivateDestructor() ?
QLatin1String("SbkDeallocWrapperWithPrivateDtor") :
- QLatin1String("object_dealloc /* PYSIDE-832: Prevent replacement of \"0\" with subtype_dealloc. */");
+ QLatin1String("Sbk_object_dealloc /* PYSIDE-832: Prevent replacement of \"0\" with subtype_dealloc. */");
tp_init.clear();
} else {
QString deallocClassName;
#include "qapp_macro.h"
#include "voidptr.h"
+#if defined(__APPLE__)
+#include <dlfcn.h>
+#endif
+
namespace {
void _destroyParentInfo(SbkObject *obj, bool keepReference);
}
extern "C"
{
+// PYSIDE-939: A general replacement for object_dealloc.
+void Sbk_object_dealloc(PyObject *self)
+{
+ if (PepRuntime_38_flag) {
+ // PYSIDE-939: Handling references correctly.
+ // This was not needed before Python 3.8 (Python issue 35810)
+ Py_DECREF(Py_TYPE(self));
+ }
+ Py_TYPE(self)->tp_free(self);
+}
+
static void SbkObjectTypeDealloc(PyObject *pyObj);
static PyObject *SbkObjectTypeTpNew(PyTypeObject *metatype, PyObject *args, PyObject *kwds);
// Need to decref the type if this is the dealloc func; if type
// is subclassed, that dealloc func will decref (see subtype_dealloc
// in typeobject.c in the python sources)
- bool needTypeDecref = (PyType_GetSlot(pyType, Py_tp_dealloc) == SbkDeallocWrapper
+ bool needTypeDecref = (false
+ || PyType_GetSlot(pyType, Py_tp_dealloc) == SbkDeallocWrapper
|| PyType_GetSlot(pyType, Py_tp_dealloc) == SbkDeallocWrapperWithPrivateDtor);
+ if (PepRuntime_38_flag) {
+ // PYSIDE-939: Additional rule: Also when a subtype is heap allocated,
+ // then the subtype_dealloc deref will be suppressed, and we need again
+ // to supply a decref.
+ needTypeDecref |= (pyType->tp_base->tp_flags & Py_TPFLAGS_HEAPTYPE) != 0;
+ }
+
+#if defined(__APPLE__)
+ // Just checking once that our assumptions are right.
+ if (false) {
+ void *p = PyType_GetSlot(pyType, Py_tp_dealloc);
+ Dl_info dl_info;
+ dladdr(p, &dl_info);
+ fprintf(stderr, "tp_dealloc is %s\n", dl_info.dli_sname);
+ }
+ // Gives one of our functions
+ // "Sbk_object_dealloc"
+ // "SbkDeallocWrapperWithPrivateDtor"
+ // "SbkDeallocQAppWrapper"
+ // "SbkDeallocWrapper"
+ // but for typedealloc_test.py we get
+ // "subtype_dealloc"
+#endif
// Ensure that the GC is no longer tracking this object to avoid a
// possible reentrancy problem. Since there are multiple steps involved
if (needTypeDecref)
Py_DECREF(pyType);
+ if (PepRuntime_38_flag) {
+ // PYSIDE-939: Handling references correctly.
+ // This was not needed before Python 3.8 (Python issue 35810)
+ Py_DECREF(pyType);
+ }
}
void SbkDeallocWrapper(PyObject *pyObj)
#ifndef Py_LIMITED_API
Py_TRASHCAN_SAFE_END(pyObj);
#endif
+ if (PepRuntime_38_flag) {
+ // PYSIDE-939: Handling references correctly.
+ // This was not needed before Python 3.8 (Python issue 35810)
+ Py_DECREF(Py_TYPE(pyObj));
+ }
}
PyObject *SbkObjectTypeTpNew(PyTypeObject *metatype, PyObject *args, PyObject *kwds)
// The meta type creates a new type when the Python programmer extends a wrapped C++ class.
auto type_new = reinterpret_cast<newfunc>(PyType_Type.tp_new);
+
+ // PYSIDE-939: This is a temporary patch that circumvents the problem
+ // with Py_TPFLAGS_METHOD_DESCRIPTOR until this is finally solved.
+ PyObject *ob_PyType_Type = reinterpret_cast<PyObject *>(&PyType_Type);
+ PyObject *mro = PyObject_GetAttr(ob_PyType_Type, Shiboken::PyName::mro());
+ auto hold = Py_TYPE(mro)->tp_flags;
+ Py_TYPE(mro)->tp_flags &= ~Py_TPFLAGS_METHOD_DESCRIPTOR;
auto *newType = reinterpret_cast<SbkObjectType *>(type_new(metatype, args, kwds));
+ Py_TYPE(mro)->tp_flags = hold;
+
if (!newType)
return nullptr;
#if PY_VERSION_HEX < 0x03000000
return self == nullptr ? nullptr : _setupNew(self, subtype);
}
-void
-object_dealloc(PyObject *self)
-{
- Py_TYPE(self)->tp_free(self);
-}
-
PyObject *
SbkDummyNew(PyTypeObject *type, PyObject *, PyObject *)
{
};
+/// PYSIDE-939: A general replacement for object_dealloc.
+LIBSHIBOKEN_API void Sbk_object_dealloc(PyObject *self);
+
/// Dealloc the python object \p pyObj and the C++ object represented by it.
LIBSHIBOKEN_API void SbkDeallocWrapper(PyObject *pyObj);
LIBSHIBOKEN_API void SbkDeallocQAppWrapper(PyObject *pyObj);
* nullptr. But the default before conversion to heaptypes was to assign
* object_dealloc. This seems to be a bug in the Limited API.
*/
-LIBSHIBOKEN_API void object_dealloc(PyObject *);
+/// PYSIDE-939: Replaced by Sbk_object_dealloc.
LIBSHIBOKEN_API PyObject *SbkDummyNew(PyTypeObject *type, PyObject *, PyObject *);
} // extern "C"
#include "pep384impl.h"
#include "autodecref.h"
+#include <stdlib.h>
+
extern "C"
{
|| probe_tp_free != check->tp_free
|| probe_tp_is_gc != check->tp_is_gc
|| probe_tp_bases != typetype->tp_bases
- || probe_tp_mro != typetype->tp_mro)
+ || probe_tp_mro != typetype->tp_mro
+ || Py_TPFLAGS_DEFAULT != (check->tp_flags & Py_TPFLAGS_DEFAULT))
Py_FatalError("The structure of type objects has changed!");
Py_DECREF(check);
Py_DECREF(probe_tp_base);
#endif // Py_LIMITED_API
}
+/*****************************************************************************
+ *
+ * Runtime support for Python 3.8 incompatibilities
+ *
+ */
+
+int PepRuntime_38_flag = 0;
+
+static void
+init_PepRuntime()
+{
+ // We expect a string of the form "\d\.\d+\."
+ const char *version = Py_GetVersion();
+ if (version[0] < '3')
+ return;
+ if (std::atoi(version + 2) >= 8)
+ PepRuntime_38_flag = 1;
+}
+
/*****************************************************************************
*
* Module Initialization
Pep384_Init()
{
check_PyTypeObject_valid();
+ init_PepRuntime();
#ifdef Py_LIMITED_API
Pep_GetVerboseFlag();
PepMethod_TypePtr = getMethodType();
Py_ssize_t tp_basicsize;
void *X03; // Py_ssize_t tp_itemsize;
void *X04; // destructor tp_dealloc;
- void *X05; // printfunc tp_print;
+ void *X05; // Py_ssize_t tp_vectorcall_offset;
void *X06; // getattrfunc tp_getattr;
void *X07; // setattrfunc tp_setattr;
void *X08; // PyAsyncMethods *tp_as_async;
void *X16; // getattrofunc tp_getattro;
void *X17; // setattrofunc tp_setattro;
void *X18; // PyBufferProcs *tp_as_buffer;
- void *X19; // unsigned long tp_flags;
+ unsigned long tp_flags;
void *X20; // const char *tp_doc;
traverseproc tp_traverse;
inquiry tp_clear;
} PyTypeObject;
+#ifndef PyObject_IS_GC
+/* Test if an object has a GC head */
+#define PyObject_IS_GC(o) \
+ (PyType_IS_GC(Py_TYPE(o)) \
+ && (Py_TYPE(o)->tp_is_gc == NULL || Py_TYPE(o)->tp_is_gc(o)))
+#endif
+
// This was a macro error in the limited API from the beginning.
// It was fixed in Python master, but did make it only in Python 3.8 .
#define PY_ISSUE33738_SOLVED 0x03080000
// But this is no problem as we check it's validity for every version.
#define PYTHON_BUFFER_VERSION_COMPATIBLE (PY_VERSION_HEX >= 0x03030000 && \
- PY_VERSION_HEX < 0x0307FFFF)
+ PY_VERSION_HEX < 0x0308FFFF)
#if !PYTHON_BUFFER_VERSION_COMPATIBLE
# error Please check the buffer compatibility for this python version!
#endif
#define PepMethodDescr_TypePtr &PyMethodDescr_Type
#endif
+/*****************************************************************************
+ *
+ * Runtime support for Python 3.8 incompatibilities
+ *
+ */
+
+#ifndef Py_TPFLAGS_METHOD_DESCRIPTOR
+/* Objects behave like an unbound method */
+#define Py_TPFLAGS_METHOD_DESCRIPTOR (1UL << 17)
+#endif
+
+extern LIBSHIBOKEN_API int PepRuntime_38_flag;
+
/*****************************************************************************
*
* Module Initialization
#ifndef Py_LIMITED_API
Py_TRASHCAN_SAFE_END(pyObj);
#endif
+ if (PepRuntime_38_flag) {
+ // PYSIDE-939: Handling references correctly.
+ // This was not needed before Python 3.8 (Python issue 35810)
+ Py_DECREF(Py_TYPE(pyObj));
+ }
}
PyObject *SbkEnumTypeTpNew(PyTypeObject *metatype, PyObject *args, PyObject *kwds)
{Py_nb_index, (void *)enum_int},
{Py_tp_richcompare, (void *)enum_richcompare},
{Py_tp_hash, (void *)enum_hash},
- {Py_tp_dealloc, (void *)object_dealloc},
+ {Py_tp_dealloc, (void *)Sbk_object_dealloc},
{0, nullptr}
};
static PyType_Spec SbkNewType_spec = {
if (PyType_Ready(type) < 0)
goto fail;
- // no ht_hached_keys in Python 2
+ // no ht_cached_keys in Python 2
// if (type->tp_dictoffset) {
// res->ht_cached_keys = _PyDict_NewKeysForClass();
// }
{Py_tp_richcompare, (void *)SbkVoidPtrObject_richcmp},
{Py_tp_init, (void *)SbkVoidPtrObject_init},
{Py_tp_new, (void *)SbkVoidPtrObject_new},
- {Py_tp_dealloc, (void *)object_dealloc},
+ {Py_tp_dealloc, (void *)Sbk_object_dealloc},
{Py_tp_methods, (void *)SbkVoidPtrObject_methods},
{0, nullptr}
};
projects / pyside2.git / commitdiff