From f1ef66137f80b1238e3ac2914290163dda802922 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 16 Aug 2021 00:15:49 +0800 Subject: [PATCH 01/12] Do not reset tp_version_tag --- Include/internal/pycore_object.h | 2 ++ Lib/test/test_type_cache.py | 41 ++++++++++++++++++++++++++++++++ Objects/typeobject.c | 16 +++++++++---- Python/clinic/sysmodule.c.h | 19 ++++++++++++++- Python/sysmodule.c | 35 ++++++++++++++++++++++++++- 5 files changed, 106 insertions(+), 7 deletions(-) create mode 100644 Lib/test/test_type_cache.py diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 744b41ae5d90d6..16e95fbf971ec3 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -182,6 +182,8 @@ extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); extern int _PyObject_InitializeDict(PyObject *obj); +extern unsigned int _PyType_ClearCache_NoResetGlobalVersionTag(void); + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py new file mode 100644 index 00000000000000..638232bcec0c22 --- /dev/null +++ b/Lib/test/test_type_cache.py @@ -0,0 +1,41 @@ +""" Tests for the internal type cache in CPython. """ +import unittest +from collections import Counter +from test import support +try: + from sys import _clear_type_cache, _get_type_version_tag +except ImportError: + _clear_type_cache = None + +msg = "requires sys._clear_type_cache and sys._get_type_version_tag" +@unittest.skipIf(_clear_type_cache is None, msg) +@support.cpython_only +class TypeCacheTests(unittest.TestCase): + def test_tp_version_tag_unique(self): + """tp_version_tag should be unique assuming no overflow, even after + clearing type cache. + """ + all_version_tags = [] + append_result = all_version_tags.append + # Note: try to avoid any method lookups within this loop, + # It will affect global version tag. + for _ in range(30): + _clear_type_cache() + class C: + # __init__ is only here to force + # C.tp_version_tag to update when initializing + # object C(). + def __init__(self): + pass + x = C() + tp_version_tag_after = _get_type_version_tag(C) + # Try to avoid self.assertNotEqual to not overflow type cache. + assert tp_version_tag_after != 0 + append_result(tp_version_tag_after) + tp_version_tag, count = Counter(all_version_tags).most_common(1)[0] + self.assertEqual(count, 1, + msg=f"tp_version_tag {tp_version_tag} was not unique") + + +if __name__ == "__main__": + support.run_unittest(TypeCacheTests) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 7ae50c453ed2f8..7087dc5e8097db 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -272,7 +272,7 @@ _PyType_InitCache(PyInterpreterState *interp) static unsigned int -_PyType_ClearCache(PyInterpreterState *interp) +_PyType_ClearCache(PyInterpreterState *interp, int reset_global_version_tag) { struct type_cache *cache = &interp->type_cache; #if MCACHE_STATS @@ -288,7 +288,7 @@ _PyType_ClearCache(PyInterpreterState *interp) #endif unsigned int cur_version_tag = next_version_tag - 1; - if (_Py_IsMainInterpreter(interp)) { + if (reset_global_version_tag && _Py_IsMainInterpreter(interp)) { next_version_tag = 0; } @@ -302,14 +302,20 @@ unsigned int PyType_ClearCache(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); - return _PyType_ClearCache(interp); + return _PyType_ClearCache(interp, 1); } +unsigned int +_PyType_ClearCache_NoResetGlobalVersionTag(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + return _PyType_ClearCache(interp, 1); +} void _PyType_Fini(PyInterpreterState *interp) { - _PyType_ClearCache(interp); + _PyType_ClearCache(interp, 1); if (_Py_IsMainInterpreter(interp)) { clear_slotdefs(); } @@ -3853,7 +3859,7 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) } #endif assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); - Py_SETREF(entry->name, Py_NewRef(name)); + Py_XSETREF(entry->name, Py_NewRef(name)); } return res; } diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 8350fbf98561a9..8e0c558acc5ed7 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -925,6 +925,19 @@ sys__clear_type_cache(PyObject *module, PyObject *Py_UNUSED(ignored)) return sys__clear_type_cache_impl(module); } +#if defined(Py_DEBUG) + +PyDoc_STRVAR(sys__get_type_version_tag__doc__, +"_get_type_version_tag($module, type, /)\n" +"--\n" +"\n" +"For internal use only: get the `tp_version_tag` of *type*."); + +#define SYS__GET_TYPE_VERSION_TAG_METHODDEF \ + {"_get_type_version_tag", (PyCFunction)sys__get_type_version_tag, METH_O, sys__get_type_version_tag__doc__}, + +#endif /* defined(Py_DEBUG) */ + PyDoc_STRVAR(sys_is_finalizing__doc__, "is_finalizing($module, /)\n" "--\n" @@ -989,7 +1002,11 @@ sys_getandroidapilevel(PyObject *module, PyObject *Py_UNUSED(ignored)) #define SYS_GETTOTALREFCOUNT_METHODDEF #endif /* !defined(SYS_GETTOTALREFCOUNT_METHODDEF) */ +#ifndef SYS__GET_TYPE_VERSION_TAG_METHODDEF + #define SYS__GET_TYPE_VERSION_TAG_METHODDEF +#endif /* !defined(SYS__GET_TYPE_VERSION_TAG_METHODDEF) */ + #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=855fc93b2347710b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8e0a9df4ee406dcb input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 5dfa917e8ffb20..b367c5e867db9b 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1937,10 +1937,42 @@ static PyObject * sys__clear_type_cache_impl(PyObject *module) /*[clinic end generated code: output=20e48ca54a6f6971 input=127f3e04a8d9b555]*/ { - PyType_ClearCache(); + // Don't reset the global tp_version_tag. It causes version tag resuse and + // cache inconsistencies. + _PyType_ClearCache_NoResetGlobalVersionTag(); Py_RETURN_NONE; } +#ifdef Py_DEBUG +/*[clinic input] +sys._get_type_version_tag + + type: object + / + +For internal use only: get the `tp_version_tag` of *type*. +[clinic start generated code]*/ + +static PyObject * +sys__get_type_version_tag(PyObject *module, PyObject *type) +/*[clinic end generated code: output=bdcfab8b295e0a76 input=6016e5a02b3c3675]*/ +{ + PyThreadState *tstate = _PyThreadState_GET(); + _Py_EnsureTstateNotNULL(tstate); + + if (!PyType_Check(type)) { + _PyErr_SetString(tstate, PyExc_TypeError, "argument must be a type"); + return NULL; + } + PyObject *res = PyLong_FromUnsignedLong(((PyTypeObject *)type)->tp_version_tag); + if (res == NULL) { + assert(_PyErr_Occurred(tstate)); + return NULL; + } + return res; +} +#endif /* Py_DEBUG */ + /*[clinic input] sys.is_finalizing @@ -1976,6 +2008,7 @@ static PyMethodDef sys_methods[] = { {"breakpointhook", (PyCFunction)(void(*)(void))sys_breakpointhook, METH_FASTCALL | METH_KEYWORDS, breakpointhook_doc}, SYS__CLEAR_TYPE_CACHE_METHODDEF + SYS__GET_TYPE_VERSION_TAG_METHODDEF SYS__CURRENT_FRAMES_METHODDEF SYS__CURRENT_EXCEPTIONS_METHODDEF SYS_DISPLAYHOOK_METHODDEF From 3a99d64a7fedfc8bd69e7f869234c2727410d104 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 16 Aug 2021 20:30:57 +0800 Subject: [PATCH 02/12] revert changes to typeobject, add test to test_sys --- Include/internal/pycore_object.h | 2 -- Lib/test/test_sys.py | 5 +++++ Objects/typeobject.c | 16 +++++----------- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 16e95fbf971ec3..744b41ae5d90d6 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -182,8 +182,6 @@ extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); extern int _PyObject_InitializeDict(PyObject *obj); -extern unsigned int _PyType_ClearCache_NoResetGlobalVersionTag(void); - #ifdef __cplusplus } #endif diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index dba4928ec261ac..3b645fbaa17e01 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -628,6 +628,11 @@ def test_sys_getwindowsversion_no_instantiation(self): def test_clear_type_cache(self): sys._clear_type_cache() + @test.support.cpython_only + def test_get_type_version_tag(self): + test.support.get_attribute(sys, "_get_type_version_tag") + self.assertIsInstance(sys._get_type_version_tag(int), int) + def test_ioencoding(self): env = dict(os.environ) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 7087dc5e8097db..7ae50c453ed2f8 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -272,7 +272,7 @@ _PyType_InitCache(PyInterpreterState *interp) static unsigned int -_PyType_ClearCache(PyInterpreterState *interp, int reset_global_version_tag) +_PyType_ClearCache(PyInterpreterState *interp) { struct type_cache *cache = &interp->type_cache; #if MCACHE_STATS @@ -288,7 +288,7 @@ _PyType_ClearCache(PyInterpreterState *interp, int reset_global_version_tag) #endif unsigned int cur_version_tag = next_version_tag - 1; - if (reset_global_version_tag && _Py_IsMainInterpreter(interp)) { + if (_Py_IsMainInterpreter(interp)) { next_version_tag = 0; } @@ -302,20 +302,14 @@ unsigned int PyType_ClearCache(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); - return _PyType_ClearCache(interp, 1); + return _PyType_ClearCache(interp); } -unsigned int -_PyType_ClearCache_NoResetGlobalVersionTag(void) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - return _PyType_ClearCache(interp, 1); -} void _PyType_Fini(PyInterpreterState *interp) { - _PyType_ClearCache(interp, 1); + _PyType_ClearCache(interp); if (_Py_IsMainInterpreter(interp)) { clear_slotdefs(); } @@ -3859,7 +3853,7 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) } #endif assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); - Py_XSETREF(entry->name, Py_NewRef(name)); + Py_SETREF(entry->name, Py_NewRef(name)); } return res; } From 852c380da83124d269e07b9c4ed4e55392b876c0 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 16 Aug 2021 20:34:40 +0800 Subject: [PATCH 03/12] remove old code --- Python/sysmodule.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index b367c5e867db9b..d2cb87edb55c19 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1937,9 +1937,7 @@ static PyObject * sys__clear_type_cache_impl(PyObject *module) /*[clinic end generated code: output=20e48ca54a6f6971 input=127f3e04a8d9b555]*/ { - // Don't reset the global tp_version_tag. It causes version tag resuse and - // cache inconsistencies. - _PyType_ClearCache_NoResetGlobalVersionTag(); + PyType_ClearCache(); Py_RETURN_NONE; } From f78d56237e0e0be32c805f982a242c37f1c55352 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 16 Aug 2021 20:36:11 +0800 Subject: [PATCH 04/12] cast to unsigned long --- Python/sysmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index d2cb87edb55c19..fb524e00421e13 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1962,7 +1962,8 @@ sys__get_type_version_tag(PyObject *module, PyObject *type) _PyErr_SetString(tstate, PyExc_TypeError, "argument must be a type"); return NULL; } - PyObject *res = PyLong_FromUnsignedLong(((PyTypeObject *)type)->tp_version_tag); + PyObject *res = PyLong_FromUnsignedLong( + (unsigned long)((PyTypeObject *)type)->tp_version_tag); if (res == NULL) { assert(_PyErr_Occurred(tstate)); return NULL; From aa2508584e25047b3d06e4fd5e0aed485b901d18 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 16 Aug 2021 20:39:21 +0800 Subject: [PATCH 05/12] test for raise TypeError on non-type --- Lib/test/test_sys.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 3b645fbaa17e01..1ec6cd30e7bd2b 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -632,6 +632,7 @@ def test_clear_type_cache(self): def test_get_type_version_tag(self): test.support.get_attribute(sys, "_get_type_version_tag") self.assertIsInstance(sys._get_type_version_tag(int), int) + self.assertRaises(TypeError, sys._get_type_version_tag, 1) def test_ioencoding(self): env = dict(os.environ) From d7ba8f06215e75c74b1191b642a8fc5133439c81 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 16 Aug 2021 23:12:20 +0800 Subject: [PATCH 06/12] Revert changes in sys module, use _testcapimodule, handle overflow --- Lib/test/test_type_cache.py | 36 ++++++++++++++++++++++-------------- Modules/_testcapimodule.c | 18 ++++++++++++++++++ Python/clinic/sysmodule.c.h | 19 +------------------ Python/sysmodule.c | 32 -------------------------------- 4 files changed, 41 insertions(+), 64 deletions(-) diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 638232bcec0c22..4784bc1edae6b7 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -2,35 +2,43 @@ import unittest from collections import Counter from test import support +from test.support import import_helper try: - from sys import _clear_type_cache, _get_type_version_tag + from sys import _clear_type_cache except ImportError: _clear_type_cache = None +# Skip this test if the _testcapi module isn't available. +type_get_version = import_helper.import_module('_testcapi').type_get_version + msg = "requires sys._clear_type_cache and sys._get_type_version_tag" -@unittest.skipIf(_clear_type_cache is None, msg) @support.cpython_only +@unittest.skipIf(_clear_type_cache is None, msg) class TypeCacheTests(unittest.TestCase): def test_tp_version_tag_unique(self): """tp_version_tag should be unique assuming no overflow, even after clearing type cache. """ - all_version_tags = [] - append_result = all_version_tags.append + # Check if global version tag has already overflowed. + Y = type('Y', (), {}) + Y.x = 1 + Y.x # Force a _PyType_Lookup, populating version tag + y_ver = type_get_version(Y) + # Overflow, or not enough left to conduct the test. + if y_ver == 0 or y_ver > 0xFFFFF000: + self.skipTest("Out of type version tags") # Note: try to avoid any method lookups within this loop, # It will affect global version tag. + all_version_tags = [] + append_result = all_version_tags.append + assertNotEqual = self.assertNotEqual for _ in range(30): _clear_type_cache() - class C: - # __init__ is only here to force - # C.tp_version_tag to update when initializing - # object C(). - def __init__(self): - pass - x = C() - tp_version_tag_after = _get_type_version_tag(C) - # Try to avoid self.assertNotEqual to not overflow type cache. - assert tp_version_tag_after != 0 + X = type('Y', (), {}) + X.x = 1 + X.x + tp_version_tag_after = type_get_version(X) + assertNotEqual(tp_version_tag_after, 0, msg="Version overflowed") append_result(tp_version_tag_after) tp_version_tag, count = Counter(all_version_tags).most_common(1)[0] self.assertEqual(count, 1, diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f338e89f426da0..cf9908128a6bb4 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5575,6 +5575,23 @@ test_fatal_error(PyObject *self, PyObject *args) Py_RETURN_NONE; } +// type->tp_version_tag +static PyObject * +type_get_version(PyObject *self, PyObject *type) +{ + if (!PyType_Check(type)) { + PyErr_SetString(PyExc_TypeError, "argument must be a type"); + return NULL; + } + PyObject *res = PyLong_FromUnsignedLong( + (unsigned long)((PyTypeObject *)type)->tp_version_tag); + if (res == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + return res; +} + static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); static PyObject *getargs_s_hash_int(PyObject *, PyObject *, PyObject*); @@ -5854,6 +5871,7 @@ static PyMethodDef TestMethods[] = { {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, {"fatal_error", test_fatal_error, METH_VARARGS, PyDoc_STR("fatal_error(message, release_gil=False): call Py_FatalError(message)")}, + {"type_get_version", type_get_version, METH_O, PyDoc_STR("type->tp_version_tag")}, {NULL, NULL} /* sentinel */ }; diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 8e0c558acc5ed7..8350fbf98561a9 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -925,19 +925,6 @@ sys__clear_type_cache(PyObject *module, PyObject *Py_UNUSED(ignored)) return sys__clear_type_cache_impl(module); } -#if defined(Py_DEBUG) - -PyDoc_STRVAR(sys__get_type_version_tag__doc__, -"_get_type_version_tag($module, type, /)\n" -"--\n" -"\n" -"For internal use only: get the `tp_version_tag` of *type*."); - -#define SYS__GET_TYPE_VERSION_TAG_METHODDEF \ - {"_get_type_version_tag", (PyCFunction)sys__get_type_version_tag, METH_O, sys__get_type_version_tag__doc__}, - -#endif /* defined(Py_DEBUG) */ - PyDoc_STRVAR(sys_is_finalizing__doc__, "is_finalizing($module, /)\n" "--\n" @@ -1002,11 +989,7 @@ sys_getandroidapilevel(PyObject *module, PyObject *Py_UNUSED(ignored)) #define SYS_GETTOTALREFCOUNT_METHODDEF #endif /* !defined(SYS_GETTOTALREFCOUNT_METHODDEF) */ -#ifndef SYS__GET_TYPE_VERSION_TAG_METHODDEF - #define SYS__GET_TYPE_VERSION_TAG_METHODDEF -#endif /* !defined(SYS__GET_TYPE_VERSION_TAG_METHODDEF) */ - #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=8e0a9df4ee406dcb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=855fc93b2347710b input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index fb524e00421e13..5dfa917e8ffb20 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1941,37 +1941,6 @@ sys__clear_type_cache_impl(PyObject *module) Py_RETURN_NONE; } -#ifdef Py_DEBUG -/*[clinic input] -sys._get_type_version_tag - - type: object - / - -For internal use only: get the `tp_version_tag` of *type*. -[clinic start generated code]*/ - -static PyObject * -sys__get_type_version_tag(PyObject *module, PyObject *type) -/*[clinic end generated code: output=bdcfab8b295e0a76 input=6016e5a02b3c3675]*/ -{ - PyThreadState *tstate = _PyThreadState_GET(); - _Py_EnsureTstateNotNULL(tstate); - - if (!PyType_Check(type)) { - _PyErr_SetString(tstate, PyExc_TypeError, "argument must be a type"); - return NULL; - } - PyObject *res = PyLong_FromUnsignedLong( - (unsigned long)((PyTypeObject *)type)->tp_version_tag); - if (res == NULL) { - assert(_PyErr_Occurred(tstate)); - return NULL; - } - return res; -} -#endif /* Py_DEBUG */ - /*[clinic input] sys.is_finalizing @@ -2007,7 +1976,6 @@ static PyMethodDef sys_methods[] = { {"breakpointhook", (PyCFunction)(void(*)(void))sys_breakpointhook, METH_FASTCALL | METH_KEYWORDS, breakpointhook_doc}, SYS__CLEAR_TYPE_CACHE_METHODDEF - SYS__GET_TYPE_VERSION_TAG_METHODDEF SYS__CURRENT_FRAMES_METHODDEF SYS__CURRENT_EXCEPTIONS_METHODDEF SYS_DISPLAYHOOK_METHODDEF From 5abf657078fff6312764c5598d64c35259a718bf Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 16 Aug 2021 23:14:08 +0800 Subject: [PATCH 07/12] revert changes in test_sys --- Lib/test/test_sys.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 1ec6cd30e7bd2b..dba4928ec261ac 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -628,12 +628,6 @@ def test_sys_getwindowsversion_no_instantiation(self): def test_clear_type_cache(self): sys._clear_type_cache() - @test.support.cpython_only - def test_get_type_version_tag(self): - test.support.get_attribute(sys, "_get_type_version_tag") - self.assertIsInstance(sys._get_type_version_tag(int), int) - self.assertRaises(TypeError, sys._get_type_version_tag, 1) - def test_ioencoding(self): env = dict(os.environ) From fe0e29c3fdcf926fdc4f0c5953f8a8c62acf2689 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 16 Aug 2021 23:15:22 +0800 Subject: [PATCH 08/12] fix message --- Lib/test/test_type_cache.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 4784bc1edae6b7..cdd7a67d3572aa 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -11,9 +11,8 @@ # Skip this test if the _testcapi module isn't available. type_get_version = import_helper.import_module('_testcapi').type_get_version -msg = "requires sys._clear_type_cache and sys._get_type_version_tag" @support.cpython_only -@unittest.skipIf(_clear_type_cache is None, msg) +@unittest.skipIf(_clear_type_cache is None, "requires sys._clear_type_cache") class TypeCacheTests(unittest.TestCase): def test_tp_version_tag_unique(self): """tp_version_tag should be unique assuming no overflow, even after From e907b5d415e6bce1efffd5d13293c7c874eae813 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 16 Aug 2021 23:45:21 +0800 Subject: [PATCH 09/12] use sys over _testcapimodule --- Lib/test/test_sys.py | 6 ++++++ Lib/test/test_type_cache.py | 15 ++++++--------- Modules/_testcapimodule.c | 18 ------------------ Python/clinic/sysmodule.c.h | 19 ++++++++++++++++++- Python/sysmodule.c | 32 ++++++++++++++++++++++++++++++++ 5 files changed, 62 insertions(+), 28 deletions(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index dba4928ec261ac..1ec6cd30e7bd2b 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -628,6 +628,12 @@ def test_sys_getwindowsversion_no_instantiation(self): def test_clear_type_cache(self): sys._clear_type_cache() + @test.support.cpython_only + def test_get_type_version_tag(self): + test.support.get_attribute(sys, "_get_type_version_tag") + self.assertIsInstance(sys._get_type_version_tag(int), int) + self.assertRaises(TypeError, sys._get_type_version_tag, 1) + def test_ioencoding(self): env = dict(os.environ) diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index cdd7a67d3572aa..2613f7d974ea4d 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -2,17 +2,14 @@ import unittest from collections import Counter from test import support -from test.support import import_helper try: - from sys import _clear_type_cache + from sys import _clear_type_cache, _get_type_version_tag except ImportError: _clear_type_cache = None -# Skip this test if the _testcapi module isn't available. -type_get_version = import_helper.import_module('_testcapi').type_get_version - +msg = "requires sys._clear_type_cache and _get_type_version_tag" @support.cpython_only -@unittest.skipIf(_clear_type_cache is None, "requires sys._clear_type_cache") +@unittest.skipIf(_clear_type_cache is None, msg) class TypeCacheTests(unittest.TestCase): def test_tp_version_tag_unique(self): """tp_version_tag should be unique assuming no overflow, even after @@ -22,11 +19,11 @@ def test_tp_version_tag_unique(self): Y = type('Y', (), {}) Y.x = 1 Y.x # Force a _PyType_Lookup, populating version tag - y_ver = type_get_version(Y) + y_ver = _get_type_version_tag (Y) # Overflow, or not enough left to conduct the test. if y_ver == 0 or y_ver > 0xFFFFF000: self.skipTest("Out of type version tags") - # Note: try to avoid any method lookups within this loop, + # Note: try to avoid any method lookups within this loop. # It will affect global version tag. all_version_tags = [] append_result = all_version_tags.append @@ -36,7 +33,7 @@ def test_tp_version_tag_unique(self): X = type('Y', (), {}) X.x = 1 X.x - tp_version_tag_after = type_get_version(X) + tp_version_tag_after = _get_type_version_tag(X) assertNotEqual(tp_version_tag_after, 0, msg="Version overflowed") append_result(tp_version_tag_after) tp_version_tag, count = Counter(all_version_tags).most_common(1)[0] diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index cf9908128a6bb4..f338e89f426da0 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5575,23 +5575,6 @@ test_fatal_error(PyObject *self, PyObject *args) Py_RETURN_NONE; } -// type->tp_version_tag -static PyObject * -type_get_version(PyObject *self, PyObject *type) -{ - if (!PyType_Check(type)) { - PyErr_SetString(PyExc_TypeError, "argument must be a type"); - return NULL; - } - PyObject *res = PyLong_FromUnsignedLong( - (unsigned long)((PyTypeObject *)type)->tp_version_tag); - if (res == NULL) { - assert(PyErr_Occurred()); - return NULL; - } - return res; -} - static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); static PyObject *getargs_s_hash_int(PyObject *, PyObject *, PyObject*); @@ -5871,7 +5854,6 @@ static PyMethodDef TestMethods[] = { {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, {"fatal_error", test_fatal_error, METH_VARARGS, PyDoc_STR("fatal_error(message, release_gil=False): call Py_FatalError(message)")}, - {"type_get_version", type_get_version, METH_O, PyDoc_STR("type->tp_version_tag")}, {NULL, NULL} /* sentinel */ }; diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 8350fbf98561a9..8e0c558acc5ed7 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -925,6 +925,19 @@ sys__clear_type_cache(PyObject *module, PyObject *Py_UNUSED(ignored)) return sys__clear_type_cache_impl(module); } +#if defined(Py_DEBUG) + +PyDoc_STRVAR(sys__get_type_version_tag__doc__, +"_get_type_version_tag($module, type, /)\n" +"--\n" +"\n" +"For internal use only: get the `tp_version_tag` of *type*."); + +#define SYS__GET_TYPE_VERSION_TAG_METHODDEF \ + {"_get_type_version_tag", (PyCFunction)sys__get_type_version_tag, METH_O, sys__get_type_version_tag__doc__}, + +#endif /* defined(Py_DEBUG) */ + PyDoc_STRVAR(sys_is_finalizing__doc__, "is_finalizing($module, /)\n" "--\n" @@ -989,7 +1002,11 @@ sys_getandroidapilevel(PyObject *module, PyObject *Py_UNUSED(ignored)) #define SYS_GETTOTALREFCOUNT_METHODDEF #endif /* !defined(SYS_GETTOTALREFCOUNT_METHODDEF) */ +#ifndef SYS__GET_TYPE_VERSION_TAG_METHODDEF + #define SYS__GET_TYPE_VERSION_TAG_METHODDEF +#endif /* !defined(SYS__GET_TYPE_VERSION_TAG_METHODDEF) */ + #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=855fc93b2347710b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8e0a9df4ee406dcb input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 5dfa917e8ffb20..fb524e00421e13 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1941,6 +1941,37 @@ sys__clear_type_cache_impl(PyObject *module) Py_RETURN_NONE; } +#ifdef Py_DEBUG +/*[clinic input] +sys._get_type_version_tag + + type: object + / + +For internal use only: get the `tp_version_tag` of *type*. +[clinic start generated code]*/ + +static PyObject * +sys__get_type_version_tag(PyObject *module, PyObject *type) +/*[clinic end generated code: output=bdcfab8b295e0a76 input=6016e5a02b3c3675]*/ +{ + PyThreadState *tstate = _PyThreadState_GET(); + _Py_EnsureTstateNotNULL(tstate); + + if (!PyType_Check(type)) { + _PyErr_SetString(tstate, PyExc_TypeError, "argument must be a type"); + return NULL; + } + PyObject *res = PyLong_FromUnsignedLong( + (unsigned long)((PyTypeObject *)type)->tp_version_tag); + if (res == NULL) { + assert(_PyErr_Occurred(tstate)); + return NULL; + } + return res; +} +#endif /* Py_DEBUG */ + /*[clinic input] sys.is_finalizing @@ -1976,6 +2007,7 @@ static PyMethodDef sys_methods[] = { {"breakpointhook", (PyCFunction)(void(*)(void))sys_breakpointhook, METH_FASTCALL | METH_KEYWORDS, breakpointhook_doc}, SYS__CLEAR_TYPE_CACHE_METHODDEF + SYS__GET_TYPE_VERSION_TAG_METHODDEF SYS__CURRENT_FRAMES_METHODDEF SYS__CURRENT_EXCEPTIONS_METHODDEF SYS_DISPLAYHOOK_METHODDEF From 8a01d9c318cf11c08c71bbfd4562c3c0f77bfea5 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 17 Aug 2021 00:50:21 +0800 Subject: [PATCH 10/12] simplify code --- Python/sysmodule.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index fb524e00421e13..a2fb8a1c5165ca 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1955,17 +1955,14 @@ static PyObject * sys__get_type_version_tag(PyObject *module, PyObject *type) /*[clinic end generated code: output=bdcfab8b295e0a76 input=6016e5a02b3c3675]*/ { - PyThreadState *tstate = _PyThreadState_GET(); - _Py_EnsureTstateNotNULL(tstate); - if (!PyType_Check(type)) { - _PyErr_SetString(tstate, PyExc_TypeError, "argument must be a type"); + PyErr_SetString(PyExc_TypeError, "argument must be a type"); return NULL; } PyObject *res = PyLong_FromUnsignedLong( (unsigned long)((PyTypeObject *)type)->tp_version_tag); if (res == NULL) { - assert(_PyErr_Occurred(tstate)); + assert(PyErr_Occurred()); return NULL; } return res; From 2a043e7a2e1d57dfff26cd03bd7537d22a46d6ad Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 17 Aug 2021 01:02:35 +0800 Subject: [PATCH 11/12] Revert to using _testcapimodule --- Lib/test/test_sys.py | 6 ------ Lib/test/test_type_cache.py | 15 +++++++++------ Modules/_testcapimodule.c | 18 ++++++++++++++++++ Python/clinic/sysmodule.c.h | 19 +------------------ Python/sysmodule.c | 29 ----------------------------- 5 files changed, 28 insertions(+), 59 deletions(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 1ec6cd30e7bd2b..dba4928ec261ac 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -628,12 +628,6 @@ def test_sys_getwindowsversion_no_instantiation(self): def test_clear_type_cache(self): sys._clear_type_cache() - @test.support.cpython_only - def test_get_type_version_tag(self): - test.support.get_attribute(sys, "_get_type_version_tag") - self.assertIsInstance(sys._get_type_version_tag(int), int) - self.assertRaises(TypeError, sys._get_type_version_tag, 1) - def test_ioencoding(self): env = dict(os.environ) diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 2613f7d974ea4d..cdd7a67d3572aa 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -2,14 +2,17 @@ import unittest from collections import Counter from test import support +from test.support import import_helper try: - from sys import _clear_type_cache, _get_type_version_tag + from sys import _clear_type_cache except ImportError: _clear_type_cache = None -msg = "requires sys._clear_type_cache and _get_type_version_tag" +# Skip this test if the _testcapi module isn't available. +type_get_version = import_helper.import_module('_testcapi').type_get_version + @support.cpython_only -@unittest.skipIf(_clear_type_cache is None, msg) +@unittest.skipIf(_clear_type_cache is None, "requires sys._clear_type_cache") class TypeCacheTests(unittest.TestCase): def test_tp_version_tag_unique(self): """tp_version_tag should be unique assuming no overflow, even after @@ -19,11 +22,11 @@ def test_tp_version_tag_unique(self): Y = type('Y', (), {}) Y.x = 1 Y.x # Force a _PyType_Lookup, populating version tag - y_ver = _get_type_version_tag (Y) + y_ver = type_get_version(Y) # Overflow, or not enough left to conduct the test. if y_ver == 0 or y_ver > 0xFFFFF000: self.skipTest("Out of type version tags") - # Note: try to avoid any method lookups within this loop. + # Note: try to avoid any method lookups within this loop, # It will affect global version tag. all_version_tags = [] append_result = all_version_tags.append @@ -33,7 +36,7 @@ def test_tp_version_tag_unique(self): X = type('Y', (), {}) X.x = 1 X.x - tp_version_tag_after = _get_type_version_tag(X) + tp_version_tag_after = type_get_version(X) assertNotEqual(tp_version_tag_after, 0, msg="Version overflowed") append_result(tp_version_tag_after) tp_version_tag, count = Counter(all_version_tags).most_common(1)[0] diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f338e89f426da0..cf9908128a6bb4 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5575,6 +5575,23 @@ test_fatal_error(PyObject *self, PyObject *args) Py_RETURN_NONE; } +// type->tp_version_tag +static PyObject * +type_get_version(PyObject *self, PyObject *type) +{ + if (!PyType_Check(type)) { + PyErr_SetString(PyExc_TypeError, "argument must be a type"); + return NULL; + } + PyObject *res = PyLong_FromUnsignedLong( + (unsigned long)((PyTypeObject *)type)->tp_version_tag); + if (res == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + return res; +} + static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); static PyObject *getargs_s_hash_int(PyObject *, PyObject *, PyObject*); @@ -5854,6 +5871,7 @@ static PyMethodDef TestMethods[] = { {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, {"fatal_error", test_fatal_error, METH_VARARGS, PyDoc_STR("fatal_error(message, release_gil=False): call Py_FatalError(message)")}, + {"type_get_version", type_get_version, METH_O, PyDoc_STR("type->tp_version_tag")}, {NULL, NULL} /* sentinel */ }; diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 8e0c558acc5ed7..8350fbf98561a9 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -925,19 +925,6 @@ sys__clear_type_cache(PyObject *module, PyObject *Py_UNUSED(ignored)) return sys__clear_type_cache_impl(module); } -#if defined(Py_DEBUG) - -PyDoc_STRVAR(sys__get_type_version_tag__doc__, -"_get_type_version_tag($module, type, /)\n" -"--\n" -"\n" -"For internal use only: get the `tp_version_tag` of *type*."); - -#define SYS__GET_TYPE_VERSION_TAG_METHODDEF \ - {"_get_type_version_tag", (PyCFunction)sys__get_type_version_tag, METH_O, sys__get_type_version_tag__doc__}, - -#endif /* defined(Py_DEBUG) */ - PyDoc_STRVAR(sys_is_finalizing__doc__, "is_finalizing($module, /)\n" "--\n" @@ -1002,11 +989,7 @@ sys_getandroidapilevel(PyObject *module, PyObject *Py_UNUSED(ignored)) #define SYS_GETTOTALREFCOUNT_METHODDEF #endif /* !defined(SYS_GETTOTALREFCOUNT_METHODDEF) */ -#ifndef SYS__GET_TYPE_VERSION_TAG_METHODDEF - #define SYS__GET_TYPE_VERSION_TAG_METHODDEF -#endif /* !defined(SYS__GET_TYPE_VERSION_TAG_METHODDEF) */ - #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=8e0a9df4ee406dcb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=855fc93b2347710b input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index a2fb8a1c5165ca..5dfa917e8ffb20 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1941,34 +1941,6 @@ sys__clear_type_cache_impl(PyObject *module) Py_RETURN_NONE; } -#ifdef Py_DEBUG -/*[clinic input] -sys._get_type_version_tag - - type: object - / - -For internal use only: get the `tp_version_tag` of *type*. -[clinic start generated code]*/ - -static PyObject * -sys__get_type_version_tag(PyObject *module, PyObject *type) -/*[clinic end generated code: output=bdcfab8b295e0a76 input=6016e5a02b3c3675]*/ -{ - if (!PyType_Check(type)) { - PyErr_SetString(PyExc_TypeError, "argument must be a type"); - return NULL; - } - PyObject *res = PyLong_FromUnsignedLong( - (unsigned long)((PyTypeObject *)type)->tp_version_tag); - if (res == NULL) { - assert(PyErr_Occurred()); - return NULL; - } - return res; -} -#endif /* Py_DEBUG */ - /*[clinic input] sys.is_finalizing @@ -2004,7 +1976,6 @@ static PyMethodDef sys_methods[] = { {"breakpointhook", (PyCFunction)(void(*)(void))sys_breakpointhook, METH_FASTCALL | METH_KEYWORDS, breakpointhook_doc}, SYS__CLEAR_TYPE_CACHE_METHODDEF - SYS__GET_TYPE_VERSION_TAG_METHODDEF SYS__CURRENT_FRAMES_METHODDEF SYS__CURRENT_EXCEPTIONS_METHODDEF SYS_DISPLAYHOOK_METHODDEF From 7c60a68f548e0dd0d2ac5507dbba1c8a1b32a9a9 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 17 Aug 2021 01:06:36 +0800 Subject: [PATCH 12/12] Apply Victor's suggestions --- Lib/test/test_type_cache.py | 7 +++---- Modules/_testcapimodule.c | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index cdd7a67d3572aa..8502f6b0584b00 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -1,6 +1,5 @@ """ Tests for the internal type cache in CPython. """ import unittest -from collections import Counter from test import support from test.support import import_helper try: @@ -11,6 +10,7 @@ # Skip this test if the _testcapi module isn't available. type_get_version = import_helper.import_module('_testcapi').type_get_version + @support.cpython_only @unittest.skipIf(_clear_type_cache is None, "requires sys._clear_type_cache") class TypeCacheTests(unittest.TestCase): @@ -39,9 +39,8 @@ def test_tp_version_tag_unique(self): tp_version_tag_after = type_get_version(X) assertNotEqual(tp_version_tag_after, 0, msg="Version overflowed") append_result(tp_version_tag_after) - tp_version_tag, count = Counter(all_version_tags).most_common(1)[0] - self.assertEqual(count, 1, - msg=f"tp_version_tag {tp_version_tag} was not unique") + self.assertEqual(len(set(all_version_tags)), 30, + msg=f"{all_version_tags} contains non-unique versions") if __name__ == "__main__": diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index cf9908128a6bb4..c35eac78ebe880 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5584,7 +5584,7 @@ type_get_version(PyObject *self, PyObject *type) return NULL; } PyObject *res = PyLong_FromUnsignedLong( - (unsigned long)((PyTypeObject *)type)->tp_version_tag); + ((PyTypeObject *)type)->tp_version_tag); if (res == NULL) { assert(PyErr_Occurred()); return NULL;