From b1b7fd5fab21f7e938de1d963160ebbdbe3b4aef Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 3 Jun 2019 23:22:30 +0100 Subject: [PATCH 1/4] bpo-37146: Allow to configure the opcode cache and fix huntleaks by presetting it to 1 --- Include/internal/pycore_pystate.h | 9 +++ Lib/test/libregrtest/runtest.py | 4 +- .../2019-06-03-23-29-35.bpo-37146.5B7ZiD.rst | 3 + Python/ceval.c | 18 +++++- Python/clinic/sysmodule.c.h | 63 ++++++++++++++++++- Python/pylifecycle.c | 3 + Python/sysmodule.c | 40 ++++++++++++ 7 files changed, 135 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-06-03-23-29-35.bpo-37146.5B7ZiD.rst diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 520a74b8a61fde..246dd480d6ae4f 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -43,6 +43,10 @@ struct _pending_calls { int last; }; +struct _opcodecache_config { + int minruns; +}; + struct _ceval_runtime_state { int recursion_limit; /* Records whether tracing is on for any thread. Counts the number @@ -60,6 +64,7 @@ struct _ceval_runtime_state { /* Request for checking signals. */ _Py_atomic_int signals_pending; struct _gil_runtime_state gil; + struct _opcodecache_config opcodeconfig; }; /* interpreter state */ @@ -312,6 +317,10 @@ PyAPI_FUNC(void) _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime); PyAPI_FUNC(void) _PyGILState_Reinit(_PyRuntimeState *runtime); +PyAPI_FUNC(void) _PyEval_SetOpcodeCacheMinRuns(int minruns); + +PyAPI_FUNC(int) _PyEval_GetOpcodeCacheMinRuns(); + #ifdef __cplusplus } #endif diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index a43b7666cd1e35..22706a2e3ef6a4 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -206,6 +206,8 @@ def _test_module(the_module): def _runtest_inner2(ns, test_name): # Load the test function, run the test function, handle huntrleaks # and findleaks to detect leaks + minruns = sys._getopcacheminruns() + sys._setopcacheminruns(1) abstest = get_abs_module(ns, test_name) @@ -243,7 +245,7 @@ def _runtest_inner2(ns, test_name): gc.garbage.clear() support.reap_children() - + sys._setopcacheminruns(1) return refleak diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-06-03-23-29-35.bpo-37146.5B7ZiD.rst b/Misc/NEWS.d/next/Core and Builtins/2019-06-03-23-29-35.bpo-37146.5B7ZiD.rst new file mode 100644 index 00000000000000..81ba8e7e84dc08 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-06-03-23-29-35.bpo-37146.5B7ZiD.rst @@ -0,0 +1,3 @@ +Added ``sys._setopcacheminruns`` and ``sys._getopcacheminruns`` to allow +configuring the minimum number of function executions to activate the global +opcode cache. diff --git a/Python/ceval.c b/Python/ceval.c index 0a4af915d6ffe0..cdb5881834758f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -103,7 +103,6 @@ static long dxp[256]; #endif /* per opcode cache */ -#define OPCACHE_MIN_RUNS 1024 /* create opcache when code executed this time */ #define OPCACHE_STATS 0 /* Enable stats */ #if OPCACHE_STATS @@ -1152,9 +1151,10 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) f->f_stacktop = NULL; /* remains NULL unless yield suspends frame */ f->f_executing = 1; - if (co->co_opcache_flag < OPCACHE_MIN_RUNS) { + int opcacheminruns = _PyEval_GetOpcodeCacheMinRuns(); + if (co->co_opcache_flag < opcacheminruns) { co->co_opcache_flag++; - if (co->co_opcache_flag == OPCACHE_MIN_RUNS) { + if (co->co_opcache_flag == opcacheminruns) { if (_PyCode_InitOpcache(co) < 0) { return NULL; } @@ -5632,3 +5632,15 @@ maybe_dtrace_line(PyFrameObject *frame, } *instr_prev = frame->f_lasti; } + +void +_PyEval_SetOpcodeCacheMinRuns(int minruns) +{ + _PyRuntime.ceval.opcodeconfig.minruns = minruns; +} + +int +_PyEval_GetOpcodeCacheMinRuns() +{ + return _PyRuntime.ceval.opcodeconfig.minruns; +} diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 6f248ff18d9d7f..b5635cf3fbd5ed 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -337,6 +337,67 @@ sys_getcheckinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) return sys_getcheckinterval_impl(module); } +PyDoc_STRVAR(sys__setopcacheminruns__doc__, +"_setopcacheminruns($module, minruns, /)\n" +"--\n" +"\n" +"Set the minimum value of runs to activate opcode cache."); + +#define SYS__SETOPCACHEMINRUNS_METHODDEF \ + {"_setopcacheminruns", (PyCFunction)sys__setopcacheminruns, METH_O, sys__setopcacheminruns__doc__}, + +static PyObject * +sys__setopcacheminruns_impl(PyObject *module, int minruns); + +static PyObject * +sys__setopcacheminruns(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int minruns; + + if (PyFloat_Check(arg)) { + PyErr_SetString(PyExc_TypeError, + "integer argument expected, got float" ); + goto exit; + } + minruns = _PyLong_AsInt(arg); + if (minruns == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = sys__setopcacheminruns_impl(module, minruns); + +exit: + return return_value; +} + +PyDoc_STRVAR(sys__getopcacheminruns__doc__, +"_getopcacheminruns($module, /)\n" +"--\n" +"\n" +"Return the current minimum value of runs to activate opcode cache."); + +#define SYS__GETOPCACHEMINRUNS_METHODDEF \ + {"_getopcacheminruns", (PyCFunction)sys__getopcacheminruns, METH_NOARGS, sys__getopcacheminruns__doc__}, + +static int +sys__getopcacheminruns_impl(PyObject *module); + +static PyObject * +sys__getopcacheminruns(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = sys__getopcacheminruns_impl(module); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(sys_setswitchinterval__doc__, "setswitchinterval($module, interval, /)\n" "--\n" @@ -1082,4 +1143,4 @@ sys_getandroidapilevel(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=43c4fde7b5783d8d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7cf1810bb97ca9a4 input=a9049054013a1b77]*/ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index fca2ee65519104..462530b9797b97 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -737,6 +737,9 @@ _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args) return status; } + // XXX: Add this properly to the commandline config + runtime->ceval.opcodeconfig.minruns = 1024; + runtime->pre_initialized = 1; return _PyStatus_OK(); } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 97bff94d8b4196..a464e0b38c7aee 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1030,6 +1030,44 @@ sys_getcheckinterval_impl(PyObject *module) return PyLong_FromLong(interp->check_interval); } +/*[clinic input] +sys._setopcacheminruns + + minruns: int + / + +Set the minimum value of runs to activate opcode cache. +[clinic start generated code]*/ + +static PyObject * +sys__setopcacheminruns_impl(PyObject *module, int minruns) +/*[clinic end generated code: output=e29d082b62b2aa8a input=169a982eee6595d5]*/ +{ + if (minruns <= 0) { + PyErr_SetString(PyExc_ValueError, + "opcache min runs must be strictly positive"); + return NULL; + } + _PyEval_SetOpcodeCacheMinRuns(minruns); + Py_RETURN_NONE; +} + + +/*[clinic input] +sys._getopcacheminruns -> int + +Return the current minimum value of runs to activate opcode cache. +[clinic start generated code]*/ + +static int +sys__getopcacheminruns_impl(PyObject *module) +/*[clinic end generated code: output=f25d23b7afd8512c input=93784822a3e883c9]*/ +{ + return _PyEval_GetOpcodeCacheMinRuns(); +} + + + /*[clinic input] sys.setswitchinterval @@ -1936,6 +1974,8 @@ static PyMethodDef sys_methods[] = { SYS_MDEBUG_METHODDEF SYS_SETCHECKINTERVAL_METHODDEF SYS_GETCHECKINTERVAL_METHODDEF + SYS__SETOPCACHEMINRUNS_METHODDEF + SYS__GETOPCACHEMINRUNS_METHODDEF SYS_SETSWITCHINTERVAL_METHODDEF SYS_GETSWITCHINTERVAL_METHODDEF SYS_SETDLOPENFLAGS_METHODDEF From bd3a951ff81eab28b77d05290a2466573c41eda2 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 3 Jun 2019 23:31:04 +0100 Subject: [PATCH 2/4] fixup! bpo-37146: Allow to configure the opcode cache and fix huntleaks by presetting it to 1 --- Lib/test/libregrtest/runtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 22706a2e3ef6a4..e31fbd9fc5e792 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -245,7 +245,7 @@ def _runtest_inner2(ns, test_name): gc.garbage.clear() support.reap_children() - sys._setopcacheminruns(1) + sys._setopcacheminruns(minruns) return refleak From d58a33f0988258c87e5055f5dbed50accd1eea97 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 3 Jun 2019 23:34:27 +0100 Subject: [PATCH 3/4] fixup! fixup! bpo-37146: Allow to configure the opcode cache and fix huntleaks by presetting it to 1 --- Include/internal/pycore_pystate.h | 9 ++------- Python/ceval.c | 4 ++-- Python/pylifecycle.c | 1 - 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 246dd480d6ae4f..09c56ed2f19ddd 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -43,10 +43,6 @@ struct _pending_calls { int last; }; -struct _opcodecache_config { - int minruns; -}; - struct _ceval_runtime_state { int recursion_limit; /* Records whether tracing is on for any thread. Counts the number @@ -64,7 +60,7 @@ struct _ceval_runtime_state { /* Request for checking signals. */ _Py_atomic_int signals_pending; struct _gil_runtime_state gil; - struct _opcodecache_config opcodeconfig; + struct int opcode_cache_minruns; }; /* interpreter state */ @@ -318,8 +314,7 @@ PyAPI_FUNC(void) _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime); PyAPI_FUNC(void) _PyGILState_Reinit(_PyRuntimeState *runtime); PyAPI_FUNC(void) _PyEval_SetOpcodeCacheMinRuns(int minruns); - -PyAPI_FUNC(int) _PyEval_GetOpcodeCacheMinRuns(); +PyAPI_FUNC(int) _PyEval_GetOpcodeCacheMinRuns(void); #ifdef __cplusplus } diff --git a/Python/ceval.c b/Python/ceval.c index cdb5881834758f..9767785ab6b3c3 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -5636,11 +5636,11 @@ maybe_dtrace_line(PyFrameObject *frame, void _PyEval_SetOpcodeCacheMinRuns(int minruns) { - _PyRuntime.ceval.opcodeconfig.minruns = minruns; + _PyRuntime.ceval.opcode_cache_minruns = minruns; } int _PyEval_GetOpcodeCacheMinRuns() { - return _PyRuntime.ceval.opcodeconfig.minruns; + return _PyRuntime.ceval.opcode_cache_minruns; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 462530b9797b97..2474698944c8cc 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -737,7 +737,6 @@ _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args) return status; } - // XXX: Add this properly to the commandline config runtime->ceval.opcodeconfig.minruns = 1024; runtime->pre_initialized = 1; From 8e6caf6f620e6ac1de038d5b8f2bb0fad45e35a1 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 3 Jun 2019 23:36:31 +0100 Subject: [PATCH 4/4] fixup! fixup! fixup! bpo-37146: Allow to configure the opcode cache and fix huntleaks by presetting it to 1 --- Lib/test/libregrtest/refleak.py | 3 +++ Lib/test/libregrtest/runtest.py | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 8d221232eb6ce7..598347e1ffc625 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -30,6 +30,8 @@ def dash_R(ns, test_name, test_func): if not hasattr(sys, 'gettotalrefcount'): raise Exception("Tracking reference leaks requires a debug build " "of Python") + minruns = sys._getopcacheminruns() + sys._setopcacheminruns(1) # Avoid false positives due to various caches # filling slowly with random data: @@ -104,6 +106,7 @@ def get_pooled_int(value): rc_before = rc_after fd_before = fd_after + sys._setopcacheminruns(minruns) if not ns.quiet: print(file=sys.stderr) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index e31fbd9fc5e792..d8f74557cb2923 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -206,8 +206,6 @@ def _test_module(the_module): def _runtest_inner2(ns, test_name): # Load the test function, run the test function, handle huntrleaks # and findleaks to detect leaks - minruns = sys._getopcacheminruns() - sys._setopcacheminruns(1) abstest = get_abs_module(ns, test_name) @@ -245,7 +243,6 @@ def _runtest_inner2(ns, test_name): gc.garbage.clear() support.reap_children() - sys._setopcacheminruns(minruns) return refleak