From 17d31bf3843c384873999a15ce683cc3654f46ae Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 08:50:28 +0900 Subject: [PATCH 001/158] gh-112087: Store memory allocation information into _PyListArray (gh-116529) --- ...-03-09-11-10-53.gh-issue-112087.nbI0Pw.rst | 1 + Objects/listobject.c | 126 ++++++++++++++++-- 2 files changed, 113 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-09-11-10-53.gh-issue-112087.nbI0Pw.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-09-11-10-53.gh-issue-112087.nbI0Pw.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-09-11-10-53.gh-issue-112087.nbI0Pw.rst new file mode 100644 index 00000000000000..07808dc325699b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-09-11-10-53.gh-issue-112087.nbI0Pw.rst @@ -0,0 +1 @@ +:class:`list` is now compatible with the implementation of :pep:`703`. diff --git a/Objects/listobject.c b/Objects/listobject.c index e013383a712bc8..c2aaab0ec3b415 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -31,6 +31,50 @@ get_list_freelist(void) } #endif +#ifdef Py_GIL_DISABLED +typedef struct { + Py_ssize_t allocated; + PyObject *ob_item[]; +} _PyListArray; + +static _PyListArray * +list_allocate_array(size_t capacity) +{ + if (capacity > PY_SSIZE_T_MAX/sizeof(PyObject*) - 1) { + return NULL; + } + _PyListArray *array = PyMem_Malloc(sizeof(_PyListArray) + capacity * sizeof(PyObject *)); + if (array == NULL) { + return NULL; + } + array->allocated = capacity; + return array; +} + +static Py_ssize_t +list_capacity(PyObject **items) +{ + _PyListArray *array = _Py_CONTAINER_OF(items, _PyListArray, ob_item); + return array->allocated; +} +#endif + +static void +free_list_items(PyObject** items, bool use_qsbr) +{ +#ifdef Py_GIL_DISABLED + _PyListArray *array = _Py_CONTAINER_OF(items, _PyListArray, ob_item); + if (use_qsbr) { + _PyMem_FreeDelayed(array); + } + else { + PyMem_Free(array); + } +#else + PyMem_Free(items); +#endif +} + /* Ensure ob_item has room for at least newsize elements, and set * ob_size to newsize. If newsize > ob_size on entry, the content * of the new slots at exit is undefined heap trash; it's the caller's @@ -47,8 +91,7 @@ get_list_freelist(void) static int list_resize(PyListObject *self, Py_ssize_t newsize) { - PyObject **items; - size_t new_allocated, num_allocated_bytes; + size_t new_allocated, target_bytes; Py_ssize_t allocated = self->allocated; /* Bypass realloc() when a previous overallocation is large enough @@ -80,9 +123,34 @@ list_resize(PyListObject *self, Py_ssize_t newsize) if (newsize == 0) new_allocated = 0; + +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(new_allocated); + if (array == NULL) { + PyErr_NoMemory(); + return -1; + } + PyObject **old_items = self->ob_item; + if (self->ob_item) { + if (new_allocated < (size_t)allocated) { + target_bytes = new_allocated * sizeof(PyObject*); + } + else { + target_bytes = allocated * sizeof(PyObject*); + } + memcpy(array->ob_item, self->ob_item, target_bytes); + } + _Py_atomic_store_ptr_release(&self->ob_item, &array->ob_item); + self->allocated = new_allocated; + Py_SET_SIZE(self, newsize); + if (old_items != NULL) { + free_list_items(old_items, _PyObject_GC_IS_SHARED(self)); + } +#else + PyObject **items; if (new_allocated <= (size_t)PY_SSIZE_T_MAX / sizeof(PyObject *)) { - num_allocated_bytes = new_allocated * sizeof(PyObject *); - items = (PyObject **)PyMem_Realloc(self->ob_item, num_allocated_bytes); + target_bytes = new_allocated * sizeof(PyObject *); + items = (PyObject **)PyMem_Realloc(self->ob_item, target_bytes); } else { // integer overflow @@ -95,12 +163,14 @@ list_resize(PyListObject *self, Py_ssize_t newsize) self->ob_item = items; Py_SET_SIZE(self, newsize); self->allocated = new_allocated; +#endif return 0; } static int list_preallocate_exact(PyListObject *self, Py_ssize_t size) { + PyObject **items; assert(self->ob_item == NULL); assert(size > 0); @@ -110,11 +180,20 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size) * allocated size up to the nearest even number. */ size = (size + 1) & ~(size_t)1; - PyObject **items = PyMem_New(PyObject*, size); +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(size); + if (array == NULL) { + PyErr_NoMemory(); + return -1; + } + items = array->ob_item; +#else + items = PyMem_New(PyObject*, size); if (items == NULL) { PyErr_NoMemory(); return -1; } +#endif self->ob_item = items; self->allocated = size; return 0; @@ -178,7 +257,17 @@ PyList_New(Py_ssize_t size) op->ob_item = NULL; } else { +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(size); + if (array == NULL) { + Py_DECREF(op); + return PyErr_NoMemory(); + } + memset(&array->ob_item, 0, size * sizeof(PyObject *)); + op->ob_item = array->ob_item; +#else op->ob_item = (PyObject **) PyMem_Calloc(size, sizeof(PyObject *)); +#endif if (op->ob_item == NULL) { Py_DECREF(op); return PyErr_NoMemory(); @@ -199,11 +288,20 @@ list_new_prealloc(Py_ssize_t size) return NULL; } assert(op->ob_item == NULL); +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(size); + if (array == NULL) { + Py_DECREF(op); + return PyErr_NoMemory(); + } + op->ob_item = array->ob_item; +#else op->ob_item = PyMem_New(PyObject *, size); if (op->ob_item == NULL) { Py_DECREF(op); return PyErr_NoMemory(); } +#endif op->allocated = size; return (PyObject *) op; } @@ -268,7 +366,7 @@ list_get_item_ref(PyListObject *op, Py_ssize_t i) if (ob_item == NULL) { return NULL; } - Py_ssize_t cap = _Py_atomic_load_ssize_relaxed(&op->allocated); + Py_ssize_t cap = list_capacity(ob_item); assert(cap != -1 && cap >= size); if (!valid_index(i, cap)) { return NULL; @@ -438,7 +536,7 @@ list_dealloc(PyObject *self) while (--i >= 0) { Py_XDECREF(op->ob_item[i]); } - PyMem_Free(op->ob_item); + free_list_items(op->ob_item, false); } #ifdef WITH_FREELISTS struct _Py_list_freelist *list_freelist = get_list_freelist(); @@ -737,12 +835,7 @@ list_clear_impl(PyListObject *a, bool is_resize) #else bool use_qsbr = false; #endif - if (use_qsbr) { - _PyMem_FreeDelayed(items); - } - else { - PyMem_Free(items); - } + free_list_items(items, use_qsbr); // Note that there is no guarantee that the list is actually empty // at this point, because XDECREF may have populated it indirectly again! } @@ -2758,7 +2851,12 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) while (--i >= 0) { Py_XDECREF(final_ob_item[i]); } - PyMem_Free(final_ob_item); +#ifdef Py_GIL_DISABLED + bool use_qsbr = _PyObject_GC_IS_SHARED(self); +#else + bool use_qsbr = false; +#endif + free_list_items(final_ob_item, use_qsbr); } return Py_XNewRef(result); } From 5b2f21faf388d8de5b388996cfd4f03430085764 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 09:45:42 +0900 Subject: [PATCH 002/158] gh-112087: Make list.sort to be thread-safe for PEP 703. (gh-116553) --- Objects/clinic/listobject.c.h | 4 +++- Objects/listobject.c | 11 +++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/listobject.c.h index a61550a49b66fc..b90dc0a0b463c6 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -268,7 +268,9 @@ list_sort(PyListObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject goto exit; } skip_optional_kwonly: + Py_BEGIN_CRITICAL_SECTION(self); return_value = list_sort_impl(self, keyfunc, reverse); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -452,4 +454,4 @@ list___reversed__(PyListObject *self, PyObject *Py_UNUSED(ignored)) { return list___reversed___impl(self); } -/*[clinic end generated code: output=26dfb2c9846348f9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a77eda9931ec0c20 input=a9049054013a1b77]*/ diff --git a/Objects/listobject.c b/Objects/listobject.c index c2aaab0ec3b415..164f363efe24f0 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2577,6 +2577,7 @@ unsafe_tuple_compare(PyObject *v, PyObject *w, MergeState *ms) * duplicated). */ /*[clinic input] +@critical_section list.sort * @@ -2596,7 +2597,7 @@ The reverse flag can be set to sort in descending order. static PyObject * list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) -/*[clinic end generated code: output=57b9f9c5e23fbe42 input=a74c4cd3ec6b5c08]*/ +/*[clinic end generated code: output=57b9f9c5e23fbe42 input=667bf25d0e3a3676]*/ { MergeState ms; Py_ssize_t nremaining; @@ -2623,7 +2624,7 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) saved_ob_item = self->ob_item; saved_allocated = self->allocated; Py_SET_SIZE(self, 0); - self->ob_item = NULL; + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item, NULL); self->allocated = -1; /* any operation will reset it to >= 0 */ if (keyfunc == NULL) { @@ -2843,8 +2844,8 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) final_ob_item = self->ob_item; i = Py_SIZE(self); Py_SET_SIZE(self, saved_ob_size); - self->ob_item = saved_ob_item; - self->allocated = saved_allocated; + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item, saved_ob_item); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->allocated, saved_allocated); if (final_ob_item != NULL) { /* we cannot use list_clear() for this because it does not guarantee that the list is really empty when it returns */ @@ -2870,7 +2871,9 @@ PyList_Sort(PyObject *v) PyErr_BadInternalCall(); return -1; } + Py_BEGIN_CRITICAL_SECTION(v); v = list_sort_impl((PyListObject *)v, NULL, 0); + Py_END_CRITICAL_SECTION(); if (v == NULL) return -1; Py_DECREF(v); From 729bfb3105c7ca2ad6150a6207786096732c3b9e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Mar 2024 20:42:40 +0100 Subject: [PATCH 003/158] gh-116417: Avoid PyFloat_AS_DOUBLE() in AC limited C API (#116568) Argument Clinic no longer calls PyFloat_AS_DOUBLE() when the usage of the limited C API is requested. --- Modules/_testclinic_limited.c | 36 +++++++++++ Modules/clinic/_testclinic_limited.c.h | 84 +++++++++++++++++++++++++- Tools/clinic/clinic.py | 54 +++++++++++------ 3 files changed, 155 insertions(+), 19 deletions(-) diff --git a/Modules/_testclinic_limited.c b/Modules/_testclinic_limited.c index ef595be0b626db..df08ff9a369b1f 100644 --- a/Modules/_testclinic_limited.c +++ b/Modules/_testclinic_limited.c @@ -72,10 +72,46 @@ my_int_sum_impl(PyObject *module, int x, int y) } +/*[clinic input] +my_float_sum -> float + + x: float + y: float + / + +[clinic start generated code]*/ + +static float +my_float_sum_impl(PyObject *module, float x, float y) +/*[clinic end generated code: output=634f59a5a419cad7 input=d4b5313bdf4dc377]*/ +{ + return x + y; +} + + +/*[clinic input] +my_double_sum -> double + + x: double + y: double + / + +[clinic start generated code]*/ + +static double +my_double_sum_impl(PyObject *module, double x, double y) +/*[clinic end generated code: output=a75576d9e4d8557f input=16b11c8aba172801]*/ +{ + return x + y; +} + + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF MY_INT_FUNC_METHODDEF MY_INT_SUM_METHODDEF + MY_FLOAT_SUM_METHODDEF + MY_DOUBLE_SUM_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_testclinic_limited.c.h b/Modules/clinic/_testclinic_limited.c.h index eff72cee24975c..690e782b839cb5 100644 --- a/Modules/clinic/_testclinic_limited.c.h +++ b/Modules/clinic/_testclinic_limited.c.h @@ -91,4 +91,86 @@ my_int_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=5cf64baf978d2288 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(my_float_sum__doc__, +"my_float_sum($module, x, y, /)\n" +"--\n" +"\n"); + +#define MY_FLOAT_SUM_METHODDEF \ + {"my_float_sum", (PyCFunction)(void(*)(void))my_float_sum, METH_FASTCALL, my_float_sum__doc__}, + +static float +my_float_sum_impl(PyObject *module, float x, float y); + +static PyObject * +my_float_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + float x; + float y; + float _return_value; + + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "my_float_sum expected 2 arguments, got %zd", nargs); + goto exit; + } + x = (float) PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + y = (float) PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + _return_value = my_float_sum_impl(module, x, y); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble((double)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(my_double_sum__doc__, +"my_double_sum($module, x, y, /)\n" +"--\n" +"\n"); + +#define MY_DOUBLE_SUM_METHODDEF \ + {"my_double_sum", (PyCFunction)(void(*)(void))my_double_sum, METH_FASTCALL, my_double_sum__doc__}, + +static double +my_double_sum_impl(PyObject *module, double x, double y); + +static PyObject * +my_double_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double x; + double y; + double _return_value; + + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "my_double_sum expected 2 arguments, got %zd", nargs); + goto exit; + } + x = PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + y = PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + _return_value = my_double_sum_impl(module, x, y); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble(_return_value); + +exit: + return return_value; +} +/*[clinic end generated code: output=bb9f6b8c5d9e6a79 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 0a8546247cc326..8353941f929eb1 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -3867,19 +3867,28 @@ class float_converter(CConverter): def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'f': - return self.format_code(""" - if (PyFloat_CheckExact({argname})) {{{{ - {paramname} = (float) (PyFloat_AS_DOUBLE({argname})); - }}}} - else - {{{{ + if not limited_capi: + return self.format_code(""" + if (PyFloat_CheckExact({argname})) {{{{ + {paramname} = (float) (PyFloat_AS_DOUBLE({argname})); + }}}} + else + {{{{ + {paramname} = (float) PyFloat_AsDouble({argname}); + if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + """, + argname=argname) + else: + return self.format_code(""" {paramname} = (float) PyFloat_AsDouble({argname}); if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ goto exit; }}}} - }}}} - """, - argname=argname) + """, + argname=argname) return super().parse_arg(argname, displayname, limited_capi=limited_capi) class double_converter(CConverter): @@ -3890,19 +3899,28 @@ class double_converter(CConverter): def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'd': - return self.format_code(""" - if (PyFloat_CheckExact({argname})) {{{{ - {paramname} = PyFloat_AS_DOUBLE({argname}); - }}}} - else - {{{{ + if not limited_capi: + return self.format_code(""" + if (PyFloat_CheckExact({argname})) {{{{ + {paramname} = PyFloat_AS_DOUBLE({argname}); + }}}} + else + {{{{ + {paramname} = PyFloat_AsDouble({argname}); + if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + """, + argname=argname) + else: + return self.format_code(""" {paramname} = PyFloat_AsDouble({argname}); if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ goto exit; }}}} - }}}} - """, - argname=argname) + """, + argname=argname) return super().parse_arg(argname, displayname, limited_capi=limited_capi) From c5fa796619a8cae5a1a8a4a043d05a99adec713d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Mar 2024 21:19:47 +0100 Subject: [PATCH 004/158] gh-116417: Fix make check-c-globals for _testlimitedcapi (#116570) * Remove unused '_testcapimodule' global in Modules/_testcapi/unicode.c. * Update c-analyzer to not use the internal C API in _testlimitedcapi.c. --- Modules/_testcapi/unicode.c | 4 ---- Tools/c-analyzer/c_parser/preprocessor/gcc.py | 1 + Tools/c-analyzer/cpython/ignored.tsv | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index a10183dddeca98..d0954bbe36ff9d 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -3,8 +3,6 @@ #include "parts.h" #include "util.h" -static struct PyModuleDef *_testcapimodule = NULL; // set at initialization - static PyObject * codec_incrementalencoder(PyObject *self, PyObject *args) { @@ -2098,8 +2096,6 @@ static PyMethodDef TestMethods[] = { int _PyTestCapi_Init_Unicode(PyObject *m) { - _testcapimodule = PyModule_GetDef(m); - if (PyModule_AddFunctions(m, TestMethods) < 0) { return -1; } diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index cc3a9be2104dbe..bb5ec5ae087a74 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -7,6 +7,7 @@ FILES_WITHOUT_INTERNAL_CAPI = frozenset(( # Modules/ '_testcapimodule.c', + '_testlimitedcapi.c', '_testclinic_limited.c', 'xxlimited.c', 'xxlimited_35.c', diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 0f212ecb528ac4..682dceaf8ae41e 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -444,7 +444,6 @@ Modules/_testcapi/heaptype.c - _testcapimodule - Modules/_testcapi/mem.c - FmData - Modules/_testcapi/mem.c - FmHook - Modules/_testcapi/structmember.c - test_structmembersType_OldAPI - -Modules/_testcapi/unicode.c - _testcapimodule - Modules/_testcapi/watchers.c - g_dict_watch_events - Modules/_testcapi/watchers.c - g_dict_watchers_installed - Modules/_testcapi/watchers.c - g_type_modified_events - From 2339e7cff745271f0e4a919573a347ab2bc1c2e9 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 10 Mar 2024 22:00:33 +0000 Subject: [PATCH 005/158] gh-116057: Use relative recursion limits when testing os.walk() and Path.walk() (#116058) Replace test.support.set_recursion_limit with test.support.infinite_recursion. --- Lib/test/test_os.py | 4 ++-- Lib/test/test_pathlib/test_pathlib.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 3b2d5fccc30f3c..ae8b405dab50fc 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -34,7 +34,7 @@ from test.support import import_helper from test.support import os_helper from test.support import socket_helper -from test.support import set_recursion_limit +from test.support import infinite_recursion from test.support import warnings_helper from platform import win32_is_iot @@ -1496,7 +1496,7 @@ def test_walk_many_open_files(self): def test_walk_above_recursion_limit(self): depth = 50 os.makedirs(os.path.join(self.walk_path, *(['d'] * depth))) - with set_recursion_limit(depth - 5): + with infinite_recursion(depth - 5): all = list(self.walk(self.walk_path)) sub2_path = self.sub2_tree[0] diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 7e44ae61a5eba7..6509c08d227346 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -15,7 +15,7 @@ from test.support import import_helper from test.support import is_emscripten, is_wasi -from test.support import set_recursion_limit +from test.support import infinite_recursion from test.support import os_helper from test.support.os_helper import TESTFN, FakePath from test.test_pathlib import test_pathlib_abc @@ -1199,7 +1199,7 @@ def test_walk_above_recursion_limit(self): path = base.joinpath(*(['d'] * directory_depth)) path.mkdir(parents=True) - with set_recursion_limit(recursion_limit): + with infinite_recursion(recursion_limit): list(base.walk()) list(base.walk(top_down=False)) @@ -1239,7 +1239,7 @@ def test_glob_above_recursion_limit(self): path = base.joinpath(*(['d'] * directory_depth)) path.mkdir(parents=True) - with set_recursion_limit(recursion_limit): + with infinite_recursion(recursion_limit): list(base.glob('**/')) def test_glob_pathlike(self): From 4704e55a71c859c5d17cc2747ba62f49da58ea2d Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 11 Mar 2024 09:38:04 +0300 Subject: [PATCH 006/158] gh-116576: Fix `Tools/scripts/sortperf.py` sorting the same list (#116577) --- Tools/scripts/sortperf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/scripts/sortperf.py b/Tools/scripts/sortperf.py index b54681524ac173..1978a6c83dbe2b 100644 --- a/Tools/scripts/sortperf.py +++ b/Tools/scripts/sortperf.py @@ -130,7 +130,8 @@ def run(self, loops: int) -> float: def _prepare_data(self, loops: int) -> list[float]: bench = BENCHMARKS[self._name] - return [bench(self._size, self._random)] * loops + data = bench(self._size, self._random) + return [data.copy() for _ in range(loops)] def add_cmdline_args(cmd: list[str], args) -> None: From 8d7fde655fbb57e393831b9f30ebba80d6da366f Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 11 Mar 2024 08:44:42 +0100 Subject: [PATCH 007/158] gh-116417: Argument Clinic: test generated Limited C API code for float args (#116573) --- Lib/test/test_clinic.py | 58 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 02a293e04b2182..cf3eeaadf0010c 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -21,9 +21,9 @@ from clinic import DSLParser -def _make_clinic(*, filename='clinic_tests'): +def _make_clinic(*, filename='clinic_tests', limited_capi=False): clang = clinic.CLanguage(filename) - c = clinic.Clinic(clang, filename=filename, limited_capi=False) + c = clinic.Clinic(clang, filename=filename, limited_capi=limited_capi) c.block_parser = clinic.BlockParser('', clang) return c @@ -3614,6 +3614,46 @@ def test_depr_multi(self): self.assertRaises(TypeError, fn, a="a", b="b", c="c", d="d", e="e", f="f", g="g") +class LimitedCAPIOutputTests(unittest.TestCase): + + def setUp(self): + self.clinic = _make_clinic(limited_capi=True) + + @staticmethod + def wrap_clinic_input(block): + return dedent(f""" + /*[clinic input] + output everything buffer + {block} + [clinic start generated code]*/ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + + def test_limited_capi_float(self): + block = self.wrap_clinic_input(""" + func + f: float + / + """) + generated = self.clinic.parse(block) + self.assertNotIn("PyFloat_AS_DOUBLE", generated) + self.assertIn("float f;", generated) + self.assertIn("f = (float) PyFloat_AsDouble", generated) + + def test_limited_capi_double(self): + block = self.wrap_clinic_input(""" + func + f: double + / + """) + generated = self.clinic.parse(block) + self.assertNotIn("PyFloat_AS_DOUBLE", generated) + self.assertIn("double f;", generated) + self.assertIn("f = PyFloat_AsDouble", generated) + + try: import _testclinic_limited except ImportError: @@ -3644,6 +3684,20 @@ def test_my_int_sum(self): with self.assertRaises(TypeError): _testclinic_limited.my_int_sum(1, "str") + def test_my_double_sum(self): + for func in ( + _testclinic_limited.my_float_sum, + _testclinic_limited.my_double_sum, + ): + with self.subTest(func=func.__name__): + self.assertEqual(func(1.0, 2.5), 3.5) + with self.assertRaises(TypeError): + func() + with self.assertRaises(TypeError): + func(1) + with self.assertRaises(TypeError): + func(1., "2") + class PermutationTests(unittest.TestCase): From 4e5df2013fc29ed8bdb71572f1d12ff36e7028d5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 11 Mar 2024 09:30:15 +0000 Subject: [PATCH 008/158] GH-116468: Use constants instead of `oparg` in stack effects when `oparg` is known to be a constant. (GH-116469) --- Include/internal/pycore_opcode_metadata.h | 10 +-- Python/bytecodes.c | 30 ++++----- Python/executor_cases.c.h | 64 +++++++++----------- Python/generated_cases.c.h | 74 +++++++++++------------ Python/optimizer_cases.c.h | 28 +++++---- 5 files changed, 98 insertions(+), 108 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index efb731f1863283..05ff78de627caf 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -116,7 +116,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case CALL_LEN: return 2 + oparg; case CALL_LIST_APPEND: - return 2 + oparg; + return 3; case CALL_METHOD_DESCRIPTOR_FAST: return 2 + oparg; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: @@ -130,11 +130,11 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case CALL_PY_WITH_DEFAULTS: return 2 + oparg; case CALL_STR_1: - return 2 + oparg; + return 3; case CALL_TUPLE_1: - return 2 + oparg; + return 3; case CALL_TYPE_1: - return 2 + oparg; + return 3; case CHECK_EG_MATCH: return 2; case CHECK_EXC_MATCH: @@ -879,7 +879,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case UNPACK_SEQUENCE_TUPLE: return oparg; case UNPACK_SEQUENCE_TWO_TUPLE: - return oparg; + return 2; case WITH_EXCEPT_START: return 5; case YIELD_VALUE: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index bf9d2a316ceea0..03e5f4e330bdd8 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -129,6 +129,8 @@ dummy_func( PyObject *top; PyObject *type; PyObject *typevars; + PyObject *val0; + PyObject *val1; int values_or_none; switch (opcode) { @@ -1223,13 +1225,13 @@ dummy_func( macro(UNPACK_SEQUENCE) = _SPECIALIZE_UNPACK_SEQUENCE + _UNPACK_SEQUENCE; - inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- values[oparg])) { + inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- val1, val0)) { + assert(oparg == 2); DEOPT_IF(!PyTuple_CheckExact(seq)); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2); - assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); DECREF_INPUTS(); } @@ -3236,39 +3238,33 @@ dummy_func( DISPATCH_INLINED(new_frame); } - inst(CALL_TYPE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { + inst(CALL_TYPE_1, (unused/1, unused/2, callable, null, arg -- res)) { assert(oparg == 1); DEOPT_IF(null != NULL); - PyObject *obj = args[0]; DEOPT_IF(callable != (PyObject *)&PyType_Type); STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable + res = Py_NewRef(Py_TYPE(arg)); + Py_DECREF(arg); } - inst(CALL_STR_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { + inst(CALL_STR_1, (unused/1, unused/2, callable, null, arg -- res)) { assert(oparg == 1); DEOPT_IF(null != NULL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type); STAT_INC(CALL, hit); - PyObject *arg = args[0]; res = PyObject_Str(arg); Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } - inst(CALL_TUPLE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { + inst(CALL_TUPLE_1, (unused/1, unused/2, callable, null, arg -- res)) { assert(oparg == 1); DEOPT_IF(null != NULL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type); STAT_INC(CALL, hit); - PyObject *arg = args[0]; res = PySequence_Tuple(arg); Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } @@ -3490,14 +3486,14 @@ dummy_func( } // This is secretly a super-instruction - tier1 inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, args[oparg] -- unused)) { + tier1 inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, arg -- unused)) { assert(oparg == 1); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.list_append); assert(self != NULL); DEOPT_IF(!PyList_Check(self)); STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { + if (_PyList_AppendTakeRef((PyListObject *)self, arg) < 0) { goto pop_1_error; // Since arg is DECREF'ed already } Py_DECREF(self); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index ccd8fb32989e92..42e884c20ba04f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1061,18 +1061,20 @@ case _UNPACK_SEQUENCE_TWO_TUPLE: { PyObject *seq; - PyObject **values; + PyObject *val1; + PyObject *val0; oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - values = &stack_pointer[-1]; + assert(oparg == 2); if (!PyTuple_CheckExact(seq)) goto deoptimize; if (PyTuple_GET_SIZE(seq) != 2) goto deoptimize; - assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); Py_DECREF(seq); - stack_pointer += -1 + oparg; + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; break; } @@ -3057,71 +3059,65 @@ /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 */ case _CALL_TYPE_1: { - PyObject **args; + PyObject *arg; PyObject *null; PyObject *callable; PyObject *res; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; assert(oparg == 1); if (null != NULL) goto deoptimize; - PyObject *obj = args[0]; if (callable != (PyObject *)&PyType_Type) goto deoptimize; STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + res = Py_NewRef(Py_TYPE(arg)); + Py_DECREF(arg); + stack_pointer[-3] = res; + stack_pointer += -2; break; } case _CALL_STR_1: { - PyObject **args; + PyObject *arg; PyObject *null; PyObject *callable; PyObject *res; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; assert(oparg == 1); if (null != NULL) goto deoptimize; if (callable != (PyObject *)&PyUnicode_Type) goto deoptimize; STAT_INC(CALL, hit); - PyObject *arg = args[0]; res = PyObject_Str(arg); Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + if (res == NULL) goto pop_3_error_tier_two; + stack_pointer[-3] = res; + stack_pointer += -2; CHECK_EVAL_BREAKER(); break; } case _CALL_TUPLE_1: { - PyObject **args; + PyObject *arg; PyObject *null; PyObject *callable; PyObject *res; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; assert(oparg == 1); if (null != NULL) goto deoptimize; if (callable != (PyObject *)&PyTuple_Type) goto deoptimize; STAT_INC(CALL, hit); - PyObject *arg = args[0]; res = PySequence_Tuple(arg); Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + if (res == NULL) goto pop_3_error_tier_two; + stack_pointer[-3] = res; + stack_pointer += -2; CHECK_EVAL_BREAKER(); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f5d125c63b10ca..53c0211be2fe6c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1472,21 +1472,21 @@ next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; + PyObject *arg; PyObject *self; PyObject *callable; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + self = stack_pointer[-2]; + callable = stack_pointer[-3]; assert(oparg == 1); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.list_append, CALL); assert(self != NULL); DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { + if (_PyList_AppendTakeRef((PyListObject *)self, arg) < 0) { goto pop_1_error; // Since arg is DECREF'ed already } Py_DECREF(self); @@ -1810,26 +1810,24 @@ next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; + PyObject *arg; PyObject *null; PyObject *callable; PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; assert(oparg == 1); DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); STAT_INC(CALL, hit); - PyObject *arg = args[0]; res = PyObject_Str(arg); Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + if (res == NULL) goto pop_3_error; + stack_pointer[-3] = res; + stack_pointer += -2; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1839,26 +1837,24 @@ next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; + PyObject *arg; PyObject *null; PyObject *callable; PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; assert(oparg == 1); DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); STAT_INC(CALL, hit); - PyObject *arg = args[0]; res = PySequence_Tuple(arg); Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + if (res == NULL) goto pop_3_error; + stack_pointer[-3] = res; + stack_pointer += -2; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1868,25 +1864,23 @@ next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; + PyObject *arg; PyObject *null; PyObject *callable; PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; assert(oparg == 1); DEOPT_IF(null != NULL, CALL); - PyObject *obj = args[0]; DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + res = Py_NewRef(Py_TYPE(arg)); + Py_DECREF(arg); + stack_pointer[-3] = res; + stack_pointer += -2; DISPATCH(); } @@ -5910,18 +5904,20 @@ INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; - PyObject **values; + PyObject *val1; + PyObject *val0; /* Skip 1 cache entry */ seq = stack_pointer[-1]; - values = &stack_pointer[-1]; + assert(oparg == 2); DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); - assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); Py_DECREF(seq); - stack_pointer += -1 + oparg; + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; DISPATCH(); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index c88b25132c0e90..fed5730d2e50c1 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -684,13 +684,15 @@ } case _UNPACK_SEQUENCE_TWO_TUPLE: { - _Py_UopsSymbol **values; - values = &stack_pointer[-1]; - for (int _i = oparg; --_i >= 0;) { - values[_i] = sym_new_unknown(ctx); - if (values[_i] == NULL) goto out_of_space; - } - stack_pointer += -1 + oparg; + _Py_UopsSymbol *val1; + _Py_UopsSymbol *val0; + val1 = sym_new_unknown(ctx); + if (val1 == NULL) goto out_of_space; + val0 = sym_new_unknown(ctx); + if (val0 == NULL) goto out_of_space; + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; break; } @@ -1632,8 +1634,8 @@ _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-3] = res; + stack_pointer += -2; break; } @@ -1641,8 +1643,8 @@ _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-3] = res; + stack_pointer += -2; break; } @@ -1650,8 +1652,8 @@ _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-3] = res; + stack_pointer += -2; break; } From d8712fa0c75ad5ea56543903fa45674ab47cc647 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 11 Mar 2024 11:57:07 +0200 Subject: [PATCH 009/158] gh-88352: Make TimedRotatingFileHandler tests more stable (GH-116409) The tests failed (with less than 1% probability) if for example the file was created at 11:46:03.999, but the record was emitted at 11:46:04.001, with atTime=11:46:04, which caused an unexpected rollover. Ensure that the tests are always run within the range of the same whole second. Also share code between test_rollover_at_midnight and test_rollover_at_weekday. --- Lib/test/test_logging.py | 55 +++++++++++++--------------------------- 1 file changed, 17 insertions(+), 38 deletions(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index d71385bf2c78d7..1dfad6e1d2609b 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -6080,46 +6080,22 @@ def test_rollover(self): print(tf.read()) self.assertTrue(found, msg=msg) - def test_rollover_at_midnight(self): - atTime = datetime.datetime.now().time() - fmt = logging.Formatter('%(asctime)s %(message)s') - for i in range(3): - fh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when='MIDNIGHT', atTime=atTime) - fh.setFormatter(fmt) - r2 = logging.makeLogRecord({'msg': f'testing1 {i}'}) - fh.emit(r2) - fh.close() - self.assertLogFile(self.fn) - with open(self.fn, encoding="utf-8") as f: - for i, line in enumerate(f): - self.assertIn(f'testing1 {i}', line) - - os.utime(self.fn, (time.time() - 1,)*2) - for i in range(2): - fh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when='MIDNIGHT', atTime=atTime) - fh.setFormatter(fmt) - r2 = logging.makeLogRecord({'msg': f'testing2 {i}'}) - fh.emit(r2) - fh.close() - rolloverDate = datetime.datetime.now() - datetime.timedelta(days=1) - otherfn = f'{self.fn}.{rolloverDate:%Y-%m-%d}' - self.assertLogFile(otherfn) - with open(self.fn, encoding="utf-8") as f: - for i, line in enumerate(f): - self.assertIn(f'testing2 {i}', line) - with open(otherfn, encoding="utf-8") as f: - for i, line in enumerate(f): - self.assertIn(f'testing1 {i}', line) - - def test_rollover_at_weekday(self): + def test_rollover_at_midnight(self, weekly=False): + os_helper.unlink(self.fn) now = datetime.datetime.now() atTime = now.time() + if not 0.1 < atTime.microsecond/1e6 < 0.9: + # The test requires all records to be emitted within + # the range of the same whole second. + time.sleep((0.1 - atTime.microsecond/1e6) % 1.0) + now = datetime.datetime.now() + atTime = now.time() + atTime = atTime.replace(microsecond=0) fmt = logging.Formatter('%(asctime)s %(message)s') + when = f'W{now.weekday()}' if weekly else 'MIDNIGHT' for i in range(3): fh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when=f'W{now.weekday()}', atTime=atTime) + self.fn, encoding="utf-8", when=when, atTime=atTime) fh.setFormatter(fmt) r2 = logging.makeLogRecord({'msg': f'testing1 {i}'}) fh.emit(r2) @@ -6129,15 +6105,15 @@ def test_rollover_at_weekday(self): for i, line in enumerate(f): self.assertIn(f'testing1 {i}', line) - os.utime(self.fn, (time.time() - 1,)*2) + os.utime(self.fn, (now.timestamp() - 1,)*2) for i in range(2): fh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when=f'W{now.weekday()}', atTime=atTime) + self.fn, encoding="utf-8", when=when, atTime=atTime) fh.setFormatter(fmt) r2 = logging.makeLogRecord({'msg': f'testing2 {i}'}) fh.emit(r2) fh.close() - rolloverDate = datetime.datetime.now() - datetime.timedelta(days=7) + rolloverDate = now - datetime.timedelta(days=7 if weekly else 1) otherfn = f'{self.fn}.{rolloverDate:%Y-%m-%d}' self.assertLogFile(otherfn) with open(self.fn, encoding="utf-8") as f: @@ -6147,6 +6123,9 @@ def test_rollover_at_weekday(self): for i, line in enumerate(f): self.assertIn(f'testing1 {i}', line) + def test_rollover_at_weekday(self): + self.test_rollover_at_midnight(weekly=True) + def test_invalid(self): assertRaises = self.assertRaises assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, From 1cc02ca063f50b8c527fbdde9957b03c145c1575 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Mar 2024 11:28:16 +0100 Subject: [PATCH 010/158] gh-116417: Move 4 limited C API test files to _testlimitedcapi (#116571) Move the following files from Modules/_testcapi/ to Modules/_testlimitedcapi/: * bytearray.c * bytes.c * pyos.c * sys.c Changes: * Replace PyBytes_AS_STRING() with PyBytes_AsString(). * Replace PyBytes_GET_SIZE() with PyBytes_Size(). * Update related test_capi tests. * Copy Modules/_testcapi/util.h to Modules/_testlimitedcapi/util.h. --- Lib/test/test_capi/test_bytearray.py | 18 +++++----- Lib/test/test_capi/test_bytes.py | 28 ++++++++-------- Lib/test/test_capi/test_sys.py | 16 ++++----- Modules/Setup.stdlib.in | 4 +-- Modules/_testcapi/parts.h | 4 --- Modules/_testcapimodule.c | 12 ------- Modules/_testlimitedcapi.c | 14 +++++++- .../bytearray.c | 0 .../{_testcapi => _testlimitedcapi}/bytes.c | 8 ++--- Modules/_testlimitedcapi/parts.h | 6 +++- .../{_testcapi => _testlimitedcapi}/pyos.c | 0 Modules/{_testcapi => _testlimitedcapi}/sys.c | 0 Modules/_testlimitedcapi/util.h | 33 +++++++++++++++++++ PCbuild/_testcapi.vcxproj | 4 --- PCbuild/_testcapi.vcxproj.filters | 12 ------- PCbuild/_testlimitedcapi.vcxproj | 8 +++-- PCbuild/_testlimitedcapi.vcxproj.filters | 14 ++++---- 17 files changed, 101 insertions(+), 80 deletions(-) rename Modules/{_testcapi => _testlimitedcapi}/bytearray.c (100%) rename Modules/{_testcapi => _testlimitedcapi}/bytes.c (95%) rename Modules/{_testcapi => _testlimitedcapi}/pyos.c (100%) rename Modules/{_testcapi => _testlimitedcapi}/sys.c (100%) create mode 100644 Modules/_testlimitedcapi/util.h diff --git a/Lib/test/test_capi/test_bytearray.py b/Lib/test/test_capi/test_bytearray.py index 833122c4e319d8..39099f6b82240f 100644 --- a/Lib/test/test_capi/test_bytearray.py +++ b/Lib/test/test_capi/test_bytearray.py @@ -1,7 +1,7 @@ import unittest from test.support import import_helper -_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX NULL = None @@ -19,7 +19,7 @@ def __bytes__(self): class CAPITest(unittest.TestCase): def test_check(self): # Test PyByteArray_Check() - check = _testcapi.bytearray_check + check = _testlimitedcapi.bytearray_check self.assertTrue(check(bytearray(b'abc'))) self.assertFalse(check(b'abc')) self.assertTrue(check(ByteArraySubclass(b'abc'))) @@ -32,7 +32,7 @@ def test_check(self): def test_checkexact(self): # Test PyByteArray_CheckExact() - check = _testcapi.bytearray_checkexact + check = _testlimitedcapi.bytearray_checkexact self.assertTrue(check(bytearray(b'abc'))) self.assertFalse(check(b'abc')) self.assertFalse(check(ByteArraySubclass(b'abc'))) @@ -45,7 +45,7 @@ def test_checkexact(self): def test_fromstringandsize(self): # Test PyByteArray_FromStringAndSize() - fromstringandsize = _testcapi.bytearray_fromstringandsize + fromstringandsize = _testlimitedcapi.bytearray_fromstringandsize self.assertEqual(fromstringandsize(b'abc'), bytearray(b'abc')) self.assertEqual(fromstringandsize(b'abc', 2), bytearray(b'ab')) @@ -62,7 +62,7 @@ def test_fromstringandsize(self): def test_fromobject(self): # Test PyByteArray_FromObject() - fromobject = _testcapi.bytearray_fromobject + fromobject = _testlimitedcapi.bytearray_fromobject self.assertEqual(fromobject(b'abc'), bytearray(b'abc')) self.assertEqual(fromobject(bytearray(b'abc')), bytearray(b'abc')) @@ -77,7 +77,7 @@ def test_fromobject(self): def test_size(self): # Test PyByteArray_Size() - size = _testcapi.bytearray_size + size = _testlimitedcapi.bytearray_size self.assertEqual(size(bytearray(b'abc')), 3) self.assertEqual(size(ByteArraySubclass(b'abc')), 3) @@ -88,7 +88,7 @@ def test_size(self): def test_asstring(self): """Test PyByteArray_AsString()""" - asstring = _testcapi.bytearray_asstring + asstring = _testlimitedcapi.bytearray_asstring self.assertEqual(asstring(bytearray(b'abc'), 4), b'abc\0') self.assertEqual(asstring(ByteArraySubclass(b'abc'), 4), b'abc\0') @@ -100,7 +100,7 @@ def test_asstring(self): def test_concat(self): """Test PyByteArray_Concat()""" - concat = _testcapi.bytearray_concat + concat = _testlimitedcapi.bytearray_concat ba = bytearray(b'abc') self.assertEqual(concat(ba, b'def'), bytearray(b'abcdef')) @@ -133,7 +133,7 @@ def test_concat(self): def test_resize(self): """Test PyByteArray_Resize()""" - resize = _testcapi.bytearray_resize + resize = _testlimitedcapi.bytearray_resize ba = bytearray(b'abcdef') self.assertEqual(resize(ba, 3), 0) diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index bb5d724ff187d4..a2ba7708f8fd26 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -1,7 +1,7 @@ import unittest from test.support import import_helper -_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX NULL = None @@ -19,7 +19,7 @@ def __bytes__(self): class CAPITest(unittest.TestCase): def test_check(self): # Test PyBytes_Check() - check = _testcapi.bytes_check + check = _testlimitedcapi.bytes_check self.assertTrue(check(b'abc')) self.assertFalse(check('abc')) self.assertFalse(check(bytearray(b'abc'))) @@ -33,7 +33,7 @@ def test_check(self): def test_checkexact(self): # Test PyBytes_CheckExact() - check = _testcapi.bytes_checkexact + check = _testlimitedcapi.bytes_checkexact self.assertTrue(check(b'abc')) self.assertFalse(check('abc')) self.assertFalse(check(bytearray(b'abc'))) @@ -47,7 +47,7 @@ def test_checkexact(self): def test_fromstringandsize(self): # Test PyBytes_FromStringAndSize() - fromstringandsize = _testcapi.bytes_fromstringandsize + fromstringandsize = _testlimitedcapi.bytes_fromstringandsize self.assertEqual(fromstringandsize(b'abc'), b'abc') self.assertEqual(fromstringandsize(b'abc', 2), b'ab') @@ -65,7 +65,7 @@ def test_fromstringandsize(self): def test_fromstring(self): # Test PyBytes_FromString() - fromstring = _testcapi.bytes_fromstring + fromstring = _testlimitedcapi.bytes_fromstring self.assertEqual(fromstring(b'abc\0def'), b'abc') self.assertEqual(fromstring(b''), b'') @@ -74,7 +74,7 @@ def test_fromstring(self): def test_fromobject(self): # Test PyBytes_FromObject() - fromobject = _testcapi.bytes_fromobject + fromobject = _testlimitedcapi.bytes_fromobject self.assertEqual(fromobject(b'abc'), b'abc') self.assertEqual(fromobject(bytearray(b'abc')), b'abc') @@ -88,7 +88,7 @@ def test_fromobject(self): def test_size(self): # Test PyBytes_Size() - size = _testcapi.bytes_size + size = _testlimitedcapi.bytes_size self.assertEqual(size(b'abc'), 3) self.assertEqual(size(BytesSubclass(b'abc')), 3) @@ -100,7 +100,7 @@ def test_size(self): def test_asstring(self): """Test PyBytes_AsString()""" - asstring = _testcapi.bytes_asstring + asstring = _testlimitedcapi.bytes_asstring self.assertEqual(asstring(b'abc', 4), b'abc\0') self.assertEqual(asstring(b'abc\0def', 8), b'abc\0def\0') @@ -111,8 +111,8 @@ def test_asstring(self): def test_asstringandsize(self): """Test PyBytes_AsStringAndSize()""" - asstringandsize = _testcapi.bytes_asstringandsize - asstringandsize_null = _testcapi.bytes_asstringandsize_null + asstringandsize = _testlimitedcapi.bytes_asstringandsize + asstringandsize_null = _testlimitedcapi.bytes_asstringandsize_null self.assertEqual(asstringandsize(b'abc', 4), (b'abc\0', 3)) self.assertEqual(asstringandsize(b'abc\0def', 8), (b'abc\0def\0', 7)) @@ -128,7 +128,7 @@ def test_asstringandsize(self): def test_repr(self): # Test PyBytes_Repr() - bytes_repr = _testcapi.bytes_repr + bytes_repr = _testlimitedcapi.bytes_repr self.assertEqual(bytes_repr(b'''abc''', 0), r"""b'abc'""") self.assertEqual(bytes_repr(b'''abc''', 1), r"""b'abc'""") @@ -149,7 +149,7 @@ def test_repr(self): def test_concat(self, concat=None): """Test PyBytes_Concat()""" if concat is None: - concat = _testcapi.bytes_concat + concat = _testlimitedcapi.bytes_concat self.assertEqual(concat(b'abc', b'def'), b'abcdef') self.assertEqual(concat(b'a\0b', b'c\0d'), b'a\0bc\0d') @@ -182,11 +182,11 @@ def test_concat(self, concat=None): def test_concatanddel(self): """Test PyBytes_ConcatAndDel()""" - self.test_concat(_testcapi.bytes_concatanddel) + self.test_concat(_testlimitedcapi.bytes_concatanddel) def test_decodeescape(self): """Test PyBytes_DecodeEscape()""" - decodeescape = _testcapi.bytes_decodeescape + decodeescape = _testlimitedcapi.bytes_decodeescape self.assertEqual(decodeescape(b'abc'), b'abc') self.assertEqual(decodeescape(br'\t\n\r\x0b\x0c\x00\\\'\"'), diff --git a/Lib/test/test_capi/test_sys.py b/Lib/test/test_capi/test_sys.py index 08bf0370fc59b5..54a8e026d883d4 100644 --- a/Lib/test/test_capi/test_sys.py +++ b/Lib/test/test_capi/test_sys.py @@ -5,9 +5,9 @@ from test.support import import_helper try: - import _testcapi + import _testlimitedcapi except ImportError: - _testcapi = None + _testlimitedcapi = None NULL = None @@ -20,10 +20,10 @@ class CAPITest(unittest.TestCase): maxDiff = None @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_sys_getobject(self): # Test PySys_GetObject() - getobject = _testcapi.sys_getobject + getobject = _testlimitedcapi.sys_getobject self.assertIs(getobject(b'stdout'), sys.stdout) with support.swap_attr(sys, '\U0001f40d', 42): @@ -38,10 +38,10 @@ def test_sys_getobject(self): # CRASHES getobject(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_sys_setobject(self): # Test PySys_SetObject() - setobject = _testcapi.sys_setobject + setobject = _testlimitedcapi.sys_setobject value = ['value'] value2 = ['value2'] @@ -70,10 +70,10 @@ def test_sys_setobject(self): # CRASHES setobject(NULL, value) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_sys_getxoptions(self): # Test PySys_GetXOptions() - getxoptions = _testcapi.sys_getxoptions + getxoptions = _testlimitedcapi.sys_getxoptions self.assertIs(getxoptions(), sys._xoptions) diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index e8eaafd8b99b98..deada66cf1a807 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -162,8 +162,8 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/bytearray.c _testcapi/bytes.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/pyos.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/sys.c _testcapi/hash.c _testcapi/time.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/heaptype_relative.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/pyos.c _testlimitedcapi/sys.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 2a043cfa73fa8b..f9bdd830775a75 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -31,8 +31,6 @@ int _PyTestCapi_Init_Vectorcall(PyObject *module); int _PyTestCapi_Init_Heaptype(PyObject *module); int _PyTestCapi_Init_Abstract(PyObject *module); -int _PyTestCapi_Init_ByteArray(PyObject *module); -int _PyTestCapi_Init_Bytes(PyObject *module); int _PyTestCapi_Init_Unicode(PyObject *module); int _PyTestCapi_Init_GetArgs(PyObject *module); int _PyTestCapi_Init_DateTime(PyObject *module); @@ -52,12 +50,10 @@ int _PyTestCapi_Init_Exceptions(PyObject *module); int _PyTestCapi_Init_Code(PyObject *module); int _PyTestCapi_Init_Buffer(PyObject *module); int _PyTestCapi_Init_PyAtomic(PyObject *module); -int _PyTestCapi_Init_PyOS(PyObject *module); int _PyTestCapi_Init_File(PyObject *module); int _PyTestCapi_Init_Codec(PyObject *module); int _PyTestCapi_Init_Immortal(PyObject *module); int _PyTestCapi_Init_GC(PyObject *module); -int _PyTestCapi_Init_Sys(PyObject *module); int _PyTestCapi_Init_Hash(PyObject *module); int _PyTestCapi_Init_Time(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b6536045e645f9..b5e646f904b2d1 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4017,12 +4017,6 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Abstract(m) < 0) { return NULL; } - if (_PyTestCapi_Init_ByteArray(m) < 0) { - return NULL; - } - if (_PyTestCapi_Init_Bytes(m) < 0) { - return NULL; - } if (_PyTestCapi_Init_Unicode(m) < 0) { return NULL; } @@ -4077,18 +4071,12 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Buffer(m) < 0) { return NULL; } - if (_PyTestCapi_Init_PyOS(m) < 0) { - return NULL; - } if (_PyTestCapi_Init_File(m) < 0) { return NULL; } if (_PyTestCapi_Init_Codec(m) < 0) { return NULL; } - if (_PyTestCapi_Init_Sys(m) < 0) { - return NULL; - } if (_PyTestCapi_Init_Immortal(m) < 0) { return NULL; } diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index da09e3f5084464..49bf6a3ea39613 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -26,11 +26,23 @@ PyInit__testlimitedcapi(void) return NULL; } - if (_PyTestCapi_Init_VectorcallLimited(mod) < 0) { + if (_PyTestCapi_Init_ByteArray(mod) < 0) { + return NULL; + } + if (_PyTestCapi_Init_Bytes(mod) < 0) { return NULL; } if (_PyTestCapi_Init_HeaptypeRelative(mod) < 0) { return NULL; } + if (_PyTestCapi_Init_PyOS(mod) < 0) { + return NULL; + } + if (_PyTestCapi_Init_Sys(mod) < 0) { + return NULL; + } + if (_PyTestCapi_Init_VectorcallLimited(mod) < 0) { + return NULL; + } return mod; } diff --git a/Modules/_testcapi/bytearray.c b/Modules/_testlimitedcapi/bytearray.c similarity index 100% rename from Modules/_testcapi/bytearray.c rename to Modules/_testlimitedcapi/bytearray.c diff --git a/Modules/_testcapi/bytes.c b/Modules/_testlimitedcapi/bytes.c similarity index 95% rename from Modules/_testcapi/bytes.c rename to Modules/_testlimitedcapi/bytes.c index da10503f6f6856..a14c4f9d4d30a8 100644 --- a/Modules/_testcapi/bytes.c +++ b/Modules/_testlimitedcapi/bytes.c @@ -160,8 +160,8 @@ bytes_concat(PyObject *Py_UNUSED(module), PyObject *args) if (new) { assert(left != NULL); assert(PyBytes_CheckExact(left)); - left = PyBytes_FromStringAndSize(PyBytes_AS_STRING(left), - PyBytes_GET_SIZE(left)); + left = PyBytes_FromStringAndSize(PyBytes_AsString(left), + PyBytes_Size(left)); if (left == NULL) { return NULL; } @@ -191,8 +191,8 @@ bytes_concatanddel(PyObject *Py_UNUSED(module), PyObject *args) if (new) { assert(left != NULL); assert(PyBytes_CheckExact(left)); - left = PyBytes_FromStringAndSize(PyBytes_AS_STRING(left), - PyBytes_GET_SIZE(left)); + left = PyBytes_FromStringAndSize(PyBytes_AsString(left), + PyBytes_Size(left)); if (left == NULL) { return NULL; } diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index 83590a7afd3228..039576d5cf4c29 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -21,7 +21,11 @@ # error "Py_BUILD_CORE macro must not be defined" #endif -int _PyTestCapi_Init_VectorcallLimited(PyObject *module); +int _PyTestCapi_Init_ByteArray(PyObject *module); +int _PyTestCapi_Init_Bytes(PyObject *module); int _PyTestCapi_Init_HeaptypeRelative(PyObject *module); +int _PyTestCapi_Init_PyOS(PyObject *module); +int _PyTestCapi_Init_Sys(PyObject *module); +int _PyTestCapi_Init_VectorcallLimited(PyObject *module); #endif // Py_TESTLIMITEDCAPI_PARTS_H diff --git a/Modules/_testcapi/pyos.c b/Modules/_testlimitedcapi/pyos.c similarity index 100% rename from Modules/_testcapi/pyos.c rename to Modules/_testlimitedcapi/pyos.c diff --git a/Modules/_testcapi/sys.c b/Modules/_testlimitedcapi/sys.c similarity index 100% rename from Modules/_testcapi/sys.c rename to Modules/_testlimitedcapi/sys.c diff --git a/Modules/_testlimitedcapi/util.h b/Modules/_testlimitedcapi/util.h new file mode 100644 index 00000000000000..f26d7656a10138 --- /dev/null +++ b/Modules/_testlimitedcapi/util.h @@ -0,0 +1,33 @@ +#define NULLABLE(x) do { \ + if (x == Py_None) { \ + x = NULL; \ + } \ + } while (0); + +#define RETURN_INT(value) do { \ + int _ret = (value); \ + if (_ret == -1) { \ + assert(PyErr_Occurred()); \ + return NULL; \ + } \ + assert(!PyErr_Occurred()); \ + return PyLong_FromLong(_ret); \ + } while (0) + +#define RETURN_SIZE(value) do { \ + Py_ssize_t _ret = (value); \ + if (_ret == -1) { \ + assert(PyErr_Occurred()); \ + return NULL; \ + } \ + assert(!PyErr_Occurred()); \ + return PyLong_FromSsize_t(_ret); \ + } while (0) + +/* Marker to check that pointer value was set. */ +static const char uninitialized[] = "uninitialized"; +#define UNINITIALIZED_PTR ((void *)uninitialized) +/* Marker to check that Py_ssize_t value was set. */ +#define UNINITIALIZED_SIZE ((Py_ssize_t)236892191) +/* Marker to check that integer value was set. */ +#define UNINITIALIZED_INT (63256717) diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 3ca4c5f86c783c..6522cb1fcf5c63 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -98,8 +98,6 @@ - - @@ -118,10 +116,8 @@ - - diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 651eb1d6ba0b7f..772a9a861517ec 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -30,12 +30,6 @@ Source Files - - Source Files - - - Source Files - Source Files @@ -90,18 +84,12 @@ Source Files - - Source Files - Source Files Source Files - - Source Files - Source Files diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index 1b27942c1da9ae..1afeacaa93396e 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -94,8 +94,12 @@ - + + + + + @@ -113,4 +117,4 @@ - \ No newline at end of file + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index c4764a55f410a0..b3eeb86185c372 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -9,12 +9,12 @@ - - Source Files - - - Source Files - + + + + + + @@ -22,4 +22,4 @@ Resource Files - \ No newline at end of file + From ffd79bea0f032df5a2e7f75e8c823a09cdc7c7a2 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 11 Mar 2024 13:58:24 +0300 Subject: [PATCH 011/158] gh-116545: Fix error handling in `mkpwent` in `pwdmodule` (#116548) --- Modules/pwdmodule.c | 61 ++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index b7034369c4731e..c59a8e41aa292a 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -64,53 +64,52 @@ static struct PyModuleDef pwdmodule; #define DEFAULT_BUFFER_SIZE 1024 -static void -sets(PyObject *v, int i, const char* val) -{ - if (val) { - PyObject *o = PyUnicode_DecodeFSDefault(val); - PyStructSequence_SET_ITEM(v, i, o); - } - else { - PyStructSequence_SET_ITEM(v, i, Py_None); - Py_INCREF(Py_None); - } -} - static PyObject * mkpwent(PyObject *module, struct passwd *p) { - int setIndex = 0; PyObject *v = PyStructSequence_New(get_pwd_state(module)->StructPwdType); - if (v == NULL) + if (v == NULL) { return NULL; + } + + int setIndex = 0; + +#define SET_STRING(VAL) \ + SET_RESULT((VAL) ? PyUnicode_DecodeFSDefault((VAL)) : Py_NewRef(Py_None)) -#define SETS(i,val) sets(v, i, val) +#define SET_RESULT(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + goto error; \ + } \ + PyStructSequence_SET_ITEM(v, setIndex++, item); \ + } while(0) - SETS(setIndex++, p->pw_name); + SET_STRING(p->pw_name); #if defined(HAVE_STRUCT_PASSWD_PW_PASSWD) && !defined(__ANDROID__) - SETS(setIndex++, p->pw_passwd); + SET_STRING(p->pw_passwd); #else - SETS(setIndex++, ""); + SET_STRING(""); #endif - PyStructSequence_SET_ITEM(v, setIndex++, _PyLong_FromUid(p->pw_uid)); - PyStructSequence_SET_ITEM(v, setIndex++, _PyLong_FromGid(p->pw_gid)); + SET_RESULT(_PyLong_FromUid(p->pw_uid)); + SET_RESULT(_PyLong_FromGid(p->pw_gid)); #if defined(HAVE_STRUCT_PASSWD_PW_GECOS) - SETS(setIndex++, p->pw_gecos); + SET_STRING(p->pw_gecos); #else - SETS(setIndex++, ""); + SET_STRING(""); #endif - SETS(setIndex++, p->pw_dir); - SETS(setIndex++, p->pw_shell); - -#undef SETS + SET_STRING(p->pw_dir); + SET_STRING(p->pw_shell); - if (PyErr_Occurred()) { - Py_XDECREF(v); - return NULL; - } +#undef SET_STRING +#undef SET_RESULT return v; + +error: + Py_DECREF(v); + return NULL; } /*[clinic input] From 817fe33a1da747c57b467f73a47b701c0b0eb911 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 11 Mar 2024 16:25:04 +0300 Subject: [PATCH 012/158] gh-116590: Fix unused `current_thread_holds_gil` function warning (#116591) --- Python/ceval_gil.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index f5c44307a513f8..edfc466d9f20ec 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -417,6 +417,7 @@ PyEval_ThreadsInitialized(void) return _PyEval_ThreadsInitialized(); } +#ifndef NDEBUG static inline int current_thread_holds_gil(struct _gil_runtime_state *gil, PyThreadState *tstate) { @@ -425,6 +426,7 @@ current_thread_holds_gil(struct _gil_runtime_state *gil, PyThreadState *tstate) } return _Py_atomic_load_int_relaxed(&gil->locked); } +#endif static void init_shared_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) From 6c4fc209e1941958164509204cdc3505130c1820 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Mon, 11 Mar 2024 22:25:55 +0900 Subject: [PATCH 013/158] gh-112536: Define MI_TSAN to 1 for --with-mimalloc and --with-thread-sanitizer (gh-116558) --- Include/internal/pycore_mimalloc.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Include/internal/pycore_mimalloc.h b/Include/internal/pycore_mimalloc.h index 3ef0154ba76d20..10d451398f1410 100644 --- a/Include/internal/pycore_mimalloc.h +++ b/Include/internal/pycore_mimalloc.h @@ -32,6 +32,10 @@ typedef enum { # define MI_DEBUG 0 #endif +#ifdef _Py_THREAD_SANITIZER +# define MI_TSAN 1 +#endif + #include "mimalloc.h" #include "mimalloc/types.h" #include "mimalloc/internal.h" From b6ae6da1bd987506b599a30e37fb452f909b5cbe Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 11 Mar 2024 13:37:48 +0000 Subject: [PATCH 014/158] GH-116596: Better determination of escaping uops. (GH-116597) --- Include/internal/pycore_opcode_metadata.h | 24 +++++++++---------- Include/internal/pycore_uop_metadata.h | 28 +++++++++++------------ Python/optimizer.c | 1 + Python/optimizer_analysis.c | 8 +++---- Tools/cases_generator/analyzer.py | 16 +++++++++++++ 5 files changed, 47 insertions(+), 30 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 05ff78de627caf..de93d4ef14de2a 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -959,7 +959,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, @@ -973,12 +973,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1008,8 +1008,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1035,7 +1035,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, - [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1075,9 +1075,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, @@ -1132,7 +1132,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 7f984d88299b62..62405a362fd7ab 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -65,7 +65,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG, [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, - [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -114,9 +114,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG, [_COPY_FREE_VARS] = HAS_ARG_FLAG, - [_BUILD_STRING] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_STRING] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_BUILD_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -146,12 +146,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_CLASS_1] = 0, [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG, [_GUARD_DORV_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_STORE_ATTR_INSTANCE_VALUE] = HAS_ESCAPES_FLAG, + [_STORE_ATTR_INSTANCE_VALUE] = 0, [_STORE_ATTR_SLOT] = HAS_ESCAPES_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_COMPARE_OP_STR] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_COMPARE_OP_STR] = HAS_ARG_FLAG, [_IS_OP] = HAS_ARG_FLAG, [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -175,19 +175,19 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ITER_NEXT_TUPLE] = 0, [_ITER_CHECK_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG, [_BEFORE_ASYNC_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BEFORE_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_EXC_INFO] = 0, [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_GUARD_KEYS_VERSION] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG, + [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = HAS_ARG_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG, [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG, [_CHECK_PEP_523] = HAS_DEOPT_FLAG, @@ -199,7 +199,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_PUSH_FRAME] = HAS_ESCAPES_FLAG, + [_PUSH_FRAME] = 0, [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CALL_STR_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -216,12 +216,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_WITH_SPEC] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, [_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, diff --git a/Python/optimizer.c b/Python/optimizer.c index 4dbfd918a51f62..aaf75b2339cd2e 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1033,6 +1033,7 @@ uop_optimize( break; } assert(_PyOpcode_uop_name[buffer[pc].opcode]); + assert(strncmp(_PyOpcode_uop_name[buffer[pc].opcode], _PyOpcode_uop_name[opcode], strlen(_PyOpcode_uop_name[opcode])) == 0); } _PyExecutorObject *executor = make_executor_from_uops(buffer, &dependencies); if (executor == NULL) { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 51ec14b718f291..9fd4b1967ecc3b 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -430,7 +430,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) int opcode = buffer[pc].opcode; switch (opcode) { case _SET_IP: - buffer[pc].opcode = NOP; + buffer[pc].opcode = _NOP; last_set_ip = pc; break; case _CHECK_VALIDITY: @@ -438,7 +438,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) may_have_escaped = false; } else { - buffer[pc].opcode = NOP; + buffer[pc].opcode = _NOP; } break; case _CHECK_VALIDITY_AND_SET_IP: @@ -447,7 +447,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) buffer[pc].opcode = _CHECK_VALIDITY; } else { - buffer[pc].opcode = NOP; + buffer[pc].opcode = _NOP; } last_set_ip = pc; break; @@ -463,7 +463,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) last->opcode == _COPY ) { last->opcode = _NOP; - buffer[pc].opcode = NOP; + buffer[pc].opcode = _NOP; } if (last->opcode == _REPLACE_WITH_TRUE) { last->opcode = _NOP; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index b0a15e6d87c2c6..27e6ba2b3fdedf 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -335,6 +335,7 @@ def is_infallible(op: parser.InstDef) -> bool: "_PyDictOrValues_IsValues", "_PyObject_DictOrValuesPointer", "_PyDictOrValues_GetValues", + "_PyDictValues_AddToInsertionOrder", "_PyObject_MakeInstanceAttributesFromDict", "Py_DECREF", "_Py_DECREF_SPECIALIZED", @@ -355,8 +356,10 @@ def is_infallible(op: parser.InstDef) -> bool: "_PyLong_IsCompact", "_PyLong_IsNonNegativeCompact", "_PyLong_CompactValue", + "_PyLong_DigitCount", "_Py_NewRef", "_Py_IsImmortal", + "PyLong_FromLong", "_Py_STR", "_PyLong_Add", "_PyLong_Multiply", @@ -368,6 +371,17 @@ def is_infallible(op: parser.InstDef) -> bool: "_Py_atomic_load_uintptr_relaxed", "_PyFrame_GetCode", "_PyThreadState_HasStackSpace", + "_PyUnicode_Equal", + "_PyFrame_SetStackPointer", + "_PyType_HasFeature", + "PyUnicode_Concat", + "_PyList_FromArraySteal", + "_PyTuple_FromArraySteal", + "PySlice_New", + "_Py_LeaveRecursiveCallPy", + "CALL_STAT_INC", + "maybe_lltrace_resume_frame", + "_PyUnicode_JoinArray", ) ESCAPING_FUNCTIONS = ( @@ -379,6 +393,8 @@ def is_infallible(op: parser.InstDef) -> bool: def makes_escaping_api_call(instr: parser.InstDef) -> bool: if "CALL_INTRINSIC" in instr.name: return True + if instr.name == "_BINARY_OP": + return True tkns = iter(instr.tokens) for tkn in tkns: if tkn.kind != lexer.IDENTIFIER: From 546eb7a3be241c5abd8a83cebbbab8c71107edcf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Mar 2024 15:20:04 +0100 Subject: [PATCH 015/158] gh-116417: Build _testinternalcapi with limited C API version 3.5 (#116598) --- Modules/_testlimitedcapi/heaptype_relative.c | 6 ++++++ Modules/_testlimitedcapi/parts.h | 5 +++-- Modules/_testlimitedcapi/vectorcall_limited.c | 6 ++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Modules/_testlimitedcapi/heaptype_relative.c b/Modules/_testlimitedcapi/heaptype_relative.c index 9878a4daf1b2cb..d0316dd4fc63b4 100644 --- a/Modules/_testlimitedcapi/heaptype_relative.c +++ b/Modules/_testlimitedcapi/heaptype_relative.c @@ -1,3 +1,9 @@ +// Need limited C API version 3.12 for PyType_FromMetaclass() +#include "pyconfig.h" // Py_GIL_DISABLED +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API ) +# define Py_LIMITED_API 0x030c0000 +#endif + #include "parts.h" #include // max_align_t #include // memset diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index 039576d5cf4c29..9bc52413382eb5 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -7,8 +7,9 @@ #include "pyconfig.h" // Py_GIL_DISABLED // Use the limited C API -#ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 // 3.12 +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API ) + // need limited C API version 3.5 for PyModule_AddFunctions() +# define Py_LIMITED_API 0x03050000 #endif // Make sure that the internal C API cannot be used. diff --git a/Modules/_testlimitedcapi/vectorcall_limited.c b/Modules/_testlimitedcapi/vectorcall_limited.c index 24aa5e991eca3f..fc1a89c9098e1b 100644 --- a/Modules/_testlimitedcapi/vectorcall_limited.c +++ b/Modules/_testlimitedcapi/vectorcall_limited.c @@ -1,5 +1,11 @@ /* Test Vectorcall in the limited API */ +// Need limited C API version 3.12 for PyObject_Vectorcall() +#include "pyconfig.h" // Py_GIL_DISABLED +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API ) +# define Py_LIMITED_API 0x030c0000 +#endif + #include "parts.h" #include "clinic/vectorcall_limited.c.h" From 2731913dd5234ff5ab630a3b7f1c98ad79d4d9df Mon Sep 17 00:00:00 2001 From: Brett Simmers Date: Mon, 11 Mar 2024 11:02:58 -0400 Subject: [PATCH 016/158] gh-116167: Allow disabling the GIL with `PYTHON_GIL=0` or `-X gil=0` (#116338) In free-threaded builds, running with `PYTHON_GIL=0` will now disable the GIL. Follow-up issues track work to re-enable the GIL when loading an incompatible extension, and to disable the GIL by default. In order to support re-enabling the GIL at runtime, all GIL-related data structures are initialized as usual, and disabling the GIL simply sets a flag that causes `take_gil()` and `drop_gil()` to return early. --- Doc/using/cmdline.rst | 18 +++++++++++ Include/cpython/initconfig.h | 3 ++ Include/internal/pycore_gil.h | 5 ++++ Include/internal/pycore_initconfig.h | 12 ++++++++ Lib/subprocess.py | 2 +- Lib/test/_test_embed_set_config.py | 14 +++++++++ Lib/test/test_cmd_line.py | 33 ++++++++++++++++++++ Lib/test/test_embed.py | 2 ++ Misc/python.man | 4 +++ Python/ceval_gil.c | 15 ++++++++++ Python/initconfig.c | 45 ++++++++++++++++++++++++++++ Python/sysmodule.c | 11 +++++++ 12 files changed, 163 insertions(+), 1 deletion(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 0a7f6363a2b628..36cddffb9eae34 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -559,6 +559,9 @@ Miscellaneous options :mod:`__main__`. This can be used to execute code early during Python initialization. Python needs to be :ref:`built in debug mode ` for this option to exist. See also :envvar:`PYTHON_PRESITE`. + * :samp:`-X gil={0,1}` forces the GIL to be disabled or enabled, + respectively. Only available in builds configured with + :option:`--disable-gil`. See also :envvar:`PYTHON_GIL`. It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -601,6 +604,9 @@ Miscellaneous options .. versionchanged:: 3.13 Added the ``-X cpu_count`` and ``-X presite`` options. + .. versionchanged:: 3.13 + Added the ``-X gil`` option. + .. _using-on-controlling-color: Controlling color @@ -1138,6 +1144,18 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_GIL + + If this variable is set to ``1``, the global interpreter lock (GIL) will be + forced on. Setting it to ``0`` forces the GIL off. + + See also the :option:`-X gil <-X>` command-line option, which takes + precedence over this variable. + + Needs Python configured with the :option:`--disable-gil` build option. + + .. versionadded:: 3.13 + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 87c059c521cbc9..5da5ef9e5431b1 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -181,6 +181,9 @@ typedef struct PyConfig { int int_max_str_digits; int cpu_count; +#ifdef Py_GIL_DISABLED + int enable_gil; +#endif /* --- Path configuration inputs ------------ */ int pathconfig_warnings; diff --git a/Include/internal/pycore_gil.h b/Include/internal/pycore_gil.h index 19b0d23a68568a..d36b4c0db010b2 100644 --- a/Include/internal/pycore_gil.h +++ b/Include/internal/pycore_gil.h @@ -20,6 +20,11 @@ extern "C" { #define FORCE_SWITCHING struct _gil_runtime_state { +#ifdef Py_GIL_DISABLED + /* Whether or not this GIL is being used. Can change from 0 to 1 at runtime + if, for example, a module that requires the GIL is loaded. */ + int enabled; +#endif /* microseconds (the Python API uses seconds, though) */ unsigned long interval; /* Last PyThreadState holding / having held the GIL. This helps us diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index c86988234f6a05..1c68161341860a 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -153,6 +153,18 @@ typedef enum { _PyConfig_INIT_ISOLATED = 3 } _PyConfigInitEnum; +typedef enum { + /* For now, this means the GIL is enabled. + + gh-116329: This will eventually change to "the GIL is disabled but can + be reenabled by loading an incompatible extension module." */ + _PyConfig_GIL_DEFAULT = -1, + + /* The GIL has been forced off or on, and will not be affected by module loading. */ + _PyConfig_GIL_DISABLE = 0, + _PyConfig_GIL_ENABLE = 1, +} _PyConfigGILEnum; + // Export for '_testembed' program PyAPI_FUNC(void) _PyConfig_InitCompatConfig(PyConfig *config); diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 20db7747d5db13..1437bf8148282c 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -350,7 +350,7 @@ def _args_from_interpreter_flags(): if dev_mode: args.extend(('-X', 'dev')) for opt in ('faulthandler', 'tracemalloc', 'importtime', - 'frozen_modules', 'showrefcount', 'utf8'): + 'frozen_modules', 'showrefcount', 'utf8', 'gil'): if opt in xoptions: value = xoptions[opt] if value is True: diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py index 75b6b7d1b39fa4..5ff521892cb6fe 100644 --- a/Lib/test/_test_embed_set_config.py +++ b/Lib/test/_test_embed_set_config.py @@ -9,6 +9,7 @@ import os import sys import unittest +from test import support from test.support import MS_WINDOWS @@ -211,6 +212,19 @@ def test_flags(self): self.set_config(use_hash_seed=1, hash_seed=123) self.assertEqual(sys.flags.hash_randomization, 1) + if support.Py_GIL_DISABLED: + self.set_config(enable_gil=-1) + self.assertEqual(sys.flags.gil, None) + self.set_config(enable_gil=0) + self.assertEqual(sys.flags.gil, 0) + self.set_config(enable_gil=1) + self.assertEqual(sys.flags.gil, 1) + else: + # Builds without Py_GIL_DISABLED don't have + # PyConfig.enable_gil. sys.flags.gil is always defined to 1, for + # consistency. + self.assertEqual(sys.flags.gil, 1) + def test_options(self): self.check(warnoptions=[]) self.check(warnoptions=["default", "ignore"]) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 6796dc6e3570e9..c633f6493cfab7 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -869,6 +869,39 @@ def test_pythondevmode_env(self): self.assertEqual(proc.stdout.rstrip(), 'True') self.assertEqual(proc.returncode, 0, proc) + @unittest.skipUnless(support.Py_GIL_DISABLED, + "PYTHON_GIL and -X gil only supported in Py_GIL_DISABLED builds") + def test_python_gil(self): + cases = [ + # (env, opt, expected, msg) + (None, None, 'None', "no options set"), + ('0', None, '0', "PYTHON_GIL=0"), + ('1', None, '1', "PYTHON_GIL=1"), + ('1', '0', '0', "-X gil=0 overrides PYTHON_GIL=1"), + (None, '0', '0', "-X gil=0"), + (None, '1', '1', "-X gil=1"), + ] + + code = "import sys; print(sys.flags.gil)" + environ = dict(os.environ) + + for env, opt, expected, msg in cases: + with self.subTest(msg, env=env, opt=opt): + environ.pop('PYTHON_GIL', None) + if env is not None: + environ['PYTHON_GIL'] = env + extra_args = [] + if opt is not None: + extra_args = ['-X', f'gil={opt}'] + + proc = subprocess.run([sys.executable, *extra_args, '-c', code], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, env=environ) + self.assertEqual(proc.returncode, 0, proc) + self.assertEqual(proc.stdout.rstrip(), expected) + self.assertEqual(proc.stderr, '') + @unittest.skipUnless(sys.platform == 'win32', 'bpo-32457 only applies on Windows') def test_argv0_normalization(self): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 55d3acf448540a..ab1d579ed12755 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -523,6 +523,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): CONFIG_COMPAT['_pystats'] = 0 if support.Py_DEBUG: CONFIG_COMPAT['run_presite'] = None + if support.Py_GIL_DISABLED: + CONFIG_COMPAT['enable_gil'] = -1 if MS_WINDOWS: CONFIG_COMPAT.update({ 'legacy_windows_stdio': 0, diff --git a/Misc/python.man b/Misc/python.man index 0f5dfa2e2289f7..4c90c0e2a998ba 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -607,6 +607,10 @@ output. Setting it to 0 deactivates this behavior. .IP PYTHON_HISTORY This environment variable can be used to set the location of a history file (on Unix, it is \fI~/.python_history\fP by default). +.IP PYTHON_GIL +If this variable is set to 1, the global interpreter lock (GIL) will be forced +on. Setting it to 0 forces the GIL off. Only available in builds configured +with \fB--disable-gil\fP. .SS Debug-mode variables Setting these variables only has an effect in a debug build of Python, that is, if Python was configured with the diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index edfc466d9f20ec..d2cd35dfa86833 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -219,6 +219,11 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) // XXX assert(tstate == NULL || !tstate->_status.cleared); struct _gil_runtime_state *gil = ceval->gil; +#ifdef Py_GIL_DISABLED + if (!gil->enabled) { + return; + } +#endif if (!_Py_atomic_load_ptr_relaxed(&gil->locked)) { Py_FatalError("drop_gil: GIL is not locked"); } @@ -294,6 +299,11 @@ take_gil(PyThreadState *tstate) assert(_PyThreadState_CheckConsistency(tstate)); PyInterpreterState *interp = tstate->interp; struct _gil_runtime_state *gil = interp->ceval.gil; +#ifdef Py_GIL_DISABLED + if (!gil->enabled) { + return; + } +#endif /* Check that _PyEval_InitThreads() was called to create the lock */ assert(gil_created(gil)); @@ -440,6 +450,11 @@ static void init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) { assert(!gil_created(gil)); +#ifdef Py_GIL_DISABLED + // gh-116329: Once it is safe to do so, change this condition to + // (enable_gil == _PyConfig_GIL_ENABLE), so the GIL is disabled by default. + gil->enabled = _PyInterpreterState_GetConfig(interp)->enable_gil != _PyConfig_GIL_DISABLE; +#endif create_gil(gil); assert(gil_created(gil)); interp->ceval.gil = gil; diff --git a/Python/initconfig.c b/Python/initconfig.c index 17c95171d9528a..e3a62e53334163 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -95,6 +95,9 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(safe_path, BOOL), SPEC(int_max_str_digits, INT), SPEC(cpu_count, INT), +#ifdef Py_GIL_DISABLED + SPEC(enable_gil, INT), +#endif SPEC(pathconfig_warnings, BOOL), SPEC(program_name, WSTR), SPEC(pythonpath_env, WSTR_OPT), @@ -278,6 +281,9 @@ static const char usage_envvars[] = "PYTHON_CPU_COUNT: Overrides the return value of os.process_cpu_count(),\n" " os.cpu_count(), and multiprocessing.cpu_count() if set to\n" " a positive integer.\n" +#ifdef Py_GIL_DISABLED +"PYTHON_GIL : When set to 0, disables the GIL.\n" +#endif "PYTHONDEVMODE : enable the development mode.\n" "PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files.\n" "PYTHONWARNDEFAULTENCODING: enable opt-in EncodingWarning for 'encoding=None'.\n" @@ -862,6 +868,9 @@ _PyConfig_InitCompatConfig(PyConfig *config) config->_is_python_build = 0; config->code_debug_ranges = 1; config->cpu_count = -1; +#ifdef Py_GIL_DISABLED + config->enable_gil = _PyConfig_GIL_DEFAULT; +#endif } @@ -1574,6 +1583,24 @@ config_wstr_to_int(const wchar_t *wstr, int *result) return 0; } +static PyStatus +config_read_gil(PyConfig *config, size_t len, wchar_t first_char) +{ +#ifdef Py_GIL_DISABLED + if (len == 1 && first_char == L'0') { + config->enable_gil = _PyConfig_GIL_DISABLE; + } + else if (len == 1 && first_char == L'1') { + config->enable_gil = _PyConfig_GIL_ENABLE; + } + else { + return _PyStatus_ERR("PYTHON_GIL / -X gil must be \"0\" or \"1\""); + } + return _PyStatus_OK(); +#else + return _PyStatus_ERR("PYTHON_GIL / -X gil are not supported by this build"); +#endif +} static PyStatus config_read_env_vars(PyConfig *config) @@ -1652,6 +1679,15 @@ config_read_env_vars(PyConfig *config) config->safe_path = 1; } + const char *gil = config_get_env(config, "PYTHON_GIL"); + if (gil != NULL) { + size_t len = strlen(gil); + status = config_read_gil(config, len, gil[0]); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } + return _PyStatus_OK(); } @@ -2207,6 +2243,15 @@ config_read(PyConfig *config, int compute_path_config) config->show_ref_count = 1; } + const wchar_t *x_gil = config_get_xoption_value(config, L"gil"); + if (x_gil != NULL) { + size_t len = wcslen(x_gil); + status = config_read_gil(config, len, x_gil[0]); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } + #ifdef Py_STATS if (config_get_xoption(config, L"pystats")) { config->_pystats = 1; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index a4161da02980a7..cd193c1581c679 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3048,6 +3048,7 @@ static PyStructSequence_Field flags_fields[] = { {"warn_default_encoding", "-X warn_default_encoding"}, {"safe_path", "-P"}, {"int_max_str_digits", "-X int_max_str_digits"}, + {"gil", "-X gil"}, {0} }; @@ -3097,6 +3098,16 @@ set_flags_from_config(PyInterpreterState *interp, PyObject *flags) SetFlag(config->warn_default_encoding); SetFlagObj(PyBool_FromLong(config->safe_path)); SetFlag(config->int_max_str_digits); +#ifdef Py_GIL_DISABLED + if (config->enable_gil == _PyConfig_GIL_DEFAULT) { + SetFlagObj(Py_NewRef(Py_None)); + } + else { + SetFlag(config->enable_gil); + } +#else + SetFlagObj(PyLong_FromLong(1)); +#endif #undef SetFlagObj #undef SetFlag return 0; From 113053a070ba753101f73553ef6435c5c6c9f3f7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Mar 2024 17:35:29 +0100 Subject: [PATCH 017/158] gh-110850: Fix _PyTime_FromSecondsDouble() API (#116606) Return 0 on success. Set an exception and return -1 on error. Fix os.timerfd_settime(): properly report exceptions on _PyTime_FromSecondsDouble() failure. No longer export _PyTime_FromSecondsDouble(). --- Include/internal/pycore_time.h | 6 ++++-- Modules/clinic/posixmodule.c.h | 24 ++++++++++++------------ Modules/posixmodule.c | 30 ++++++++++++++++++++---------- Python/pytime.c | 11 ++++------- 4 files changed, 40 insertions(+), 31 deletions(-) diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index 40b28e0ba221ae..138d60fdb69806 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -132,8 +132,10 @@ PyAPI_FUNC(int) _PyTime_ObjectToTimespec( PyAPI_FUNC(PyTime_t) _PyTime_FromSeconds(int seconds); // Create a timestamp from a number of seconds in double. -// Export for '_socket' shared extension. -PyAPI_FUNC(PyTime_t) _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round); +extern int _PyTime_FromSecondsDouble( + double seconds, + _PyTime_round_t round, + PyTime_t *result); // Macro to create a timestamp from a number of seconds, no integer overflow. // Only use the macro for small values, prefer _PyTime_FromSeconds(). diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index b49d64d4281889..b7338d138e91ce 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -6279,8 +6279,8 @@ PyDoc_STRVAR(os_timerfd_settime__doc__, {"timerfd_settime", _PyCFunction_CAST(os_timerfd_settime), METH_FASTCALL|METH_KEYWORDS, os_timerfd_settime__doc__}, static PyObject * -os_timerfd_settime_impl(PyObject *module, int fd, int flags, double initial, - double interval); +os_timerfd_settime_impl(PyObject *module, int fd, int flags, + double initial_double, double interval_double); static PyObject * os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -6315,8 +6315,8 @@ os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; int fd; int flags = 0; - double initial = 0.0; - double interval = 0.0; + double initial_double = 0.0; + double interval_double = 0.0; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -6339,12 +6339,12 @@ os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py } if (args[2]) { if (PyFloat_CheckExact(args[2])) { - initial = PyFloat_AS_DOUBLE(args[2]); + initial_double = PyFloat_AS_DOUBLE(args[2]); } else { - initial = PyFloat_AsDouble(args[2]); - if (initial == -1.0 && PyErr_Occurred()) { + initial_double = PyFloat_AsDouble(args[2]); + if (initial_double == -1.0 && PyErr_Occurred()) { goto exit; } } @@ -6353,17 +6353,17 @@ os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py } } if (PyFloat_CheckExact(args[3])) { - interval = PyFloat_AS_DOUBLE(args[3]); + interval_double = PyFloat_AS_DOUBLE(args[3]); } else { - interval = PyFloat_AsDouble(args[3]); - if (interval == -1.0 && PyErr_Occurred()) { + interval_double = PyFloat_AsDouble(args[3]); + if (interval_double == -1.0 && PyErr_Occurred()) { goto exit; } } skip_optional_kwonly: - return_value = os_timerfd_settime_impl(module, fd, flags, initial, interval); + return_value = os_timerfd_settime_impl(module, fd, flags, initial_double, interval_double); exit: return return_value; @@ -12588,4 +12588,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=268af5cbc8baa9d4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2965306970f31c5d input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 940a9cc8955e11..f3403e1675276b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -10483,30 +10483,40 @@ os.timerfd_settime * flags: int = 0 0 or a bit mask of TFD_TIMER_ABSTIME or TFD_TIMER_CANCEL_ON_SET. - initial: double = 0.0 + initial as initial_double: double = 0.0 The initial expiration time, in seconds. - interval: double = 0.0 + interval as interval_double: double = 0.0 The timer's interval, in seconds. Alter a timer file descriptor's internal timer in seconds. [clinic start generated code]*/ static PyObject * -os_timerfd_settime_impl(PyObject *module, int fd, int flags, double initial, - double interval) -/*[clinic end generated code: output=0dda31115317adb9 input=6c24e47e7a4d799e]*/ +os_timerfd_settime_impl(PyObject *module, int fd, int flags, + double initial_double, double interval_double) +/*[clinic end generated code: output=df4c1bce6859224e input=81d2c0d7e936e8a7]*/ { - struct itimerspec new_value; - struct itimerspec old_value; - int result; - if (_PyTime_AsTimespec(_PyTime_FromSecondsDouble(initial, _PyTime_ROUND_FLOOR), &new_value.it_value) < 0) { + PyTime_t initial, interval; + if (_PyTime_FromSecondsDouble(initial_double, _PyTime_ROUND_FLOOR, + &initial) < 0) { + return NULL; + } + if (_PyTime_FromSecondsDouble(interval_double, _PyTime_ROUND_FLOOR, + &interval) < 0) { + return NULL; + } + + struct itimerspec new_value, old_value; + if (_PyTime_AsTimespec(initial, &new_value.it_value) < 0) { PyErr_SetString(PyExc_ValueError, "invalid initial value"); return NULL; } - if (_PyTime_AsTimespec(_PyTime_FromSecondsDouble(interval, _PyTime_ROUND_FLOOR), &new_value.it_interval) < 0) { + if (_PyTime_AsTimespec(interval, &new_value.it_interval) < 0) { PyErr_SetString(PyExc_ValueError, "invalid interval value"); return NULL; } + + int result; Py_BEGIN_ALLOW_THREADS result = timerfd_settime(fd, flags, &new_value, &old_value); Py_END_ALLOW_THREADS diff --git a/Python/pytime.c b/Python/pytime.c index 90ef2eeb546f7f..70d92ca00ee28e 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -565,6 +565,7 @@ pytime_from_double(PyTime_t *tp, double value, _PyTime_round_t round, /* See comments in pytime_double_to_denominator */ if (!((double)PyTime_MIN <= d && d < -(double)PyTime_MIN)) { pytime_time_t_overflow(); + *tp = 0; return -1; } PyTime_t ns = (PyTime_t)d; @@ -652,14 +653,10 @@ _PyTime_AsLong(PyTime_t ns) return PyLong_FromLongLong((long long)ns); } -PyTime_t -_PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round) +int +_PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round, PyTime_t *result) { - PyTime_t tp; - if(pytime_from_double(&tp, seconds, round, SEC_TO_NS) < 0) { - return -1; - } - return tp; + return pytime_from_double(result, seconds, round, SEC_TO_NS); } From 05070f40bbc3384c36c8b3dab76345ba92098d42 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 11 Mar 2024 11:59:09 -0700 Subject: [PATCH 018/158] GH-115976: Add WASI to CI (GH-116516) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/build.yml | 10 ++++ .github/workflows/reusable-wasi.yml | 71 +++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 .github/workflows/reusable-wasi.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 20d1fad40ecafe..ae14046935b97b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -301,6 +301,14 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py + build_wasi: + name: 'WASI' + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + uses: ./.github/workflows/reusable-wasi.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + test_hypothesis: name: "Hypothesis tests on Ubuntu" runs-on: ubuntu-20.04 @@ -525,6 +533,7 @@ jobs: - build_ubuntu - build_ubuntu_free_threading - build_ubuntu_ssltests + - build_wasi - build_windows - build_windows_free_threading - test_hypothesis @@ -558,6 +567,7 @@ jobs: build_ubuntu, build_ubuntu_free_threading, build_ubuntu_ssltests, + build_wasi, build_windows, build_windows_free_threading, build_asan, diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml new file mode 100644 index 00000000000000..995e669c228b5c --- /dev/null +++ b/.github/workflows/reusable-wasi.yml @@ -0,0 +1,71 @@ +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + +jobs: + build_wasi_reusable: + name: 'build and test' + timeout-minutes: 60 + runs-on: ubuntu-20.04 + env: + WASMTIME_VERSION: 18.0.2 + WASI_SDK_VERSION: 20 + WASI_SDK_PATH: /opt/wasi-sdk + CROSS_BUILD_PYTHON: cross-build/build + CROSS_BUILD_WASI: cross-build/wasm32-wasi + steps: + - uses: actions/checkout@v4 + # No problem resolver registered as one doesn't currently exist for Clang. + - name: "Install wasmtime" + uses: jcbhmr/setup-wasmtime@v2 + with: + wasmtime-version: ${{ env.WASMTIME_VERSION }} + - name: "Restore WASI SDK" + id: cache-wasi-sdk + uses: actions/cache@v4 + with: + path: ${{ env.WASI_SDK_PATH }} + key: ${{ runner.os }}-wasi-sdk-${{ env.WASI_SDK_VERSION }} + - name: "Install WASI SDK" + if: steps.cache-wasi-sdk.outputs.cache-hit != 'true' + run: | + mkdir ${{ env.WASI_SDK_PATH }} && \ + curl -s -S --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${{ env.WASI_SDK_VERSION }}/wasi-sdk-${{ env.WASI_SDK_VERSION }}.0-linux.tar.gz | \ + tar --strip-components 1 --directory ${{ env.WASI_SDK_PATH }} --extract --gunzip + - name: "Configure ccache action" + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: "Add ccache to PATH" + run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: "Install Python" + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: "Restore Python build config.cache" + uses: actions/cache@v4 + with: + path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + - name: "Configure build Python" + run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug + - name: "Make build Python" + run: python3 Tools/wasm/wasi.py make-build-python + - name: "Restore host config.cache" + uses: actions/cache@v4 + with: + path: ${{ env.CROSS_BUILD_WASI }}/config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-wasi-sdk-${{ env.WASI_SDK_VERSION }}-${{ inputs.config_hash }} + - name: "Configure host" + # `--with-pydebug` inferred from configure-build-python + run: python3 Tools/wasm/wasi.py configure-host -- --config-cache + - name: "Make host" + run: python3 Tools/wasm/wasi.py make-host + - name: "Display build info" + run: make --directory ${{ env.CROSS_BUILD_WASI }} pythoninfo + - name: "Test" + run: make --directory ${{ env.CROSS_BUILD_WASI }} test From 9f983e00ec55b87a098a4c8229fe5bb9acb9f3ac Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 11 Mar 2024 15:14:20 -0400 Subject: [PATCH 019/158] gh-116515: Clear thread-local state before tstate_delete_common() (#116517) This moves `current_fast_clear()` up so that the current thread state is `NULL` while running `tstate_delete_common()`. This doesn't fix any bugs, but it means that we are more consistent that `_PyThreadState_GET() != NULL` means that the thread is "attached". --- Python/pystate.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/pystate.c b/Python/pystate.c index 1418d034ca2fe9..635616c5648c18 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1609,6 +1609,7 @@ tstate_delete_common(PyThreadState *tstate) { assert(tstate->_status.cleared && !tstate->_status.finalized); assert(tstate->state != _Py_THREAD_ATTACHED); + tstate_verify_not_active(tstate); PyInterpreterState *interp = tstate->interp; if (interp == NULL) { @@ -1687,8 +1688,8 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate) _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); #endif tstate_set_detached(tstate, _Py_THREAD_DETACHED); - tstate_delete_common(tstate); current_fast_clear(tstate->interp->runtime); + tstate_delete_common(tstate); _PyEval_ReleaseLock(tstate->interp, NULL); free_threadstate((_PyThreadStateImpl *)tstate); } From 872c0714fcdc168ce4a69bdd0346f2d5dd488ec2 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 11 Mar 2024 19:25:39 +0000 Subject: [PATCH 020/158] gh-71052: Change Android's `sys.platform` from "linux" to "android" Co-authored-by: Erlend E. Aasland --- Doc/library/sys.rst | 45 +++++++++---------- Lib/ctypes/__init__.py | 2 +- Lib/multiprocessing/util.py | 6 +-- Lib/shutil.py | 3 +- Lib/test/support/__init__.py | 2 +- Lib/test/support/os_helper.py | 2 +- Lib/test/test_asyncio/test_subprocess.py | 3 +- Lib/test/test_c_locale_coercion.py | 21 +++++---- Lib/test/test_fcntl.py | 4 +- Lib/test/test_logging.py | 4 +- Lib/test/test_mmap.py | 2 +- Lib/test/test_os.py | 7 ++- Lib/test/test_resource.py | 2 +- Lib/test/test_socket.py | 17 ++++--- Lib/test/test_ssl.py | 2 +- Lib/test/test_sys.py | 5 +-- Lib/test/test_sysconfig.py | 22 ++++++--- Lib/test/test_tarfile.py | 2 +- Lib/test/test_time.py | 2 +- Lib/uuid.py | 2 +- ...4-03-01-16-44-19.gh-issue-71052.Hs-9EP.rst | 1 + Misc/platform_triplet.c | 14 +++++- configure | 1 + configure.ac | 1 + 24 files changed, 94 insertions(+), 78 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-03-01-16-44-19.gh-issue-71052.Hs-9EP.rst diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 380ba1090b39b3..087a3454c33272 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1367,47 +1367,42 @@ always available. .. data:: platform - This string contains a platform identifier that can be used to append - platform-specific components to :data:`sys.path`, for instance. - - For Unix systems, except on Linux and AIX, this is the lowercased OS name as - returned by ``uname -s`` with the first part of the version as returned by - ``uname -r`` appended, e.g. ``'sunos5'`` or ``'freebsd8'``, *at the time - when Python was built*. Unless you want to test for a specific system - version, it is therefore recommended to use the following idiom:: - - if sys.platform.startswith('freebsd'): - # FreeBSD-specific code here... - elif sys.platform.startswith('linux'): - # Linux-specific code here... - elif sys.platform.startswith('aix'): - # AIX-specific code here... - - For other systems, the values are: + A string containing a platform identifier. Known values are: ================ =========================== System ``platform`` value ================ =========================== AIX ``'aix'`` + Android ``'android'`` Emscripten ``'emscripten'`` + iOS ``'ios'`` Linux ``'linux'`` - WASI ``'wasi'`` + macOS ``'darwin'`` Windows ``'win32'`` Windows/Cygwin ``'cygwin'`` - macOS ``'darwin'`` + WASI ``'wasi'`` ================ =========================== + On Unix systems not listed in the table, the value is the lowercased OS name + as returned by ``uname -s``, with the first part of the version as returned by + ``uname -r`` appended, e.g. ``'sunos5'`` or ``'freebsd8'``, *at the time + when Python was built*. Unless you want to test for a specific system + version, it is therefore recommended to use the following idiom:: + + if sys.platform.startswith('freebsd'): + # FreeBSD-specific code here... + .. versionchanged:: 3.3 On Linux, :data:`sys.platform` doesn't contain the major version anymore. - It is always ``'linux'``, instead of ``'linux2'`` or ``'linux3'``. Since - older Python versions include the version number, it is recommended to - always use the ``startswith`` idiom presented above. + It is always ``'linux'``, instead of ``'linux2'`` or ``'linux3'``. .. versionchanged:: 3.8 On AIX, :data:`sys.platform` doesn't contain the major version anymore. - It is always ``'aix'``, instead of ``'aix5'`` or ``'aix7'``. Since - older Python versions include the version number, it is recommended to - always use the ``startswith`` idiom presented above. + It is always ``'aix'``, instead of ``'aix5'`` or ``'aix7'``. + + .. versionchanged:: 3.13 + On Android, :data:`sys.platform` now returns ``'android'`` rather than + ``'linux'``. .. seealso:: diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index d54ee05b15f5bd..f63e31a3fb0107 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -468,7 +468,7 @@ def LoadLibrary(self, name): if _os.name == "nt": pythonapi = PyDLL("python dll", None, _sys.dllhandle) -elif hasattr(_sys, "getandroidapilevel"): +elif _sys.platform == "android": pythonapi = PyDLL("libpython%d.%d.so" % _sys.version_info[:2]) elif _sys.platform == "cygwin": pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 32871850ddec8b..75dde02d88c533 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -102,11 +102,7 @@ def log_to_stderr(level=None): # Abstract socket support def _platform_supports_abstract_sockets(): - if sys.platform == "linux": - return True - if hasattr(sys, 'getandroidapilevel'): - return True - return False + return sys.platform in ("linux", "android") def is_abstract_socket_namespace(address): diff --git a/Lib/shutil.py b/Lib/shutil.py index c19ea0607208af..f8be82d97616d3 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -47,7 +47,8 @@ COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 # This should never be removed, see rationale in: # https://bugs.python.org/issue43743#msg393429 -_USE_CP_SENDFILE = hasattr(os, "sendfile") and sys.platform.startswith("linux") +_USE_CP_SENDFILE = (hasattr(os, "sendfile") + and sys.platform.startswith(("linux", "android"))) _HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS # CMD defaults in Windows 10 diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 505d4bea83fe8d..af43446c26120e 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -520,7 +520,7 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): # Is not actually used in tests, but is kept for compatibility. is_jython = sys.platform.startswith('java') -is_android = hasattr(sys, 'getandroidapilevel') +is_android = sys.platform == "android" if sys.platform not in {"win32", "vxworks", "ios", "tvos", "watchos"}: unix_shell = '/system/bin/sh' if is_android else '/bin/sh' diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index ffa5fc52e9b2bd..8071c248b9b67e 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -612,7 +612,7 @@ def __fspath__(self): def fd_count(): """Count the number of open file descriptors. """ - if sys.platform.startswith(('linux', 'freebsd', 'emscripten')): + if sys.platform.startswith(('linux', 'android', 'freebsd', 'emscripten')): fd_path = "/proc/self/fd" elif sys.platform == "darwin": fd_path = "/dev/fd" diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 890c0acc1720dc..cf1a1985338e40 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -480,7 +480,8 @@ async def empty_error(): self.assertEqual(output, None) self.assertEqual(exitcode, 0) - @unittest.skipIf(sys.platform != 'linux', "Don't have /dev/stdin") + @unittest.skipIf(sys.platform not in ('linux', 'android'), + "Don't have /dev/stdin") def test_devstdin_input(self): async def devstdin_input(message): diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py index 7334a325ba22f0..e4b0b8c451fd45 100644 --- a/Lib/test/test_c_locale_coercion.py +++ b/Lib/test/test_c_locale_coercion.py @@ -26,17 +26,16 @@ TARGET_LOCALES = ["C.UTF-8", "C.utf8", "UTF-8"] # Apply some platform dependent overrides -if sys.platform.startswith("linux"): - if support.is_android: - # Android defaults to using UTF-8 for all system interfaces - EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8" - EXPECTED_C_LOCALE_FS_ENCODING = "utf-8" - else: - # Linux distros typically alias the POSIX locale directly to the C - # locale. - # TODO: Once https://bugs.python.org/issue30672 is addressed, we'll be - # able to check this case unconditionally - EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX") +if sys.platform == "android": + # Android defaults to using UTF-8 for all system interfaces + EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8" + EXPECTED_C_LOCALE_FS_ENCODING = "utf-8" +elif sys.platform.startswith("linux"): + # Linux distros typically alias the POSIX locale directly to the C + # locale. + # TODO: Once https://bugs.python.org/issue30672 is addressed, we'll be + # able to check this case unconditionally + EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX") elif sys.platform.startswith("aix"): # AIX uses iso8859-1 in the C locale, other *nix platforms use ASCII EXPECTED_C_LOCALE_STREAM_ENCODING = "iso8859-1" diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 8b4ed4a9e3a4fe..5fae0de7423c87 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -129,8 +129,8 @@ def test_fcntl_bad_file_overflow(self): fcntl.fcntl(BadFile(INT_MIN - 1), fcntl.F_SETFL, os.O_NONBLOCK) @unittest.skipIf( - any(platform.machine().startswith(name) for name in {"arm", "aarch"}) - and platform.system() in {"Linux", "Android"}, + platform.machine().startswith(("arm", "aarch")) + and platform.system() in ("Linux", "Android"), "ARM Linux returns EINVAL for F_NOTIFY DN_MULTISHOT") def test_fcntl_64_bit(self): # Issue #1309352: fcntl shouldn't fail when the third arg fits in a diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 1dfad6e1d2609b..32bb5171a3f757 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -603,7 +603,7 @@ def test_name(self): def test_builtin_handlers(self): # We can't actually *use* too many handlers in the tests, # but we can try instantiating them with various options - if sys.platform in ('linux', 'darwin'): + if sys.platform in ('linux', 'android', 'darwin'): for existing in (True, False): fn = make_temp_file() if not existing: @@ -667,7 +667,7 @@ def test_path_objects(self): (logging.handlers.RotatingFileHandler, (pfn, 'a')), (logging.handlers.TimedRotatingFileHandler, (pfn, 'h')), ) - if sys.platform in ('linux', 'darwin'): + if sys.platform in ('linux', 'android', 'darwin'): cases += ((logging.handlers.WatchedFileHandler, (pfn, 'w')),) for cls, args in cases: h = cls(*args, encoding="utf-8") diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index ac759757d24659..ee86227e026b67 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -837,7 +837,7 @@ def test_flush_return_value(self): mm.write(b'python') result = mm.flush() self.assertIsNone(result) - if sys.platform.startswith('linux'): + if sys.platform.startswith(('linux', 'android')): # 'offset' must be a multiple of mmap.PAGESIZE on Linux. # See bpo-34754 for details. self.assertRaises(OSError, mm.flush, 1, len(b'python')) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index ae8b405dab50fc..fc886f967c11bf 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3533,9 +3533,8 @@ def test_set_get_priority(self): class TestSendfile(unittest.IsolatedAsyncioTestCase): DATA = b"12345abcde" * 16 * 1024 # 160 KiB - SUPPORT_HEADERS_TRAILERS = not sys.platform.startswith("linux") and \ - not sys.platform.startswith("solaris") and \ - not sys.platform.startswith("sunos") + SUPPORT_HEADERS_TRAILERS = ( + not sys.platform.startswith(("linux", "android", "solaris", "sunos"))) requires_headers_trailers = unittest.skipUnless(SUPPORT_HEADERS_TRAILERS, 'requires headers and trailers support') requires_32b = unittest.skipUnless(sys.maxsize < 2**32, @@ -5256,7 +5255,7 @@ def test_fork(self): else: assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") - @unittest.skipUnless(sys.platform in ("linux", "darwin"), + @unittest.skipUnless(sys.platform in ("linux", "android", "darwin"), "Only Linux and macOS detect this today.") def test_fork_warns_when_non_python_thread_exists(self): code = """if 1: diff --git a/Lib/test/test_resource.py b/Lib/test/test_resource.py index 317e7ca8f8c853..d23d3623235f38 100644 --- a/Lib/test/test_resource.py +++ b/Lib/test/test_resource.py @@ -138,7 +138,7 @@ def test_pagesize(self): self.assertIsInstance(pagesize, int) self.assertGreaterEqual(pagesize, 0) - @unittest.skipUnless(sys.platform == 'linux', 'test requires Linux') + @unittest.skipUnless(sys.platform in ('linux', 'android'), 'Linux only') def test_linux_constants(self): for attr in ['MSGQUEUE', 'NICE', 'RTPRIO', 'RTTIME', 'SIGPENDING']: with contextlib.suppress(AttributeError): diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index b936e9ae91daca..a7e657f5718524 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1199,8 +1199,8 @@ def testGetServBy(self): # I've ordered this by protocols that have both a tcp and udp # protocol, at least for modern Linuxes. if ( - sys.platform.startswith(('freebsd', 'netbsd', 'gnukfreebsd')) - or sys.platform == 'linux' + sys.platform.startswith( + ('linux', 'android', 'freebsd', 'netbsd', 'gnukfreebsd')) or is_apple ): # avoid the 'echo' service on this platform, as there is an @@ -1218,8 +1218,7 @@ def testGetServBy(self): raise OSError # Try same call with optional protocol omitted # Issue #26936: Android getservbyname() was broken before API 23. - if (not hasattr(sys, 'getandroidapilevel') or - sys.getandroidapilevel() >= 23): + if (not support.is_android) or sys.getandroidapilevel() >= 23: port2 = socket.getservbyname(service) eq(port, port2) # Try udp, but don't barf if it doesn't exist @@ -1577,8 +1576,7 @@ def testGetaddrinfo(self): # port can be a string service name such as "http", a numeric # port number or None # Issue #26936: Android getaddrinfo() was broken before API level 23. - if (not hasattr(sys, 'getandroidapilevel') or - sys.getandroidapilevel() >= 23): + if (not support.is_android) or sys.getandroidapilevel() >= 23: socket.getaddrinfo(HOST, "http") socket.getaddrinfo(HOST, 80) socket.getaddrinfo(HOST, None) @@ -3196,7 +3194,7 @@ def _testSendmsgTimeout(self): # Linux supports MSG_DONTWAIT when sending, but in general, it # only works when receiving. Could add other platforms if they # support it too. - @skipWithClientIf(sys.platform not in {"linux"}, + @skipWithClientIf(sys.platform not in {"linux", "android"}, "MSG_DONTWAIT not known to work on this platform when " "sending") def testSendmsgDontWait(self): @@ -5634,7 +5632,7 @@ def test_setblocking_invalidfd(self): sock.setblocking(False) -@unittest.skipUnless(sys.platform == 'linux', 'Linux specific test') +@unittest.skipUnless(sys.platform in ('linux', 'android'), 'Linux specific test') class TestLinuxAbstractNamespace(unittest.TestCase): UNIX_PATH_MAX = 108 @@ -5759,7 +5757,8 @@ def testUnencodableAddr(self): self.addCleanup(os_helper.unlink, path) self.assertEqual(self.sock.getsockname(), path) - @unittest.skipIf(sys.platform == 'linux', 'Linux specific test') + @unittest.skipIf(sys.platform in ('linux', 'android'), + 'Linux behavior is tested by TestLinuxAbstractNamespace') def testEmptyAddress(self): # Test that binding empty address fails. self.assertRaises(OSError, self.sock.bind, "") diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index cdf96c808b86ac..489cb5e23ba57e 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4914,7 +4914,7 @@ def run(self): pass # closed, protocol error, etc. def non_linux_skip_if_other_okay_error(self, err): - if sys.platform == "linux": + if sys.platform in ("linux", "android"): return # Expect the full test setup to always work on Linux. if (isinstance(err, ConnectionResetError) or (isinstance(err, OSError) and err.errno == errno.EINVAL) or diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 38dcabd84d8170..37c16cd1047885 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -668,7 +668,7 @@ def test_thread_info(self): self.assertEqual(len(info), 3) self.assertIn(info.name, ('nt', 'pthread', 'pthread-stubs', 'solaris', None)) self.assertIn(info.lock, ('semaphore', 'mutex+cond', None)) - if sys.platform.startswith(("linux", "freebsd")): + if sys.platform.startswith(("linux", "android", "freebsd")): self.assertEqual(info.name, "pthread") elif sys.platform == "win32": self.assertEqual(info.name, "nt") @@ -1101,8 +1101,7 @@ def __del__(self): self.assertEqual(stdout.rstrip(), b"") self.assertEqual(stderr.rstrip(), b"") - @unittest.skipUnless(hasattr(sys, 'getandroidapilevel'), - 'need sys.getandroidapilevel()') + @unittest.skipUnless(sys.platform == "android", "Android only") def test_getandroidapilevel(self): level = sys.getandroidapilevel() self.assertIsInstance(level, int) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index bb87bf00dc2d1a..c8315bbc8b727d 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -1,3 +1,5 @@ +import platform +import re import unittest import sys import os @@ -516,12 +518,9 @@ def test_EXT_SUFFIX_in_vars(self): vars = sysconfig.get_config_vars() self.assertEqual(vars['EXT_SUFFIX'], _imp.extension_suffixes()[0]) - @unittest.skipUnless(sys.platform == 'linux' and - hasattr(sys.implementation, '_multiarch'), - 'multiarch-specific test') - def test_triplet_in_ext_suffix(self): + @unittest.skipUnless(sys.platform == 'linux', 'Linux-specific test') + def test_linux_ext_suffix(self): ctypes = import_module('ctypes') - import platform, re machine = platform.machine() suffix = sysconfig.get_config_var('EXT_SUFFIX') if re.match('(aarch64|arm|mips|ppc|powerpc|s390|sparc)', machine): @@ -534,6 +533,19 @@ def test_triplet_in_ext_suffix(self): self.assertTrue(suffix.endswith(expected_suffixes), f'unexpected suffix {suffix!r}') + @unittest.skipUnless(sys.platform == 'android', 'Android-specific test') + def test_android_ext_suffix(self): + machine = platform.machine() + suffix = sysconfig.get_config_var('EXT_SUFFIX') + expected_triplet = { + "x86_64": "x86_64-linux-android", + "i686": "i686-linux-android", + "aarch64": "aarch64-linux-android", + "armv7l": "arm-linux-androideabi", + }[machine] + self.assertTrue(suffix.endswith(f"-{expected_triplet}.so"), + f"{machine=}, {suffix=}") + @unittest.skipUnless(sys.platform == 'darwin', 'OS X-specific test') def test_osx_ext_suffix(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index a047780fdd7e17..39541faa237b24 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1186,7 +1186,7 @@ def _fs_supports_holes(): # # The function returns False if page size is larger than 4 KiB. # For example, ppc64 uses pages of 64 KiB. - if sys.platform.startswith("linux"): + if sys.platform.startswith(("linux", "android")): # Linux evidentially has 512 byte st_blocks units. name = os.path.join(TEMPDIR, "sparse-test") with open(name, "wb") as fobj: diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index a0aeea515afbd6..fb234b7bc5962a 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -511,7 +511,7 @@ def test_process_time(self): def test_thread_time(self): if not hasattr(time, 'thread_time'): - if sys.platform.startswith(('linux', 'win')): + if sys.platform.startswith(('linux', 'android', 'win')): self.fail("time.thread_time() should be available on %r" % (sys.platform,)) else: diff --git a/Lib/uuid.py b/Lib/uuid.py index d4e486da15a8d7..da2f1d50563095 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -62,7 +62,7 @@ import platform _platform_system = platform.system() _AIX = _platform_system == 'AIX' - _LINUX = _platform_system == 'Linux' + _LINUX = _platform_system in ('Linux', 'Android') _MAC_DELIM = b':' _MAC_OMITS_LEADING_ZEROES = False diff --git a/Misc/NEWS.d/next/Build/2024-03-01-16-44-19.gh-issue-71052.Hs-9EP.rst b/Misc/NEWS.d/next/Build/2024-03-01-16-44-19.gh-issue-71052.Hs-9EP.rst new file mode 100644 index 00000000000000..187475f56b4415 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-03-01-16-44-19.gh-issue-71052.Hs-9EP.rst @@ -0,0 +1 @@ +Change Android's :data:`sys.platform` from ``"linux"`` to ``"android"``. diff --git a/Misc/platform_triplet.c b/Misc/platform_triplet.c index 0b912e332510a6..06b03bfa9a266a 100644 --- a/Misc/platform_triplet.c +++ b/Misc/platform_triplet.c @@ -12,8 +12,20 @@ #undef powerpc #undef sparc #undef unix + #if defined(__ANDROID__) - # Android is not a multiarch system. +# if defined(__x86_64__) +PLATFORM_TRIPLET=x86_64-linux-android +# elif defined(__i386__) +PLATFORM_TRIPLET=i686-linux-android +# elif defined(__aarch64__) +PLATFORM_TRIPLET=aarch64-linux-android +# elif defined(__arm__) +PLATFORM_TRIPLET=arm-linux-androideabi +# else +# error unknown Android platform +# endif + #elif defined(__linux__) /* * BEGIN of Linux block diff --git a/configure b/configure index 5e8c9af7befde0..267a5183379ce0 100755 --- a/configure +++ b/configure @@ -4071,6 +4071,7 @@ then case $MACHDEP in aix*) MACHDEP="aix";; + linux-android*) MACHDEP="android";; linux*) MACHDEP="linux";; cygwin*) MACHDEP="cygwin";; darwin*) MACHDEP="darwin";; diff --git a/configure.ac b/configure.ac index e615259dcb1fca..681d081f571ea5 100644 --- a/configure.ac +++ b/configure.ac @@ -362,6 +362,7 @@ then case $MACHDEP in aix*) MACHDEP="aix";; + linux-android*) MACHDEP="android";; linux*) MACHDEP="linux";; cygwin*) MACHDEP="cygwin";; darwin*) MACHDEP="darwin";; From 1d0d49a7e86257ff95b4de0685e6997d7533993c Mon Sep 17 00:00:00 2001 From: "Pierre Ossman (ThinLinc team)" Date: Mon, 11 Mar 2024 20:43:30 +0100 Subject: [PATCH 021/158] gh-113538: Add asycio.Server.{close,abort}_clients (#114432) These give applications the option of more forcefully terminating client connections for asyncio servers. Useful when terminating a service and there is limited time to wait for clients to finish up their work. --- Doc/library/asyncio-eventloop.rst | 25 +++++ Doc/whatsnew/3.13.rst | 5 + Lib/asyncio/base_events.py | 25 +++-- Lib/asyncio/events.py | 8 ++ Lib/asyncio/proactor_events.py | 4 +- Lib/asyncio/selector_events.py | 6 +- Lib/test/test_asyncio/test_server.py | 96 +++++++++++++++++-- ...-01-22-15-50-58.gh-issue-113538.v2wrwg.rst | 3 + 8 files changed, 152 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 06c5c877ccc173..d6ed817b13676f 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -1641,6 +1641,31 @@ Do not instantiate the :class:`Server` class directly. coroutine to wait until the server is closed (and no more connections are active). + .. method:: close_clients() + + Close all existing incoming client connections. + + Calls :meth:`~asyncio.BaseTransport.close` on all associated + transports. + + :meth:`close` should be called before :meth:`close_clients` when + closing the server to avoid races with new clients connecting. + + .. versionadded:: 3.13 + + .. method:: abort_clients() + + Close all existing incoming client connections immediately, + without waiting for pending operations to complete. + + Calls :meth:`~asyncio.WriteTransport.abort` on all associated + transports. + + :meth:`close` should be called before :meth:`abort_clients` when + closing the server to avoid races with new clients connecting. + + .. versionadded:: 3.13 + .. method:: get_loop() Return the event loop associated with the server object. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 51939909000960..95e8ff3eb2e575 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -270,6 +270,11 @@ asyncio the buffer size. (Contributed by Jamie Phan in :gh:`115199`.) +* Add :meth:`asyncio.Server.close_clients` and + :meth:`asyncio.Server.abort_clients` methods which allow to more + forcefully close an asyncio server. + (Contributed by Pierre Ossman in :gh:`113538`.) + base64 --- diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 6c5cf28e7c59d4..f0e690b61a73dd 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -279,7 +279,9 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, ssl_handshake_timeout, ssl_shutdown_timeout=None): self._loop = loop self._sockets = sockets - self._active_count = 0 + # Weak references so we don't break Transport's ability to + # detect abandoned transports + self._clients = weakref.WeakSet() self._waiters = [] self._protocol_factory = protocol_factory self._backlog = backlog @@ -292,14 +294,13 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, def __repr__(self): return f'<{self.__class__.__name__} sockets={self.sockets!r}>' - def _attach(self): + def _attach(self, transport): assert self._sockets is not None - self._active_count += 1 + self._clients.add(transport) - def _detach(self): - assert self._active_count > 0 - self._active_count -= 1 - if self._active_count == 0 and self._sockets is None: + def _detach(self, transport): + self._clients.discard(transport) + if len(self._clients) == 0 and self._sockets is None: self._wakeup() def _wakeup(self): @@ -348,9 +349,17 @@ def close(self): self._serving_forever_fut.cancel() self._serving_forever_fut = None - if self._active_count == 0: + if len(self._clients) == 0: self._wakeup() + def close_clients(self): + for transport in self._clients.copy(): + transport.close() + + def abort_clients(self): + for transport in self._clients.copy(): + transport.abort() + async def start_serving(self): self._start_serving() # Skip one loop iteration so that all 'loop.add_reader' diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 680749325025db..be495469a0558b 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -175,6 +175,14 @@ def close(self): """Stop serving. This leaves existing connections open.""" raise NotImplementedError + def close_clients(self): + """Close all active connections.""" + raise NotImplementedError + + def abort_clients(self): + """Close all active connections immediately.""" + raise NotImplementedError + def get_loop(self): """Get the event loop the Server object is attached to.""" raise NotImplementedError diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index a512db6367b20a..397a8cda757895 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -63,7 +63,7 @@ def __init__(self, loop, sock, protocol, waiter=None, self._called_connection_lost = False self._eof_written = False if self._server is not None: - self._server._attach() + self._server._attach(self) self._loop.call_soon(self._protocol.connection_made, self) if waiter is not None: # only wake up the waiter when connection_made() has been called @@ -167,7 +167,7 @@ def _call_connection_lost(self, exc): self._sock = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None self._called_connection_lost = True diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 8e888d26ea0737..f94bf10b4225e7 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -791,7 +791,7 @@ def __init__(self, loop, sock, protocol, extra=None, server=None): self._paused = False # Set when pause_reading() called if self._server is not None: - self._server._attach() + self._server._attach(self) loop._transports[self._sock_fd] = self def __repr__(self): @@ -868,6 +868,8 @@ def __del__(self, _warn=warnings.warn): if self._sock is not None: _warn(f"unclosed transport {self!r}", ResourceWarning, source=self) self._sock.close() + if self._server is not None: + self._server._detach(self) def _fatal_error(self, exc, message='Fatal error on transport'): # Should be called from exception handler only. @@ -906,7 +908,7 @@ def _call_connection_lost(self, exc): self._loop = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None def get_write_buffer_size(self): diff --git a/Lib/test/test_asyncio/test_server.py b/Lib/test/test_asyncio/test_server.py index 918faac909b9bf..0c55661bfb88f4 100644 --- a/Lib/test/test_asyncio/test_server.py +++ b/Lib/test/test_asyncio/test_server.py @@ -125,8 +125,12 @@ async def main(srv): class TestServer2(unittest.IsolatedAsyncioTestCase): async def test_wait_closed_basic(self): - async def serve(*args): - pass + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) self.addCleanup(srv.close) @@ -137,7 +141,8 @@ async def serve(*args): self.assertFalse(task1.done()) # active count != 0, not closed: should block - srv._attach() + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) task2 = asyncio.create_task(srv.wait_closed()) await asyncio.sleep(0) self.assertFalse(task1.done()) @@ -152,7 +157,8 @@ async def serve(*args): self.assertFalse(task2.done()) self.assertFalse(task3.done()) - srv._detach() + wr.close() + await wr.wait_closed() # active count == 0, closed: should unblock await task1 await task2 @@ -161,8 +167,12 @@ async def serve(*args): async def test_wait_closed_race(self): # Test a regression in 3.12.0, should be fixed in 3.12.1 - async def serve(*args): - pass + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) self.addCleanup(srv.close) @@ -170,13 +180,83 @@ async def serve(*args): task = asyncio.create_task(srv.wait_closed()) await asyncio.sleep(0) self.assertFalse(task.done()) - srv._attach() + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) loop = asyncio.get_running_loop() loop.call_soon(srv.close) - loop.call_soon(srv._detach) + loop.call_soon(wr.close) await srv.wait_closed() + async def test_close_clients(self): + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() + + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) + self.addCleanup(wr.close) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + + srv.close() + srv.close_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) + + async def test_abort_clients(self): + async def serve(rd, wr): + nonlocal s_rd, s_wr + s_rd = rd + s_wr = wr + await wr.wait_closed() + + s_rd = s_wr = None + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (c_rd, c_wr) = await asyncio.open_connection(addr[0], addr[1], limit=4096) + self.addCleanup(c_wr.close) + + # Limit the socket buffers so we can reliably overfill them + s_sock = s_wr.get_extra_info('socket') + s_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 65536) + c_sock = c_wr.get_extra_info('socket') + c_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 65536) + + # Get the reader in to a paused state by sending more than twice + # the configured limit + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + while c_wr.transport.is_reading(): + await asyncio.sleep(0) + + # Get the writer in a waiting state by sending data until the + # socket buffers are full on both server and client sockets and + # the kernel stops accepting more data + s_wr.write(b'a' * c_sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF)) + s_wr.write(b'a' * s_sock.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF)) + self.assertNotEqual(s_wr.transport.get_write_buffer_size(), 0) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + srv.close() + srv.abort_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) # Test the various corner cases of Unix server socket removal diff --git a/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst b/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst new file mode 100644 index 00000000000000..5c59af98e136bb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst @@ -0,0 +1,3 @@ +Add :meth:`asyncio.Server.close_clients` and +:meth:`asyncio.Server.abort_clients` methods which allow to more forcefully +close an asyncio server. From 34920f36917de0d4e658cf94992d53a5a7f27f51 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 11 Mar 2024 20:39:17 +0000 Subject: [PATCH 022/158] gh-71052: Use `raise_signal` in `ThreadSignals.test_signals` (#116423) Use `raise_signal` rather than `kill` in `ThreadSignals.test_signals` --- Lib/test/test_threadsignals.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_threadsignals.py b/Lib/test/test_threadsignals.py index 6a53d655015cdb..bf241ada90ea35 100644 --- a/Lib/test/test_threadsignals.py +++ b/Lib/test/test_threadsignals.py @@ -32,39 +32,28 @@ def handle_signals(sig,frame): # a function that will be spawned as a separate thread. def send_signals(): - os.kill(process_pid, signal.SIGUSR1) - os.kill(process_pid, signal.SIGUSR2) + # We use `raise_signal` rather than `kill` because: + # * It verifies that a signal delivered to a background thread still has + # its Python-level handler called on the main thread. + # * It ensures the signal is handled before the thread exits. + signal.raise_signal(signal.SIGUSR1) + signal.raise_signal(signal.SIGUSR2) signalled_all.release() @threading_helper.requires_working_threading() -@unittest.skipUnless(hasattr(signal, "alarm"), "test requires signal.alarm") class ThreadSignals(unittest.TestCase): def test_signals(self): with threading_helper.wait_threads_exit(): # Test signal handling semantics of threads. - # We spawn a thread, have the thread send two signals, and + # We spawn a thread, have the thread send itself two signals, and # wait for it to finish. Check that we got both signals # and that they were run by the main thread. signalled_all.acquire() self.spawnSignallingThread() signalled_all.acquire() - # the signals that we asked the kernel to send - # will come back, but we don't know when. - # (it might even be after the thread exits - # and might be out of order.) If we haven't seen - # the signals yet, send yet another signal and - # wait for it return. - if signal_blackboard[signal.SIGUSR1]['tripped'] == 0 \ - or signal_blackboard[signal.SIGUSR2]['tripped'] == 0: - try: - signal.alarm(1) - signal.pause() - finally: - signal.alarm(0) - self.assertEqual( signal_blackboard[signal.SIGUSR1]['tripped'], 1) self.assertEqual( signal_blackboard[signal.SIGUSR1]['tripped_by'], thread.get_ident()) From 3c0dcef9808e34744096769b15bad4f1f97569f0 Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Mon, 11 Mar 2024 13:42:01 -0700 Subject: [PATCH 023/158] gh-116040: [Enum] fix test_empty_names test (GH-116508) * and fix _not_given usage --- Lib/enum.py | 4 +--- Lib/test/test_enum.py | 18 +++++++----------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/Lib/enum.py b/Lib/enum.py index 3499cb0b71547c..bcf7aae949ffdb 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -165,8 +165,6 @@ def _dedent(text): class _not_given: def __repr__(self): return('') - def __bool__(self): - return False _not_given = _not_given() class _auto_null: @@ -727,7 +725,7 @@ def __call__(cls, value, names=_not_given, *values, module=None, qualname=None, ) return cls._create_( class_name=value, - names=names or None, + names=None if names is _not_given else names, module=module, qualname=qualname, type=type, diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 0a44b61e9049ed..a83aca406cbd83 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -3431,17 +3431,13 @@ def test_no_members(self): Flag(7) def test_empty_names(self): - for nothing, e_type in ( - ('', None), - ('', int), - ([], None), - ([], int), - ({}, None), - ({}, int), - ): - empty_enum = Enum('empty_enum', nothing, type=e_type) - self.assertEqual(len(empty_enum), 0) - self.assertRaises(TypeError, 'has no members', empty_enum, 0) + for nothing in '', [], {}: + for e_type in None, int: + empty_enum = Enum('empty_enum', nothing, type=e_type) + self.assertEqual(len(empty_enum), 0) + self.assertRaisesRegex(TypeError, 'has no members', empty_enum, 0) + self.assertRaisesRegex(TypeError, '.int. object is not iterable', Enum, 'bad_enum', names=0) + self.assertRaisesRegex(TypeError, '.int. object is not iterable', Enum, 'bad_enum', 0, type=int) class TestOrder(unittest.TestCase): From 44f9a84b67c97c94f0d581ffd63b24b73fb79610 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 11 Mar 2024 14:27:00 -0700 Subject: [PATCH 024/158] gh-90095: Make .pdbrc work properly and add some reasonable tests (#110496) --- Lib/pdb.py | 47 ++---- Lib/test/test_pdb.py | 149 ++++++++++-------- ...3-10-07-06-15-13.gh-issue-90095.gWn1ka.rst | 1 + 3 files changed, 101 insertions(+), 96 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-10-07-06-15-13.gh-issue-90095.gWn1ka.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index 519c1ccd5640a1..f4d19386703de0 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -363,26 +363,9 @@ def setup(self, f, tb): self._chained_exceptions[self._chained_exception_index], ) - return self.execRcLines() - - # Can be executed earlier than 'setup' if desired - def execRcLines(self): - if not self.rcLines: - return - # local copy because of recursion - rcLines = self.rcLines - rcLines.reverse() - # execute every line only once - self.rcLines = [] - while rcLines: - line = rcLines.pop().strip() - if line and line[0] != '#': - if self.onecmd(line): - # if onecmd returns True, the command wants to exit - # from the interaction, save leftover rc lines - # to execute before next interaction - self.rcLines += reversed(rcLines) - return True + if self.rcLines: + self.cmdqueue = self.rcLines + self.rcLines = [] # Override Bdb methods @@ -571,12 +554,10 @@ def interaction(self, frame, tb_or_exc): if isinstance(tb_or_exc, BaseException): assert tb is not None, "main exception must have a traceback" with self._hold_exceptions(_chained_exceptions): - if self.setup(frame, tb): - # no interaction desired at this time (happens if .pdbrc contains - # a command like "continue") - self.forget() - return - self.print_stack_entry(self.stack[self.curindex]) + self.setup(frame, tb) + # if we have more commands to process, do not show the stack entry + if not self.cmdqueue: + self.print_stack_entry(self.stack[self.curindex]) self._cmdloop() self.forget() @@ -712,7 +693,7 @@ def precmd(self, line): if marker >= 0: # queue up everything after marker next = line[marker+2:].lstrip() - self.cmdqueue.append(next) + self.cmdqueue.insert(0, next) line = line[:marker].rstrip() # Replace all the convenience variables @@ -737,13 +718,12 @@ def handle_command_def(self, line): """Handles one command line during command list definition.""" cmd, arg, line = self.parseline(line) if not cmd: - return + return False if cmd == 'silent': self.commands_silent[self.commands_bnum] = True - return # continue to handle other cmd def in the cmd list + return False # continue to handle other cmd def in the cmd list elif cmd == 'end': - self.cmdqueue = [] - return 1 # end of cmd list + return True # end of cmd list cmdlist = self.commands[self.commands_bnum] if arg: cmdlist.append(cmd+' '+arg) @@ -757,9 +737,8 @@ def handle_command_def(self, line): # one of the resuming commands if func.__name__ in self.commands_resuming: self.commands_doprompt[self.commands_bnum] = False - self.cmdqueue = [] - return 1 - return + return True + return False # interface abstraction functions diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 3dd275caf43200..44728542787423 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2618,13 +2618,29 @@ def _run_pdb(self, pdb_args, commands, def run_pdb_script(self, script, commands, expected_returncode=0, - extra_env=None): + extra_env=None, + pdbrc=None, + remove_home=False): """Run 'script' lines with pdb and the pdb 'commands'.""" filename = 'main.py' with open(filename, 'w') as f: f.write(textwrap.dedent(script)) + + if pdbrc is not None: + with open('.pdbrc', 'w') as f: + f.write(textwrap.dedent(pdbrc)) + self.addCleanup(os_helper.unlink, '.pdbrc') self.addCleanup(os_helper.unlink, filename) - return self._run_pdb([filename], commands, expected_returncode, extra_env) + + homesave = None + if remove_home: + homesave = os.environ.pop('HOME', None) + try: + stdout, stderr = self._run_pdb([filename], commands, expected_returncode, extra_env) + finally: + if homesave is not None: + os.environ['HOME'] = homesave + return stdout, stderr def run_pdb_module(self, script, commands): """Runs the script code as part of a module""" @@ -2904,37 +2920,80 @@ def test_issue26053(self): self.assertRegex(res, "Restarting .* with arguments:\na b c") self.assertRegex(res, "Restarting .* with arguments:\nd e f") - def test_readrc_kwarg(self): + def test_pdbrc_basic(self): script = textwrap.dedent(""" - import pdb; pdb.Pdb(readrc=False).set_trace() + a = 1 + b = 2 + """) - print('hello') + pdbrc = textwrap.dedent(""" + # Comments should be fine + n + p f"{a+8=}" """) - save_home = os.environ.pop('HOME', None) - try: - with os_helper.temp_cwd(): - with open('.pdbrc', 'w') as f: - f.write("invalid\n") - - with open('main.py', 'w') as f: - f.write(script) - - cmd = [sys.executable, 'main.py'] - proc = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - with proc: - stdout, stderr = proc.communicate(b'q\n') - self.assertNotIn(b"NameError: name 'invalid' is not defined", - stdout) + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("a+8=9", stdout) - finally: - if save_home is not None: - os.environ['HOME'] = save_home + def test_pdbrc_alias(self): + script = textwrap.dedent(""" + class A: + def __init__(self): + self.attr = 1 + a = A() + b = 2 + """) + + pdbrc = textwrap.dedent(""" + alias pi for k in %1.__dict__.keys(): print(f"%1.{k} = {%1.__dict__[k]}") + until 6 + pi a + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("a.attr = 1", stdout) + + def test_pdbrc_semicolon(self): + script = textwrap.dedent(""" + class A: + def __init__(self): + self.attr = 1 + a = A() + b = 2 + """) + + pdbrc = textwrap.dedent(""" + b 5;;c;;n + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("-> b = 2", stdout) + + def test_pdbrc_commands(self): + script = textwrap.dedent(""" + class A: + def __init__(self): + self.attr = 1 + a = A() + b = 2 + """) + + pdbrc = textwrap.dedent(""" + b 6 + commands 1 ;; p a;; end + c + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("<__main__.A object at", stdout) + + def test_readrc_kwarg(self): + script = textwrap.dedent(""" + print('hello') + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc='invalid', remove_home=True) + self.assertIn("NameError: name 'invalid' is not defined", stdout) def test_readrc_homedir(self): save_home = os.environ.pop("HOME", None) @@ -2949,40 +3008,6 @@ def test_readrc_homedir(self): if save_home is not None: os.environ["HOME"] = save_home - def test_read_pdbrc_with_ascii_encoding(self): - script = textwrap.dedent(""" - import pdb; pdb.Pdb().set_trace() - print('hello') - """) - save_home = os.environ.pop('HOME', None) - try: - with os_helper.temp_cwd(): - with open('.pdbrc', 'w', encoding='utf-8') as f: - f.write("Fran\u00E7ais") - - with open('main.py', 'w', encoding='utf-8') as f: - f.write(script) - - cmd = [sys.executable, 'main.py'] - env = {'PYTHONIOENCODING': 'ascii'} - if sys.platform == 'win32': - env['PYTHONLEGACYWINDOWSSTDIO'] = 'non-empty-string' - proc = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - env={**os.environ, **env} - ) - with proc: - stdout, stderr = proc.communicate(b'c\n') - self.assertIn(b"UnicodeEncodeError: \'ascii\' codec can\'t encode character " - b"\'\\xe7\' in position 21: ordinal not in range(128)", stderr) - - finally: - if save_home is not None: - os.environ['HOME'] = save_home - def test_header(self): stdout = StringIO() header = 'Nobody expects... blah, blah, blah' diff --git a/Misc/NEWS.d/next/Library/2023-10-07-06-15-13.gh-issue-90095.gWn1ka.rst b/Misc/NEWS.d/next/Library/2023-10-07-06-15-13.gh-issue-90095.gWn1ka.rst new file mode 100644 index 00000000000000..d71442ef642b6a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-07-06-15-13.gh-issue-90095.gWn1ka.rst @@ -0,0 +1 @@ +Make .pdbrc and -c work with any valid pdb commands. From 3e45030076bf2cfab41c4456c73fb212b7322c60 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 11 Mar 2024 18:01:08 -0400 Subject: [PATCH 025/158] gh-116563: Update tutorial error example (#116569) There now may be multiple carets pointing at a token rather than just a character. Fix the sentence about possible causes. --- Doc/tutorial/errors.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 4058ebe8efdb42..0b9acd00fdc6bd 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -20,12 +20,12 @@ complaint you get while you are still learning Python:: >>> while True print('Hello world') File "", line 1 while True print('Hello world') - ^ + ^^^^^ SyntaxError: invalid syntax -The parser repeats the offending line and displays a little 'arrow' pointing at -the earliest point in the line where the error was detected. The error is -caused by (or at least detected at) the token *preceding* the arrow: in the +The parser repeats the offending line and displays little 'arrow's pointing +at the token in the line where the error was detected. The error may be +caused by the absence of a token *before* the indicated token. In the example, the error is detected at the function :func:`print`, since a colon (``':'``) is missing before it. File name and line number are printed so you know where to look in case the input came from a script. From 06e29a224fac9edeba55422d2e60f2fbb88dddce Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Mon, 11 Mar 2024 15:41:53 -0700 Subject: [PATCH 026/158] gh-116600: [Enum] fix global Flag repr (GH-116615) * and fix global flag repr * Update Misc/NEWS.d/next/Library/2024-03-11-12-11-10.gh-issue-116600.FcNBy_.rst Co-authored-by: Kirill Podoprigora --- Lib/enum.py | 2 +- Lib/test/test_enum.py | 2 ++ .../next/Library/2024-03-11-12-11-10.gh-issue-116600.FcNBy_.rst | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-11-12-11-10.gh-issue-116600.FcNBy_.rst diff --git a/Lib/enum.py b/Lib/enum.py index bcf7aae949ffdb..5c5e711f9b078f 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1680,7 +1680,7 @@ def global_flag_repr(self): cls_name = self.__class__.__name__ if self._name_ is None: return "%s.%s(%r)" % (module, cls_name, self._value_) - if _is_single_bit(self): + if _is_single_bit(self._value_): return '%s.%s' % (module, self._name_) if self._boundary_ is not FlagBoundary.KEEP: return '|'.join(['%s.%s' % (module, name) for name in self.name.split('|')]) diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index a83aca406cbd83..6418d243db65ce 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -4063,6 +4063,8 @@ def test_global_repr_conform1(self): @reraise_if_not_enum(NoName) def test_global_enum_str(self): + self.assertEqual(repr(NoName.ONE), 'test_enum.ONE') + self.assertEqual(repr(NoName(0)), 'test_enum.NoName(0)') self.assertEqual(str(NoName.ONE & NoName.TWO), 'NoName(0)') self.assertEqual(str(NoName(0)), 'NoName(0)') diff --git a/Misc/NEWS.d/next/Library/2024-03-11-12-11-10.gh-issue-116600.FcNBy_.rst b/Misc/NEWS.d/next/Library/2024-03-11-12-11-10.gh-issue-116600.FcNBy_.rst new file mode 100644 index 00000000000000..e9148ba9290e7b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-11-12-11-10.gh-issue-116600.FcNBy_.rst @@ -0,0 +1 @@ +Fix :func:`repr` for global :class:`~enum.Flag` members. From 2b67fc57f6e97c8389fe970ed232c1ad484113e1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Mar 2024 23:42:18 +0100 Subject: [PATCH 027/158] gh-108494: Fix Argument Clinic LIMITED_CAPI_REGEX (#116610) Accept spaces in "# define Py_LIMITED_API 0x030d0000". --- Modules/_ctypes/_ctypes_test.c | 5 ++--- Modules/_multiprocessing/posixshmem.c | 5 ++--- Modules/_scproxy.c | 5 ++--- Modules/_stat.c | 5 ++--- Modules/_testclinic_limited.c | 5 ++--- Modules/_testimportmultiple.c | 3 +-- Modules/_uuidmodule.c | 5 ++--- Modules/errnomodule.c | 5 ++--- Modules/resource.c | 5 ++--- Modules/xxlimited.c | 5 ++--- Modules/xxlimited_35.c | 4 ++-- PC/winsound.c | 5 ++--- Tools/clinic/clinic.py | 5 +++-- 13 files changed, 26 insertions(+), 36 deletions(-) diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index ecc60417790417..1dd3ef19052470 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1,8 +1,7 @@ +// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030c0000 #endif // gh-85283: On Windows, Py_LIMITED_API requires Py_BUILD_CORE to not attempt diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index 4ab15fa6573665..d332a4e9d9ea0b 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -2,11 +2,10 @@ posixshmem - A Python extension that provides shm_open() and shm_unlink() */ +// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030c0000 #endif #include diff --git a/Modules/_scproxy.c b/Modules/_scproxy.c index fe82e918677f9a..042738b4ab83a2 100644 --- a/Modules/_scproxy.c +++ b/Modules/_scproxy.c @@ -3,11 +3,10 @@ * using the SystemConfiguration framework. */ +// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030c0000 #endif #include diff --git a/Modules/_stat.c b/Modules/_stat.c index b43e79453f5b2f..8059ec2f1f066d 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -11,11 +11,10 @@ * */ +// Need limited C API version 3.13 for PyModule_Add() on Windows #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.13 for PyModule_Add() on Windows -#define Py_LIMITED_API 0x030d0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" diff --git a/Modules/_testclinic_limited.c b/Modules/_testclinic_limited.c index df08ff9a369b1f..1a73c04aecb8af 100644 --- a/Modules/_testclinic_limited.c +++ b/Modules/_testclinic_limited.c @@ -4,11 +4,10 @@ #undef Py_BUILD_CORE_MODULE #undef Py_BUILD_CORE_BUILTIN +// For now, AC only supports the limited C API version 3.13 #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// For now, only limited C API 3.13 is supported -#define Py_LIMITED_API 0x030d0000 +# define Py_LIMITED_API 0x030d0000 #endif /* Always enable assertions */ diff --git a/Modules/_testimportmultiple.c b/Modules/_testimportmultiple.c index 7e6556ad400cde..a65ca513a12516 100644 --- a/Modules/_testimportmultiple.c +++ b/Modules/_testimportmultiple.c @@ -5,9 +5,8 @@ */ #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -#define Py_LIMITED_API 0x03020000 +# define Py_LIMITED_API 0x03020000 #endif #include diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index 4b6852c0d0ec73..052cb9fef3b21c 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -3,11 +3,10 @@ * DCE compatible Universally Unique Identifier library. */ +// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030c0000 #endif #include "Python.h" diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 1100e9f6094352..97e5f0180d76fb 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -1,10 +1,9 @@ /* Errno module */ +// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030c0000 #endif #include "Python.h" diff --git a/Modules/resource.c b/Modules/resource.c index 19020b8cc1b6db..8ee07bd0c8054c 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -1,8 +1,7 @@ +// Need limited C API version 3.13 for PySys_Audit() #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.13 for PySys_Audit() -#define Py_LIMITED_API 0x030d0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 0bb5e12d7c3dd9..3357b8076b67b1 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -62,11 +62,10 @@ pass */ +// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030c0000 #endif #include "Python.h" diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index 754a368f77e940..52690d9d10a81f 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -5,10 +5,10 @@ * See the xxlimited module for an extension module template. */ +// Test the limited C API version 3.5 #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -#define Py_LIMITED_API 0x03050000 +# define Py_LIMITED_API 0x03050000 #endif #include "Python.h" diff --git a/PC/winsound.c b/PC/winsound.c index 7e4ebd90f50c2e..a6b2dac6ac1466 100644 --- a/PC/winsound.c +++ b/PC/winsound.c @@ -35,11 +35,10 @@ winsound.PlaySound(None, 0) */ +// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030c0000 #endif #include diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 8353941f929eb1..893f4cc12ed084 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -66,8 +66,9 @@ # -# match '#define Py_LIMITED_API' -LIMITED_CAPI_REGEX = re.compile(r'#define +Py_LIMITED_API') +# Match '#define Py_LIMITED_API'. +# Match '# define Py_LIMITED_API 0x030d0000' (without the version). +LIMITED_CAPI_REGEX = re.compile(r'# *define +Py_LIMITED_API') class Sentinels(enum.Enum): From ba13215eb1ec20b6af10e3fcee7a725bd7a7f83e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 11 Mar 2024 17:31:49 -0700 Subject: [PATCH 028/158] gh-113538: Revert "gh-113538: Add asycio.Server.{close,abort}_clients (#114432)" (#116632) Revert "gh-113538: Add asycio.Server.{close,abort}_clients (#114432)" Reason: The new test doesn't always pass: https://github.com/python/cpython/pull/116423#issuecomment-1989425489 This reverts commit 1d0d49a7e86257ff95b4de0685e6997d7533993c. --- Doc/library/asyncio-eventloop.rst | 25 ----- Doc/whatsnew/3.13.rst | 5 - Lib/asyncio/base_events.py | 25 ++--- Lib/asyncio/events.py | 8 -- Lib/asyncio/proactor_events.py | 4 +- Lib/asyncio/selector_events.py | 6 +- Lib/test/test_asyncio/test_server.py | 96 ++----------------- ...-01-22-15-50-58.gh-issue-113538.v2wrwg.rst | 3 - 8 files changed, 20 insertions(+), 152 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index d6ed817b13676f..06c5c877ccc173 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -1641,31 +1641,6 @@ Do not instantiate the :class:`Server` class directly. coroutine to wait until the server is closed (and no more connections are active). - .. method:: close_clients() - - Close all existing incoming client connections. - - Calls :meth:`~asyncio.BaseTransport.close` on all associated - transports. - - :meth:`close` should be called before :meth:`close_clients` when - closing the server to avoid races with new clients connecting. - - .. versionadded:: 3.13 - - .. method:: abort_clients() - - Close all existing incoming client connections immediately, - without waiting for pending operations to complete. - - Calls :meth:`~asyncio.WriteTransport.abort` on all associated - transports. - - :meth:`close` should be called before :meth:`abort_clients` when - closing the server to avoid races with new clients connecting. - - .. versionadded:: 3.13 - .. method:: get_loop() Return the event loop associated with the server object. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 95e8ff3eb2e575..51939909000960 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -270,11 +270,6 @@ asyncio the buffer size. (Contributed by Jamie Phan in :gh:`115199`.) -* Add :meth:`asyncio.Server.close_clients` and - :meth:`asyncio.Server.abort_clients` methods which allow to more - forcefully close an asyncio server. - (Contributed by Pierre Ossman in :gh:`113538`.) - base64 --- diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index f0e690b61a73dd..6c5cf28e7c59d4 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -279,9 +279,7 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, ssl_handshake_timeout, ssl_shutdown_timeout=None): self._loop = loop self._sockets = sockets - # Weak references so we don't break Transport's ability to - # detect abandoned transports - self._clients = weakref.WeakSet() + self._active_count = 0 self._waiters = [] self._protocol_factory = protocol_factory self._backlog = backlog @@ -294,13 +292,14 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, def __repr__(self): return f'<{self.__class__.__name__} sockets={self.sockets!r}>' - def _attach(self, transport): + def _attach(self): assert self._sockets is not None - self._clients.add(transport) + self._active_count += 1 - def _detach(self, transport): - self._clients.discard(transport) - if len(self._clients) == 0 and self._sockets is None: + def _detach(self): + assert self._active_count > 0 + self._active_count -= 1 + if self._active_count == 0 and self._sockets is None: self._wakeup() def _wakeup(self): @@ -349,17 +348,9 @@ def close(self): self._serving_forever_fut.cancel() self._serving_forever_fut = None - if len(self._clients) == 0: + if self._active_count == 0: self._wakeup() - def close_clients(self): - for transport in self._clients.copy(): - transport.close() - - def abort_clients(self): - for transport in self._clients.copy(): - transport.abort() - async def start_serving(self): self._start_serving() # Skip one loop iteration so that all 'loop.add_reader' diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index be495469a0558b..680749325025db 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -175,14 +175,6 @@ def close(self): """Stop serving. This leaves existing connections open.""" raise NotImplementedError - def close_clients(self): - """Close all active connections.""" - raise NotImplementedError - - def abort_clients(self): - """Close all active connections immediately.""" - raise NotImplementedError - def get_loop(self): """Get the event loop the Server object is attached to.""" raise NotImplementedError diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 397a8cda757895..a512db6367b20a 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -63,7 +63,7 @@ def __init__(self, loop, sock, protocol, waiter=None, self._called_connection_lost = False self._eof_written = False if self._server is not None: - self._server._attach(self) + self._server._attach() self._loop.call_soon(self._protocol.connection_made, self) if waiter is not None: # only wake up the waiter when connection_made() has been called @@ -167,7 +167,7 @@ def _call_connection_lost(self, exc): self._sock = None server = self._server if server is not None: - server._detach(self) + server._detach() self._server = None self._called_connection_lost = True diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index f94bf10b4225e7..8e888d26ea0737 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -791,7 +791,7 @@ def __init__(self, loop, sock, protocol, extra=None, server=None): self._paused = False # Set when pause_reading() called if self._server is not None: - self._server._attach(self) + self._server._attach() loop._transports[self._sock_fd] = self def __repr__(self): @@ -868,8 +868,6 @@ def __del__(self, _warn=warnings.warn): if self._sock is not None: _warn(f"unclosed transport {self!r}", ResourceWarning, source=self) self._sock.close() - if self._server is not None: - self._server._detach(self) def _fatal_error(self, exc, message='Fatal error on transport'): # Should be called from exception handler only. @@ -908,7 +906,7 @@ def _call_connection_lost(self, exc): self._loop = None server = self._server if server is not None: - server._detach(self) + server._detach() self._server = None def get_write_buffer_size(self): diff --git a/Lib/test/test_asyncio/test_server.py b/Lib/test/test_asyncio/test_server.py index 0c55661bfb88f4..918faac909b9bf 100644 --- a/Lib/test/test_asyncio/test_server.py +++ b/Lib/test/test_asyncio/test_server.py @@ -125,12 +125,8 @@ async def main(srv): class TestServer2(unittest.IsolatedAsyncioTestCase): async def test_wait_closed_basic(self): - async def serve(rd, wr): - try: - await rd.read() - finally: - wr.close() - await wr.wait_closed() + async def serve(*args): + pass srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) self.addCleanup(srv.close) @@ -141,8 +137,7 @@ async def serve(rd, wr): self.assertFalse(task1.done()) # active count != 0, not closed: should block - addr = srv.sockets[0].getsockname() - (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) + srv._attach() task2 = asyncio.create_task(srv.wait_closed()) await asyncio.sleep(0) self.assertFalse(task1.done()) @@ -157,8 +152,7 @@ async def serve(rd, wr): self.assertFalse(task2.done()) self.assertFalse(task3.done()) - wr.close() - await wr.wait_closed() + srv._detach() # active count == 0, closed: should unblock await task1 await task2 @@ -167,12 +161,8 @@ async def serve(rd, wr): async def test_wait_closed_race(self): # Test a regression in 3.12.0, should be fixed in 3.12.1 - async def serve(rd, wr): - try: - await rd.read() - finally: - wr.close() - await wr.wait_closed() + async def serve(*args): + pass srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) self.addCleanup(srv.close) @@ -180,83 +170,13 @@ async def serve(rd, wr): task = asyncio.create_task(srv.wait_closed()) await asyncio.sleep(0) self.assertFalse(task.done()) - addr = srv.sockets[0].getsockname() - (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) + srv._attach() loop = asyncio.get_running_loop() loop.call_soon(srv.close) - loop.call_soon(wr.close) + loop.call_soon(srv._detach) await srv.wait_closed() - async def test_close_clients(self): - async def serve(rd, wr): - try: - await rd.read() - finally: - wr.close() - await wr.wait_closed() - - srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) - self.addCleanup(srv.close) - - addr = srv.sockets[0].getsockname() - (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) - self.addCleanup(wr.close) - - task = asyncio.create_task(srv.wait_closed()) - await asyncio.sleep(0) - self.assertFalse(task.done()) - - srv.close() - srv.close_clients() - await asyncio.sleep(0) - await asyncio.sleep(0) - self.assertTrue(task.done()) - - async def test_abort_clients(self): - async def serve(rd, wr): - nonlocal s_rd, s_wr - s_rd = rd - s_wr = wr - await wr.wait_closed() - - s_rd = s_wr = None - srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) - self.addCleanup(srv.close) - - addr = srv.sockets[0].getsockname() - (c_rd, c_wr) = await asyncio.open_connection(addr[0], addr[1], limit=4096) - self.addCleanup(c_wr.close) - - # Limit the socket buffers so we can reliably overfill them - s_sock = s_wr.get_extra_info('socket') - s_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 65536) - c_sock = c_wr.get_extra_info('socket') - c_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 65536) - - # Get the reader in to a paused state by sending more than twice - # the configured limit - s_wr.write(b'a' * 4096) - s_wr.write(b'a' * 4096) - s_wr.write(b'a' * 4096) - while c_wr.transport.is_reading(): - await asyncio.sleep(0) - - # Get the writer in a waiting state by sending data until the - # socket buffers are full on both server and client sockets and - # the kernel stops accepting more data - s_wr.write(b'a' * c_sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF)) - s_wr.write(b'a' * s_sock.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF)) - self.assertNotEqual(s_wr.transport.get_write_buffer_size(), 0) - - task = asyncio.create_task(srv.wait_closed()) - await asyncio.sleep(0) - self.assertFalse(task.done()) - srv.close() - srv.abort_clients() - await asyncio.sleep(0) - await asyncio.sleep(0) - self.assertTrue(task.done()) # Test the various corner cases of Unix server socket removal diff --git a/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst b/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst deleted file mode 100644 index 5c59af98e136bb..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add :meth:`asyncio.Server.close_clients` and -:meth:`asyncio.Server.abort_clients` methods which allow to more forcefully -close an asyncio server. From 3cc5ae5c2c6e729ca2750ed490dad56faa7c342d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 12 Mar 2024 01:46:53 +0100 Subject: [PATCH 029/158] gh-85283: Convert grp extension to the limited C API (#116611) posixmodule.h: remove check on the limited C API, since these helpers are not part of the public C API. --- .../pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - .../internal/pycore_runtime_init_generated.h | 1 - .../internal/pycore_unicodeobject_generated.h | 3 - Modules/clinic/grpmodule.c.h | 86 +++---------------- Modules/grpmodule.c | 12 +-- Modules/posixmodule.h | 2 - Tools/c-analyzer/cpython/ignored.tsv | 2 + 8 files changed, 20 insertions(+), 88 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index e8b62bd0dcd369..d2287687181450 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -983,7 +983,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hi)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hour)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ignore)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index b41f5c8952021e..fb9ec44d3f52aa 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -472,7 +472,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(hi) STRUCT_FOR_ID(hook) STRUCT_FOR_ID(hour) - STRUCT_FOR_ID(id) STRUCT_FOR_ID(ident) STRUCT_FOR_ID(identity_hint) STRUCT_FOR_ID(ignore) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 016eae02a2103d..658bf8030f661d 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -981,7 +981,6 @@ extern "C" { INIT_ID(hi), \ INIT_ID(hook), \ INIT_ID(hour), \ - INIT_ID(id), \ INIT_ID(ident), \ INIT_ID(identity_hint), \ INIT_ID(ignore), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 64c4cf8c077056..d72353d56eae60 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1257,9 +1257,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(hour); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(id); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(ident); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Modules/clinic/grpmodule.c.h b/Modules/clinic/grpmodule.c.h index 1da061f3fd8eb1..cc0ad210f42743 100644 --- a/Modules/clinic/grpmodule.c.h +++ b/Modules/clinic/grpmodule.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif -#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() - PyDoc_STRVAR(grp_getgrgid__doc__, "getgrgid($module, /, id)\n" "--\n" @@ -17,48 +11,21 @@ PyDoc_STRVAR(grp_getgrgid__doc__, "If id is not valid, raise KeyError."); #define GRP_GETGRGID_METHODDEF \ - {"getgrgid", _PyCFunction_CAST(grp_getgrgid), METH_FASTCALL|METH_KEYWORDS, grp_getgrgid__doc__}, + {"getgrgid", (PyCFunction)(void(*)(void))grp_getgrgid, METH_VARARGS|METH_KEYWORDS, grp_getgrgid__doc__}, static PyObject * grp_getgrgid_impl(PyObject *module, PyObject *id); static PyObject * -grp_getgrgid(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +grp_getgrgid(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(id), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"id", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "getgrgid", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; + static char *_keywords[] = {"id", NULL}; PyObject *id; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O:getgrgid", _keywords, + &id)) goto exit; - } - id = args[0]; return_value = grp_getgrgid_impl(module, id); exit: @@ -74,52 +41,21 @@ PyDoc_STRVAR(grp_getgrnam__doc__, "If name is not valid, raise KeyError."); #define GRP_GETGRNAM_METHODDEF \ - {"getgrnam", _PyCFunction_CAST(grp_getgrnam), METH_FASTCALL|METH_KEYWORDS, grp_getgrnam__doc__}, + {"getgrnam", (PyCFunction)(void(*)(void))grp_getgrnam, METH_VARARGS|METH_KEYWORDS, grp_getgrnam__doc__}, static PyObject * grp_getgrnam_impl(PyObject *module, PyObject *name); static PyObject * -grp_getgrnam(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +grp_getgrnam(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"name", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "getgrnam", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; + static char *_keywords[] = {"name", NULL}; PyObject *name; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { - goto exit; - } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("getgrnam", "argument 'name'", "str", args[0]); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "U:getgrnam", _keywords, + &name)) goto exit; - } - name = args[0]; return_value = grp_getgrnam_impl(module, name); exit: @@ -146,4 +82,4 @@ grp_getgrall(PyObject *module, PyObject *Py_UNUSED(ignored)) { return grp_getgrall_impl(module); } -/*[clinic end generated code: output=2f7011384604d38d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=81f180beb67fc585 input=a9049054013a1b77]*/ diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index 9756f1c2fb15b1..a1fa6cf20f71fd 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -1,15 +1,17 @@ - /* UNIX group file access module */ -// clinic/grpmodule.c.h uses internal pycore_modsupport.h API -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 +// Need limited C API version 3.13 for PyMem_RawRealloc() +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +#define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" #include "posixmodule.h" +#include // ERANGE #include // getgrgid_r() +#include // memcpy() #include // sysconf() #include "clinic/grpmodule.c.h" @@ -88,7 +90,7 @@ mkgrent(PyObject *module, struct group *p) Py_DECREF(x); } -#define SET(i,val) PyStructSequence_SET_ITEM(v, i, val) +#define SET(i,val) PyStructSequence_SetItem(v, i, val) SET(setIndex++, PyUnicode_DecodeFSDefault(p->gr_name)); if (p->gr_passwd) SET(setIndex++, PyUnicode_DecodeFSDefault(p->gr_passwd)); diff --git a/Modules/posixmodule.h b/Modules/posixmodule.h index 8827ce153fed8c..be732db04faf94 100644 --- a/Modules/posixmodule.h +++ b/Modules/posixmodule.h @@ -2,7 +2,6 @@ #ifndef Py_POSIXMODULE_H #define Py_POSIXMODULE_H -#ifndef Py_LIMITED_API #ifdef __cplusplus extern "C" { #endif @@ -34,5 +33,4 @@ extern int _Py_Sigset_Converter(PyObject *, void *); #ifdef __cplusplus } #endif -#endif // !Py_LIMITED_API #endif // !Py_POSIXMODULE_H diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 682dceaf8ae41e..0024e2683052c8 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -740,3 +740,5 @@ Modules/expat/xmlrole.c - error - Modules/_io/_iomodule.c - _PyIO_Module - Modules/_sqlite/module.c - _sqlite3module - Modules/clinic/md5module.c.h _md5_md5 _keywords - +Modules/clinic/grpmodule.c.h grp_getgrgid _keywords - +Modules/clinic/grpmodule.c.h grp_getgrnam _keywords - From 4fa95c6ec392b9fc80ad720cc4a8bd2786fc2835 Mon Sep 17 00:00:00 2001 From: Christopher Chavez Date: Tue, 12 Mar 2024 00:37:07 -0500 Subject: [PATCH 030/158] gh-116145: Update macOS installer to Tcl/Tk 8.6.14 (GH-116151) --- Mac/BuildScript/backport_gh92603_fix.patch | 82 ------------------- Mac/BuildScript/build-installer.py | 8 +- ...-02-29-20-52-23.gh-issue-116145.ygafim.rst | 1 + 3 files changed, 5 insertions(+), 86 deletions(-) delete mode 100644 Mac/BuildScript/backport_gh92603_fix.patch create mode 100644 Misc/NEWS.d/next/macOS/2024-02-29-20-52-23.gh-issue-116145.ygafim.rst diff --git a/Mac/BuildScript/backport_gh92603_fix.patch b/Mac/BuildScript/backport_gh92603_fix.patch deleted file mode 100644 index 9a37b029650340..00000000000000 --- a/Mac/BuildScript/backport_gh92603_fix.patch +++ /dev/null @@ -1,82 +0,0 @@ -Accepted upstream for release in Tk 8.6.14: -https://core.tcl-lang.org/tk/info/cf3830280b - ---- tk8.6.13/macosx/tkMacOSXWindowEvent.c.orig -+++ tk8.6.13-patched/macosx/tkMacOSXWindowEvent.c -@@ -239,8 +239,8 @@ extern NSString *NSWindowDidOrderOffScreenNotification; - if (winPtr) { - TKContentView *view = [window contentView]; - --#if MAC_OS_X_VERSION_MAX_ALLOWED >= 101500 -- if (@available(macOS 10.15, *)) { -+#if MAC_OS_X_VERSION_MAX_ALLOWED >= 101400 -+ if (@available(macOS 10.14, *)) { - [view viewDidChangeEffectiveAppearance]; - } - #endif -@@ -1237,29 +1237,8 @@ static const char *const accentNames[] = { - } else if (effectiveAppearanceName == NSAppearanceNameDarkAqua) { - TkSendVirtualEvent(tkwin, "DarkAqua", NULL); - } -- if ([NSApp macOSVersion] < 101500) { -- -- /* -- * Mojave cannot handle the KVO shenanigans that we need for the -- * highlight and accent color notifications. -- */ -- -- return; -- } - if (!defaultColor) { - defaultColor = [NSApp macOSVersion] < 110000 ? "Blue" : "Multicolor"; -- preferences = [[NSUserDefaults standardUserDefaults] retain]; -- -- /* -- * AppKit calls this method when the user changes the Accent Color -- * but not when the user changes the Highlight Color. So we register -- * to receive KVO notifications for Highlight Color as well. -- */ -- -- [preferences addObserver:self -- forKeyPath:@"AppleHighlightColor" -- options:NSKeyValueObservingOptionNew -- context:NULL]; - } - NSString *accent = [preferences stringForKey:@"AppleAccentColor"]; - NSArray *words = [[preferences stringForKey:@"AppleHighlightColor"] ---- tk8.6.13/macosx/tkMacOSXWm.c.orig -+++ tk8.6.13-patched/macosx/tkMacOSXWm.c -@@ -1289,6 +1289,11 @@ TkWmDeadWindow( - [NSApp _setMainWindow:nil]; - } - [deadNSWindow close]; -+#if MAC_OS_X_VERSION_MAX_ALLOWED >= 101400 -+ NSUserDefaults *preferences = [NSUserDefaults standardUserDefaults]; -+ [preferences removeObserver:deadNSWindow.contentView -+ forKeyPath:@"AppleHighlightColor"]; -+#endif - [deadNSWindow release]; - - #if DEBUG_ZOMBIES > 1 -@@ -6763,6 +6768,21 @@ TkMacOSXMakeRealWindowExist( - } - TKContentView *contentView = [[TKContentView alloc] - initWithFrame:NSZeroRect]; -+#if MAC_OS_X_VERSION_MAX_ALLOWED >= 101400 -+ NSUserDefaults *preferences = [NSUserDefaults standardUserDefaults]; -+ -+ /* -+ * AppKit calls the viewDidChangeEffectiveAppearance method when the -+ * user changes the Accent Color but not when the user changes the -+ * Highlight Color. So we register to receive KVO notifications for -+ * Highlight Color as well. -+ */ -+ -+ [preferences addObserver:contentView -+ forKeyPath:@"AppleHighlightColor" -+ options:NSKeyValueObservingOptionNew -+ context:NULL]; -+#endif - [window setContentView:contentView]; - [contentView release]; - [window setDelegate:NSApp]; diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 0af90563cbbb2b..b2f8c77e0c1a73 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -264,11 +264,11 @@ def library_recipes(): tk_patches = ['backport_gh71383_fix.patch', 'tk868_on_10_8_10_9.patch', 'backport_gh110950_fix.patch'] else: - tcl_tk_ver='8.6.13' - tcl_checksum='43a1fae7412f61ff11de2cfd05d28cfc3a73762f354a417c62370a54e2caf066' + tcl_tk_ver='8.6.14' + tcl_checksum='5880225babf7954c58d4fb0f5cf6279104ce1cd6aa9b71e9a6322540e1c4de66' - tk_checksum='2e65fa069a23365440a3c56c556b8673b5e32a283800d8d9b257e3f584ce0675' - tk_patches = ['backport_gh92603_fix.patch', 'backport_gh71383_fix.patch', 'backport_gh110950_fix.patch'] + tk_checksum='8ffdb720f47a6ca6107eac2dd877e30b0ef7fac14f3a84ebbd0b3612cee41a94' + tk_patches = [] base_url = "https://prdownloads.sourceforge.net/tcl/{what}{version}-src.tar.gz" diff --git a/Misc/NEWS.d/next/macOS/2024-02-29-20-52-23.gh-issue-116145.ygafim.rst b/Misc/NEWS.d/next/macOS/2024-02-29-20-52-23.gh-issue-116145.ygafim.rst new file mode 100644 index 00000000000000..bc0a2e09dde1bb --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2024-02-29-20-52-23.gh-issue-116145.ygafim.rst @@ -0,0 +1 @@ +Update macOS installer to Tcl/Tk 8.6.14. From d308d33e098d8e176f1e5169225d3cf800ed6aa1 Mon Sep 17 00:00:00 2001 From: Mehdi Drissi Date: Mon, 11 Mar 2024 23:11:56 -0700 Subject: [PATCH 031/158] gh-89547: Support for nesting special forms like Final (#116096) --- Lib/test/test_typing.py | 40 +++++++++++++------ Lib/typing.py | 4 +- Misc/ACKS | 1 + ...4-02-28-17-50-42.gh-issue-89547.GetF38.rst | 1 + 4 files changed, 32 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-02-28-17-50-42.gh-issue-89547.GetF38.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 912384ab6bfe84..a9942b44f29ed9 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4655,8 +4655,6 @@ def test_fail_with_bare_union(self): List[Union] with self.assertRaises(TypeError): Tuple[Optional] - with self.assertRaises(TypeError): - ClassVar[ClassVar[int]] with self.assertRaises(TypeError): List[ClassVar[int]] @@ -6014,16 +6012,6 @@ class F: for clazz in [C, D, E, F]: self.assertEqual(get_type_hints(clazz), expected_result) - def test_nested_classvar_fails_forward_ref_check(self): - class E: - foo: 'typing.ClassVar[typing.ClassVar[int]]' = 7 - class F: - foo: ClassVar['ClassVar[int]'] = 7 - - for clazz in [E, F]: - with self.assertRaises(TypeError): - get_type_hints(clazz) - def test_meta_no_type_check(self): depr_msg = ( "'typing.no_type_check_decorator' is deprecated " @@ -8716,6 +8704,34 @@ class C: self.assertEqual(get_type_hints(C, globals())['classvar'], ClassVar[int]) self.assertEqual(get_type_hints(C, globals())['const'], Final[int]) + def test_special_forms_nesting(self): + # These are uncommon types and are to ensure runtime + # is lax on validation. See gh-89547 for more context. + class CF: + x: ClassVar[Final[int]] + + class FC: + x: Final[ClassVar[int]] + + class ACF: + x: Annotated[ClassVar[Final[int]], "a decoration"] + + class CAF: + x: ClassVar[Annotated[Final[int], "a decoration"]] + + class AFC: + x: Annotated[Final[ClassVar[int]], "a decoration"] + + class FAC: + x: Final[Annotated[ClassVar[int], "a decoration"]] + + self.assertEqual(get_type_hints(CF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(FC, globals())['x'], Final[ClassVar[int]]) + self.assertEqual(get_type_hints(ACF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(CAF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(AFC, globals())['x'], Final[ClassVar[int]]) + self.assertEqual(get_type_hints(FAC, globals())['x'], Final[ClassVar[int]]) + def test_cannot_subclass(self): with self.assertRaisesRegex(TypeError, "Cannot subclass .*Annotated"): class C(Annotated): diff --git a/Lib/typing.py b/Lib/typing.py index cca9525d632ea5..b2350433953ccd 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -653,7 +653,7 @@ class Starship: Note that ClassVar is not a class itself, and should not be used with isinstance() or issubclass(). """ - item = _type_check(parameters, f'{self} accepts only single type.') + item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) return _GenericAlias(self, (item,)) @_SpecialForm @@ -675,7 +675,7 @@ class FastConnector(Connection): There is no runtime checking of these properties. """ - item = _type_check(parameters, f'{self} accepts only single type.') + item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) return _GenericAlias(self, (item,)) @_SpecialForm diff --git a/Misc/ACKS b/Misc/ACKS index f01c7a70a65dc5..03e458d170d2b8 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -470,6 +470,7 @@ Allen Downey Cesar Douady Dean Draayer Fred L. Drake, Jr. +Mehdi Drissi Derk Drukker John DuBois Paul Dubois diff --git a/Misc/NEWS.d/next/Library/2024-02-28-17-50-42.gh-issue-89547.GetF38.rst b/Misc/NEWS.d/next/Library/2024-02-28-17-50-42.gh-issue-89547.GetF38.rst new file mode 100644 index 00000000000000..7be4591b83a8f3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-28-17-50-42.gh-issue-89547.GetF38.rst @@ -0,0 +1 @@ +Add support for nested typing special forms like Final[ClassVar[int]]. From f8147d01da44da2434496d868c86c2785f7244cd Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 12 Mar 2024 13:10:00 +0300 Subject: [PATCH 032/158] gh-116541: Handle errors correctly in `_pystatvfs_fromstructstatvfs` (#116542) --- Modules/posixmodule.c | 68 +++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f3403e1675276b..19e925730a5110 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -12980,46 +12980,50 @@ _pystatvfs_fromstructstatvfs(PyObject *module, struct statvfs st) { if (v == NULL) return NULL; + int pos = 0; + +#define SET_RESULT(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + Py_DECREF(v); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(v, pos++, item); \ + } while(0) + #if !defined(HAVE_LARGEFILE_SUPPORT) - PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long) st.f_bsize)); - PyStructSequence_SET_ITEM(v, 1, PyLong_FromLong((long) st.f_frsize)); - PyStructSequence_SET_ITEM(v, 2, PyLong_FromLong((long) st.f_blocks)); - PyStructSequence_SET_ITEM(v, 3, PyLong_FromLong((long) st.f_bfree)); - PyStructSequence_SET_ITEM(v, 4, PyLong_FromLong((long) st.f_bavail)); - PyStructSequence_SET_ITEM(v, 5, PyLong_FromLong((long) st.f_files)); - PyStructSequence_SET_ITEM(v, 6, PyLong_FromLong((long) st.f_ffree)); - PyStructSequence_SET_ITEM(v, 7, PyLong_FromLong((long) st.f_favail)); - PyStructSequence_SET_ITEM(v, 8, PyLong_FromLong((long) st.f_flag)); - PyStructSequence_SET_ITEM(v, 9, PyLong_FromLong((long) st.f_namemax)); + SET_RESULT(PyLong_FromLong((long) st.f_bsize)); + SET_RESULT(PyLong_FromLong((long) st.f_frsize)); + SET_RESULT(PyLong_FromLong((long) st.f_blocks)); + SET_RESULT(PyLong_FromLong((long) st.f_bfree)); + SET_RESULT(PyLong_FromLong((long) st.f_bavail)); + SET_RESULT(PyLong_FromLong((long) st.f_files)); + SET_RESULT(PyLong_FromLong((long) st.f_ffree)); + SET_RESULT(PyLong_FromLong((long) st.f_favail)); + SET_RESULT(PyLong_FromLong((long) st.f_flag)); + SET_RESULT(PyLong_FromLong((long) st.f_namemax)); #else - PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long) st.f_bsize)); - PyStructSequence_SET_ITEM(v, 1, PyLong_FromLong((long) st.f_frsize)); - PyStructSequence_SET_ITEM(v, 2, - PyLong_FromLongLong((long long) st.f_blocks)); - PyStructSequence_SET_ITEM(v, 3, - PyLong_FromLongLong((long long) st.f_bfree)); - PyStructSequence_SET_ITEM(v, 4, - PyLong_FromLongLong((long long) st.f_bavail)); - PyStructSequence_SET_ITEM(v, 5, - PyLong_FromLongLong((long long) st.f_files)); - PyStructSequence_SET_ITEM(v, 6, - PyLong_FromLongLong((long long) st.f_ffree)); - PyStructSequence_SET_ITEM(v, 7, - PyLong_FromLongLong((long long) st.f_favail)); - PyStructSequence_SET_ITEM(v, 8, PyLong_FromLong((long) st.f_flag)); - PyStructSequence_SET_ITEM(v, 9, PyLong_FromLong((long) st.f_namemax)); + SET_RESULT(PyLong_FromLong((long) st.f_bsize)); + SET_RESULT(PyLong_FromLong((long) st.f_frsize)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_blocks)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_bfree)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_bavail)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_files)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_ffree)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_favail)); + SET_RESULT(PyLong_FromLong((long) st.f_flag)); + SET_RESULT(PyLong_FromLong((long) st.f_namemax)); #endif /* The _ALL_SOURCE feature test macro defines f_fsid as a structure * (issue #32390). */ #if defined(_AIX) && defined(_ALL_SOURCE) - PyStructSequence_SET_ITEM(v, 10, PyLong_FromUnsignedLong(st.f_fsid.val[0])); + SET_RESULT(PyLong_FromUnsignedLong(st.f_fsid.val[0])); #else - PyStructSequence_SET_ITEM(v, 10, PyLong_FromUnsignedLong(st.f_fsid)); + SET_RESULT(PyLong_FromUnsignedLong(st.f_fsid)); #endif - if (PyErr_Occurred()) { - Py_DECREF(v); - return NULL; - } + +#undef SET_RESULT return v; } From eb947cdc1374842a32fa82249ba3c688abf252dc Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 12 Mar 2024 13:50:06 +0300 Subject: [PATCH 033/158] =?UTF-8?q?gh-110819:=20Fix=20=E2=80=98kind?= =?UTF-8?q?=E2=80=99=20may=20be=20used=20uninitialized=20warning=20in=20`l?= =?UTF-8?q?ongobject`=20(#116599)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Objects/longobject.c | 63 ++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 28 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 2d1c6ad788e281..cc2fe11f31c430 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1965,7 +1965,9 @@ long_to_decimal_string_internal(PyObject *aa, digit *pout, *pin, rem, tenpow; int negative; int d; - int kind; + + // writer or bytes_writer can be used, but not both at the same time. + assert(writer == NULL || bytes_writer == NULL); a = (PyLongObject *)aa; if (a == NULL || !PyLong_Check(a)) { @@ -2078,7 +2080,6 @@ long_to_decimal_string_internal(PyObject *aa, Py_DECREF(scratch); return -1; } - kind = writer->kind; } else if (bytes_writer) { *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, strlen); @@ -2093,7 +2094,6 @@ long_to_decimal_string_internal(PyObject *aa, Py_DECREF(scratch); return -1; } - kind = PyUnicode_KIND(str); } #define WRITE_DIGITS(p) \ @@ -2141,19 +2141,23 @@ long_to_decimal_string_internal(PyObject *aa, WRITE_DIGITS(p); assert(p == *bytes_str); } - else if (kind == PyUnicode_1BYTE_KIND) { - Py_UCS1 *p; - WRITE_UNICODE_DIGITS(Py_UCS1); - } - else if (kind == PyUnicode_2BYTE_KIND) { - Py_UCS2 *p; - WRITE_UNICODE_DIGITS(Py_UCS2); - } else { - Py_UCS4 *p; - assert (kind == PyUnicode_4BYTE_KIND); - WRITE_UNICODE_DIGITS(Py_UCS4); + int kind = writer ? writer->kind : PyUnicode_KIND(str); + if (kind == PyUnicode_1BYTE_KIND) { + Py_UCS1 *p; + WRITE_UNICODE_DIGITS(Py_UCS1); + } + else if (kind == PyUnicode_2BYTE_KIND) { + Py_UCS2 *p; + WRITE_UNICODE_DIGITS(Py_UCS2); + } + else { + assert (kind == PyUnicode_4BYTE_KIND); + Py_UCS4 *p; + WRITE_UNICODE_DIGITS(Py_UCS4); + } } + #undef WRITE_DIGITS #undef WRITE_UNICODE_DIGITS @@ -2194,11 +2198,12 @@ long_format_binary(PyObject *aa, int base, int alternate, PyObject *v = NULL; Py_ssize_t sz; Py_ssize_t size_a; - int kind; int negative; int bits; assert(base == 2 || base == 8 || base == 16); + // writer or bytes_writer can be used, but not both at the same time. + assert(writer == NULL || bytes_writer == NULL); if (a == NULL || !PyLong_Check(a)) { PyErr_BadInternalCall(); return -1; @@ -2246,7 +2251,6 @@ long_format_binary(PyObject *aa, int base, int alternate, if (writer) { if (_PyUnicodeWriter_Prepare(writer, sz, 'x') == -1) return -1; - kind = writer->kind; } else if (bytes_writer) { *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, sz); @@ -2257,7 +2261,6 @@ long_format_binary(PyObject *aa, int base, int alternate, v = PyUnicode_New(sz, 'x'); if (v == NULL) return -1; - kind = PyUnicode_KIND(v); } #define WRITE_DIGITS(p) \ @@ -2318,19 +2321,23 @@ long_format_binary(PyObject *aa, int base, int alternate, WRITE_DIGITS(p); assert(p == *bytes_str); } - else if (kind == PyUnicode_1BYTE_KIND) { - Py_UCS1 *p; - WRITE_UNICODE_DIGITS(Py_UCS1); - } - else if (kind == PyUnicode_2BYTE_KIND) { - Py_UCS2 *p; - WRITE_UNICODE_DIGITS(Py_UCS2); - } else { - Py_UCS4 *p; - assert (kind == PyUnicode_4BYTE_KIND); - WRITE_UNICODE_DIGITS(Py_UCS4); + int kind = writer ? writer->kind : PyUnicode_KIND(v); + if (kind == PyUnicode_1BYTE_KIND) { + Py_UCS1 *p; + WRITE_UNICODE_DIGITS(Py_UCS1); + } + else if (kind == PyUnicode_2BYTE_KIND) { + Py_UCS2 *p; + WRITE_UNICODE_DIGITS(Py_UCS2); + } + else { + assert (kind == PyUnicode_4BYTE_KIND); + Py_UCS4 *p; + WRITE_UNICODE_DIGITS(Py_UCS4); + } } + #undef WRITE_DIGITS #undef WRITE_UNICODE_DIGITS From 02918aa96117781261cb1a564e37a861b01eb883 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 12 Mar 2024 12:00:49 +0000 Subject: [PATCH 034/158] gh-116604: Correctly honor the gc status when calling _Py_RunGC (#116628) --- Lib/test/test_gc.py | 25 +++++++++++++++++++ ...-03-11-22-24-59.gh-issue-116604.LCEzAT.rst | 3 +++ Python/gc.c | 4 +++ 3 files changed, 32 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-24-59.gh-issue-116604.LCEzAT.rst diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index dd09643788d62f..2aea025fcc140a 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1397,6 +1397,31 @@ def __del__(self): # empty __dict__. self.assertEqual(x, None) + def test_indirect_calls_with_gc_disabled(self): + junk = [] + i = 0 + detector = GC_Detector() + while not detector.gc_happened: + i += 1 + if i > 10000: + self.fail("gc didn't happen after 10000 iterations") + junk.append([]) # this will eventually trigger gc + + try: + gc.disable() + junk = [] + i = 0 + detector = GC_Detector() + while not detector.gc_happened: + i += 1 + if i > 10000: + break + junk.append([]) # this may eventually trigger gc (if it is enabled) + + self.assertEqual(i, 10001) + finally: + gc.enable() + class PythonFinalizationTests(unittest.TestCase): def test_ast_fini(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-24-59.gh-issue-116604.LCEzAT.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-24-59.gh-issue-116604.LCEzAT.rst new file mode 100644 index 00000000000000..516edfa9e6cedf --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-24-59.gh-issue-116604.LCEzAT.rst @@ -0,0 +1,3 @@ +Respect the status of the garbage collector when indirect calls are made via +:c:func:`PyErr_CheckSignals` and the evaluation breaker. Patch by Pablo +Galindo diff --git a/Python/gc.c b/Python/gc.c index ea3b596d1713df..6b3316b642ea9e 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1805,6 +1805,10 @@ _PyObject_GC_Link(PyObject *op) void _Py_RunGC(PyThreadState *tstate) { + GCState *gcstate = get_gc_state(); + if (!gcstate->enabled) { + return; + } gc_collect_main(tstate, GENERATION_AUTO, _Py_GC_REASON_HEAP); } From 3b7fe117fab91371f6b621e9efd02f3925f5d53b Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 12 Mar 2024 14:44:39 +0100 Subject: [PATCH 035/158] gh-116616: Use relaxed atomic ops to access socket module defaulttimeout (#116623) Co-authored-by: Sam Gross --- Modules/socketmodule.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index cd9a803648be71..7720d59e46590e 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -1054,8 +1054,8 @@ init_sockobject(socket_state *state, PySocketSockObject *s, else #endif { - s->sock_timeout = state->defaulttimeout; - if (state->defaulttimeout >= 0) { + s->sock_timeout = _Py_atomic_load_int64_relaxed(&state->defaulttimeout); + if (s->sock_timeout >= 0) { if (internal_setblocking(s, 0) == -1) { return -1; } @@ -6913,11 +6913,12 @@ static PyObject * socket_getdefaulttimeout(PyObject *self, PyObject *Py_UNUSED(ignored)) { socket_state *state = get_module_state(self); - if (state->defaulttimeout < 0) { + PyTime_t timeout = _Py_atomic_load_int64_relaxed(&state->defaulttimeout); + if (timeout < 0) { Py_RETURN_NONE; } else { - double seconds = PyTime_AsSecondsDouble(state->defaulttimeout); + double seconds = PyTime_AsSecondsDouble(timeout); return PyFloat_FromDouble(seconds); } } @@ -6938,7 +6939,7 @@ socket_setdefaulttimeout(PyObject *self, PyObject *arg) return NULL; socket_state *state = get_module_state(self); - state->defaulttimeout = timeout; + _Py_atomic_store_int64_relaxed(&state->defaulttimeout, timeout); Py_RETURN_NONE; } From 3265087c07c261d1b5f526953682def334a52d56 Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Tue, 12 Mar 2024 09:05:30 -0500 Subject: [PATCH 036/158] Fix code comment regarding DK_ENTRIES (GH-113960) fix code comment regarding dict entries --- Include/internal/pycore_dict.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index cd171a4384db5d..ef59960dbab071 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -167,7 +167,7 @@ struct _dictkeysobject { char dk_indices[]; /* char is required to avoid strict aliasing. */ /* "PyDictKeyEntry or PyDictUnicodeEntry dk_entries[USABLE_FRACTION(DK_SIZE(dk))];" array follows: - see the DK_ENTRIES() macro */ + see the DK_ENTRIES() / DK_UNICODE_ENTRIES() functions below */ }; /* This must be no more than 250, for the prefix size to fit in one byte. */ From df4784b3b7519d137ca6a1aeb500ef59e24a7f9b Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 12 Mar 2024 17:49:39 +0300 Subject: [PATCH 037/158] gh-116127: PEP-705: Add `ReadOnly` support for `TypedDict` (#116350) Co-authored-by: Jelle Zijlstra --- Doc/library/typing.rst | 39 +++++++++ Doc/whatsnew/3.13.rst | 4 + Lib/test/test_typing.py | 65 ++++++++++++++- Lib/typing.py | 83 ++++++++++++++++--- ...-03-05-14-34-22.gh-issue-116127.5uktu3.rst | 2 + 5 files changed, 182 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-05-14-34-22.gh-issue-116127.5uktu3.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index da70757ea74452..3db5f06803607f 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1274,6 +1274,26 @@ These can be used as types in annotations. They all support subscription using .. versionadded:: 3.11 +.. data:: ReadOnly + + A special typing construct to mark an item of a :class:`TypedDict` as read-only. + + For example:: + + class Movie(TypedDict): + title: ReadOnly[str] + year: int + + def mutate_movie(m: Movie) -> None: + m["year"] = 1992 # allowed + m["title"] = "The Matrix" # typechecker error + + There is no runtime checking for this property. + + See :class:`TypedDict` and :pep:`705` for more details. + + .. versionadded:: 3.13 + .. data:: Annotated Special typing form to add context-specific metadata to an annotation. @@ -2454,6 +2474,22 @@ types. ``__required_keys__`` and ``__optional_keys__`` rely on may not work properly, and the values of the attributes may be incorrect. + Support for :data:`ReadOnly` is reflected in the following attributes:: + + .. attribute:: __readonly_keys__ + + A :class:`frozenset` containing the names of all read-only keys. Keys + are read-only if they carry the :data:`ReadOnly` qualifier. + + .. versionadded:: 3.13 + + .. attribute:: __mutable_keys__ + + A :class:`frozenset` containing the names of all mutable keys. Keys + are mutable if they do not carry the :data:`ReadOnly` qualifier. + + .. versionadded:: 3.13 + See :pep:`589` for more examples and detailed rules of using ``TypedDict``. .. versionadded:: 3.8 @@ -2468,6 +2504,9 @@ types. .. versionchanged:: 3.13 Removed support for the keyword-argument method of creating ``TypedDict``\ s. + .. versionchanged:: 3.13 + Support for the :data:`ReadOnly` qualifier was added. + .. deprecated-removed:: 3.13 3.15 When using the functional syntax to create a TypedDict class, failing to pass a value to the 'fields' parameter (``TD = TypedDict("TD")``) is diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 51939909000960..d78f219ed3a746 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -602,6 +602,10 @@ typing check whether a class is a :class:`typing.Protocol`. (Contributed by Jelle Zijlstra in :gh:`104873`.) +* Add :data:`typing.ReadOnly`, a special typing construct to mark + an item of a :class:`typing.TypedDict` as read-only for type checkers. + See :pep:`705` for more details. + unicodedata ----------- diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index a9942b44f29ed9..54c7b976185585 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -31,7 +31,7 @@ from typing import dataclass_transform from typing import no_type_check, no_type_check_decorator from typing import Type -from typing import NamedTuple, NotRequired, Required, TypedDict +from typing import NamedTuple, NotRequired, Required, ReadOnly, TypedDict from typing import IO, TextIO, BinaryIO from typing import Pattern, Match from typing import Annotated, ForwardRef @@ -8322,6 +8322,69 @@ class T4(TypedDict, Generic[S]): pass self.assertEqual(klass.__optional_keys__, set()) self.assertIsInstance(klass(), dict) + def test_readonly_inheritance(self): + class Base1(TypedDict): + a: ReadOnly[int] + + class Child1(Base1): + b: str + + self.assertEqual(Child1.__readonly_keys__, frozenset({'a'})) + self.assertEqual(Child1.__mutable_keys__, frozenset({'b'})) + + class Base2(TypedDict): + a: ReadOnly[int] + + class Child2(Base2): + b: str + + self.assertEqual(Child1.__readonly_keys__, frozenset({'a'})) + self.assertEqual(Child1.__mutable_keys__, frozenset({'b'})) + + def test_cannot_make_mutable_key_readonly(self): + class Base(TypedDict): + a: int + + with self.assertRaises(TypeError): + class Child(Base): + a: ReadOnly[int] + + def test_can_make_readonly_key_mutable(self): + class Base(TypedDict): + a: ReadOnly[int] + + class Child(Base): + a: int + + self.assertEqual(Child.__readonly_keys__, frozenset()) + self.assertEqual(Child.__mutable_keys__, frozenset({'a'})) + + def test_combine_qualifiers(self): + class AllTheThings(TypedDict): + a: Annotated[Required[ReadOnly[int]], "why not"] + b: Required[Annotated[ReadOnly[int], "why not"]] + c: ReadOnly[NotRequired[Annotated[int, "why not"]]] + d: NotRequired[Annotated[int, "why not"]] + + self.assertEqual(AllTheThings.__required_keys__, frozenset({'a', 'b'})) + self.assertEqual(AllTheThings.__optional_keys__, frozenset({'c', 'd'})) + self.assertEqual(AllTheThings.__readonly_keys__, frozenset({'a', 'b', 'c'})) + self.assertEqual(AllTheThings.__mutable_keys__, frozenset({'d'})) + + self.assertEqual( + get_type_hints(AllTheThings, include_extras=False), + {'a': int, 'b': int, 'c': int, 'd': int}, + ) + self.assertEqual( + get_type_hints(AllTheThings, include_extras=True), + { + 'a': Annotated[Required[ReadOnly[int]], 'why not'], + 'b': Required[Annotated[ReadOnly[int], 'why not']], + 'c': ReadOnly[NotRequired[Annotated[int, 'why not']]], + 'd': NotRequired[Annotated[int, 'why not']], + }, + ) + class RequiredTests(BaseTestCase): diff --git a/Lib/typing.py b/Lib/typing.py index b2350433953ccd..533b64062834d2 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -144,6 +144,7 @@ 'override', 'ParamSpecArgs', 'ParamSpecKwargs', + 'ReadOnly', 'Required', 'reveal_type', 'runtime_checkable', @@ -2301,7 +2302,7 @@ def _strip_annotations(t): """Strip the annotations from a given type.""" if isinstance(t, _AnnotatedAlias): return _strip_annotations(t.__origin__) - if hasattr(t, "__origin__") and t.__origin__ in (Required, NotRequired): + if hasattr(t, "__origin__") and t.__origin__ in (Required, NotRequired, ReadOnly): return _strip_annotations(t.__args__[0]) if isinstance(t, _GenericAlias): stripped_args = tuple(_strip_annotations(a) for a in t.__args__) @@ -2922,6 +2923,28 @@ def _namedtuple_mro_entries(bases): NamedTuple.__mro_entries__ = _namedtuple_mro_entries +def _get_typeddict_qualifiers(annotation_type): + while True: + annotation_origin = get_origin(annotation_type) + if annotation_origin is Annotated: + annotation_args = get_args(annotation_type) + if annotation_args: + annotation_type = annotation_args[0] + else: + break + elif annotation_origin is Required: + yield Required + (annotation_type,) = get_args(annotation_type) + elif annotation_origin is NotRequired: + yield NotRequired + (annotation_type,) = get_args(annotation_type) + elif annotation_origin is ReadOnly: + yield ReadOnly + (annotation_type,) = get_args(annotation_type) + else: + break + + class _TypedDictMeta(type): def __new__(cls, name, bases, ns, total=True): """Create a new typed dict class object. @@ -2955,6 +2978,8 @@ def __new__(cls, name, bases, ns, total=True): } required_keys = set() optional_keys = set() + readonly_keys = set() + mutable_keys = set() for base in bases: annotations.update(base.__dict__.get('__annotations__', {})) @@ -2967,18 +2992,15 @@ def __new__(cls, name, bases, ns, total=True): required_keys -= base_optional optional_keys |= base_optional + readonly_keys.update(base.__dict__.get('__readonly_keys__', ())) + mutable_keys.update(base.__dict__.get('__mutable_keys__', ())) + annotations.update(own_annotations) for annotation_key, annotation_type in own_annotations.items(): - annotation_origin = get_origin(annotation_type) - if annotation_origin is Annotated: - annotation_args = get_args(annotation_type) - if annotation_args: - annotation_type = annotation_args[0] - annotation_origin = get_origin(annotation_type) - - if annotation_origin is Required: + qualifiers = set(_get_typeddict_qualifiers(annotation_type)) + if Required in qualifiers: is_required = True - elif annotation_origin is NotRequired: + elif NotRequired in qualifiers: is_required = False else: is_required = total @@ -2990,6 +3012,17 @@ def __new__(cls, name, bases, ns, total=True): optional_keys.add(annotation_key) required_keys.discard(annotation_key) + if ReadOnly in qualifiers: + if annotation_key in mutable_keys: + raise TypeError( + f"Cannot override mutable key {annotation_key!r}" + " with read-only key" + ) + readonly_keys.add(annotation_key) + else: + mutable_keys.add(annotation_key) + readonly_keys.discard(annotation_key) + assert required_keys.isdisjoint(optional_keys), ( f"Required keys overlap with optional keys in {name}:" f" {required_keys=}, {optional_keys=}" @@ -2997,6 +3030,8 @@ def __new__(cls, name, bases, ns, total=True): tp_dict.__annotations__ = annotations tp_dict.__required_keys__ = frozenset(required_keys) tp_dict.__optional_keys__ = frozenset(optional_keys) + tp_dict.__readonly_keys__ = frozenset(readonly_keys) + tp_dict.__mutable_keys__ = frozenset(mutable_keys) tp_dict.__total__ = total return tp_dict @@ -3055,6 +3090,14 @@ class Point2D(TypedDict): y: NotRequired[int] # the "y" key can be omitted See PEP 655 for more details on Required and NotRequired. + + The ReadOnly special form can be used + to mark individual keys as immutable for type checkers:: + + class DatabaseUser(TypedDict): + id: ReadOnly[int] # the "id" key must not be modified + username: str # the "username" key can be changed + """ if fields is _sentinel or fields is None: import warnings @@ -3131,6 +3174,26 @@ class Movie(TypedDict): return _GenericAlias(self, (item,)) +@_SpecialForm +def ReadOnly(self, parameters): + """A special typing construct to mark an item of a TypedDict as read-only. + + For example:: + + class Movie(TypedDict): + title: ReadOnly[str] + year: int + + def mutate_movie(m: Movie) -> None: + m["year"] = 1992 # allowed + m["title"] = "The Matrix" # typechecker error + + There is no runtime checking for this property. + """ + item = _type_check(parameters, f'{self._name} accepts only a single type.') + return _GenericAlias(self, (item,)) + + class NewType: """NewType creates simple unique types with almost zero runtime overhead. diff --git a/Misc/NEWS.d/next/Library/2024-03-05-14-34-22.gh-issue-116127.5uktu3.rst b/Misc/NEWS.d/next/Library/2024-03-05-14-34-22.gh-issue-116127.5uktu3.rst new file mode 100644 index 00000000000000..59edde9811e2f5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-05-14-34-22.gh-issue-116127.5uktu3.rst @@ -0,0 +1,2 @@ +:mod:`typing`: implement :pep:`705` which adds :data:`typing.ReadOnly` +support to :class:`typing.TypedDict`. From 5d72b753889977fa6d2d015499de03f94e16b035 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 12 Mar 2024 13:12:02 -0400 Subject: [PATCH 038/158] gh-116604: Check for `gcstate->enabled` in _Py_RunGC in free-threaded build (#116663) This isn't strictly necessary because the implementation of `gc_should_collect` already checks `gcstate->enabled` in the free-threaded build, but it seems like a good idea until the common pieces of gc.c and gc_free_threading.c are refactored out. --- Python/gc_free_threading.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 59e76012f8fc50..2b13d1f005dd97 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1603,6 +1603,10 @@ _PyObject_GC_Link(PyObject *op) void _Py_RunGC(PyThreadState *tstate) { + GCState *gcstate = get_gc_state(); + if (!gcstate->enabled) { + return; + } gc_collect_main(tstate, 0, _Py_GC_REASON_HEAP); } From f6e7a6ce651b43c6e060608a4bb20685f39e9eaa Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 12 Mar 2024 18:31:35 +0100 Subject: [PATCH 039/158] gh-116656: Fix test_capi test_py_config_isoloated_per_interpreter() (#116658) Don't parse argv when setting the configuration, to avoid SystemExit if parsing argv fails. --- Lib/test/test_capi/test_misc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 64c0da6e3cc06e..c1395ab00077cb 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1797,6 +1797,7 @@ def test_py_config_isoloated_per_interpreter(self): # double checked at the time this test was written. config = _testinternalcapi.get_config() config['int_max_str_digits'] = 55555 + config['parse_argv'] = 0 _testinternalcapi.set_config(config) sub_value = _testinternalcapi.get_config()['int_max_str_digits'] assert sub_value == 55555, sub_value From bb66600558cb8d5dd9a56f562bd9531eb1e1685f Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 12 Mar 2024 19:49:26 +0200 Subject: [PATCH 040/158] CI: Process stale issues twice per day (#116636) --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 07608fe91b4dbe..32299d0fc47c01 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,7 +2,7 @@ name: Mark stale pull requests on: schedule: - - cron: "0 0 * * *" + - cron: "0 */12 * * *" permissions: pull-requests: write From 076d169ebbe59f7035eaa28d33d517bcb375f342 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Tue, 12 Mar 2024 18:34:05 +0100 Subject: [PATCH 041/158] Python 3.13.0a5 --- Include/patchlevel.h | 4 +- Lib/pydoc_data/topics.py | 106 +- Misc/NEWS.d/3.13.0a5.rst | 1163 +++++++++++++++++ ...-02-13-14-52-59.gh-issue-114099.zjXsQr.rst | 2 - ...-02-21-11-58-30.gh-issue-115737.dpNl2T.rst | 2 - ...-02-21-18-22-49.gh-issue-111225.Z8C3av.rst | 1 - ...-02-24-12-50-43.gh-issue-115350.naQA6y.rst | 1 - ...-02-26-13-13-53.gh-issue-114099.8lpX-7.rst | 1 - ...4-02-26-14-54-58.gh-issue-71052.XvFay1.rst | 1 - ...-02-29-15-12-31.gh-issue-116117.eENkQK.rst | 2 - ...4-03-01-16-44-19.gh-issue-71052.Hs-9EP.rst | 1 - ...-03-04-12-43-42.gh-issue-116313.cLLb8S.rst | 1 - ...-11-15-09-24-51.gh-issue-111418.FYYetY.rst | 2 - ...-02-16-15-56-53.gh-issue-114626.ie2esA.rst | 4 - ...2-09-04-16-51-56.gh-issue-96497.HTBuIL.rst | 2 - ...-02-13-11-36-50.gh-issue-101860.CKCMbC.rst | 1 - ...-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst | 8 - ...-07-16-15-02-47.gh-issue-104090.oMjNa9.rst | 2 - ...-01-28-02-46-12.gh-issue-112433.FUX-nT.rst | 1 - ...-02-08-16-01-18.gh-issue-115154.ji96FV.rst | 2 - ...-02-09-18-59-22.gh-issue-112175.qglugr.rst | 1 - ...-02-12-23-29-17.gh-issue-115323.3t6687.rst | 2 - ...-02-14-23-50-43.gh-issue-115347.VkHvQC.rst | 2 - ...-02-20-12-46-20.gh-issue-115700.KLJ5r4.rst | 1 - ...-02-20-18-49-02.gh-issue-115733.51Zb85.rst | 1 - ...-02-22-11-33-20.gh-issue-115778.jksd1D.rst | 1 - ...-02-22-16-17-53.gh-issue-115823.c1TreJ.rst | 3 - ...-03-04-10-19-51.gh-issue-116296.gvtxyU.rst | 1 - ...-03-05-22-00-58.gh-issue-116381.0Nq9iO.rst | 1 - ...-03-09-11-10-53.gh-issue-112087.nbI0Pw.rst | 1 - ...-03-11-22-24-59.gh-issue-116604.LCEzAT.rst | 3 - ...-02-08-08-51-37.gh-issue-109653.QHLW4w.rst | 1 - ...-02-14-20-17-04.gh-issue-115399.fb9a0R.rst | 1 - ...3-12-09-11-04-26.gh-issue-88516.SIIvfs.rst | 2 - .../2019-04-06-23-50-59.bpo-33775.0yhMDc.rst | 1 - .../2020-05-29-18-08-54.bpo-40818.Ij8ffq.rst | 3 - .../2020-07-13-23-59-42.bpo-41122.8P_Brh.rst | 3 - .../2020-12-15-22-30-49.bpo-42125.UGyseY.rst | 2 - .../2021-05-03-11-04-12.bpo-43952.Me7fJe.rst | 2 - .../2021-08-24-20-47-37.bpo-44865.c3BhZS.rst | 1 - .../2022-01-14-10-50-17.bpo-31116.0bduV9.rst | 1 - ...2-05-25-17-49-04.gh-issue-93205.DjhFVR.rst | 1 - ...2-08-26-15-50-53.gh-issue-96310.0NssDh.rst | 2 - ...2-11-22-23-17-43.gh-issue-95782.an_and.rst | 4 - ...-01-09-14-08-02.gh-issue-100884.DcmdLl.rst | 2 - ...-01-12-14-16-01.gh-issue-100985.GT5Fvd.rst | 2 - ...3-02-14-17-19-59.gh-issue-72249.fv35wU.rst | 2 - ...-03-03-09-05-42.gh-issue-102389.ucmo0_.rst | 1 - ...3-04-02-21-20-35.gh-issue-60346.7mjgua.rst | 1 - ...-05-01-22-28-57.gh-issue-104061.vxfBXf.rst | 1 - ...3-05-17-21-33-21.gh-issue-69990.Blvz9G.rst | 1 - ...3-07-12-14-52-04.gh-issue-57141.L2k8Xb.rst | 3 - ...-08-02-01-17-32.gh-issue-107155.Mj1K9L.rst | 3 - ...-08-05-08-41-58.gh-issue-107625.cVSHCT.rst | 4 - ...3-10-07-06-15-13.gh-issue-90095.gWn1ka.rst | 1 - ...-11-07-10-22-06.gh-issue-111775.IoVxfX.rst | 2 - ...-11-20-16-15-44.gh-issue-112281.gH4EVk.rst | 2 - ...11-24-23-40-00.gh-issue-107361.v54gh46.rst | 2 - ...-01-26-16-42-31.gh-issue-114610.S18Vuz.rst | 4 - ...-01-29-13-46-41.gh-issue-114709.SQ998l.rst | 5 - ...-01-30-23-28-29.gh-issue-114763.BRjKkg.rst | 3 - ...-02-09-12-22-47.gh-issue-113812.wOraaG.rst | 3 - ...-02-09-19-41-48.gh-issue-115197.20wkWH.rst | 2 - ...-02-10-17-18-49.gh-issue-115256.41Fy9P.rst | 5 - ...-02-11-20-12-39.gh-issue-113942.i72sMJ.rst | 2 - ...-02-12-11-42-48.gh-issue-103092.sGMKr0.rst | 1 - ...-02-15-19-11-49.gh-issue-101293.898b8l.rst | 4 - ...-02-15-23-42-54.gh-issue-112006.4wxcK-.rst | 3 - ...-02-16-16-40-10.gh-issue-112720.io6_Ac.rst | 2 - ...-02-17-18-47-12.gh-issue-115618.napiNp.rst | 3 - ...-02-18-12-18-12.gh-issue-111358.9yJUMD.rst | 2 - ...-02-19-15-52-30.gh-issue-114914.M5-1d8.rst | 2 - ...-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst | 2 - ...-02-20-07-38-15.gh-issue-112364.EX7uGI.rst | 1 - ...-02-20-16-42-54.gh-issue-115712.EXVMXw.rst | 4 - ...4-02-20-22-02-34.gh-issue-67044.QF9_Ru.rst | 2 - ...4-02-21-17-54-59.gh-issue-74668.JT-Q8W.rst | 3 - ...-02-22-11-29-27.gh-issue-115809.9H1DhB.rst | 4 - ...-02-22-12-10-18.gh-issue-115714.P2JsU1.rst | 4 - ...-02-23-11-08-31.gh-issue-115532.zVd3gK.rst | 1 - ...-02-24-18-48-14.gh-issue-115886.rgM6AF.rst | 2 - ...-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst | 4 - ...4-02-27-13-05-51.gh-issue-75988.In6LlB.rst | 1 - ...4-02-27-20-11-29.gh-issue-85644.3rgcBm.rst | 2 - ...-02-28-12-14-31.gh-issue-115821.YO2vKA.rst | 2 - ...-02-28-13-10-17.gh-issue-116040.wDidHd.rst | 1 - ...4-02-28-17-04-28.gh-issue-65824.gG8KR1.rst | 1 - ...4-02-28-17-50-42.gh-issue-89547.GetF38.rst | 1 - ...4-02-29-17-06-54.gh-issue-76511.WqjRLP.rst | 4 - ...4-02-29-20-06-06.gh-issue-87115.FVMiOR.rst | 1 - ...4-03-01-11-57-32.gh-issue-88352.bZ68rw.rst | 6 - ...-03-01-14-22-08.gh-issue-115978.r2ePTo.rst | 4 - ...-03-02-11-31-49.gh-issue-115957.C-3Z_U.rst | 1 - ...-03-05-02-09-18.gh-issue-116325.FmlBYv.rst | 2 - ...-03-05-14-34-22.gh-issue-116127.5uktu3.rst | 2 - ...-03-05-20-53-34.gh-issue-116143.sww6Zl.rst | 3 - ...-03-07-21-57-50.gh-issue-116349.fD2pbP.rst | 3 - ...-03-11-12-11-10.gh-issue-116600.FcNBy_.rst | 1 - ...-01-26-22-14-09.gh-issue-114572.t1QMQD.rst | 4 - ...-02-18-03-14-40.gh-issue-115398.tzvxH8.rst | 8 - ...-02-12-22-35-01.gh-issue-115376.n9vubZ.rst | 1 - ...-02-13-18-24-04.gh-issue-115420.-dlzfI.rst | 2 - ...-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst | 2 - ...-02-17-08-25-01.gh-issue-115596.RGPCrR.rst | 2 - ...-02-18-14-20-52.gh-issue-115122.3rGNo9.rst | 2 - ...-02-20-15-47-41.gh-issue-115720.w8i8UG.rst | 2 - ...-02-22-00-17-06.gh-issue-115796.d4hpKy.rst | 2 - ...4-02-25-15-58-28.gh-issue-71052.lxBjqY.rst | 2 - ...4-02-25-16-28-26.gh-issue-71052.lSb9EC.rst | 1 - .../2021-09-05-02-47-48.bpo-45101.60Zqmt.rst | 1 - ...-02-12-19-28-08.gh-issue-100176.Kzs4Zw.rst | 1 - ...-02-15-23-16-31.gh-issue-115543.otrWnw.rst | 3 - ...-02-21-23-48-59.gh-issue-115554.02mpQC.rst | 6 - ...-02-23-11-43-43.gh-issue-115582.sk1XPi.rst | 3 - ...-02-27-23-21-55.gh-issue-116012.B9_IwM.rst | 1 - ...-02-29-20-52-23.gh-issue-116145.ygafim.rst | 1 - README.rst | 2 +- 117 files changed, 1231 insertions(+), 293 deletions(-) create mode 100644 Misc/NEWS.d/3.13.0a5.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-02-13-14-52-59.gh-issue-114099.zjXsQr.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-02-21-11-58-30.gh-issue-115737.dpNl2T.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-02-21-18-22-49.gh-issue-111225.Z8C3av.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-02-26-13-13-53.gh-issue-114099.8lpX-7.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-02-26-14-54-58.gh-issue-71052.XvFay1.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-02-29-15-12-31.gh-issue-116117.eENkQK.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-03-01-16-44-19.gh-issue-71052.Hs-9EP.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-03-04-12-43-42.gh-issue-116313.cLLb8S.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-11-15-09-24-51.gh-issue-111418.FYYetY.rst delete mode 100644 Misc/NEWS.d/next/C API/2024-02-16-15-56-53.gh-issue-114626.ie2esA.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-09-04-16-51-56.gh-issue-96497.HTBuIL.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-13-11-36-50.gh-issue-101860.CKCMbC.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-07-16-15-02-47.gh-issue-104090.oMjNa9.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-01-28-02-46-12.gh-issue-112433.FUX-nT.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-08-16-01-18.gh-issue-115154.ji96FV.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-09-18-59-22.gh-issue-112175.qglugr.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-12-23-29-17.gh-issue-115323.3t6687.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-14-23-50-43.gh-issue-115347.VkHvQC.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-20-12-46-20.gh-issue-115700.KLJ5r4.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-20-18-49-02.gh-issue-115733.51Zb85.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-22-11-33-20.gh-issue-115778.jksd1D.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-22-16-17-53.gh-issue-115823.c1TreJ.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-04-10-19-51.gh-issue-116296.gvtxyU.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-05-22-00-58.gh-issue-116381.0Nq9iO.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-09-11-10-53.gh-issue-112087.nbI0Pw.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-24-59.gh-issue-116604.LCEzAT.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2024-02-08-08-51-37.gh-issue-109653.QHLW4w.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2024-02-14-20-17-04.gh-issue-115399.fb9a0R.rst delete mode 100644 Misc/NEWS.d/next/IDLE/2023-12-09-11-04-26.gh-issue-88516.SIIvfs.rst delete mode 100644 Misc/NEWS.d/next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-05-29-18-08-54.bpo-40818.Ij8ffq.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-07-13-23-59-42.bpo-41122.8P_Brh.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-12-15-22-30-49.bpo-42125.UGyseY.rst delete mode 100644 Misc/NEWS.d/next/Library/2021-05-03-11-04-12.bpo-43952.Me7fJe.rst delete mode 100644 Misc/NEWS.d/next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-01-14-10-50-17.bpo-31116.0bduV9.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-05-25-17-49-04.gh-issue-93205.DjhFVR.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-08-26-15-50-53.gh-issue-96310.0NssDh.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-11-22-23-17-43.gh-issue-95782.an_and.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-09-14-08-02.gh-issue-100884.DcmdLl.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-12-14-16-01.gh-issue-100985.GT5Fvd.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-14-17-19-59.gh-issue-72249.fv35wU.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-03-09-05-42.gh-issue-102389.ucmo0_.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-02-21-20-35.gh-issue-60346.7mjgua.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-01-22-28-57.gh-issue-104061.vxfBXf.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-17-21-33-21.gh-issue-69990.Blvz9G.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-07-12-14-52-04.gh-issue-57141.L2k8Xb.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-08-02-01-17-32.gh-issue-107155.Mj1K9L.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-08-05-08-41-58.gh-issue-107625.cVSHCT.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-10-07-06-15-13.gh-issue-90095.gWn1ka.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-11-07-10-22-06.gh-issue-111775.IoVxfX.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-11-20-16-15-44.gh-issue-112281.gH4EVk.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-11-24-23-40-00.gh-issue-107361.v54gh46.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-01-26-16-42-31.gh-issue-114610.S18Vuz.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-01-29-13-46-41.gh-issue-114709.SQ998l.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-01-30-23-28-29.gh-issue-114763.BRjKkg.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-09-12-22-47.gh-issue-113812.wOraaG.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-09-19-41-48.gh-issue-115197.20wkWH.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-10-17-18-49.gh-issue-115256.41Fy9P.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-12-11-42-48.gh-issue-103092.sGMKr0.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-15-19-11-49.gh-issue-101293.898b8l.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-15-23-42-54.gh-issue-112006.4wxcK-.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-16-16-40-10.gh-issue-112720.io6_Ac.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-17-18-47-12.gh-issue-115618.napiNp.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-18-12-18-12.gh-issue-111358.9yJUMD.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-19-15-52-30.gh-issue-114914.M5-1d8.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-20-07-38-15.gh-issue-112364.EX7uGI.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-20-16-42-54.gh-issue-115712.EXVMXw.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-20-22-02-34.gh-issue-67044.QF9_Ru.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-21-17-54-59.gh-issue-74668.JT-Q8W.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-22-11-29-27.gh-issue-115809.9H1DhB.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-22-12-10-18.gh-issue-115714.P2JsU1.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-23-11-08-31.gh-issue-115532.zVd3gK.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-24-18-48-14.gh-issue-115886.rgM6AF.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-27-13-05-51.gh-issue-75988.In6LlB.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-27-20-11-29.gh-issue-85644.3rgcBm.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-28-13-10-17.gh-issue-116040.wDidHd.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-28-17-04-28.gh-issue-65824.gG8KR1.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-28-17-50-42.gh-issue-89547.GetF38.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-29-17-06-54.gh-issue-76511.WqjRLP.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-29-20-06-06.gh-issue-87115.FVMiOR.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-01-11-57-32.gh-issue-88352.bZ68rw.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-01-14-22-08.gh-issue-115978.r2ePTo.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-02-11-31-49.gh-issue-115957.C-3Z_U.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-05-02-09-18.gh-issue-116325.FmlBYv.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-05-14-34-22.gh-issue-116127.5uktu3.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-05-20-53-34.gh-issue-116143.sww6Zl.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-07-21-57-50.gh-issue-116349.fD2pbP.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-11-12-11-10.gh-issue-116600.FcNBy_.rst delete mode 100644 Misc/NEWS.d/next/Security/2024-01-26-22-14-09.gh-issue-114572.t1QMQD.rst delete mode 100644 Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-02-12-22-35-01.gh-issue-115376.n9vubZ.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-02-13-18-24-04.gh-issue-115420.-dlzfI.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-02-17-08-25-01.gh-issue-115596.RGPCrR.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-02-18-14-20-52.gh-issue-115122.3rGNo9.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-02-20-15-47-41.gh-issue-115720.w8i8UG.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-02-22-00-17-06.gh-issue-115796.d4hpKy.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-02-25-15-58-28.gh-issue-71052.lxBjqY.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-02-25-16-28-26.gh-issue-71052.lSb9EC.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-02-12-19-28-08.gh-issue-100176.Kzs4Zw.rst delete mode 100644 Misc/NEWS.d/next/Windows/2024-02-15-23-16-31.gh-issue-115543.otrWnw.rst delete mode 100644 Misc/NEWS.d/next/Windows/2024-02-21-23-48-59.gh-issue-115554.02mpQC.rst delete mode 100644 Misc/NEWS.d/next/Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst delete mode 100644 Misc/NEWS.d/next/Windows/2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst delete mode 100644 Misc/NEWS.d/next/macOS/2024-02-29-20-52-23.gh-issue-116145.ygafim.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index ae2d22f80817d5..ea22ad2856c869 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 13 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 4 +#define PY_RELEASE_SERIAL 5 /* Version as a string */ -#define PY_VERSION "3.13.0a4+" +#define PY_VERSION "3.13.0a5" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index a49c38a51d39f2..05045ac8c945c8 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Thu Feb 15 14:30:52 2024 +# Autogenerated by Sphinx on Tue Mar 12 18:35:04 2024 # as part of the release process. topics = {'assert': 'The "assert" statement\n' '**********************\n' @@ -4724,7 +4724,7 @@ 'reflection,\n' ' and "__eq__()" and "__ne__()" are their own reflection. ' 'If the\n' - ' operands are of different types, and right operand’s ' + ' operands are of different types, and the right operand’s ' 'type is a\n' ' direct or indirect subclass of the left operand’s type, ' 'the\n' @@ -4734,6 +4734,11 @@ 'is not\n' ' considered.\n' '\n' + ' When no appropriate method returns any value other than\n' + ' "NotImplemented", the "==" and "!=" operators will fall ' + 'back to\n' + ' "is" and "is not", respectively.\n' + '\n' 'object.__hash__(self)\n' '\n' ' Called by built-in function "hash()" and for operations ' @@ -5223,15 +5228,15 @@ 'overridden by\n' 'the local file.\n' '\n' - 'Changed in version 3.11: ".pdbrc" is now read with "\'utf-8\'" ' - 'encoding.\n' - 'Previously, it was read with the system locale encoding.\n' - '\n' 'Changed in version 3.2: ".pdbrc" can now contain commands that\n' 'continue debugging, such as "continue" or "next". Previously, ' 'these\n' 'commands had no effect.\n' '\n' + 'Changed in version 3.11: ".pdbrc" is now read with "\'utf-8\'" ' + 'encoding.\n' + 'Previously, it was read with the system locale encoding.\n' + '\n' 'h(elp) [command]\n' '\n' ' Without argument, print the list of available commands. With ' @@ -8756,7 +8761,7 @@ '"__rsub__()"\n' ' method, "type(y).__rsub__(y, x)" is called if ' '"type(x).__sub__(x,\n' - ' y)" returns *NotImplemented*.\n' + ' y)" returns "NotImplemented".\n' '\n' ' Note that ternary "pow()" will not try calling ' '"__rpow__()" (the\n' @@ -8799,14 +8804,18 @@ 'the result\n' ' (which could be, but does not have to be, *self*). If a ' 'specific\n' - ' method is not defined, the augmented assignment falls ' - 'back to the\n' - ' normal methods. For instance, if *x* is an instance of ' - 'a class\n' - ' with an "__iadd__()" method, "x += y" is equivalent to ' - '"x =\n' - ' x.__iadd__(y)" . Otherwise, "x.__add__(y)" and ' - '"y.__radd__(x)" are\n' + ' method is not defined, or if that method returns ' + '"NotImplemented",\n' + ' the augmented assignment falls back to the normal ' + 'methods. For\n' + ' instance, if *x* is an instance of a class with an ' + '"__iadd__()"\n' + ' method, "x += y" is equivalent to "x = x.__iadd__(y)" . ' + 'If\n' + ' "__iadd__()" does not exist, or if "x.__iadd__(y)" ' + 'returns\n' + ' "NotImplemented", "x.__add__(y)" and "y.__radd__(x)" ' + 'are\n' ' considered, as with the evaluation of "x + y". In ' 'certain\n' ' situations, augmented assignment can result in ' @@ -8887,7 +8896,7 @@ 'Every object has an identity, a type and a value. An object’s\n' '*identity* never changes once it has been created; you may think ' 'of it\n' - 'as the object’s address in memory. The ‘"is"’ operator compares ' + 'as the object’s address in memory. The "is" operator compares ' 'the\n' 'identity of two objects; the "id()" function returns an integer\n' 'representing its identity.\n' @@ -8952,7 +8961,7 @@ 'Note that the use of the implementation’s tracing or debugging\n' 'facilities may keep objects alive that would normally be ' 'collectable.\n' - 'Also note that catching an exception with a ‘"try"…"except"’ ' + 'Also note that catching an exception with a "try"…"except" ' 'statement\n' 'may keep objects alive.\n' '\n' @@ -8967,8 +8976,9 @@ 'release the external resource, usually a "close()" method. ' 'Programs\n' 'are strongly recommended to explicitly close such objects. The\n' - '‘"try"…"finally"’ statement and the ‘"with"’ statement provide\n' - 'convenient ways to do this.\n' + '"try"…"finally" statement and the "with" statement provide ' + 'convenient\n' + 'ways to do this.\n' '\n' 'Some objects contain references to other objects; these are ' 'called\n' @@ -9345,10 +9355,7 @@ 'The try statement.\n' '\n' 'Changed in version 3.3: "None" is now permitted as "Y" in "raise X\n' - 'from Y".\n' - '\n' - 'New in version 3.3: The "__suppress_context__" attribute to ' - 'suppress\n' + 'from Y".Added the "__suppress_context__" attribute to suppress\n' 'automatic display of the exception context.\n' '\n' 'Changed in version 3.11: If the traceback of the active exception ' @@ -10133,8 +10140,8 @@ 'reflection,\n' ' and "__eq__()" and "__ne__()" are their own reflection. ' 'If the\n' - ' operands are of different types, and right operand’s type ' - 'is a\n' + ' operands are of different types, and the right operand’s ' + 'type is a\n' ' direct or indirect subclass of the left operand’s type, ' 'the\n' ' reflected method of the right operand has priority, ' @@ -10143,6 +10150,11 @@ 'is not\n' ' considered.\n' '\n' + ' When no appropriate method returns any value other than\n' + ' "NotImplemented", the "==" and "!=" operators will fall ' + 'back to\n' + ' "is" and "is not", respectively.\n' + '\n' 'object.__hash__(self)\n' '\n' ' Called by built-in function "hash()" and for operations ' @@ -11682,7 +11694,7 @@ '"__rsub__()"\n' ' method, "type(y).__rsub__(y, x)" is called if ' '"type(x).__sub__(x,\n' - ' y)" returns *NotImplemented*.\n' + ' y)" returns "NotImplemented".\n' '\n' ' Note that ternary "pow()" will not try calling ' '"__rpow__()" (the\n' @@ -11725,14 +11737,17 @@ 'the result\n' ' (which could be, but does not have to be, *self*). If a ' 'specific\n' - ' method is not defined, the augmented assignment falls ' - 'back to the\n' - ' normal methods. For instance, if *x* is an instance of a ' - 'class\n' - ' with an "__iadd__()" method, "x += y" is equivalent to "x ' - '=\n' - ' x.__iadd__(y)" . Otherwise, "x.__add__(y)" and ' - '"y.__radd__(x)" are\n' + ' method is not defined, or if that method returns ' + '"NotImplemented",\n' + ' the augmented assignment falls back to the normal ' + 'methods. For\n' + ' instance, if *x* is an instance of a class with an ' + '"__iadd__()"\n' + ' method, "x += y" is equivalent to "x = x.__iadd__(y)" . ' + 'If\n' + ' "__iadd__()" does not exist, or if "x.__iadd__(y)" ' + 'returns\n' + ' "NotImplemented", "x.__add__(y)" and "y.__radd__(x)" are\n' ' considered, as with the evaluation of "x + y". In ' 'certain\n' ' situations, augmented assignment can result in unexpected ' @@ -13080,9 +13095,8 @@ '\n' 'New in version 3.3: The "\'rb\'" prefix of raw bytes literals has ' 'been\n' - 'added as a synonym of "\'br\'".\n' - '\n' - 'New in version 3.3: Support for the unicode legacy literal\n' + 'added as a synonym of "\'br\'".Support for the unicode legacy ' + 'literal\n' '("u\'value\'") was reintroduced to simplify the maintenance of ' 'dual\n' 'Python 2.x and 3.x codebases. See **PEP 414** for more ' @@ -14765,6 +14779,17 @@ 'tools.\n' ' The PEP that introduced the "co_lines()" method.\n' '\n' + 'codeobject.replace(**kwargs)\n' + '\n' + ' Return a copy of the code object with new values for the ' + 'specified\n' + ' fields.\n' + '\n' + ' Code objects are also supported by the generic function\n' + ' "copy.replace()".\n' + '\n' + ' New in version 3.8.\n' + '\n' '\n' 'Frame objects\n' '-------------\n' @@ -16109,7 +16134,7 @@ '\n' ' For sorting examples and a brief sorting tutorial, see ' 'Sorting\n' - ' HOW TO.\n' + ' Techniques.\n' '\n' ' **CPython implementation detail:** While a list is being ' 'sorted,\n' @@ -16324,9 +16349,8 @@ 'objects\n' 'based on the sequence of values they define (instead of ' 'comparing\n' - 'based on object identity).\n' - '\n' - 'New in version 3.3: The "start", "stop" and "step" attributes.\n' + 'based on object identity).Added the "start", "stop" and "step"\n' + 'attributes.\n' '\n' 'See also:\n' '\n' diff --git a/Misc/NEWS.d/3.13.0a5.rst b/Misc/NEWS.d/3.13.0a5.rst new file mode 100644 index 00000000000000..fb0163eed67aeb --- /dev/null +++ b/Misc/NEWS.d/3.13.0a5.rst @@ -0,0 +1,1163 @@ +.. date: 2024-02-18-03-14-40 +.. gh-issue: 115398 +.. nonce: tzvxH8 +.. release date: 2024-03-12 +.. section: Security + +Allow controlling Expat >=2.6.0 reparse deferral (CVE-2023-52425) by adding +five new methods: + +* :meth:`xml.etree.ElementTree.XMLParser.flush` +* :meth:`xml.etree.ElementTree.XMLPullParser.flush` +* :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` +* :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` +* :meth:`xml.sax.expatreader.ExpatParser.flush` + +.. + +.. date: 2024-01-26-22-14-09 +.. gh-issue: 114572 +.. nonce: t1QMQD +.. section: Security + +:meth:`ssl.SSLContext.cert_store_stats` and +:meth:`ssl.SSLContext.get_ca_certs` now correctly lock access to the +certificate store, when the :class:`ssl.SSLContext` is shared across +multiple threads. + +.. + +.. date: 2024-03-11-22-24-59 +.. gh-issue: 116604 +.. nonce: LCEzAT +.. section: Core and Builtins + +Respect the status of the garbage collector when indirect calls are made via +:c:func:`PyErr_CheckSignals` and the evaluation breaker. Patch by Pablo +Galindo + +.. + +.. date: 2024-03-09-11-10-53 +.. gh-issue: 112087 +.. nonce: nbI0Pw +.. section: Core and Builtins + +:class:`list` is now compatible with the implementation of :pep:`703`. + +.. + +.. date: 2024-03-05-22-00-58 +.. gh-issue: 116381 +.. nonce: 0Nq9iO +.. section: Core and Builtins + +Add specialization for ``CONTAINS_OP``. + +.. + +.. date: 2024-03-04-10-19-51 +.. gh-issue: 116296 +.. nonce: gvtxyU +.. section: Core and Builtins + +Fix possible refleak in :meth:`!object.__reduce__` internal error handling. + +.. + +.. date: 2024-02-22-16-17-53 +.. gh-issue: 115823 +.. nonce: c1TreJ +.. section: Core and Builtins + +Properly calculate error ranges in the parser when raising +:exc:`SyntaxError` exceptions caused by invalid byte sequences. Patch by +Pablo Galindo + +.. + +.. date: 2024-02-22-11-33-20 +.. gh-issue: 115778 +.. nonce: jksd1D +.. section: Core and Builtins + +Add ``tierN`` annotation for instruction definition in interpreter DSL. + +.. + +.. date: 2024-02-20-18-49-02 +.. gh-issue: 115733 +.. nonce: 51Zb85 +.. section: Core and Builtins + +Fix crash when calling ``next()`` on exhausted list iterators. + +.. + +.. date: 2024-02-20-12-46-20 +.. gh-issue: 115700 +.. nonce: KLJ5r4 +.. section: Core and Builtins + +The regen-cases build stage now works on Windows. + +.. + +.. date: 2024-02-14-23-50-43 +.. gh-issue: 115347 +.. nonce: VkHvQC +.. section: Core and Builtins + +Fix bug where docstring was replaced by a redundant NOP when Python is run +with ``-OO``. + +.. + +.. date: 2024-02-12-23-29-17 +.. gh-issue: 115323 +.. nonce: 3t6687 +.. section: Core and Builtins + +Make error message more meaningful for when :meth:`bytearray.extend` is +called with a :class:`str` object. + +.. + +.. date: 2024-02-09-18-59-22 +.. gh-issue: 112175 +.. nonce: qglugr +.. section: Core and Builtins + +Every ``PyThreadState`` now has its own ``eval_breaker``, allowing specific +threads to be interrupted. + +.. + +.. date: 2024-02-08-16-01-18 +.. gh-issue: 115154 +.. nonce: ji96FV +.. section: Core and Builtins + +Fix a bug that was causing the :func:`tokenize.untokenize` function to +handle unicode named literals incorrectly. Patch by Pablo Galindo + +.. + +.. date: 2024-01-28-02-46-12 +.. gh-issue: 112433 +.. nonce: FUX-nT +.. section: Core and Builtins + +Add ability to force alignment of :mod:`ctypes.Structure` by way of the new +``_align_`` attribute on the class. + +.. + +.. date: 2023-07-16-15-02-47 +.. gh-issue: 104090 +.. nonce: oMjNa9 +.. section: Core and Builtins + +The multiprocessing resource tracker now exits with non-zero status code if +a resource leak was detected. It still exits with status code 0 otherwise. + +.. + +.. date: 2023-06-16-21-29-06 +.. gh-issue: 105858 +.. nonce: Q7h0EV +.. section: Core and Builtins + +Improve the constructors for :mod:`ast` nodes. Arguments of list types now +default to an empty list if omitted, and optional fields default to +``None``. AST nodes now have an ``__annotations__`` attribute with the +expected types of their attributes. Passing unrecognized extra arguments to +AST nodes is deprecated and will become an error in Python 3.15. Omitting a +required argument to an AST node is deprecated and will become an error in +Python 3.15. Patch by Jelle Zijlstra. + +.. + +.. date: 2023-02-13-11-36-50 +.. gh-issue: 101860 +.. nonce: CKCMbC +.. section: Core and Builtins + +Expose ``__name__`` attribute on property. + +.. + +.. date: 2022-09-04-16-51-56 +.. gh-issue: 96497 +.. nonce: HTBuIL +.. section: Core and Builtins + +Fix incorrect resolution of mangled class variables used in assignment +expressions in comprehensions. + +.. + +.. date: 2024-03-11-12-11-10 +.. gh-issue: 116600 +.. nonce: FcNBy_ +.. section: Library + +Fix :func:`repr` for global :class:`~enum.Flag` members. + +.. + +.. date: 2024-03-07-21-57-50 +.. gh-issue: 116349 +.. nonce: fD2pbP +.. section: Library + +:func:`platform.java_ver` is deprecated and will be removed in 3.15. It was +largely untested, had a confusing API, and was only useful for Jython +support. + +.. + +.. date: 2024-03-05-20-53-34 +.. gh-issue: 116143 +.. nonce: sww6Zl +.. section: Library + +Fix a race in pydoc ``_start_server``, eliminating a window in which +``_start_server`` can return a thread that is "serving" but without a +``docserver`` set. + +.. + +.. date: 2024-03-05-14-34-22 +.. gh-issue: 116127 +.. nonce: 5uktu3 +.. section: Library + +:mod:`typing`: implement :pep:`705` which adds :data:`typing.ReadOnly` +support to :class:`typing.TypedDict`. + +.. + +.. date: 2024-03-05-02-09-18 +.. gh-issue: 116325 +.. nonce: FmlBYv +.. section: Library + +:mod:`typing`: raise :exc:`SyntaxError` instead of :exc:`AttributeError` on +forward references as empty strings. + +.. + +.. date: 2024-03-02-11-31-49 +.. gh-issue: 115957 +.. nonce: C-3Z_U +.. section: Library + +When ``asyncio.TaskGroup.create_task`` is called on an inactive +``asyncio.TaskGroup``, the given coroutine will be closed (which prevents a +``RuntimeWarning``). + +.. + +.. date: 2024-03-01-14-22-08 +.. gh-issue: 115978 +.. nonce: r2ePTo +.. section: Library + +Disable preadv(), readv(), pwritev(), and writev() on WASI. + +Under wasmtime for WASI 0.2, these functions don't pass test_posix +(https://github.com/bytecodealliance/wasmtime/issues/7830). + +.. + +.. date: 2024-03-01-11-57-32 +.. gh-issue: 88352 +.. nonce: bZ68rw +.. section: Library + +Fix the computation of the next rollover time in the +:class:`logging.TimedRotatingFileHandler` handler. :meth:`!computeRollover` +now always returns a timestamp larger than the specified time and works +correctly during the DST change. :meth:`!doRollover` no longer overwrite the +already rolled over file, saving from data loss when run at midnight or +during repeated time at the DST change. + +.. + +.. date: 2024-02-29-20-06-06 +.. gh-issue: 87115 +.. nonce: FVMiOR +.. section: Library + +Set ``__main__.__spec__`` to ``None`` when running a script with :mod:`pdb` + +.. + +.. date: 2024-02-29-17-06-54 +.. gh-issue: 76511 +.. nonce: WqjRLP +.. section: Library + +Fix UnicodeEncodeError in :meth:`email.Message.as_string` that results when +a message that claims to be in the ascii character set actually has +non-ascii characters. Non-ascii characters are now replaced with the U+FFFD +replacement character, like in the ``replace`` error handler. + +.. + +.. date: 2024-02-28-17-50-42 +.. gh-issue: 89547 +.. nonce: GetF38 +.. section: Library + +Add support for nested typing special forms like Final[ClassVar[int]]. + +.. + +.. date: 2024-02-28-17-04-28 +.. gh-issue: 65824 +.. nonce: gG8KR1 +.. section: Library + +Improve the ``less`` prompt in :mod:`pydoc`. + +.. + +.. date: 2024-02-28-13-10-17 +.. gh-issue: 116040 +.. nonce: wDidHd +.. section: Library + +[Enum] fix by-value calls when second value is falsey; e.g. Cardinal(1, 0) + +.. + +.. date: 2024-02-28-12-14-31 +.. gh-issue: 115821 +.. nonce: YO2vKA +.. section: Library + +[Enum] Improve error message when calling super().__new__() in custom +__new__. + +.. + +.. date: 2024-02-27-20-11-29 +.. gh-issue: 85644 +.. nonce: 3rgcBm +.. section: Library + +Use the ``XDG_CURRENT_DESKTOP`` environment variable in :mod:`webbrowser` to +check desktop. Prefer it to the deprecated ``GNOME_DESKTOP_SESSION_ID`` for +GNOME detection. + +.. + +.. date: 2024-02-27-13-05-51 +.. gh-issue: 75988 +.. nonce: In6LlB +.. section: Library + +Fixed :func:`unittest.mock.create_autospec` to pass the call through to the +wrapped object to return the real result. + +.. + +.. date: 2024-02-25-19-20-05 +.. gh-issue: 115881 +.. nonce: ro_Kuw +.. section: Library + +Fix issue where :func:`ast.parse` would incorrectly flag conditional context +managers (such as ``with (x() if y else z()): ...``) as invalid syntax if +``feature_version=(3, 8)`` was passed. This reverts changes to the grammar +made as part of gh-94949. + +.. + +.. date: 2024-02-24-18-48-14 +.. gh-issue: 115886 +.. nonce: rgM6AF +.. section: Library + +Fix silent truncation of the name with an embedded null character in +:class:`multiprocessing.shared_memory.SharedMemory`. + +.. + +.. date: 2024-02-23-11-08-31 +.. gh-issue: 115532 +.. nonce: zVd3gK +.. section: Library + +Add kernel density estimation to the statistics module. + +.. + +.. date: 2024-02-22-12-10-18 +.. gh-issue: 115714 +.. nonce: P2JsU1 +.. section: Library + +On WASI, the :mod:`time` module no longer get process time using ``times()`` +or ``CLOCK_PROCESS_CPUTIME_ID``, system API is that is unreliable and is +likely to be removed from WASI. The affected clock functions fall back to +calling ``clock()``. + +.. + +.. date: 2024-02-22-11-29-27 +.. gh-issue: 115809 +.. nonce: 9H1DhB +.. section: Library + +Improve algorithm for computing which rolled-over log files to delete in +:class:`logging.TimedRotatingFileHandler`. It is now reliable for handlers +without ``namer`` and with arbitrary deterministic ``namer`` that leaves the +datetime part in the file name unmodified. + +.. + +.. date: 2024-02-21-17-54-59 +.. gh-issue: 74668 +.. nonce: JT-Q8W +.. section: Library + +:mod:`urllib.parse` functions :func:`~urllib.parse.parse_qs` and +:func:`~urllib.parse.parse_qsl` now support bytes arguments containing raw +and percent-encoded non-ASCII data. + +.. + +.. date: 2024-02-20-22-02-34 +.. gh-issue: 67044 +.. nonce: QF9_Ru +.. section: Library + +:func:`csv.writer` now always quotes or escapes ``'\r'`` and ``'\n'``, +regardless of *lineterminator* value. + +.. + +.. date: 2024-02-20-16-42-54 +.. gh-issue: 115712 +.. nonce: EXVMXw +.. section: Library + +Restore support of space delimiter with ``skipinitialspace=True`` in +:mod:`csv`. :func:`csv.writer()` now quotes empty fields if delimiter is a +space and skipinitialspace is true and raises exception if quoting is not +possible. + +.. + +.. date: 2024-02-20-07-38-15 +.. gh-issue: 112364 +.. nonce: EX7uGI +.. section: Library + +Fixed :func:`ast.unparse` to handle format_spec with ``"``, ``'`` or ``\\``. +Patched by Frank Hoffmann. + +.. + +.. date: 2024-02-19-16-53-48 +.. gh-issue: 112997 +.. nonce: sYBXRZ +.. section: Library + +Stop logging potentially sensitive callback arguments in :mod:`asyncio` +unless debug mode is active. + +.. + +.. date: 2024-02-19-15-52-30 +.. gh-issue: 114914 +.. nonce: M5-1d8 +.. section: Library + +Fix an issue where an abandoned :class:`StreamWriter` would not be garbage +collected. + +.. + +.. date: 2024-02-18-12-18-12 +.. gh-issue: 111358 +.. nonce: 9yJUMD +.. section: Library + +Fix a bug in :meth:`asyncio.BaseEventLoop.shutdown_default_executor` to +ensure the timeout passed to the coroutine behaves as expected. + +.. + +.. date: 2024-02-17-18-47-12 +.. gh-issue: 115618 +.. nonce: napiNp +.. section: Library + +Fix improper decreasing the reference count for ``None`` argument in +:class:`property` methods :meth:`~property.getter`, :meth:`~property.setter` +and :meth:`~property.deleter`. + +.. + +.. date: 2024-02-16-16-40-10 +.. gh-issue: 112720 +.. nonce: io6_Ac +.. section: Library + +Refactor :class:`dis.ArgResolver` to make it possible to subclass and change +the way jump args are interpreted. + +.. + +.. date: 2024-02-15-23-42-54 +.. gh-issue: 112006 +.. nonce: 4wxcK- +.. section: Library + +Fix :func:`inspect.unwrap` for types with the ``__wrapper__`` data +descriptor. Fix :meth:`inspect.Signature.from_callable` for builtins +:func:`classmethod` and :func:`staticmethod`. + +.. + +.. date: 2024-02-15-19-11-49 +.. gh-issue: 101293 +.. nonce: 898b8l +.. section: Library + +Support callables with the ``__call__()`` method and types with +``__new__()`` and ``__init__()`` methods set to class methods, static +methods, bound methods, partial functions, and other types of methods and +descriptors in :meth:`inspect.Signature.from_callable`. + +.. + +.. date: 2024-02-12-11-42-48 +.. gh-issue: 103092 +.. nonce: sGMKr0 +.. section: Library + +Isolate :mod:`_lsprof` (apply :pep:`687`). + +.. + +.. date: 2024-02-11-20-12-39 +.. gh-issue: 113942 +.. nonce: i72sMJ +.. section: Library + +:mod:`pydoc` no longer skips global functions implemented as builtin +methods, such as :class:`~type.MethodDescriptorType` and +:class:`~type.WrapperDescriptorType`. + +.. + +.. date: 2024-02-10-17-18-49 +.. gh-issue: 115256 +.. nonce: 41Fy9P +.. section: Library + +Added DeprecationWarning when accessing the tarfile attribute of TarInfo +objects. The attribute is never used internally and is only attached to +TarInfos when the tarfile is opened in write-mode, not read-mode. The +attribute creates an unnecessary reference cycle which may cause corruption +when not closing the handle after writing a tarfile. + +.. + +.. date: 2024-02-09-19-41-48 +.. gh-issue: 115197 +.. nonce: 20wkWH +.. section: Library + +``urllib.request`` no longer resolves the hostname before checking it +against the system's proxy bypass list on macOS and Windows. + +.. + +.. date: 2024-02-09-12-22-47 +.. gh-issue: 113812 +.. nonce: wOraaG +.. section: Library + +:meth:`DatagramTransport.sendto` will now send zero-length datagrams if +called with an empty bytes object. The transport flow control also now +accounts for the datagram header when calculating the buffer size. + +.. + +.. date: 2024-01-30-23-28-29 +.. gh-issue: 114763 +.. nonce: BRjKkg +.. section: Library + +Protect modules loaded with :class:`importlib.util.LazyLoader` from race +conditions when multiple threads try to access attributes before the loading +is complete. + +.. + +.. date: 2024-01-29-13-46-41 +.. gh-issue: 114709 +.. nonce: SQ998l +.. section: Library + +:func:`posixpath.commonpath()` now raises a :exc:`ValueError` exception when +passed an empty iterable. Previously, :exc:`IndexError` was raised. + +:func:`posixpath.commonpath()` now raises a :exc:`TypeError` exception when +passed ``None``. Previously, :exc:`ValueError` was raised. + +.. + +.. date: 2024-01-26-16-42-31 +.. gh-issue: 114610 +.. nonce: S18Vuz +.. section: Library + +Fix bug where :meth:`pathlib.PurePath.with_stem` converted a non-empty path +suffix to a stem when given an empty *stem* argument. It now raises +:exc:`ValueError`, just like :meth:`pathlib.PurePath.with_suffix` does when +called on a path with an empty stem, given a non-empty *suffix* argument. + +.. + +.. date: 2023-11-24-23-40-00 +.. gh-issue: 107361 +.. nonce: v54gh46 +.. section: Library + +Add :data:`ssl.VERIFY_X509_PARTIAL_CHAIN` and :data:`VERIFY_X509_STRICT` to +the default SSL context created with :func:`ssl.create_default_context`. + +.. + +.. date: 2023-11-20-16-15-44 +.. gh-issue: 112281 +.. nonce: gH4EVk +.. section: Library + +Allow creating :ref:`union of types` for +:class:`typing.Annotated` with unhashable metadata. + +.. + +.. date: 2023-11-07-10-22-06 +.. gh-issue: 111775 +.. nonce: IoVxfX +.. section: Library + +Fix :meth:`importlib.resources.simple.ResourceHandle.open` for text mode, +added missed ``stream`` argument. + +.. + +.. date: 2023-10-07-06-15-13 +.. gh-issue: 90095 +.. nonce: gWn1ka +.. section: Library + +Make .pdbrc and -c work with any valid pdb commands. + +.. + +.. date: 2023-08-05-08-41-58 +.. gh-issue: 107625 +.. nonce: cVSHCT +.. section: Library + +Raise :exc:`configparser.ParsingError` from +:meth:`~configparser.ConfigParser.read` and +:meth:`~configparser.ConfigParser.read_file` methods of +:class:`configparser.ConfigParser` if a key without a corresponding value is +continued (that is, followed by an indented line). + +.. + +.. date: 2023-08-02-01-17-32 +.. gh-issue: 107155 +.. nonce: Mj1K9L +.. section: Library + +Fix incorrect output of ``help(x)`` where ``x`` is a :keyword:`lambda` +function, which has an ``__annotations__`` dictionary attribute with a +``"return"`` key. + +.. + +.. date: 2023-07-12-14-52-04 +.. gh-issue: 57141 +.. nonce: L2k8Xb +.. section: Library + +Add option for *non-shallow* comparisons to :class:`filecmp.dircmp` like +:func:`filecmp.cmp`. Original patch by Steven Ward. Enhanced by Tobias +Rautenkranz + +.. + +.. date: 2023-05-17-21-33-21 +.. gh-issue: 69990 +.. nonce: Blvz9G +.. section: Library + +:meth:`Profile.print_stats` has been improved to accept multiple sort +arguments. Patched by Chiu-Hsiang Hsu and Furkan Onder. + +.. + +.. date: 2023-05-01-22-28-57 +.. gh-issue: 104061 +.. nonce: vxfBXf +.. section: Library + +Add :data:`socket.SO_BINDTOIFINDEX` constant. + +.. + +.. date: 2023-04-02-21-20-35 +.. gh-issue: 60346 +.. nonce: 7mjgua +.. section: Library + +Fix ArgumentParser inconsistent with parse_known_args. + +.. + +.. date: 2023-03-03-09-05-42 +.. gh-issue: 102389 +.. nonce: ucmo0_ +.. section: Library + +Add ``windows_31j`` to aliases for ``cp932`` codec + +.. + +.. date: 2023-02-14-17-19-59 +.. gh-issue: 72249 +.. nonce: fv35wU +.. section: Library + +:func:`functools.partial`s of :func:`repr` has been improved to include the +:term:`module` name. Patched by Furkan Onder and Anilyka Barry. + +.. + +.. date: 2023-01-12-14-16-01 +.. gh-issue: 100985 +.. nonce: GT5Fvd +.. section: Library + +Update HTTPSConnection to consistently wrap IPv6 Addresses when using a +proxy. + +.. + +.. date: 2023-01-09-14-08-02 +.. gh-issue: 100884 +.. nonce: DcmdLl +.. section: Library + +email: fix misfolding of comma in address-lists over multiple lines in +combination with unicode encoding. + +.. + +.. date: 2022-11-22-23-17-43 +.. gh-issue: 95782 +.. nonce: an_and +.. section: Library + +Fix :func:`io.BufferedReader.tell`, :func:`io.BufferedReader.seek`, +:func:`_pyio.BufferedReader.tell`, :func:`io.BufferedRandom.tell`, +:func:`io.BufferedRandom.seek` and :func:`_pyio.BufferedRandom.tell` being +able to return negative offsets. + +.. + +.. date: 2022-08-26-15-50-53 +.. gh-issue: 96310 +.. nonce: 0NssDh +.. section: Library + +Fix a traceback in :mod:`argparse` when all options in a mutually exclusive +group are suppressed. + +.. + +.. date: 2022-05-25-17-49-04 +.. gh-issue: 93205 +.. nonce: DjhFVR +.. section: Library + +Fixed a bug in :class:`logging.handlers.TimedRotatingFileHandler` where +multiple rotating handler instances pointing to files with the same name but +different extensions would conflict and not delete the correct files. + +.. + +.. bpo: 31116 +.. date: 2022-01-14-10-50-17 +.. nonce: 0bduV9 +.. section: Library + +Add Z85 encoding to ``base64``. + +.. + +.. bpo: 44865 +.. date: 2021-08-24-20-47-37 +.. nonce: c3BhZS +.. section: Library + +Add missing call to localization function in :mod:`argparse`. + +.. + +.. bpo: 43952 +.. date: 2021-05-03-11-04-12 +.. nonce: Me7fJe +.. section: Library + +Fix :meth:`multiprocessing.connection.Listener.accept()` to accept empty +bytes as authkey. Not accepting empty bytes as key causes it to hang +indefinitely. + +.. + +.. bpo: 42125 +.. date: 2020-12-15-22-30-49 +.. nonce: UGyseY +.. section: Library + +linecache: get module name from ``__spec__`` if available. This allows +getting source code for the ``__main__`` module when a custom loader is +used. + +.. + +.. bpo: 41122 +.. date: 2020-07-13-23-59-42 +.. nonce: 8P_Brh +.. section: Library + +Failing to pass arguments properly to :func:`functools.singledispatchmethod` +now throws a TypeError instead of hitting an index out of bounds internally. + +.. + +.. bpo: 40818 +.. date: 2020-05-29-18-08-54 +.. nonce: Ij8ffq +.. section: Library + +The asyncio REPL now runs :data:`sys.__interactivehook__` on startup. The +default implementation of :data:`sys.__interactivehook__` provides +auto-completion to the asyncio REPL. Patch contributed by Rémi Lapeyre. + +.. + +.. bpo: 33775 +.. date: 2019-04-06-23-50-59 +.. nonce: 0yhMDc +.. section: Library + +Add 'default' and 'version' help text for localization in argparse. + +.. + +.. date: 2024-02-14-20-17-04 +.. gh-issue: 115399 +.. nonce: fb9a0R +.. section: Documentation + +Document CVE-2023-52425 of Expat <2.6.0 under "XML vulnerabilities". + +.. + +.. date: 2024-02-08-08-51-37 +.. gh-issue: 109653 +.. nonce: QHLW4w +.. section: Documentation + +Improve import time of :mod:`uuid` on Linux. + +.. + +.. date: 2024-02-25-16-28-26 +.. gh-issue: 71052 +.. nonce: lSb9EC +.. section: Tests + +Add test exclusions to support running the test suite on Android. + +.. + +.. date: 2024-02-25-15-58-28 +.. gh-issue: 71052 +.. nonce: lxBjqY +.. section: Tests + +Enable ``test_concurrent_futures`` on platforms that support threading but +not multiprocessing. + +.. + +.. date: 2024-02-22-00-17-06 +.. gh-issue: 115796 +.. nonce: d4hpKy +.. section: Tests + +Make '_testinternalcapi.assemble_code_object' construct the exception table +for the code object. + +.. + +.. date: 2024-02-20-15-47-41 +.. gh-issue: 115720 +.. nonce: w8i8UG +.. section: Tests + +Leak tests (``-R``, ``--huntrleaks``) now show a summary of the number of +leaks found in each iteration. + +.. + +.. date: 2024-02-18-14-20-52 +.. gh-issue: 115122 +.. nonce: 3rGNo9 +.. section: Tests + +Add ``--bisect`` option to regrtest test runner: run failed tests with +``test.bisect_cmd`` to identify failing tests. Patch by Victor Stinner. + +.. + +.. date: 2024-02-17-08-25-01 +.. gh-issue: 115596 +.. nonce: RGPCrR +.. section: Tests + +Fix ``ProgramPriorityTests`` in ``test_os`` permanently changing the process +priority. + +.. + +.. date: 2024-02-16-13-04-28 +.. gh-issue: 115556 +.. nonce: rjaQ9w +.. section: Tests + +On Windows, commas passed in arguments to ``Tools\buildbot\test.bat`` and +``PCbuild\\rt.bat`` are now properly handled. + +.. + +.. date: 2024-02-13-18-24-04 +.. gh-issue: 115420 +.. nonce: -dlzfI +.. section: Tests + +Fix translation of exception hander targets by +``_testinternalcapi.optimize_cfg``. + +.. + +.. date: 2024-02-12-22-35-01 +.. gh-issue: 115376 +.. nonce: n9vubZ +.. section: Tests + +Fix segfault in ``_testinternalcapi.compiler_codegen`` on bad input. + +.. + +.. date: 2024-03-04-12-43-42 +.. gh-issue: 116313 +.. nonce: cLLb8S +.. section: Build + +Get WASI builds to work under wasmtime 18 w/ WASI 0.2/preview2 primitives. + +.. + +.. date: 2024-03-01-16-44-19 +.. gh-issue: 71052 +.. nonce: Hs-9EP +.. section: Build + +Change Android's :data:`sys.platform` from ``"linux"`` to ``"android"``. + +.. + +.. date: 2024-02-29-15-12-31 +.. gh-issue: 116117 +.. nonce: eENkQK +.. section: Build + +Backport ``libb2``'s PR #42 to fix compiling CPython on 32-bit Windows with +``clang-cl``. + +.. + +.. date: 2024-02-26-14-54-58 +.. gh-issue: 71052 +.. nonce: XvFay1 +.. section: Build + +Fix several Android build issues + +.. + +.. date: 2024-02-26-13-13-53 +.. gh-issue: 114099 +.. nonce: 8lpX-7 +.. section: Build + +A testbed project was added to run the test suite on iOS. + +.. + +.. date: 2024-02-24-12-50-43 +.. gh-issue: 115350 +.. nonce: naQA6y +.. section: Build + +Fix building ctypes module with -DWIN32_LEAN_AND_MEAN defined + +.. + +.. date: 2024-02-21-18-22-49 +.. gh-issue: 111225 +.. nonce: Z8C3av +.. section: Build + +Link extension modules against libpython on Android. + +.. + +.. date: 2024-02-21-11-58-30 +.. gh-issue: 115737 +.. nonce: dpNl2T +.. section: Build + +The install name for libPython is now correctly set for non-framework macOS +builds. + +.. + +.. date: 2024-02-13-14-52-59 +.. gh-issue: 114099 +.. nonce: zjXsQr +.. section: Build + +Makefile targets were added to support compiling an iOS-compatible framework +build. + +.. + +.. date: 2024-02-27-23-21-55 +.. gh-issue: 116012 +.. nonce: B9_IwM +.. section: Windows + +Ensure the value of ``GetLastError()`` is preserved across GIL operations. + +.. + +.. date: 2024-02-23-11-43-43 +.. gh-issue: 115582 +.. nonce: sk1XPi +.. section: Windows + +Building extensions intended for free-threaded builds of CPython now require +compiling with ``/DPy_GIL_DISABLED`` manually when using a regular install. +This is expected to change in future releases. + +.. + +.. date: 2024-02-21-23-48-59 +.. gh-issue: 115554 +.. nonce: 02mpQC +.. section: Windows + +The installer now has more strict rules about updating the :ref:`launcher`. +In general, most users only have a single launcher installed and will see no +difference. When multiple launchers have been installed, the option to +install the launcher is disabled until all but one have been removed. +Downgrading the launcher (which was never allowed) is now more obviously +blocked. + +.. + +.. date: 2024-02-15-23-16-31 +.. gh-issue: 115543 +.. nonce: otrWnw +.. section: Windows + +:ref:`launcher` can now detect Python 3.13 when installed from the Microsoft +Store, and will install Python 3.12 by default when +:envvar:`PYLAUNCHER_ALLOW_INSTALL` is set. + +.. + +.. date: 2024-02-29-20-52-23 +.. gh-issue: 116145 +.. nonce: ygafim +.. section: macOS + +Update macOS installer to Tcl/Tk 8.6.14. + +.. + +.. date: 2023-12-09-11-04-26 +.. gh-issue: 88516 +.. nonce: SIIvfs +.. section: IDLE + +On macOS show a proxy icon in the title bar of editor windows to match +platform behaviour. + +.. + +.. date: 2023-02-12-19-28-08 +.. gh-issue: 100176 +.. nonce: Kzs4Zw +.. section: Tools/Demos + +Remove outdated Tools/{io,cc,string}bench + +.. + +.. bpo: 45101 +.. date: 2021-09-05-02-47-48 +.. nonce: 60Zqmt +.. section: Tools/Demos + +Add consistency in usage message IO between 2 versions of python-config. + +.. + +.. date: 2024-02-16-15-56-53 +.. gh-issue: 114626 +.. nonce: ie2esA +.. section: C API + +Add again ``_PyCFunctionFastWithKeywords`` name, removed in Python 3.13 +alpha 4 by mistake. Keep the old private ``_PyCFunctionFastWithKeywords`` +name (Python 3.7) as an alias to the new public name +``PyCFunctionFastWithKeywords`` (Python 3.13a4). Patch by Victor Stinner. + +.. + +.. date: 2023-11-15-09-24-51 +.. gh-issue: 111418 +.. nonce: FYYetY +.. section: C API + +Add :c:macro:`PyHASH_MODULUS`, :c:macro:`PyHASH_BITS`, :c:macro:`PyHASH_INF` +and :c:macro:`PyHASH_IMAG` C macros. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Build/2024-02-13-14-52-59.gh-issue-114099.zjXsQr.rst b/Misc/NEWS.d/next/Build/2024-02-13-14-52-59.gh-issue-114099.zjXsQr.rst deleted file mode 100644 index e2858bd71d28cb..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-02-13-14-52-59.gh-issue-114099.zjXsQr.rst +++ /dev/null @@ -1,2 +0,0 @@ -Makefile targets were added to support compiling an iOS-compatible framework -build. diff --git a/Misc/NEWS.d/next/Build/2024-02-21-11-58-30.gh-issue-115737.dpNl2T.rst b/Misc/NEWS.d/next/Build/2024-02-21-11-58-30.gh-issue-115737.dpNl2T.rst deleted file mode 100644 index 112f65258dd84b..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-02-21-11-58-30.gh-issue-115737.dpNl2T.rst +++ /dev/null @@ -1,2 +0,0 @@ -The install name for libPython is now correctly set for non-framework macOS -builds. diff --git a/Misc/NEWS.d/next/Build/2024-02-21-18-22-49.gh-issue-111225.Z8C3av.rst b/Misc/NEWS.d/next/Build/2024-02-21-18-22-49.gh-issue-111225.Z8C3av.rst deleted file mode 100644 index 8cdeba46ba2313..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-02-21-18-22-49.gh-issue-111225.Z8C3av.rst +++ /dev/null @@ -1 +0,0 @@ -Link extension modules against libpython on Android. diff --git a/Misc/NEWS.d/next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst b/Misc/NEWS.d/next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst deleted file mode 100644 index 5492a804255838..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst +++ /dev/null @@ -1 +0,0 @@ -Fix building ctypes module with -DWIN32_LEAN_AND_MEAN defined diff --git a/Misc/NEWS.d/next/Build/2024-02-26-13-13-53.gh-issue-114099.8lpX-7.rst b/Misc/NEWS.d/next/Build/2024-02-26-13-13-53.gh-issue-114099.8lpX-7.rst deleted file mode 100644 index 12b6b39a0a2300..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-02-26-13-13-53.gh-issue-114099.8lpX-7.rst +++ /dev/null @@ -1 +0,0 @@ -A testbed project was added to run the test suite on iOS. diff --git a/Misc/NEWS.d/next/Build/2024-02-26-14-54-58.gh-issue-71052.XvFay1.rst b/Misc/NEWS.d/next/Build/2024-02-26-14-54-58.gh-issue-71052.XvFay1.rst deleted file mode 100644 index bda91335814936..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-02-26-14-54-58.gh-issue-71052.XvFay1.rst +++ /dev/null @@ -1 +0,0 @@ -Fix several Android build issues diff --git a/Misc/NEWS.d/next/Build/2024-02-29-15-12-31.gh-issue-116117.eENkQK.rst b/Misc/NEWS.d/next/Build/2024-02-29-15-12-31.gh-issue-116117.eENkQK.rst deleted file mode 100644 index 22477b343c06f0..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-02-29-15-12-31.gh-issue-116117.eENkQK.rst +++ /dev/null @@ -1,2 +0,0 @@ -Backport ``libb2``'s PR #42 to fix compiling CPython on 32-bit Windows -with ``clang-cl``. diff --git a/Misc/NEWS.d/next/Build/2024-03-01-16-44-19.gh-issue-71052.Hs-9EP.rst b/Misc/NEWS.d/next/Build/2024-03-01-16-44-19.gh-issue-71052.Hs-9EP.rst deleted file mode 100644 index 187475f56b4415..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-03-01-16-44-19.gh-issue-71052.Hs-9EP.rst +++ /dev/null @@ -1 +0,0 @@ -Change Android's :data:`sys.platform` from ``"linux"`` to ``"android"``. diff --git a/Misc/NEWS.d/next/Build/2024-03-04-12-43-42.gh-issue-116313.cLLb8S.rst b/Misc/NEWS.d/next/Build/2024-03-04-12-43-42.gh-issue-116313.cLLb8S.rst deleted file mode 100644 index 61501549060024..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-03-04-12-43-42.gh-issue-116313.cLLb8S.rst +++ /dev/null @@ -1 +0,0 @@ -Get WASI builds to work under wasmtime 18 w/ WASI 0.2/preview2 primitives. diff --git a/Misc/NEWS.d/next/C API/2023-11-15-09-24-51.gh-issue-111418.FYYetY.rst b/Misc/NEWS.d/next/C API/2023-11-15-09-24-51.gh-issue-111418.FYYetY.rst deleted file mode 100644 index 5f76ec1443fb44..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-11-15-09-24-51.gh-issue-111418.FYYetY.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :c:macro:`PyHASH_MODULUS`, :c:macro:`PyHASH_BITS`, :c:macro:`PyHASH_INF` -and :c:macro:`PyHASH_IMAG` C macros. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/C API/2024-02-16-15-56-53.gh-issue-114626.ie2esA.rst b/Misc/NEWS.d/next/C API/2024-02-16-15-56-53.gh-issue-114626.ie2esA.rst deleted file mode 100644 index 763f4cee6d3f0b..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-02-16-15-56-53.gh-issue-114626.ie2esA.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add again ``_PyCFunctionFastWithKeywords`` name, removed in Python 3.13 -alpha 4 by mistake. Keep the old private ``_PyCFunctionFastWithKeywords`` -name (Python 3.7) as an alias to the new public name -``PyCFunctionFastWithKeywords`` (Python 3.13a4). Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-04-16-51-56.gh-issue-96497.HTBuIL.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-04-16-51-56.gh-issue-96497.HTBuIL.rst deleted file mode 100644 index 6881dde2e6cf44..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-09-04-16-51-56.gh-issue-96497.HTBuIL.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix incorrect resolution of mangled class variables used in assignment -expressions in comprehensions. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-11-36-50.gh-issue-101860.CKCMbC.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-11-36-50.gh-issue-101860.CKCMbC.rst deleted file mode 100644 index 5a274353466973..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-11-36-50.gh-issue-101860.CKCMbC.rst +++ /dev/null @@ -1 +0,0 @@ -Expose ``__name__`` attribute on property. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst deleted file mode 100644 index 9338f662374c9a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst +++ /dev/null @@ -1,8 +0,0 @@ -Improve the constructors for :mod:`ast` nodes. Arguments of list types now -default to an empty list if omitted, and optional fields default to ``None``. -AST nodes now have an -``__annotations__`` attribute with the expected types of their attributes. -Passing unrecognized extra arguments to AST nodes is deprecated and will -become an error in Python 3.15. Omitting a required argument to an AST node -is deprecated and will become an error in Python 3.15. Patch by Jelle -Zijlstra. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-16-15-02-47.gh-issue-104090.oMjNa9.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-16-15-02-47.gh-issue-104090.oMjNa9.rst deleted file mode 100644 index e581d291d047ae..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-16-15-02-47.gh-issue-104090.oMjNa9.rst +++ /dev/null @@ -1,2 +0,0 @@ -The multiprocessing resource tracker now exits with non-zero status code if a resource -leak was detected. It still exits with status code 0 otherwise. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-28-02-46-12.gh-issue-112433.FUX-nT.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-28-02-46-12.gh-issue-112433.FUX-nT.rst deleted file mode 100644 index fdd11bdf4241b9..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-28-02-46-12.gh-issue-112433.FUX-nT.rst +++ /dev/null @@ -1 +0,0 @@ -Add ability to force alignment of :mod:`ctypes.Structure` by way of the new ``_align_`` attribute on the class. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-08-16-01-18.gh-issue-115154.ji96FV.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-08-16-01-18.gh-issue-115154.ji96FV.rst deleted file mode 100644 index 045596bfcdca43..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-08-16-01-18.gh-issue-115154.ji96FV.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug that was causing the :func:`tokenize.untokenize` function to -handle unicode named literals incorrectly. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-09-18-59-22.gh-issue-112175.qglugr.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-09-18-59-22.gh-issue-112175.qglugr.rst deleted file mode 100644 index 6d919134bf4d9c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-09-18-59-22.gh-issue-112175.qglugr.rst +++ /dev/null @@ -1 +0,0 @@ -Every ``PyThreadState`` now has its own ``eval_breaker``, allowing specific threads to be interrupted. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-12-23-29-17.gh-issue-115323.3t6687.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-12-23-29-17.gh-issue-115323.3t6687.rst deleted file mode 100644 index 171855608fbc6a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-12-23-29-17.gh-issue-115323.3t6687.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make error message more meaningful for when :meth:`bytearray.extend` is -called with a :class:`str` object. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-14-23-50-43.gh-issue-115347.VkHvQC.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-14-23-50-43.gh-issue-115347.VkHvQC.rst deleted file mode 100644 index 16f357a944ed7a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-14-23-50-43.gh-issue-115347.VkHvQC.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix bug where docstring was replaced by a redundant NOP when Python is run -with ``-OO``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-20-12-46-20.gh-issue-115700.KLJ5r4.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-20-12-46-20.gh-issue-115700.KLJ5r4.rst deleted file mode 100644 index 5b7b8e410b5063..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-20-12-46-20.gh-issue-115700.KLJ5r4.rst +++ /dev/null @@ -1 +0,0 @@ -The regen-cases build stage now works on Windows. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-20-18-49-02.gh-issue-115733.51Zb85.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-20-18-49-02.gh-issue-115733.51Zb85.rst deleted file mode 100644 index 5cbb292065b5da..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-20-18-49-02.gh-issue-115733.51Zb85.rst +++ /dev/null @@ -1 +0,0 @@ -Fix crash when calling ``next()`` on exhausted list iterators. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-22-11-33-20.gh-issue-115778.jksd1D.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-22-11-33-20.gh-issue-115778.jksd1D.rst deleted file mode 100644 index 023f6aa3026733..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-22-11-33-20.gh-issue-115778.jksd1D.rst +++ /dev/null @@ -1 +0,0 @@ -Add ``tierN`` annotation for instruction definition in interpreter DSL. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-22-16-17-53.gh-issue-115823.c1TreJ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-22-16-17-53.gh-issue-115823.c1TreJ.rst deleted file mode 100644 index 8cda4c9343d4d7..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-22-16-17-53.gh-issue-115823.c1TreJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Properly calculate error ranges in the parser when raising -:exc:`SyntaxError` exceptions caused by invalid byte sequences. Patch by -Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-04-10-19-51.gh-issue-116296.gvtxyU.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-04-10-19-51.gh-issue-116296.gvtxyU.rst deleted file mode 100644 index 0781e9282205d1..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-04-10-19-51.gh-issue-116296.gvtxyU.rst +++ /dev/null @@ -1 +0,0 @@ -Fix possible refleak in :meth:`!object.__reduce__` internal error handling. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-05-22-00-58.gh-issue-116381.0Nq9iO.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-05-22-00-58.gh-issue-116381.0Nq9iO.rst deleted file mode 100644 index 8b4e63d1d9c9e3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-05-22-00-58.gh-issue-116381.0Nq9iO.rst +++ /dev/null @@ -1 +0,0 @@ -Add specialization for ``CONTAINS_OP``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-09-11-10-53.gh-issue-112087.nbI0Pw.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-09-11-10-53.gh-issue-112087.nbI0Pw.rst deleted file mode 100644 index 07808dc325699b..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-09-11-10-53.gh-issue-112087.nbI0Pw.rst +++ /dev/null @@ -1 +0,0 @@ -:class:`list` is now compatible with the implementation of :pep:`703`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-24-59.gh-issue-116604.LCEzAT.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-24-59.gh-issue-116604.LCEzAT.rst deleted file mode 100644 index 516edfa9e6cedf..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-24-59.gh-issue-116604.LCEzAT.rst +++ /dev/null @@ -1,3 +0,0 @@ -Respect the status of the garbage collector when indirect calls are made via -:c:func:`PyErr_CheckSignals` and the evaluation breaker. Patch by Pablo -Galindo diff --git a/Misc/NEWS.d/next/Documentation/2024-02-08-08-51-37.gh-issue-109653.QHLW4w.rst b/Misc/NEWS.d/next/Documentation/2024-02-08-08-51-37.gh-issue-109653.QHLW4w.rst deleted file mode 100644 index 97e0c8df326d69..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2024-02-08-08-51-37.gh-issue-109653.QHLW4w.rst +++ /dev/null @@ -1 +0,0 @@ -Improve import time of :mod:`uuid` on Linux. diff --git a/Misc/NEWS.d/next/Documentation/2024-02-14-20-17-04.gh-issue-115399.fb9a0R.rst b/Misc/NEWS.d/next/Documentation/2024-02-14-20-17-04.gh-issue-115399.fb9a0R.rst deleted file mode 100644 index 587aea802168bd..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2024-02-14-20-17-04.gh-issue-115399.fb9a0R.rst +++ /dev/null @@ -1 +0,0 @@ -Document CVE-2023-52425 of Expat <2.6.0 under "XML vulnerabilities". diff --git a/Misc/NEWS.d/next/IDLE/2023-12-09-11-04-26.gh-issue-88516.SIIvfs.rst b/Misc/NEWS.d/next/IDLE/2023-12-09-11-04-26.gh-issue-88516.SIIvfs.rst deleted file mode 100644 index b6dea5029bf353..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2023-12-09-11-04-26.gh-issue-88516.SIIvfs.rst +++ /dev/null @@ -1,2 +0,0 @@ -On macOS show a proxy icon in the title bar of editor windows to match -platform behaviour. diff --git a/Misc/NEWS.d/next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst b/Misc/NEWS.d/next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst deleted file mode 100644 index 2a663ac7940dcb..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst +++ /dev/null @@ -1 +0,0 @@ -Add 'default' and 'version' help text for localization in argparse. diff --git a/Misc/NEWS.d/next/Library/2020-05-29-18-08-54.bpo-40818.Ij8ffq.rst b/Misc/NEWS.d/next/Library/2020-05-29-18-08-54.bpo-40818.Ij8ffq.rst deleted file mode 100644 index 27f6a6daa52f53..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-05-29-18-08-54.bpo-40818.Ij8ffq.rst +++ /dev/null @@ -1,3 +0,0 @@ -The asyncio REPL now runs :data:`sys.__interactivehook__` on startup. The -default implementation of :data:`sys.__interactivehook__` provides -auto-completion to the asyncio REPL. Patch contributed by Rémi Lapeyre. diff --git a/Misc/NEWS.d/next/Library/2020-07-13-23-59-42.bpo-41122.8P_Brh.rst b/Misc/NEWS.d/next/Library/2020-07-13-23-59-42.bpo-41122.8P_Brh.rst deleted file mode 100644 index 76568d407449f5..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-07-13-23-59-42.bpo-41122.8P_Brh.rst +++ /dev/null @@ -1,3 +0,0 @@ -Failing to pass arguments properly to :func:`functools.singledispatchmethod` -now throws a TypeError instead of hitting an index out of bounds -internally. diff --git a/Misc/NEWS.d/next/Library/2020-12-15-22-30-49.bpo-42125.UGyseY.rst b/Misc/NEWS.d/next/Library/2020-12-15-22-30-49.bpo-42125.UGyseY.rst deleted file mode 100644 index 49d4462e257702..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-12-15-22-30-49.bpo-42125.UGyseY.rst +++ /dev/null @@ -1,2 +0,0 @@ -linecache: get module name from ``__spec__`` if available. This allows getting -source code for the ``__main__`` module when a custom loader is used. diff --git a/Misc/NEWS.d/next/Library/2021-05-03-11-04-12.bpo-43952.Me7fJe.rst b/Misc/NEWS.d/next/Library/2021-05-03-11-04-12.bpo-43952.Me7fJe.rst deleted file mode 100644 index e164619e44a301..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-03-11-04-12.bpo-43952.Me7fJe.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`multiprocessing.connection.Listener.accept()` to accept empty bytes -as authkey. Not accepting empty bytes as key causes it to hang indefinitely. diff --git a/Misc/NEWS.d/next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst b/Misc/NEWS.d/next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst deleted file mode 100644 index ecdb26cdd6edd6..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst +++ /dev/null @@ -1 +0,0 @@ -Add missing call to localization function in :mod:`argparse`. diff --git a/Misc/NEWS.d/next/Library/2022-01-14-10-50-17.bpo-31116.0bduV9.rst b/Misc/NEWS.d/next/Library/2022-01-14-10-50-17.bpo-31116.0bduV9.rst deleted file mode 100644 index d77a96b442bcbb..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-01-14-10-50-17.bpo-31116.0bduV9.rst +++ /dev/null @@ -1 +0,0 @@ -Add Z85 encoding to ``base64``. diff --git a/Misc/NEWS.d/next/Library/2022-05-25-17-49-04.gh-issue-93205.DjhFVR.rst b/Misc/NEWS.d/next/Library/2022-05-25-17-49-04.gh-issue-93205.DjhFVR.rst deleted file mode 100644 index 4a280b93d93347..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-05-25-17-49-04.gh-issue-93205.DjhFVR.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug in :class:`logging.handlers.TimedRotatingFileHandler` where multiple rotating handler instances pointing to files with the same name but different extensions would conflict and not delete the correct files. diff --git a/Misc/NEWS.d/next/Library/2022-08-26-15-50-53.gh-issue-96310.0NssDh.rst b/Misc/NEWS.d/next/Library/2022-08-26-15-50-53.gh-issue-96310.0NssDh.rst deleted file mode 100644 index f8efb0002e104a..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-26-15-50-53.gh-issue-96310.0NssDh.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a traceback in :mod:`argparse` when all options in a mutually exclusive -group are suppressed. diff --git a/Misc/NEWS.d/next/Library/2022-11-22-23-17-43.gh-issue-95782.an_and.rst b/Misc/NEWS.d/next/Library/2022-11-22-23-17-43.gh-issue-95782.an_and.rst deleted file mode 100644 index 123c3944aa3a3a..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-22-23-17-43.gh-issue-95782.an_and.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix :func:`io.BufferedReader.tell`, :func:`io.BufferedReader.seek`, -:func:`_pyio.BufferedReader.tell`, :func:`io.BufferedRandom.tell`, -:func:`io.BufferedRandom.seek` and :func:`_pyio.BufferedRandom.tell` -being able to return negative offsets. diff --git a/Misc/NEWS.d/next/Library/2023-01-09-14-08-02.gh-issue-100884.DcmdLl.rst b/Misc/NEWS.d/next/Library/2023-01-09-14-08-02.gh-issue-100884.DcmdLl.rst deleted file mode 100644 index 2a388178810835..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-09-14-08-02.gh-issue-100884.DcmdLl.rst +++ /dev/null @@ -1,2 +0,0 @@ -email: fix misfolding of comma in address-lists over multiple lines in -combination with unicode encoding. diff --git a/Misc/NEWS.d/next/Library/2023-01-12-14-16-01.gh-issue-100985.GT5Fvd.rst b/Misc/NEWS.d/next/Library/2023-01-12-14-16-01.gh-issue-100985.GT5Fvd.rst deleted file mode 100644 index 8d8693a5edb3d4..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-12-14-16-01.gh-issue-100985.GT5Fvd.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update HTTPSConnection to consistently wrap IPv6 Addresses when using a -proxy. diff --git a/Misc/NEWS.d/next/Library/2023-02-14-17-19-59.gh-issue-72249.fv35wU.rst b/Misc/NEWS.d/next/Library/2023-02-14-17-19-59.gh-issue-72249.fv35wU.rst deleted file mode 100644 index 10cc5a4e7528dd..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-14-17-19-59.gh-issue-72249.fv35wU.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`functools.partial`s of :func:`repr` has been improved to include the -:term:`module` name. Patched by Furkan Onder and Anilyka Barry. diff --git a/Misc/NEWS.d/next/Library/2023-03-03-09-05-42.gh-issue-102389.ucmo0_.rst b/Misc/NEWS.d/next/Library/2023-03-03-09-05-42.gh-issue-102389.ucmo0_.rst deleted file mode 100644 index 8c11567d79ba7b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-03-09-05-42.gh-issue-102389.ucmo0_.rst +++ /dev/null @@ -1 +0,0 @@ -Add ``windows_31j`` to aliases for ``cp932`` codec diff --git a/Misc/NEWS.d/next/Library/2023-04-02-21-20-35.gh-issue-60346.7mjgua.rst b/Misc/NEWS.d/next/Library/2023-04-02-21-20-35.gh-issue-60346.7mjgua.rst deleted file mode 100644 index c15bd6ed11d17f..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-02-21-20-35.gh-issue-60346.7mjgua.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ArgumentParser inconsistent with parse_known_args. diff --git a/Misc/NEWS.d/next/Library/2023-05-01-22-28-57.gh-issue-104061.vxfBXf.rst b/Misc/NEWS.d/next/Library/2023-05-01-22-28-57.gh-issue-104061.vxfBXf.rst deleted file mode 100644 index e15a811f904352..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-01-22-28-57.gh-issue-104061.vxfBXf.rst +++ /dev/null @@ -1 +0,0 @@ -Add :data:`socket.SO_BINDTOIFINDEX` constant. diff --git a/Misc/NEWS.d/next/Library/2023-05-17-21-33-21.gh-issue-69990.Blvz9G.rst b/Misc/NEWS.d/next/Library/2023-05-17-21-33-21.gh-issue-69990.Blvz9G.rst deleted file mode 100644 index b0cdf44f7b9e39..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-17-21-33-21.gh-issue-69990.Blvz9G.rst +++ /dev/null @@ -1 +0,0 @@ -:meth:`Profile.print_stats` has been improved to accept multiple sort arguments. Patched by Chiu-Hsiang Hsu and Furkan Onder. diff --git a/Misc/NEWS.d/next/Library/2023-07-12-14-52-04.gh-issue-57141.L2k8Xb.rst b/Misc/NEWS.d/next/Library/2023-07-12-14-52-04.gh-issue-57141.L2k8Xb.rst deleted file mode 100644 index b8a1236ec3edb0..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-07-12-14-52-04.gh-issue-57141.L2k8Xb.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add option for *non-shallow* comparisons to :class:`filecmp.dircmp` like -:func:`filecmp.cmp`. Original patch by Steven Ward. Enhanced by -Tobias Rautenkranz diff --git a/Misc/NEWS.d/next/Library/2023-08-02-01-17-32.gh-issue-107155.Mj1K9L.rst b/Misc/NEWS.d/next/Library/2023-08-02-01-17-32.gh-issue-107155.Mj1K9L.rst deleted file mode 100644 index 8362dc0fcfaa74..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-08-02-01-17-32.gh-issue-107155.Mj1K9L.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix incorrect output of ``help(x)`` where ``x`` is a :keyword:`lambda` -function, which has an ``__annotations__`` dictionary attribute with a -``"return"`` key. diff --git a/Misc/NEWS.d/next/Library/2023-08-05-08-41-58.gh-issue-107625.cVSHCT.rst b/Misc/NEWS.d/next/Library/2023-08-05-08-41-58.gh-issue-107625.cVSHCT.rst deleted file mode 100644 index bf779c454e6388..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-08-05-08-41-58.gh-issue-107625.cVSHCT.rst +++ /dev/null @@ -1,4 +0,0 @@ -Raise :exc:`configparser.ParsingError` from :meth:`~configparser.ConfigParser.read` -and :meth:`~configparser.ConfigParser.read_file` methods of -:class:`configparser.ConfigParser` if a key without a corresponding value -is continued (that is, followed by an indented line). diff --git a/Misc/NEWS.d/next/Library/2023-10-07-06-15-13.gh-issue-90095.gWn1ka.rst b/Misc/NEWS.d/next/Library/2023-10-07-06-15-13.gh-issue-90095.gWn1ka.rst deleted file mode 100644 index d71442ef642b6a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-10-07-06-15-13.gh-issue-90095.gWn1ka.rst +++ /dev/null @@ -1 +0,0 @@ -Make .pdbrc and -c work with any valid pdb commands. diff --git a/Misc/NEWS.d/next/Library/2023-11-07-10-22-06.gh-issue-111775.IoVxfX.rst b/Misc/NEWS.d/next/Library/2023-11-07-10-22-06.gh-issue-111775.IoVxfX.rst deleted file mode 100644 index 2a3bdd640ea67d..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-07-10-22-06.gh-issue-111775.IoVxfX.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`importlib.resources.simple.ResourceHandle.open` for text mode, -added missed ``stream`` argument. diff --git a/Misc/NEWS.d/next/Library/2023-11-20-16-15-44.gh-issue-112281.gH4EVk.rst b/Misc/NEWS.d/next/Library/2023-11-20-16-15-44.gh-issue-112281.gH4EVk.rst deleted file mode 100644 index 01f6689bb471cd..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-20-16-15-44.gh-issue-112281.gH4EVk.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allow creating :ref:`union of types` for -:class:`typing.Annotated` with unhashable metadata. diff --git a/Misc/NEWS.d/next/Library/2023-11-24-23-40-00.gh-issue-107361.v54gh46.rst b/Misc/NEWS.d/next/Library/2023-11-24-23-40-00.gh-issue-107361.v54gh46.rst deleted file mode 100644 index 731c5d998854d7..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-24-23-40-00.gh-issue-107361.v54gh46.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :data:`ssl.VERIFY_X509_PARTIAL_CHAIN` and :data:`VERIFY_X509_STRICT` -to the default SSL context created with :func:`ssl.create_default_context`. diff --git a/Misc/NEWS.d/next/Library/2024-01-26-16-42-31.gh-issue-114610.S18Vuz.rst b/Misc/NEWS.d/next/Library/2024-01-26-16-42-31.gh-issue-114610.S18Vuz.rst deleted file mode 100644 index 519aede72aaf29..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-26-16-42-31.gh-issue-114610.S18Vuz.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix bug where :meth:`pathlib.PurePath.with_stem` converted a non-empty path -suffix to a stem when given an empty *stem* argument. It now raises -:exc:`ValueError`, just like :meth:`pathlib.PurePath.with_suffix` does when -called on a path with an empty stem, given a non-empty *suffix* argument. diff --git a/Misc/NEWS.d/next/Library/2024-01-29-13-46-41.gh-issue-114709.SQ998l.rst b/Misc/NEWS.d/next/Library/2024-01-29-13-46-41.gh-issue-114709.SQ998l.rst deleted file mode 100644 index ca0d7902c73d1c..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-29-13-46-41.gh-issue-114709.SQ998l.rst +++ /dev/null @@ -1,5 +0,0 @@ -:func:`posixpath.commonpath()` now raises a :exc:`ValueError` exception when -passed an empty iterable. Previously, :exc:`IndexError` was raised. - -:func:`posixpath.commonpath()` now raises a :exc:`TypeError` exception when -passed ``None``. Previously, :exc:`ValueError` was raised. diff --git a/Misc/NEWS.d/next/Library/2024-01-30-23-28-29.gh-issue-114763.BRjKkg.rst b/Misc/NEWS.d/next/Library/2024-01-30-23-28-29.gh-issue-114763.BRjKkg.rst deleted file mode 100644 index e8bdb83dde61fb..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-30-23-28-29.gh-issue-114763.BRjKkg.rst +++ /dev/null @@ -1,3 +0,0 @@ -Protect modules loaded with :class:`importlib.util.LazyLoader` from race -conditions when multiple threads try to access attributes before the loading -is complete. diff --git a/Misc/NEWS.d/next/Library/2024-02-09-12-22-47.gh-issue-113812.wOraaG.rst b/Misc/NEWS.d/next/Library/2024-02-09-12-22-47.gh-issue-113812.wOraaG.rst deleted file mode 100644 index 7ef7bc891cd885..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-09-12-22-47.gh-issue-113812.wOraaG.rst +++ /dev/null @@ -1,3 +0,0 @@ -:meth:`DatagramTransport.sendto` will now send zero-length datagrams if -called with an empty bytes object. The transport flow control also now -accounts for the datagram header when calculating the buffer size. diff --git a/Misc/NEWS.d/next/Library/2024-02-09-19-41-48.gh-issue-115197.20wkWH.rst b/Misc/NEWS.d/next/Library/2024-02-09-19-41-48.gh-issue-115197.20wkWH.rst deleted file mode 100644 index e6ca3cc525d74a..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-09-19-41-48.gh-issue-115197.20wkWH.rst +++ /dev/null @@ -1,2 +0,0 @@ -``urllib.request`` no longer resolves the hostname before checking it -against the system's proxy bypass list on macOS and Windows. diff --git a/Misc/NEWS.d/next/Library/2024-02-10-17-18-49.gh-issue-115256.41Fy9P.rst b/Misc/NEWS.d/next/Library/2024-02-10-17-18-49.gh-issue-115256.41Fy9P.rst deleted file mode 100644 index 8cde053d862298..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-10-17-18-49.gh-issue-115256.41Fy9P.rst +++ /dev/null @@ -1,5 +0,0 @@ -Added DeprecationWarning when accessing the tarfile attribute of TarInfo -objects. The attribute is never used internally and is only attached to -TarInfos when the tarfile is opened in write-mode, not read-mode. The -attribute creates an unnecessary reference cycle which may cause -corruption when not closing the handle after writing a tarfile. diff --git a/Misc/NEWS.d/next/Library/2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst b/Misc/NEWS.d/next/Library/2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst deleted file mode 100644 index 2da43a43b78798..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`pydoc` no longer skips global functions implemented as builtin methods, -such as :class:`~type.MethodDescriptorType` and :class:`~type.WrapperDescriptorType`. diff --git a/Misc/NEWS.d/next/Library/2024-02-12-11-42-48.gh-issue-103092.sGMKr0.rst b/Misc/NEWS.d/next/Library/2024-02-12-11-42-48.gh-issue-103092.sGMKr0.rst deleted file mode 100644 index 47701396c81737..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-12-11-42-48.gh-issue-103092.sGMKr0.rst +++ /dev/null @@ -1 +0,0 @@ -Isolate :mod:`_lsprof` (apply :pep:`687`). diff --git a/Misc/NEWS.d/next/Library/2024-02-15-19-11-49.gh-issue-101293.898b8l.rst b/Misc/NEWS.d/next/Library/2024-02-15-19-11-49.gh-issue-101293.898b8l.rst deleted file mode 100644 index 98365d2edbc4b5..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-15-19-11-49.gh-issue-101293.898b8l.rst +++ /dev/null @@ -1,4 +0,0 @@ -Support callables with the ``__call__()`` method and types with -``__new__()`` and ``__init__()`` methods set to class methods, static -methods, bound methods, partial functions, and other types of methods and -descriptors in :meth:`inspect.Signature.from_callable`. diff --git a/Misc/NEWS.d/next/Library/2024-02-15-23-42-54.gh-issue-112006.4wxcK-.rst b/Misc/NEWS.d/next/Library/2024-02-15-23-42-54.gh-issue-112006.4wxcK-.rst deleted file mode 100644 index 32af2bd24e54f2..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-15-23-42-54.gh-issue-112006.4wxcK-.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :func:`inspect.unwrap` for types with the ``__wrapper__`` data -descriptor. Fix :meth:`inspect.Signature.from_callable` for builtins -:func:`classmethod` and :func:`staticmethod`. diff --git a/Misc/NEWS.d/next/Library/2024-02-16-16-40-10.gh-issue-112720.io6_Ac.rst b/Misc/NEWS.d/next/Library/2024-02-16-16-40-10.gh-issue-112720.io6_Ac.rst deleted file mode 100644 index 32916ede4dee35..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-16-16-40-10.gh-issue-112720.io6_Ac.rst +++ /dev/null @@ -1,2 +0,0 @@ -Refactor :class:`dis.ArgResolver` to make it possible to subclass and change -the way jump args are interpreted. diff --git a/Misc/NEWS.d/next/Library/2024-02-17-18-47-12.gh-issue-115618.napiNp.rst b/Misc/NEWS.d/next/Library/2024-02-17-18-47-12.gh-issue-115618.napiNp.rst deleted file mode 100644 index cb4b147d5dc663..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-17-18-47-12.gh-issue-115618.napiNp.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix improper decreasing the reference count for ``None`` argument in -:class:`property` methods :meth:`~property.getter`, :meth:`~property.setter` -and :meth:`~property.deleter`. diff --git a/Misc/NEWS.d/next/Library/2024-02-18-12-18-12.gh-issue-111358.9yJUMD.rst b/Misc/NEWS.d/next/Library/2024-02-18-12-18-12.gh-issue-111358.9yJUMD.rst deleted file mode 100644 index 2e895f8f181ce7..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-18-12-18-12.gh-issue-111358.9yJUMD.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug in :meth:`asyncio.BaseEventLoop.shutdown_default_executor` to -ensure the timeout passed to the coroutine behaves as expected. diff --git a/Misc/NEWS.d/next/Library/2024-02-19-15-52-30.gh-issue-114914.M5-1d8.rst b/Misc/NEWS.d/next/Library/2024-02-19-15-52-30.gh-issue-114914.M5-1d8.rst deleted file mode 100644 index f7d392c8bceea3..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-19-15-52-30.gh-issue-114914.M5-1d8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an issue where an abandoned :class:`StreamWriter` would not be garbage -collected. diff --git a/Misc/NEWS.d/next/Library/2024-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst b/Misc/NEWS.d/next/Library/2024-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst deleted file mode 100644 index 4f97b2d6085ba0..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Stop logging potentially sensitive callback arguments in :mod:`asyncio` -unless debug mode is active. diff --git a/Misc/NEWS.d/next/Library/2024-02-20-07-38-15.gh-issue-112364.EX7uGI.rst b/Misc/NEWS.d/next/Library/2024-02-20-07-38-15.gh-issue-112364.EX7uGI.rst deleted file mode 100644 index 6af71e60ec2a8e..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-20-07-38-15.gh-issue-112364.EX7uGI.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed :func:`ast.unparse` to handle format_spec with ``"``, ``'`` or ``\\``. Patched by Frank Hoffmann. diff --git a/Misc/NEWS.d/next/Library/2024-02-20-16-42-54.gh-issue-115712.EXVMXw.rst b/Misc/NEWS.d/next/Library/2024-02-20-16-42-54.gh-issue-115712.EXVMXw.rst deleted file mode 100644 index 8b19064dba779d..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-20-16-42-54.gh-issue-115712.EXVMXw.rst +++ /dev/null @@ -1,4 +0,0 @@ -Restore support of space delimiter with ``skipinitialspace=True`` in -:mod:`csv`. :func:`csv.writer()` now quotes empty fields if delimiter is a -space and skipinitialspace is true and raises exception if quoting is not -possible. diff --git a/Misc/NEWS.d/next/Library/2024-02-20-22-02-34.gh-issue-67044.QF9_Ru.rst b/Misc/NEWS.d/next/Library/2024-02-20-22-02-34.gh-issue-67044.QF9_Ru.rst deleted file mode 100644 index 095e69b6cadab6..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-20-22-02-34.gh-issue-67044.QF9_Ru.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`csv.writer` now always quotes or escapes ``'\r'`` and ``'\n'``, -regardless of *lineterminator* value. diff --git a/Misc/NEWS.d/next/Library/2024-02-21-17-54-59.gh-issue-74668.JT-Q8W.rst b/Misc/NEWS.d/next/Library/2024-02-21-17-54-59.gh-issue-74668.JT-Q8W.rst deleted file mode 100644 index f4a6e6d7b51b41..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-21-17-54-59.gh-issue-74668.JT-Q8W.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`urllib.parse` functions :func:`~urllib.parse.parse_qs` and -:func:`~urllib.parse.parse_qsl` now support bytes arguments containing raw -and percent-encoded non-ASCII data. diff --git a/Misc/NEWS.d/next/Library/2024-02-22-11-29-27.gh-issue-115809.9H1DhB.rst b/Misc/NEWS.d/next/Library/2024-02-22-11-29-27.gh-issue-115809.9H1DhB.rst deleted file mode 100644 index 2a47efbae5c84d..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-22-11-29-27.gh-issue-115809.9H1DhB.rst +++ /dev/null @@ -1,4 +0,0 @@ -Improve algorithm for computing which rolled-over log files to delete in -:class:`logging.TimedRotatingFileHandler`. It is now reliable for handlers -without ``namer`` and with arbitrary deterministic ``namer`` that leaves the -datetime part in the file name unmodified. diff --git a/Misc/NEWS.d/next/Library/2024-02-22-12-10-18.gh-issue-115714.P2JsU1.rst b/Misc/NEWS.d/next/Library/2024-02-22-12-10-18.gh-issue-115714.P2JsU1.rst deleted file mode 100644 index fb626344c87fdb..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-22-12-10-18.gh-issue-115714.P2JsU1.rst +++ /dev/null @@ -1,4 +0,0 @@ -On WASI, the :mod:`time` module no longer get process time using ``times()`` -or ``CLOCK_PROCESS_CPUTIME_ID``, system API is that is unreliable and is -likely to be removed from WASI. The affected clock functions fall back to -calling ``clock()``. diff --git a/Misc/NEWS.d/next/Library/2024-02-23-11-08-31.gh-issue-115532.zVd3gK.rst b/Misc/NEWS.d/next/Library/2024-02-23-11-08-31.gh-issue-115532.zVd3gK.rst deleted file mode 100644 index fb36c0b2a4fabd..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-23-11-08-31.gh-issue-115532.zVd3gK.rst +++ /dev/null @@ -1 +0,0 @@ -Add kernel density estimation to the statistics module. diff --git a/Misc/NEWS.d/next/Library/2024-02-24-18-48-14.gh-issue-115886.rgM6AF.rst b/Misc/NEWS.d/next/Library/2024-02-24-18-48-14.gh-issue-115886.rgM6AF.rst deleted file mode 100644 index 9688f713d5ba7b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-24-18-48-14.gh-issue-115886.rgM6AF.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix silent truncation of the name with an embedded null character in -:class:`multiprocessing.shared_memory.SharedMemory`. diff --git a/Misc/NEWS.d/next/Library/2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst b/Misc/NEWS.d/next/Library/2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst deleted file mode 100644 index 99bccb265ff80c..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix issue where :func:`ast.parse` would incorrectly flag conditional context -managers (such as ``with (x() if y else z()): ...``) as invalid syntax if -``feature_version=(3, 8)`` was passed. This reverts changes to the -grammar made as part of gh-94949. diff --git a/Misc/NEWS.d/next/Library/2024-02-27-13-05-51.gh-issue-75988.In6LlB.rst b/Misc/NEWS.d/next/Library/2024-02-27-13-05-51.gh-issue-75988.In6LlB.rst deleted file mode 100644 index 682b7cfa06b868..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-27-13-05-51.gh-issue-75988.In6LlB.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed :func:`unittest.mock.create_autospec` to pass the call through to the wrapped object to return the real result. diff --git a/Misc/NEWS.d/next/Library/2024-02-27-20-11-29.gh-issue-85644.3rgcBm.rst b/Misc/NEWS.d/next/Library/2024-02-27-20-11-29.gh-issue-85644.3rgcBm.rst deleted file mode 100644 index 818f7046229878..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-27-20-11-29.gh-issue-85644.3rgcBm.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use the ``XDG_CURRENT_DESKTOP`` environment variable in :mod:`webbrowser` to check desktop. -Prefer it to the deprecated ``GNOME_DESKTOP_SESSION_ID`` for GNOME detection. diff --git a/Misc/NEWS.d/next/Library/2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst b/Misc/NEWS.d/next/Library/2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst deleted file mode 100644 index 7512a09a37cd46..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst +++ /dev/null @@ -1,2 +0,0 @@ -[Enum] Improve error message when calling super().__new__() in custom -__new__. diff --git a/Misc/NEWS.d/next/Library/2024-02-28-13-10-17.gh-issue-116040.wDidHd.rst b/Misc/NEWS.d/next/Library/2024-02-28-13-10-17.gh-issue-116040.wDidHd.rst deleted file mode 100644 index 907b58b3a5c206..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-28-13-10-17.gh-issue-116040.wDidHd.rst +++ /dev/null @@ -1 +0,0 @@ -[Enum] fix by-value calls when second value is falsey; e.g. Cardinal(1, 0) diff --git a/Misc/NEWS.d/next/Library/2024-02-28-17-04-28.gh-issue-65824.gG8KR1.rst b/Misc/NEWS.d/next/Library/2024-02-28-17-04-28.gh-issue-65824.gG8KR1.rst deleted file mode 100644 index 7bc6ced120a7be..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-28-17-04-28.gh-issue-65824.gG8KR1.rst +++ /dev/null @@ -1 +0,0 @@ -Improve the ``less`` prompt in :mod:`pydoc`. diff --git a/Misc/NEWS.d/next/Library/2024-02-28-17-50-42.gh-issue-89547.GetF38.rst b/Misc/NEWS.d/next/Library/2024-02-28-17-50-42.gh-issue-89547.GetF38.rst deleted file mode 100644 index 7be4591b83a8f3..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-28-17-50-42.gh-issue-89547.GetF38.rst +++ /dev/null @@ -1 +0,0 @@ -Add support for nested typing special forms like Final[ClassVar[int]]. diff --git a/Misc/NEWS.d/next/Library/2024-02-29-17-06-54.gh-issue-76511.WqjRLP.rst b/Misc/NEWS.d/next/Library/2024-02-29-17-06-54.gh-issue-76511.WqjRLP.rst deleted file mode 100644 index da62f8a2450711..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-29-17-06-54.gh-issue-76511.WqjRLP.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix UnicodeEncodeError in :meth:`email.Message.as_string` that results when -a message that claims to be in the ascii character set actually has non-ascii -characters. Non-ascii characters are now replaced with the U+FFFD replacement -character, like in the ``replace`` error handler. diff --git a/Misc/NEWS.d/next/Library/2024-02-29-20-06-06.gh-issue-87115.FVMiOR.rst b/Misc/NEWS.d/next/Library/2024-02-29-20-06-06.gh-issue-87115.FVMiOR.rst deleted file mode 100644 index 844340583cd456..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-29-20-06-06.gh-issue-87115.FVMiOR.rst +++ /dev/null @@ -1 +0,0 @@ -Set ``__main__.__spec__`` to ``None`` when running a script with :mod:`pdb` diff --git a/Misc/NEWS.d/next/Library/2024-03-01-11-57-32.gh-issue-88352.bZ68rw.rst b/Misc/NEWS.d/next/Library/2024-03-01-11-57-32.gh-issue-88352.bZ68rw.rst deleted file mode 100644 index 8ad4ff7cb52414..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-01-11-57-32.gh-issue-88352.bZ68rw.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fix the computation of the next rollover time in the -:class:`logging.TimedRotatingFileHandler` handler. :meth:`!computeRollover` -now always returns a timestamp larger than the specified time and works -correctly during the DST change. :meth:`!doRollover` no longer overwrite the -already rolled over file, saving from data loss when run at midnight or -during repeated time at the DST change. diff --git a/Misc/NEWS.d/next/Library/2024-03-01-14-22-08.gh-issue-115978.r2ePTo.rst b/Misc/NEWS.d/next/Library/2024-03-01-14-22-08.gh-issue-115978.r2ePTo.rst deleted file mode 100644 index 2adac31ac6c21d..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-01-14-22-08.gh-issue-115978.r2ePTo.rst +++ /dev/null @@ -1,4 +0,0 @@ -Disable preadv(), readv(), pwritev(), and writev() on WASI. - -Under wasmtime for WASI 0.2, these functions don't pass test_posix -(https://github.com/bytecodealliance/wasmtime/issues/7830). diff --git a/Misc/NEWS.d/next/Library/2024-03-02-11-31-49.gh-issue-115957.C-3Z_U.rst b/Misc/NEWS.d/next/Library/2024-03-02-11-31-49.gh-issue-115957.C-3Z_U.rst deleted file mode 100644 index 72988cc63b03cf..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-02-11-31-49.gh-issue-115957.C-3Z_U.rst +++ /dev/null @@ -1 +0,0 @@ -When ``asyncio.TaskGroup.create_task`` is called on an inactive ``asyncio.TaskGroup``, the given coroutine will be closed (which prevents a ``RuntimeWarning``). diff --git a/Misc/NEWS.d/next/Library/2024-03-05-02-09-18.gh-issue-116325.FmlBYv.rst b/Misc/NEWS.d/next/Library/2024-03-05-02-09-18.gh-issue-116325.FmlBYv.rst deleted file mode 100644 index aec4ee79b59cf2..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-05-02-09-18.gh-issue-116325.FmlBYv.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`typing`: raise :exc:`SyntaxError` instead of :exc:`AttributeError` -on forward references as empty strings. diff --git a/Misc/NEWS.d/next/Library/2024-03-05-14-34-22.gh-issue-116127.5uktu3.rst b/Misc/NEWS.d/next/Library/2024-03-05-14-34-22.gh-issue-116127.5uktu3.rst deleted file mode 100644 index 59edde9811e2f5..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-05-14-34-22.gh-issue-116127.5uktu3.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`typing`: implement :pep:`705` which adds :data:`typing.ReadOnly` -support to :class:`typing.TypedDict`. diff --git a/Misc/NEWS.d/next/Library/2024-03-05-20-53-34.gh-issue-116143.sww6Zl.rst b/Misc/NEWS.d/next/Library/2024-03-05-20-53-34.gh-issue-116143.sww6Zl.rst deleted file mode 100644 index 07aa312ee25f3b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-05-20-53-34.gh-issue-116143.sww6Zl.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a race in pydoc ``_start_server``, eliminating a window in which -``_start_server`` can return a thread that is "serving" but without a -``docserver`` set. diff --git a/Misc/NEWS.d/next/Library/2024-03-07-21-57-50.gh-issue-116349.fD2pbP.rst b/Misc/NEWS.d/next/Library/2024-03-07-21-57-50.gh-issue-116349.fD2pbP.rst deleted file mode 100644 index 89eb4197348410..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-07-21-57-50.gh-issue-116349.fD2pbP.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`platform.java_ver` is deprecated and will be removed in 3.15. -It was largely untested, had a confusing API, -and was only useful for Jython support. diff --git a/Misc/NEWS.d/next/Library/2024-03-11-12-11-10.gh-issue-116600.FcNBy_.rst b/Misc/NEWS.d/next/Library/2024-03-11-12-11-10.gh-issue-116600.FcNBy_.rst deleted file mode 100644 index e9148ba9290e7b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-11-12-11-10.gh-issue-116600.FcNBy_.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :func:`repr` for global :class:`~enum.Flag` members. diff --git a/Misc/NEWS.d/next/Security/2024-01-26-22-14-09.gh-issue-114572.t1QMQD.rst b/Misc/NEWS.d/next/Security/2024-01-26-22-14-09.gh-issue-114572.t1QMQD.rst deleted file mode 100644 index b4f9fe64db0615..00000000000000 --- a/Misc/NEWS.d/next/Security/2024-01-26-22-14-09.gh-issue-114572.t1QMQD.rst +++ /dev/null @@ -1,4 +0,0 @@ -:meth:`ssl.SSLContext.cert_store_stats` and -:meth:`ssl.SSLContext.get_ca_certs` now correctly lock access to the -certificate store, when the :class:`ssl.SSLContext` is shared across -multiple threads. diff --git a/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst b/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst deleted file mode 100644 index a40fcd35ef99ae..00000000000000 --- a/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst +++ /dev/null @@ -1,8 +0,0 @@ -Allow controlling Expat >=2.6.0 reparse deferral (CVE-2023-52425) by adding -five new methods: - -* :meth:`xml.etree.ElementTree.XMLParser.flush` -* :meth:`xml.etree.ElementTree.XMLPullParser.flush` -* :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` -* :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` -* :meth:`xml.sax.expatreader.ExpatParser.flush` diff --git a/Misc/NEWS.d/next/Tests/2024-02-12-22-35-01.gh-issue-115376.n9vubZ.rst b/Misc/NEWS.d/next/Tests/2024-02-12-22-35-01.gh-issue-115376.n9vubZ.rst deleted file mode 100644 index e09d78a9c4b189..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-12-22-35-01.gh-issue-115376.n9vubZ.rst +++ /dev/null @@ -1 +0,0 @@ -Fix segfault in ``_testinternalcapi.compiler_codegen`` on bad input. diff --git a/Misc/NEWS.d/next/Tests/2024-02-13-18-24-04.gh-issue-115420.-dlzfI.rst b/Misc/NEWS.d/next/Tests/2024-02-13-18-24-04.gh-issue-115420.-dlzfI.rst deleted file mode 100644 index 1442ada3490fa0..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-13-18-24-04.gh-issue-115420.-dlzfI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix translation of exception hander targets by -``_testinternalcapi.optimize_cfg``. diff --git a/Misc/NEWS.d/next/Tests/2024-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst b/Misc/NEWS.d/next/Tests/2024-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst deleted file mode 100644 index c2811b133d9314..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst +++ /dev/null @@ -1,2 +0,0 @@ -On Windows, commas passed in arguments to ``Tools\buildbot\test.bat`` and -``PCbuild\\rt.bat`` are now properly handled. diff --git a/Misc/NEWS.d/next/Tests/2024-02-17-08-25-01.gh-issue-115596.RGPCrR.rst b/Misc/NEWS.d/next/Tests/2024-02-17-08-25-01.gh-issue-115596.RGPCrR.rst deleted file mode 100644 index 2bcb8b9ac6bcd4..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-17-08-25-01.gh-issue-115596.RGPCrR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``ProgramPriorityTests`` in ``test_os`` permanently changing the process -priority. diff --git a/Misc/NEWS.d/next/Tests/2024-02-18-14-20-52.gh-issue-115122.3rGNo9.rst b/Misc/NEWS.d/next/Tests/2024-02-18-14-20-52.gh-issue-115122.3rGNo9.rst deleted file mode 100644 index e187a40a40516b..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-18-14-20-52.gh-issue-115122.3rGNo9.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add ``--bisect`` option to regrtest test runner: run failed tests with -``test.bisect_cmd`` to identify failing tests. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2024-02-20-15-47-41.gh-issue-115720.w8i8UG.rst b/Misc/NEWS.d/next/Tests/2024-02-20-15-47-41.gh-issue-115720.w8i8UG.rst deleted file mode 100644 index a03ee11d974251..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-20-15-47-41.gh-issue-115720.w8i8UG.rst +++ /dev/null @@ -1,2 +0,0 @@ -Leak tests (``-R``, ``--huntrleaks``) now show a summary of the number of -leaks found in each iteration. diff --git a/Misc/NEWS.d/next/Tests/2024-02-22-00-17-06.gh-issue-115796.d4hpKy.rst b/Misc/NEWS.d/next/Tests/2024-02-22-00-17-06.gh-issue-115796.d4hpKy.rst deleted file mode 100644 index a40be74f73908e..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-22-00-17-06.gh-issue-115796.d4hpKy.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make '_testinternalcapi.assemble_code_object' construct the exception table -for the code object. diff --git a/Misc/NEWS.d/next/Tests/2024-02-25-15-58-28.gh-issue-71052.lxBjqY.rst b/Misc/NEWS.d/next/Tests/2024-02-25-15-58-28.gh-issue-71052.lxBjqY.rst deleted file mode 100644 index 8bac68b5aea78e..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-25-15-58-28.gh-issue-71052.lxBjqY.rst +++ /dev/null @@ -1,2 +0,0 @@ -Enable ``test_concurrent_futures`` on platforms that support threading but not -multiprocessing. diff --git a/Misc/NEWS.d/next/Tests/2024-02-25-16-28-26.gh-issue-71052.lSb9EC.rst b/Misc/NEWS.d/next/Tests/2024-02-25-16-28-26.gh-issue-71052.lSb9EC.rst deleted file mode 100644 index 9d3467ca7e7d40..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-25-16-28-26.gh-issue-71052.lSb9EC.rst +++ /dev/null @@ -1 +0,0 @@ -Add test exclusions to support running the test suite on Android. diff --git a/Misc/NEWS.d/next/Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst b/Misc/NEWS.d/next/Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst deleted file mode 100644 index 48a09da7822915..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst +++ /dev/null @@ -1 +0,0 @@ -Add consistency in usage message IO between 2 versions of python-config. diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-02-12-19-28-08.gh-issue-100176.Kzs4Zw.rst b/Misc/NEWS.d/next/Tools-Demos/2023-02-12-19-28-08.gh-issue-100176.Kzs4Zw.rst deleted file mode 100644 index 1a9fc76d93f297..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2023-02-12-19-28-08.gh-issue-100176.Kzs4Zw.rst +++ /dev/null @@ -1 +0,0 @@ -Remove outdated Tools/{io,cc,string}bench diff --git a/Misc/NEWS.d/next/Windows/2024-02-15-23-16-31.gh-issue-115543.otrWnw.rst b/Misc/NEWS.d/next/Windows/2024-02-15-23-16-31.gh-issue-115543.otrWnw.rst deleted file mode 100644 index ebd15c83b83491..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-02-15-23-16-31.gh-issue-115543.otrWnw.rst +++ /dev/null @@ -1,3 +0,0 @@ -:ref:`launcher` can now detect Python 3.13 when installed from the Microsoft -Store, and will install Python 3.12 by default when -:envvar:`PYLAUNCHER_ALLOW_INSTALL` is set. diff --git a/Misc/NEWS.d/next/Windows/2024-02-21-23-48-59.gh-issue-115554.02mpQC.rst b/Misc/NEWS.d/next/Windows/2024-02-21-23-48-59.gh-issue-115554.02mpQC.rst deleted file mode 100644 index b3c078b578205e..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-02-21-23-48-59.gh-issue-115554.02mpQC.rst +++ /dev/null @@ -1,6 +0,0 @@ -The installer now has more strict rules about updating the :ref:`launcher`. -In general, most users only have a single launcher installed and will see no -difference. When multiple launchers have been installed, the option to -install the launcher is disabled until all but one have been removed. -Downgrading the launcher (which was never allowed) is now more obviously -blocked. diff --git a/Misc/NEWS.d/next/Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst b/Misc/NEWS.d/next/Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst deleted file mode 100644 index f2e82bf6a3e028..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst +++ /dev/null @@ -1,3 +0,0 @@ -Building extensions intended for free-threaded builds of CPython now require -compiling with ``/DPy_GIL_DISABLED`` manually when using a regular install. This -is expected to change in future releases. diff --git a/Misc/NEWS.d/next/Windows/2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst b/Misc/NEWS.d/next/Windows/2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst deleted file mode 100644 index a55e5b1c7b566d..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst +++ /dev/null @@ -1 +0,0 @@ -Ensure the value of ``GetLastError()`` is preserved across GIL operations. diff --git a/Misc/NEWS.d/next/macOS/2024-02-29-20-52-23.gh-issue-116145.ygafim.rst b/Misc/NEWS.d/next/macOS/2024-02-29-20-52-23.gh-issue-116145.ygafim.rst deleted file mode 100644 index bc0a2e09dde1bb..00000000000000 --- a/Misc/NEWS.d/next/macOS/2024-02-29-20-52-23.gh-issue-116145.ygafim.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to Tcl/Tk 8.6.14. diff --git a/README.rst b/README.rst index fd3e6dd1771ae1..46167b38eab566 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.13.0 alpha 4 +This is Python version 3.13.0 alpha 5 ===================================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg From a2548077614f81f25a2c3465dabb7a0a3885c40c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 12 Mar 2024 17:36:21 -0400 Subject: [PATCH 042/158] gh-116307: Proper fix for 'mod' leaking across importlib tests (#116680) gh-116307: Create a new import helper 'isolated modules' and use that instead of 'Clean Import' to ensure that tests from importlib_resources don't leave modules in sys.modules. --- Lib/test/support/import_helper.py | 12 ++++++++++++ Lib/test/test_importlib/resources/test_files.py | 2 +- .../2024-03-06-11-00-36.gh-issue-116307.Uij0t_.rst | 3 +++ 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tests/2024-03-06-11-00-36.gh-issue-116307.Uij0t_.rst diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 3d804f2b590108..29c6f535b40342 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -268,6 +268,18 @@ def modules_cleanup(oldmodules): sys.modules.update(oldmodules) +@contextlib.contextmanager +def isolated_modules(): + """ + Save modules on entry and cleanup on exit. + """ + (saved,) = modules_setup() + try: + yield + finally: + modules_cleanup(saved) + + def mock_register_at_fork(func): # bpo-30599: Mock os.register_at_fork() when importing the random module, # since this function doesn't allow to unregister callbacks and would leak diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 1450cfb310926a..26c8b04e44c3b9 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -70,7 +70,7 @@ def setUp(self): self.addCleanup(self.fixtures.close) self.site_dir = self.fixtures.enter_context(os_helper.temp_dir()) self.fixtures.enter_context(import_helper.DirsOnSysPath(self.site_dir)) - self.fixtures.enter_context(import_helper.CleanImport()) + self.fixtures.enter_context(import_helper.isolated_modules()) class ModulesFilesTests(SiteDir, unittest.TestCase): diff --git a/Misc/NEWS.d/next/Tests/2024-03-06-11-00-36.gh-issue-116307.Uij0t_.rst b/Misc/NEWS.d/next/Tests/2024-03-06-11-00-36.gh-issue-116307.Uij0t_.rst new file mode 100644 index 00000000000000..0bc4be94789f21 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-03-06-11-00-36.gh-issue-116307.Uij0t_.rst @@ -0,0 +1,3 @@ +Added import helper ``isolated_modules`` as ``CleanImport`` does not remove +modules imported during the context. Use it in importlib.resources tests to +avoid leaving ``mod`` around to impede importlib.metadata tests. From 290330714b0af5b46b1df6cff08410bd2f73607b Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 12 Mar 2024 15:17:53 -0700 Subject: [PATCH 043/158] gh-89547: typing.rst: Add note about change in behavior with ClassVar/Final (#116686) --- Doc/library/typing.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 3db5f06803607f..713ad1c83546d1 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1233,6 +1233,10 @@ These can be used as types in annotations. They all support subscription using .. versionadded:: 3.5.3 + .. versionchanged:: 3.13 + + :data:`ClassVar` can now be nested in :data:`Final` and vice versa. + .. data:: Final Special typing construct to indicate final names to type checkers. @@ -1256,6 +1260,10 @@ These can be used as types in annotations. They all support subscription using .. versionadded:: 3.8 + .. versionchanged:: 3.13 + + :data:`Final` can now be nested in :data:`ClassVar` and vice versa. + .. data:: Required Special typing construct to mark a :class:`TypedDict` key as required. From 126186674ed3d6abd0f87e817100b5ec7290e146 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 12 Mar 2024 17:19:58 -0500 Subject: [PATCH 044/158] Beef-up tests for the itertool docs. (gh-116679) --- Doc/library/itertools.rst | 112 +++++++++++++++++++++++++++++++++++--- 1 file changed, 103 insertions(+), 9 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 2ee39fd9104df8..debb4138888e99 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -998,7 +998,7 @@ The following recipes have a more mathematical flavor: def sum_of_squares(it): "Add up the squares of the input values." - # sum_of_squares([10, 20, 30]) -> 1400 + # sum_of_squares([10, 20, 30]) --> 1400 return math.sumprod(*tee(it)) def reshape(matrix, cols): @@ -1019,17 +1019,16 @@ The following recipes have a more mathematical flavor: def convolve(signal, kernel): """Discrete linear convolution of two iterables. + Equivalent to polynomial multiplication. - The kernel is fully consumed before the calculations begin. - The signal is consumed lazily and can be infinite. - - Convolutions are mathematically commutative. - If the signal and kernel are swapped, - the output will be the same. + Convolutions are mathematically commutative; however, the inputs are + evaluated differently. The signal is consumed lazily and can be + infinite. The kernel is fully consumed before the calculations begin. Article: https://betterexplained.com/articles/intuitive-convolution/ Video: https://www.youtube.com/watch?v=KuXjwB4LzSA """ + # convolve([1, -1, -20], [1, -3]) --> 1 -4 -17 60 # convolve(data, [0.25, 0.25, 0.25, 0.25]) --> Moving average (blur) # convolve(data, [1/2, 0, -1/2]) --> 1st derivative estimate # convolve(data, [1, -2, 1]) --> 2nd derivative estimate @@ -1067,7 +1066,7 @@ The following recipes have a more mathematical flavor: f(x) = x³ -4x² -17x + 60 f'(x) = 3x² -8x -17 """ - # polynomial_derivative([1, -4, -17, 60]) -> [3, -8, -17] + # polynomial_derivative([1, -4, -17, 60]) --> [3, -8, -17] n = len(coefficients) powers = reversed(range(1, n)) return list(map(operator.mul, coefficients, powers)) @@ -1169,6 +1168,12 @@ The following recipes have a more mathematical flavor: >>> take(10, count()) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> # Verify that the input is consumed lazily + >>> it = iter('abcdef') + >>> take(3, it) + ['a', 'b', 'c'] + >>> list(it) + ['d', 'e', 'f'] >>> list(prepend(1, [2, 3, 4])) [1, 2, 3, 4] @@ -1181,25 +1186,45 @@ The following recipes have a more mathematical flavor: >>> list(tail(3, 'ABCDEFG')) ['E', 'F', 'G'] + >>> # Verify the input is consumed greedily + >>> input_iterator = iter('ABCDEFG') + >>> output_iterator = tail(3, input_iterator) + >>> list(input_iterator) + [] >>> it = iter(range(10)) >>> consume(it, 3) + >>> # Verify the input is consumed lazily >>> next(it) 3 + >>> # Verify the input is consumed completely >>> consume(it) >>> next(it, 'Done') 'Done' >>> nth('abcde', 3) 'd' - >>> nth('abcde', 9) is None True + >>> # Verify that the input is consumed lazily + >>> it = iter('abcde') + >>> nth(it, 2) + 'c' + >>> list(it) + ['d', 'e'] >>> [all_equal(s) for s in ('', 'A', 'AAAA', 'AAAB', 'AAABA')] [True, True, True, False, False] >>> [all_equal(s, key=str.casefold) for s in ('', 'A', 'AaAa', 'AAAB', 'AAABA')] [True, True, True, False, False] + >>> # Verify that the input is consumed lazily and that only + >>> # one element of a second equivalence class is used to disprove + >>> # the assertion that all elements are equal. + >>> it = iter('aaabbbccc') + >>> all_equal(it) + False + >>> ''.join(it) + 'bbccc' >>> quantify(range(99), lambda x: x%2==0) 50 @@ -1222,6 +1247,11 @@ The following recipes have a more mathematical flavor: >>> list(ncycles('abc', 3)) ['a', 'b', 'c', 'a', 'b', 'c', 'a', 'b', 'c'] + >>> # Verify greedy consumption of input iterator + >>> input_iterator = iter('abc') + >>> output_iterator = ncycles(input_iterator, 3) + >>> list(input_iterator) + [] >>> sum_of_squares([10, 20, 30]) 1400 @@ -1248,12 +1278,22 @@ The following recipes have a more mathematical flavor: >>> list(transpose([(1, 2, 3), (11, 22, 33)])) [(1, 11), (2, 22), (3, 33)] + >>> # Verify that the inputs are consumed lazily + >>> input1 = iter([1, 2, 3]) + >>> input2 = iter([11, 22, 33]) + >>> output_iterator = transpose([input1, input2]) + >>> next(output_iterator) + (1, 11) + >>> list(zip(input1, input2)) + [(2, 22), (3, 33)] >>> list(matmul([(7, 5), (3, 5)], [[2, 5], [7, 9]])) [(49, 80), (41, 60)] >>> list(matmul([[2, 5], [7, 9], [3, 4]], [[7, 11, 5, 4, 9], [3, 5, 2, 6, 3]])) [(29, 47, 20, 38, 33), (76, 122, 53, 82, 90), (33, 53, 23, 36, 39)] + >>> list(convolve([1, -1, -20], [1, -3])) == [1, -4, -17, 60] + True >>> data = [20, 40, 24, 32, 20, 28, 16] >>> list(convolve(data, [0.25, 0.25, 0.25, 0.25])) [5.0, 15.0, 21.0, 29.0, 29.0, 26.0, 24.0, 16.0, 11.0, 4.0] @@ -1261,6 +1301,18 @@ The following recipes have a more mathematical flavor: [20, 20, -16, 8, -12, 8, -12, -16] >>> list(convolve(data, [1, -2, 1])) [20, 0, -36, 24, -20, 20, -20, -4, 16] + >>> # Verify signal is consumed lazily and the kernel greedily + >>> signal_iterator = iter([10, 20, 30, 40, 50]) + >>> kernel_iterator = iter([1, 2, 3]) + >>> output_iterator = convolve(signal_iterator, kernel_iterator) + >>> list(kernel_iterator) + [] + >>> next(output_iterator) + 10 + >>> next(output_iterator) + 40 + >>> list(signal_iterator) + [30, 40, 50] >>> from fractions import Fraction >>> from decimal import Decimal @@ -1348,6 +1400,17 @@ The following recipes have a more mathematical flavor: >>> # Test list input. Lists do not support None for the stop argument >>> list(iter_index(list('AABCADEAF'), 'A')) [0, 1, 4, 7] + >>> # Verify that input is consumed lazily + >>> input_iterator = iter('AABCADEAF') + >>> output_iterator = iter_index(input_iterator, 'A') + >>> next(output_iterator) + 0 + >>> next(output_iterator) + 1 + >>> next(output_iterator) + 4 + >>> ''.join(input_iterator) + 'DEAF' >>> list(sieve(30)) [2, 3, 5, 7, 11, 13, 17, 19, 23, 29] @@ -1499,6 +1562,17 @@ The following recipes have a more mathematical flavor: [0, 2, 4, 6, 8] >>> list(odds) [1, 3, 5, 7, 9] + >>> # Verify that the input is consumed lazily + >>> input_iterator = iter(range(10)) + >>> evens, odds = partition(is_odd, input_iterator) + >>> next(odds) + 1 + >>> next(odds) + 3 + >>> next(evens) + 0 + >>> list(input_iterator) + [4, 5, 6, 7, 8, 9] >>> list(subslices('ABCD')) ['A', 'AB', 'ABC', 'ABCD', 'B', 'BC', 'BCD', 'C', 'CD', 'D'] @@ -1518,6 +1592,13 @@ The following recipes have a more mathematical flavor: ['A', 'B', 'C', 'D'] >>> list(unique_everseen('ABBcCAD', str.casefold)) ['A', 'B', 'c', 'D'] + >>> # Verify that the input is consumed lazily + >>> input_iterator = iter('AAAABBBCCDAABBB') + >>> output_iterator = unique_everseen(input_iterator) + >>> next(output_iterator) + 'A' + >>> ''.join(input_iterator) + 'AAABBBCCDAABBB' >>> list(unique_justseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D', 'A', 'B'] @@ -1525,6 +1606,13 @@ The following recipes have a more mathematical flavor: ['A', 'B', 'C', 'A', 'D'] >>> list(unique_justseen('ABBcCAD', str.casefold)) ['A', 'B', 'c', 'A', 'D'] + >>> # Verify that the input is consumed lazily + >>> input_iterator = iter('AAAABBBCCDAABBB') + >>> output_iterator = unique_justseen(input_iterator) + >>> next(output_iterator) + 'A' + >>> ''.join(input_iterator) + 'AAABBBCCDAABBB' >>> d = dict(a=1, b=2, c=3) >>> it = iter_except(d.popitem, KeyError) @@ -1545,6 +1633,12 @@ The following recipes have a more mathematical flavor: >>> first_true('ABC0DEF1', '9', str.isdigit) '0' + >>> # Verify that inputs are consumed lazily + >>> it = iter('ABC0DEF1') + >>> first_true(it, predicate=str.isdigit) + '0' + >>> ''.join(it) + 'DEF1' .. testcode:: From 3325699ffa3c633084f3e3fd94c4f3843066db85 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Wed, 13 Mar 2024 07:28:23 +0900 Subject: [PATCH 045/158] gh-116621: Set manual critical section for list.extend (gh-116657) --- Objects/clinic/listobject.c.h | 23 +----- Objects/listobject.c | 138 +++++++++++++++++++++++----------- 2 files changed, 97 insertions(+), 64 deletions(-) diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/listobject.c.h index b90dc0a0b463c6..588e021fb71fd3 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -125,29 +125,14 @@ list_append(PyListObject *self, PyObject *object) return return_value; } -PyDoc_STRVAR(py_list_extend__doc__, +PyDoc_STRVAR(list_extend__doc__, "extend($self, iterable, /)\n" "--\n" "\n" "Extend list by appending elements from the iterable."); -#define PY_LIST_EXTEND_METHODDEF \ - {"extend", (PyCFunction)py_list_extend, METH_O, py_list_extend__doc__}, - -static PyObject * -py_list_extend_impl(PyListObject *self, PyObject *iterable); - -static PyObject * -py_list_extend(PyListObject *self, PyObject *iterable) -{ - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION2(self, iterable); - return_value = py_list_extend_impl(self, iterable); - Py_END_CRITICAL_SECTION2(); - - return return_value; -} +#define LIST_EXTEND_METHODDEF \ + {"extend", (PyCFunction)list_extend, METH_O, list_extend__doc__}, PyDoc_STRVAR(list_pop__doc__, "pop($self, index=-1, /)\n" @@ -454,4 +439,4 @@ list___reversed__(PyListObject *self, PyObject *Py_UNUSED(ignored)) { return list___reversed___impl(self); } -/*[clinic end generated code: output=a77eda9931ec0c20 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=854957a1d4a89bbd input=a9049054013a1b77]*/ diff --git a/Objects/listobject.c b/Objects/listobject.c index 164f363efe24f0..759902c06b4ef3 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -10,6 +10,7 @@ #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() #include "pycore_tuple.h" // _PyTuple_FromArray() +#include "pycore_setobject.h" // _PySet_NextEntry() #include /*[clinic input] @@ -994,26 +995,28 @@ PyList_SetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) return list_ass_slice((PyListObject *)a, ilow, ihigh, v); } -static PyObject * +static int list_inplace_repeat_lock_held(PyListObject *self, Py_ssize_t n) { Py_ssize_t input_size = PyList_GET_SIZE(self); if (input_size == 0 || n == 1) { - return Py_NewRef(self); + return 0; } if (n < 1) { list_clear(self); - return Py_NewRef(self); + return 0; } if (input_size > PY_SSIZE_T_MAX / n) { - return PyErr_NoMemory(); + PyErr_NoMemory(); + return -1; } Py_ssize_t output_size = input_size * n; - if (list_resize(self, output_size) < 0) - return NULL; + if (list_resize(self, output_size) < 0) { + return -1; + } PyObject **items = self->ob_item; for (Py_ssize_t j = 0; j < input_size; j++) { @@ -1021,8 +1024,7 @@ list_inplace_repeat_lock_held(PyListObject *self, Py_ssize_t n) } _Py_memory_repeat((char *)items, sizeof(PyObject *)*output_size, sizeof(PyObject *)*input_size); - - return Py_NewRef(self); + return 0; } static PyObject * @@ -1031,7 +1033,12 @@ list_inplace_repeat(PyObject *_self, Py_ssize_t n) PyObject *ret; PyListObject *self = (PyListObject *) _self; Py_BEGIN_CRITICAL_SECTION(self); - ret = list_inplace_repeat_lock_held(self, n); + if (list_inplace_repeat_lock_held(self, n) < 0) { + ret = NULL; + } + else { + ret = Py_NewRef(self); + } Py_END_CRITICAL_SECTION(); return ret; } @@ -1179,7 +1186,7 @@ list_extend_fast(PyListObject *self, PyObject *iterable) } static int -list_extend_iter(PyListObject *self, PyObject *iterable) +list_extend_iter_lock_held(PyListObject *self, PyObject *iterable) { PyObject *it = PyObject_GetIter(iterable); if (it == NULL) { @@ -1253,45 +1260,78 @@ list_extend_iter(PyListObject *self, PyObject *iterable) return -1; } - static int -list_extend(PyListObject *self, PyObject *iterable) +list_extend_lock_held(PyListObject *self, PyObject *iterable) { - // Special cases: - // 1) lists and tuples which can use PySequence_Fast ops - // 2) extending self to self requires making a copy first - if (PyList_CheckExact(iterable) - || PyTuple_CheckExact(iterable) - || (PyObject *)self == iterable) - { - iterable = PySequence_Fast(iterable, "argument must be iterable"); - if (!iterable) { - return -1; - } - - int res = list_extend_fast(self, iterable); - Py_DECREF(iterable); - return res; - } - else { - return list_extend_iter(self, iterable); + PyObject *seq = PySequence_Fast(iterable, "argument must be iterable"); + if (!seq) { + return -1; } -} + int res = list_extend_fast(self, seq); + Py_DECREF(seq); + return res; +} -PyObject * -_PyList_Extend(PyListObject *self, PyObject *iterable) +static int +list_extend_set(PyListObject *self, PySetObject *other) { - if (list_extend(self, iterable) < 0) { - return NULL; + Py_ssize_t m = Py_SIZE(self); + Py_ssize_t n = PySet_GET_SIZE(other); + if (list_resize(self, m + n) < 0) { + return -1; } - Py_RETURN_NONE; + /* populate the end of self with iterable's items */ + Py_ssize_t setpos = 0; + Py_hash_t hash; + PyObject *key; + PyObject **dest = self->ob_item + m; + while (_PySet_NextEntry((PyObject *)other, &setpos, &key, &hash)) { + Py_INCREF(key); + *dest = key; + dest++; + } + Py_SET_SIZE(self, m + n); + return 0; } +static int +_list_extend(PyListObject *self, PyObject *iterable) +{ + // Special case: + // lists and tuples which can use PySequence_Fast ops + // TODO(@corona10): Add more special cases for other types. + int res = -1; + if ((PyObject *)self == iterable) { + Py_BEGIN_CRITICAL_SECTION(self); + res = list_inplace_repeat_lock_held(self, 2); + Py_END_CRITICAL_SECTION(); + } + else if (PyList_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION2(self, iterable); + res = list_extend_lock_held(self, iterable); + Py_END_CRITICAL_SECTION2(); + } + else if (PyTuple_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION(self); + res = list_extend_lock_held(self, iterable); + Py_END_CRITICAL_SECTION(); + } + else if (PyAnySet_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION2(self, iterable); + res = list_extend_set(self, (PySetObject *)iterable); + Py_END_CRITICAL_SECTION2(); + } + else { + Py_BEGIN_CRITICAL_SECTION(self); + res = list_extend_iter_lock_held(self, iterable); + Py_END_CRITICAL_SECTION(); + } + return res; +} /*[clinic input] -@critical_section self iterable -list.extend as py_list_extend +list.extend as list_extend iterable: object / @@ -1300,12 +1340,20 @@ Extend list by appending elements from the iterable. [clinic start generated code]*/ static PyObject * -py_list_extend_impl(PyListObject *self, PyObject *iterable) -/*[clinic end generated code: output=a2f115ceace2c845 input=1d42175414e1a5f3]*/ +list_extend(PyListObject *self, PyObject *iterable) +/*[clinic end generated code: output=630fb3bca0c8e789 input=979da7597a515791]*/ { - return _PyList_Extend(self, iterable); + if (_list_extend(self, iterable) < 0) { + return NULL; + } + Py_RETURN_NONE; } +PyObject * +_PyList_Extend(PyListObject *self, PyObject *iterable) +{ + return list_extend(self, iterable); +} int PyList_Extend(PyObject *self, PyObject *iterable) @@ -1314,7 +1362,7 @@ PyList_Extend(PyObject *self, PyObject *iterable) PyErr_BadInternalCall(); return -1; } - return list_extend((PyListObject*)self, iterable); + return _list_extend((PyListObject*)self, iterable); } @@ -1334,7 +1382,7 @@ static PyObject * list_inplace_concat(PyObject *_self, PyObject *other) { PyListObject *self = (PyListObject *)_self; - if (list_extend(self, other) < 0) { + if (_list_extend(self, other) < 0) { return NULL; } return Py_NewRef(self); @@ -3168,7 +3216,7 @@ list___init___impl(PyListObject *self, PyObject *iterable) list_clear(self); } if (iterable != NULL) { - if (list_extend(self, iterable) < 0) { + if (_list_extend(self, iterable) < 0) { return -1; } } @@ -3229,7 +3277,7 @@ static PyMethodDef list_methods[] = { LIST_COPY_METHODDEF LIST_APPEND_METHODDEF LIST_INSERT_METHODDEF - PY_LIST_EXTEND_METHODDEF + LIST_EXTEND_METHODDEF LIST_POP_METHODDEF LIST_REMOVE_METHODDEF LIST_INDEX_METHODDEF From 149f7f7ae28944579792d22607532006977177c9 Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Wed, 13 Mar 2024 04:16:42 +0530 Subject: [PATCH 046/158] Add `typing.NamedTuple` in glossary section for named tuples (#108327) --- Doc/glossary.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index f656e32514c717..72fb09ef6207c9 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -841,10 +841,11 @@ Glossary Some named tuples are built-in types (such as the above examples). Alternatively, a named tuple can be created from a regular class definition that inherits from :class:`tuple` and that defines named - fields. Such a class can be written by hand or it can be created with - the factory function :func:`collections.namedtuple`. The latter - technique also adds some extra methods that may not be found in - hand-written or built-in named tuples. + fields. Such a class can be written by hand, or it can be created by + inheriting :class:`typing.NamedTuple`, or with the factory function + :func:`collections.namedtuple`. The latter techniques also add some + extra methods that may not be found in hand-written or built-in named + tuples. namespace The place where a variable is stored. Namespaces are implemented as From a53cc3f49463e50cb3e2b839b3a82e6bf7f73fee Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 12 Mar 2024 16:35:28 -0700 Subject: [PATCH 047/158] GH-116098: Remove dead frame object creation code (GH-116687) --- Lib/test/test_frame.py | 65 ------------------------------------------ Python/frame.c | 27 ++++++------------ 2 files changed, 9 insertions(+), 83 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index f8812c281c2deb..8e744a1223e86f 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -293,71 +293,6 @@ def gen(): """) assert_python_ok("-c", code) - @support.cpython_only - @unittest.skipIf(Py_GIL_DISABLED, "test requires precise GC scheduling") - def test_sneaky_frame_object(self): - - def trace(frame, event, arg): - """ - Don't actually do anything, just force a frame object to be created. - """ - - def callback(phase, info): - """ - Yo dawg, I heard you like frames, so I'm allocating a frame while - you're allocating a frame, so you can have a frame while you have a - frame! - """ - nonlocal sneaky_frame_object - sneaky_frame_object = sys._getframe().f_back.f_back - # We're done here: - gc.callbacks.remove(callback) - - def f(): - while True: - yield - - old_threshold = gc.get_threshold() - old_callbacks = gc.callbacks[:] - old_enabled = gc.isenabled() - old_trace = sys.gettrace() - try: - # Stop the GC for a second while we set things up: - gc.disable() - # Create a paused generator: - g = f() - next(g) - # Move all objects to the oldest generation, and tell the GC to run - # on the *very next* allocation: - gc.collect() - gc.set_threshold(1, 0, 0) - sys._clear_internal_caches() - # Okay, so here's the nightmare scenario: - # - We're tracing the resumption of a generator, which creates a new - # frame object. - # - The allocation of this frame object triggers a collection - # *before* the frame object is actually created. - # - During the collection, we request the exact same frame object. - # This test does it with a GC callback, but in real code it would - # likely be a trace function, weakref callback, or finalizer. - # - The collection finishes, and the original frame object is - # created. We now have two frame objects fighting over ownership - # of the same interpreter frame! - sys.settrace(trace) - gc.callbacks.append(callback) - sneaky_frame_object = None - gc.enable() - next(g) - # g.gi_frame should be the frame object from the callback (the - # one that was *requested* second, but *created* first): - self.assertIs(g.gi_frame, sneaky_frame_object) - finally: - gc.set_threshold(*old_threshold) - gc.callbacks[:] = old_callbacks - sys.settrace(old_trace) - if old_enabled: - gc.enable() - @support.cpython_only @threading_helper.requires_working_threading() def test_sneaky_frame_object_teardown(self): diff --git a/Python/frame.c b/Python/frame.c index ddf6ef6ba5465c..f88a8f0d73d3f8 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -37,24 +37,15 @@ _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame) return NULL; } PyErr_SetRaisedException(exc); - if (frame->frame_obj) { - // GH-97002: How did we get into this horrible situation? Most likely, - // allocating f triggered a GC collection, which ran some code that - // *also* created the same frame... while we were in the middle of - // creating it! See test_sneaky_frame_object in test_frame.py for a - // concrete example. - // - // Regardless, just throw f away and use that frame instead, since it's - // already been exposed to user code. It's actually a bit tricky to do - // this, since we aren't backed by a real _PyInterpreterFrame anymore. - // Just pretend that we have an owned, cleared frame so frame_dealloc - // doesn't make the situation worse: - f->f_frame = (_PyInterpreterFrame *)f->_f_frame_data; - f->f_frame->owner = FRAME_CLEARED; - f->f_frame->frame_obj = f; - Py_DECREF(f); - return frame->frame_obj; - } + + // GH-97002: There was a time when a frame object could be created when we + // are allocating the new frame object f above, so frame->frame_obj would + // be assigned already. That path does not exist anymore. We won't call any + // Python code in this function and garbage collection will not run. + // Notice that _PyFrame_New_NoTrack() can potentially raise a MemoryError, + // but it won't allocate a traceback until the frame unwinds, so we are safe + // here. + assert(frame->frame_obj == NULL); assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT); assert(frame->owner != FRAME_CLEARED); f->f_frame = frame; From 128fbdf97b64901fdc45db19c10f2c71351d41e8 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Wed, 13 Mar 2024 00:46:17 +0100 Subject: [PATCH 048/158] Post 3.13.0a5 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index ea22ad2856c869..942922bd0df698 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 5 /* Version as a string */ -#define PY_VERSION "3.13.0a5" +#define PY_VERSION "3.13.0a5+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. From 7d1abe9502641a3602e9773aebc29ee56d8f40ae Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 12 Mar 2024 20:11:58 -0400 Subject: [PATCH 049/158] gh-116682: stdout may be empty in test_cancel_futures_wait_false (#116683) If the `shutdown()` call happens before the worker thread starts executing the task, then nothing will be printed to stdout. --- Lib/test/test_concurrent_futures/test_shutdown.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_concurrent_futures/test_shutdown.py b/Lib/test/test_concurrent_futures/test_shutdown.py index 45dab7a75fdd50..7a4065afd46fc8 100644 --- a/Lib/test/test_concurrent_futures/test_shutdown.py +++ b/Lib/test/test_concurrent_futures/test_shutdown.py @@ -247,7 +247,9 @@ def test_cancel_futures_wait_false(self): # Errors in atexit hooks don't change the process exit code, check # stderr manually. self.assertFalse(err) - self.assertEqual(out.strip(), b"apple") + # gh-116682: stdout may be empty if shutdown happens before task + # starts executing. + self.assertIn(out.strip(), [b"apple", b""]) class ProcessPoolShutdownTest(ExecutorShutdownTest): From bf121d6a694bea4fe9864a19879fe0c70c4e0656 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Tue, 12 Mar 2024 19:59:42 -0500 Subject: [PATCH 050/158] GH-116554: Relax list.sort()'s notion of "descending" runs (#116578) * GH-116554: Relax list.sort()'s notion of "descending" run Rewrote `count_run()` so that sub-runs of equal elements no longer end a descending run. Both ascending and descending runs can have arbitrarily many sub-runs of arbitrarily many equal elements now. This is tricky, because we only use ``<`` comparisons, so checking for equality doesn't come "for free". Surprisingly, it turned out there's a very cheap (one comparison) way to determine whether an ascending run consisted of all-equal elements. That sealed the deal. In addition, after a descending run is reversed in-place, we now go on to see whether it can be extended by an ascending run that just happens to be adjacent. This succeeds in finding at least one additional element to append about half the time, and so appears to more than repay its cost (the savings come from getting to skip a binary search, when a short run is artificially forced to length MIINRUN later, for each new element `count_run()` can add to the initial run). While these have been in the back of my mind for years, a question on StackOverflow pushed it to action: https://stackoverflow.com/questions/78108792/ They were wondering why it took about 4x longer to sort a list like: [999_999, 999_999, ..., 2, 2, 1, 1, 0, 0] than "similar" lists. Of course that runs very much faster after this patch. Co-authored-by: Alex Waygood Co-authored-by: Pieter Eendebak --- Lib/test/test_sort.py | 21 +++ ...-03-11-00-45-39.gh-issue-116554.gYumG5.rst | 1 + Objects/listobject.c | 157 ++++++++++++------ Objects/listsort.txt | 43 +++-- 4 files changed, 156 insertions(+), 66 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-11-00-45-39.gh-issue-116554.gYumG5.rst diff --git a/Lib/test/test_sort.py b/Lib/test/test_sort.py index 3b6ad4d17b0416..2a7cfb7affaa21 100644 --- a/Lib/test/test_sort.py +++ b/Lib/test/test_sort.py @@ -128,6 +128,27 @@ def bad_key(x): x = [e for e, i in augmented] # a stable sort of s check("stability", x, s) + def test_small_stability(self): + from itertools import product + from operator import itemgetter + + # Exhaustively test stability across all lists of small lengths + # and only a few distinct elements. + # This can provoke edge cases that randomization is unlikely to find. + # But it can grow very expensive quickly, so don't overdo it. + NELTS = 3 + MAXSIZE = 9 + + pick0 = itemgetter(0) + for length in range(MAXSIZE + 1): + # There are NELTS ** length distinct lists. + for t in product(range(NELTS), repeat=length): + xs = list(zip(t, range(length))) + # Stability forced by index in each element. + forced = sorted(xs) + # Use key= to hide the index from compares. + native = sorted(xs, key=pick0) + self.assertEqual(forced, native) #============================================================================== class TestBugs(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-11-00-45-39.gh-issue-116554.gYumG5.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-11-00-45-39.gh-issue-116554.gYumG5.rst new file mode 100644 index 00000000000000..82f92789de0a39 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-11-00-45-39.gh-issue-116554.gYumG5.rst @@ -0,0 +1 @@ +``list.sort()`` now exploits more cases of partial ordering, particularly those with long descending runs with sub-runs of equal values. Those are recognized as single runs now (previously, each block of repeated values caused a new run to be created). diff --git a/Objects/listobject.c b/Objects/listobject.c index 759902c06b4ef3..7179d5929e2148 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -1618,10 +1618,11 @@ struct s_MergeState { /* binarysort is the best method for sorting small arrays: it does few compares, but can do data movement quadratic in the number of elements. - [lo, hi) is a contiguous slice of a list, and is sorted via + [lo.keys, hi) is a contiguous slice of a list of keys, and is sorted via binary insertion. This sort is stable. - On entry, must have lo <= start <= hi, and that [lo, start) is already - sorted (pass start == lo if you don't know!). + On entry, must have lo.keys <= start <= hi, and that + [lo.keys, start) is already sorted (pass start == lo.keys if you don't + know!). If islt() complains return -1, else 0. Even in case of error, the output slice will be some permutation of the input (nothing is lost or duplicated). @@ -1634,7 +1635,7 @@ binarysort(MergeState *ms, sortslice lo, PyObject **hi, PyObject **start) PyObject *pivot; assert(lo.keys <= start && start <= hi); - /* assert [lo, start) is sorted */ + /* assert [lo.keys, start) is sorted */ if (lo.keys == start) ++start; for (; start < hi; ++start) { @@ -1643,9 +1644,9 @@ binarysort(MergeState *ms, sortslice lo, PyObject **hi, PyObject **start) r = start; pivot = *r; /* Invariants: - * pivot >= all in [lo, l). + * pivot >= all in [lo.keys, l). * pivot < all in [r, start). - * The second is vacuously true at the start. + * These are vacuously true at the start. */ assert(l < r); do { @@ -1656,7 +1657,7 @@ binarysort(MergeState *ms, sortslice lo, PyObject **hi, PyObject **start) l = p+1; } while (l < r); assert(l == r); - /* The invariants still hold, so pivot >= all in [lo, l) and + /* The invariants still hold, so pivot >= all in [lo.keys, l) and pivot < all in [l, start), so pivot belongs at l. Note that if there are elements equal to pivot, l points to the first slot after them -- that's why this sort is stable. @@ -1671,7 +1672,7 @@ binarysort(MergeState *ms, sortslice lo, PyObject **hi, PyObject **start) p = start + offset; pivot = *p; l += offset; - for (p = start + offset; p > l; --p) + for ( ; p > l; --p) *p = *(p-1); *l = pivot; } @@ -1682,56 +1683,115 @@ binarysort(MergeState *ms, sortslice lo, PyObject **hi, PyObject **start) return -1; } -/* -Return the length of the run beginning at lo, in the slice [lo, hi). lo < hi -is required on entry. "A run" is the longest ascending sequence, with - - lo[0] <= lo[1] <= lo[2] <= ... - -or the longest descending sequence, with - - lo[0] > lo[1] > lo[2] > ... +static void +sortslice_reverse(sortslice *s, Py_ssize_t n) +{ + reverse_slice(s->keys, &s->keys[n]); + if (s->values != NULL) + reverse_slice(s->values, &s->values[n]); +} -Boolean *descending is set to 0 in the former case, or to 1 in the latter. -For its intended use in a stable mergesort, the strictness of the defn of -"descending" is needed so that the caller can safely reverse a descending -sequence without violating stability (strict > ensures there are no equal -elements to get out of order). +/* +Return the length of the run beginning at slo->keys, spanning no more than +nremaining elements. The run beginning there may be ascending or descending, +but the function permutes it in place, if needed, so that it's always ascending +upon return. Returns -1 in case of error. */ static Py_ssize_t -count_run(MergeState *ms, PyObject **lo, PyObject **hi, int *descending) +count_run(MergeState *ms, sortslice *slo, Py_ssize_t nremaining) { - Py_ssize_t k; + Py_ssize_t k; /* used by IFLT macro expansion */ Py_ssize_t n; + PyObject ** const lo = slo->keys; - assert(lo < hi); - *descending = 0; - ++lo; - if (lo == hi) - return 1; - - n = 2; - IFLT(*lo, *(lo-1)) { - *descending = 1; - for (lo = lo+1; lo < hi; ++lo, ++n) { - IFLT(*lo, *(lo-1)) - ; - else - break; - } + /* In general, as things go on we've established that the slice starts + with a monotone run of n elements, starting at lo. */ + + /* We're n elements into the slice, and the most recent neq+1 elments are + * all equal. This reverses them in-place, and resets neq for reuse. + */ +#define REVERSE_LAST_NEQ \ + if (neq) { \ + sortslice slice = *slo; \ + ++neq; \ + sortslice_advance(&slice, n - neq); \ + sortslice_reverse(&slice, neq); \ + neq = 0; \ + } + + /* Sticking to only __lt__ compares is confusing and error-prone. But in + * this routine, almost all uses of IFLT can be captured by tiny macros + * giving mnemonic names to the intent. Note that inline functions don't + * work for this (IFLT expands to code including `goto fail`). + */ +#define IF_NEXT_LARGER IFLT(lo[n-1], lo[n]) +#define IF_NEXT_SMALLER IFLT(lo[n], lo[n-1]) + + assert(nremaining); + /* try ascending run first */ + for (n = 1; n < nremaining; ++n) { + IF_NEXT_SMALLER + break; } - else { - for (lo = lo+1; lo < hi; ++lo, ++n) { - IFLT(*lo, *(lo-1)) + if (n == nremaining) + return n; + /* lo[n] is strictly less */ + /* If n is 1 now, then the first compare established it's a descending + * run, so fall through to the descending case. But if n > 1, there are + * n elements in an ascending run terminated by the strictly less lo[n]. + * If the first key < lo[n-1], *somewhere* along the way the sequence + * increased, so we're done (there is no descending run). + * Else first key >= lo[n-1], which implies that the entire ascending run + * consists of equal elements. In that case, this is a descending run, + * and we reverse the all-equal prefix in-place. + */ + if (n > 1) { + IFLT(lo[0], lo[n-1]) + return n; + sortslice_reverse(slo, n); + } + ++n; /* in all cases it's been established that lo[n] has been resolved */ + + /* Finish descending run. All-squal subruns are reversed in-place on the + * fly. Their original order will be restored at the end by the whole-slice + * reversal. + */ + Py_ssize_t neq = 0; + for ( ; n < nremaining; ++n) { + IF_NEXT_SMALLER { + /* This ends the most recent run of equal elments, but still in + * the "descending" direction. + */ + REVERSE_LAST_NEQ + } + else { + IF_NEXT_LARGER /* descending run is over */ break; + else /* not x < y and not y < x implies x == y */ + ++neq; } } + REVERSE_LAST_NEQ + sortslice_reverse(slo, n); /* transform to ascending run */ + + /* And after reversing, it's possible this can be extended by a + * naturally increasing suffix; e.g., [3, 2, 3, 4, 1] makes an + * ascending run from the first 4 elements. + */ + for ( ; n < nremaining; ++n) { + IF_NEXT_SMALLER + break; + } return n; fail: return -1; + +#undef REVERSE_LAST_NEQ +#undef IF_NEXT_SMALLER +#undef IF_NEXT_LARGER } /* @@ -2449,14 +2509,6 @@ merge_compute_minrun(Py_ssize_t n) return n + r; } -static void -reverse_sortslice(sortslice *s, Py_ssize_t n) -{ - reverse_slice(s->keys, &s->keys[n]); - if (s->values != NULL) - reverse_slice(s->values, &s->values[n]); -} - /* Here we define custom comparison functions to optimize for the cases one commonly * encounters in practice: homogeneous lists, often of one of the basic types. */ @@ -2824,15 +2876,12 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) */ minrun = merge_compute_minrun(nremaining); do { - int descending; Py_ssize_t n; /* Identify next run. */ - n = count_run(&ms, lo.keys, lo.keys + nremaining, &descending); + n = count_run(&ms, &lo, nremaining); if (n < 0) goto fail; - if (descending) - reverse_sortslice(&lo, n); /* If short, extend to min(minrun, nremaining). */ if (n < minrun) { const Py_ssize_t force = nremaining <= minrun ? diff --git a/Objects/listsort.txt b/Objects/listsort.txt index 32a59e510f0754..4f84e2c87da7f1 100644 --- a/Objects/listsort.txt +++ b/Objects/listsort.txt @@ -212,24 +212,43 @@ A detailed description of timsort follows. Runs ---- -count_run() returns the # of elements in the next run. A run is either -"ascending", which means non-decreasing: +count_run() returns the # of elements in the next run, and, if it's a +descending run, reverses it in-place. A run is either "ascending", which +means non-decreasing: a0 <= a1 <= a2 <= ... -or "descending", which means strictly decreasing: +or "descending", which means non-increasing: - a0 > a1 > a2 > ... + a0 >= a1 >= a2 >= ... Note that a run is always at least 2 long, unless we start at the array's -last element. - -The definition of descending is strict, because the main routine reverses -a descending run in-place, transforming a descending run into an ascending -run. Reversal is done via the obvious fast "swap elements starting at each -end, and converge at the middle" method, and that can violate stability if -the slice contains any equal elements. Using a strict definition of -descending ensures that a descending run contains distinct elements. +last element. If all elements in the array are equal, it can be viewed as +both ascending and descending. Upon return, the run count_run() identifies +is always ascending. + +Reversal is done via the obvious fast "swap elements starting at each +end, and converge at the middle" method. That can violate stability if +the slice contains any equal elements. For that reason, for a long time +the code used strict inequality (">" rather than ">=") in its definition +of descending. + +Removing that restriction required some complication: when processing a +descending run, all-equal sub-runs of elements are reversed in-place, on the +fly. Their original relative order is restored "by magic" via the final +"reverse the entire run" step. + +This makes processing descending runs a little more costly. We only use +`__lt__` comparisons, so that `x == y` has to be deduced from +`not x < y and not y < x`. But so long as a run remains strictly decreasing, +only one of those compares needs to be done per loop iteration. So the primsry +extra cost is paid only when there are equal elements, and they get some +compensating benefit by not needing to end the descending run. + +There's one more trick added since the original: after reversing a descending +run, it's possible that it can be extended by an adjacent ascending run. For +example, given [3, 2, 1, 3, 4, 5, 0], the 3-element descending prefix is +reversed in-place, and then extended by [3, 4, 5]. If an array is random, it's very unlikely we'll see long runs. If a natural run contains less than minrun elements (see next section), the main loop From 93a687a3731caf9b27be0cd021b0337010441366 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 12 Mar 2024 21:33:42 -0500 Subject: [PATCH 051/158] Minor clarity improvement for the iter_index() recipe. Also add value subsequence tests. (gh-116696) --- Doc/library/itertools.rst | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index debb4138888e99..36e555d37d45de 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -902,18 +902,19 @@ which incur interpreter overhead. # iter_index('AABCADEAF', 'A') --> 0 1 4 7 seq_index = getattr(iterable, 'index', None) if seq_index is None: - # Slow path for general iterables + # Path for general iterables it = islice(iterable, start, stop) for i, element in enumerate(it, start): if element is value or element == value: yield i else: - # Fast path for sequences + # Path for sequences with an index() method stop = len(iterable) if stop is None else stop - i = start - 1 + i = start try: while True: - yield (i := seq_index(value, i+1, stop)) + yield (i := seq_index(value, i, stop)) + i += 1 except ValueError: pass @@ -1412,6 +1413,22 @@ The following recipes have a more mathematical flavor: >>> ''.join(input_iterator) 'DEAF' + >>> # Verify that the target value can be a sequence. + >>> seq = [[10, 20], [30, 40], 30, 40, [30, 40], 50] + >>> target = [30, 40] + >>> list(iter_index(seq, target)) + [1, 4] + + >>> # Verify faithfulness to type specific index() method behaviors. + >>> # For example, bytes and str perform subsequence searches + >>> # that do not match the general behavior specified + >>> # in collections.abc.Sequence.index(). + >>> seq = 'abracadabra' + >>> target = 'ab' + >>> list(iter_index(seq, target)) + [0, 7] + + >>> list(sieve(30)) [2, 3, 5, 7, 11, 13, 17, 19, 23, 29] >>> small_primes = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97] From 3f1b6efee95c06f8912bcea4031afacdbc0d5684 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 13 Mar 2024 05:19:33 +0100 Subject: [PATCH 052/158] Docs: fix broken links (#116651) --- Doc/faq/extending.rst | 2 +- Doc/library/math.rst | 2 +- Doc/library/pathlib.rst | 2 +- Doc/library/sqlite3.rst | 2 +- Doc/library/venv.rst | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 2a8b976925d042..1cff2c4091df06 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -50,7 +50,7 @@ to learn Python's C API. If you need to interface to some C or C++ library for which no Python extension currently exists, you can try wrapping the library's data types and functions with a tool such as `SWIG `_. `SIP -`__, `CXX +`__, `CXX `_ `Boost `_, or `Weave `_ are also diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 3c850317f60858..93755be717e2ef 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -592,7 +592,7 @@ Special functions The :func:`erf` function can be used to compute traditional statistical functions such as the `cumulative standard normal distribution - `_:: + `_:: def phi(x): 'Cumulative distribution function for the standard normal distribution' diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 4b461a5d4a2949..9041f37bd668fa 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -7,7 +7,7 @@ .. versionadded:: 3.4 -**Source code:** :source:`Lib/pathlib.py` +**Source code:** :source:`Lib/pathlib/` .. index:: single: path; operations diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 87d5ef1e42ca3a..e76dc91bf2d875 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1135,7 +1135,7 @@ Connection objects .. versionchanged:: 3.12 Added the *entrypoint* parameter. - .. _Loading an Extension: https://www.sqlite.org/loadext.html#loading_an_extension_ + .. _Loading an Extension: https://www.sqlite.org/loadext.html#loading_an_extension .. method:: iterdump(*, filter=None) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 2e7ff345a06234..a4273f97b7a8db 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -54,7 +54,7 @@ See :pep:`405` for more background on Python virtual environments. .. seealso:: `Python Packaging User Guide: Creating and using virtual environments - `__ + `__ .. include:: ../includes/wasm-notavail.rst From 43986f55671ba2f7b08f8c5cea69aa136a093697 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Wed, 13 Mar 2024 01:30:39 -0400 Subject: [PATCH 053/158] gh-111307: Update design FAQ 'switch' entry (#115899) --- Doc/faq/design.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 300e1b6cc40a58..c8beb64e39bc1a 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -259,9 +259,11 @@ is evaluated in all cases. Why isn't there a switch or case statement in Python? ----------------------------------------------------- -You can do this easily enough with a sequence of ``if... elif... elif... else``. -For literal values, or constants within a namespace, you can also use a -``match ... case`` statement. +In general, structured switch statements execute one block of code +when an expression has a particular value or set of values. +Since Python 3.10 one can easily match literal values, or constants +within a namespace, with a ``match ... case`` statement. +An older alternative is a sequence of ``if... elif... elif... else``. For cases where you need to choose from a very large number of possibilities, you can create a dictionary mapping case values to functions to call. For @@ -290,6 +292,9 @@ It's suggested that you use a prefix for the method names, such as ``visit_`` in this example. Without such a prefix, if values are coming from an untrusted source, an attacker would be able to call any method on your object. +Imitating switch with fallthrough, as with C's switch-case-default, +is possible, much harder, and less needed. + Can't you emulate threads in the interpreter instead of relying on an OS-specific thread implementation? -------------------------------------------------------------------------------------------------------- From 27df81d5643f32be6ae84a00c5cf84b58e849b21 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 13 Mar 2024 09:41:37 +0300 Subject: [PATCH 054/158] gh-115264: Fix `test_functools` with `-00` mode (#115276) --- Lib/test/test_functools.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 4eb322644fc541..1a6d8afe6ed6fe 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -2696,7 +2696,10 @@ def static_func(arg: int) -> str: A().static_func ): with self.subTest(meth=meth): - self.assertEqual(meth.__doc__, 'My function docstring') + self.assertEqual(meth.__doc__, + ('My function docstring' + if support.HAVE_DOCSTRINGS + else None)) self.assertEqual(meth.__annotations__['arg'], int) self.assertEqual(A.func.__name__, 'func') @@ -2785,7 +2788,10 @@ def decorated_classmethod(cls, arg: int) -> str: WithSingleDispatch().decorated_classmethod ): with self.subTest(meth=meth): - self.assertEqual(meth.__doc__, 'My function docstring') + self.assertEqual(meth.__doc__, + ('My function docstring' + if support.HAVE_DOCSTRINGS + else None)) self.assertEqual(meth.__annotations__['arg'], int) self.assertEqual( @@ -3128,7 +3134,10 @@ def test_access_from_class(self): self.assertIsInstance(CachedCostItem.cost, py_functools.cached_property) def test_doc(self): - self.assertEqual(CachedCostItem.cost.__doc__, "The cost of the item.") + self.assertEqual(CachedCostItem.cost.__doc__, + ("The cost of the item." + if support.HAVE_DOCSTRINGS + else None)) def test_module(self): self.assertEqual(CachedCostItem.cost.__module__, CachedCostItem.__module__) From ee0dbbc04504e0e0f1455e2bab8801ce0a682afd Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 13 Mar 2024 09:46:48 +0300 Subject: [PATCH 055/158] gh-116491: Improve `test_win32_ver` (#116506) --- Lib/test/test_platform.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index bbd0c06efed2c9..9f8aeeea257311 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -326,8 +326,36 @@ def test_java_ver(self): res = platform.java_ver() self.assertEqual(len(res), 4) + @unittest.skipUnless(support.MS_WINDOWS, 'This test only makes sense on Windows') def test_win32_ver(self): - res = platform.win32_ver() + release1, version1, csd1, ptype1 = 'a', 'b', 'c', 'd' + res = platform.win32_ver(release1, version1, csd1, ptype1) + self.assertEqual(len(res), 4) + release, version, csd, ptype = res + if release: + # Currently, release names always come from internal dicts, + # but this could change over time. For now, we just check that + # release is something different from what we have passed. + self.assertNotEqual(release, release1) + if version: + # It is rather hard to test explicit version without + # going deep into the details. + self.assertIn('.', version) + for v in version.split('.'): + int(v) # should not fail + if csd: + self.assertTrue(csd.startswith('SP'), msg=csd) + if ptype: + if os.cpu_count() > 1: + self.assertIn('Multiprocessor', ptype) + else: + self.assertIn('Uniprocessor', ptype) + + @unittest.skipIf(support.MS_WINDOWS, 'This test only makes sense on non Windows') + def test_win32_ver_on_non_windows(self): + release, version, csd, ptype = 'a', '1.0', 'c', 'd' + res = platform.win32_ver(release, version, csd, ptype) + self.assertSequenceEqual(res, (release, version, csd, ptype), seq_type=tuple) def test_mac_ver(self): res = platform.mac_ver() From e82f6dfae54170559fe46000fe6db9fee2a21a96 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 13 Mar 2024 02:12:30 -0500 Subject: [PATCH 056/158] Modernize roundrobin() recipe and improve variable names (gh-116710) --- Doc/library/itertools.rst | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 36e555d37d45de..26d5cdaff8c14f 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -932,31 +932,25 @@ which incur interpreter overhead. # grouper('ABCDEFG', 3, fillvalue='x') --> ABC DEF Gxx # grouper('ABCDEFG', 3, incomplete='strict') --> ABC DEF ValueError # grouper('ABCDEFG', 3, incomplete='ignore') --> ABC DEF - args = [iter(iterable)] * n + iterators = [iter(iterable)] * n match incomplete: case 'fill': - return zip_longest(*args, fillvalue=fillvalue) + return zip_longest(*iterators, fillvalue=fillvalue) case 'strict': - return zip(*args, strict=True) + return zip(*iterators, strict=True) case 'ignore': - return zip(*args) + return zip(*iterators) case _: raise ValueError('Expected fill, strict, or ignore') def roundrobin(*iterables): "Visit input iterables in a cycle until each is exhausted." # roundrobin('ABC', 'D', 'EF') --> A D E B F C - # Recipe credited to George Sakkis - num_active = len(iterables) - nexts = cycle(iter(it).__next__ for it in iterables) - while num_active: - try: - for next in nexts: - yield next() - except StopIteration: - # Remove the iterator we just exhausted from the cycle. - num_active -= 1 - nexts = cycle(islice(nexts, num_active)) + # Algorithm credited to George Sakkis + iterators = map(iter, iterables) + for num_active in range(len(iterables), 0, -1): + iterators = cycle(islice(iterators, num_active)) + yield from map(next, iterators) def partition(predicate, iterable): """Partition entries into false entries and true entries. @@ -997,10 +991,10 @@ The following recipes have a more mathematical flavor: s = list(iterable) return chain.from_iterable(combinations(s, r) for r in range(len(s)+1)) - def sum_of_squares(it): + def sum_of_squares(iterable): "Add up the squares of the input values." # sum_of_squares([10, 20, 30]) --> 1400 - return math.sumprod(*tee(it)) + return math.sumprod(*tee(iterable)) def reshape(matrix, cols): "Reshape a 2-D matrix to have a given number of columns." @@ -1570,6 +1564,9 @@ The following recipes have a more mathematical flavor: >>> list(roundrobin('abc', 'd', 'ef')) ['a', 'd', 'e', 'b', 'f', 'c'] + >>> ranges = [range(5, 1000), range(4, 3000), range(0), range(3, 2000), range(2, 5000), range(1, 3500)] + >>> collections.Counter(roundrobin(ranges)) == collections.Counter(ranges) + True >>> def is_odd(x): ... return x % 2 == 1 From ba82a241ac7ddee7cad18e9994f8dd560c68df02 Mon Sep 17 00:00:00 2001 From: Laurie O Date: Wed, 13 Mar 2024 17:21:30 +1000 Subject: [PATCH 057/158] gh-96471: Add ShutDown to queue.py '__all__' (#116699) --- Lib/queue.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/queue.py b/Lib/queue.py index 18fad8df08e469..387ce5425879a4 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -10,7 +10,15 @@ except ImportError: SimpleQueue = None -__all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue', 'SimpleQueue'] +__all__ = [ + 'Empty', + 'Full', + 'ShutDown', + 'Queue', + 'PriorityQueue', + 'LifoQueue', + 'SimpleQueue', +] try: From 8332e85b2f079e8b9334666084d1f8495cff25c1 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Wed, 13 Mar 2024 01:28:01 -0700 Subject: [PATCH 058/158] gh-116626: Emit `CALL` events for all `INSTRUMENTED_CALL_FUNCTION_EX` (GH-116627) --- Lib/test/test_monitoring.py | 15 ++++++++++ ...-03-11-22-05-56.gh-issue-116626.GsyczB.rst | 1 + Python/bytecodes.c | 29 ++++++++++--------- Python/generated_cases.c.h | 28 +++++++++--------- 4 files changed, 45 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-05-56.gh-issue-116626.GsyczB.rst diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 1e77eb6a2eea4c..11fb2d87f6c995 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1807,6 +1807,21 @@ def test_gh108976(self): sys.monitoring.set_events(0, E.LINE | E.INSTRUCTION) sys.monitoring.set_events(0, 0) + def test_call_function_ex(self): + def f(a, b): + return a + b + args = (1, 2) + + call_data = [] + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.set_events(0, 0) + sys.monitoring.register_callback(0, E.CALL, lambda code, offset, callable, arg0: call_data.append((callable, arg0))) + sys.monitoring.set_events(0, E.CALL) + f(*args) + sys.monitoring.set_events(0, 0) + self.assertEqual(call_data[0], (f, 1)) + class TestOptimizer(MonitoringTestBase, unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-05-56.gh-issue-116626.GsyczB.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-05-56.gh-issue-116626.GsyczB.rst new file mode 100644 index 00000000000000..5b18d04cca64b5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-05-56.gh-issue-116626.GsyczB.rst @@ -0,0 +1 @@ +Ensure ``INSTRUMENTED_CALL_FUNCTION_EX`` always emits :monitoring-event:`CALL` diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 03e5f4e330bdd8..ec05e40bd23fcb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3729,9 +3729,7 @@ dummy_func( } assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && - !PyFunction_Check(func) && !PyMethod_Check(func) - ) { + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : Py_None; int err = _Py_call_instrumentation_2args( @@ -3739,17 +3737,20 @@ dummy_func( frame, this_instr, func, arg); if (err) GOTO_ERROR(error); result = PyObject_Call(func, callargs, kwargs); - if (result == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - if (err < 0) { - Py_CLEAR(result); + + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + if (err < 0) { + Py_CLEAR(result); + } } } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 53c0211be2fe6c..72892725fb25d8 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1205,9 +1205,7 @@ } assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && - !PyFunction_Check(func) && !PyMethod_Check(func) - ) { + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : Py_None; int err = _Py_call_instrumentation_2args( @@ -1215,17 +1213,19 @@ frame, this_instr, func, arg); if (err) GOTO_ERROR(error); result = PyObject_Call(func, callargs, kwargs); - if (result == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - if (err < 0) { - Py_CLEAR(result); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + if (err < 0) { + Py_CLEAR(result); + } } } } From aa7bcf284f006434b07839d82f325618f7a5c06c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 13 Mar 2024 11:40:28 +0200 Subject: [PATCH 059/158] gh-116401: Fix blocking os.fwalk() and shutil.rmtree() on opening a named pipe (GH-116421) --- Lib/os.py | 4 +- Lib/shutil.py | 4 +- Lib/test/test_glob.py | 12 +++ Lib/test/test_os.py | 82 ++++++++++++++++++- Lib/test/test_shutil.py | 17 ++++ ...-03-06-18-30-37.gh-issue-116401.3Wcda2.rst | 2 + 6 files changed, 113 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-06-18-30-37.gh-issue-116401.3Wcda2.rst diff --git a/Lib/os.py b/Lib/os.py index 7f38e14e7bdd96..7661ce68ca3be2 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -475,7 +475,7 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= # lstat()/open()/fstat() trick. if not follow_symlinks: orig_st = stat(top, follow_symlinks=False, dir_fd=dir_fd) - topfd = open(top, O_RDONLY, dir_fd=dir_fd) + topfd = open(top, O_RDONLY | O_NONBLOCK, dir_fd=dir_fd) try: if (follow_symlinks or (st.S_ISDIR(orig_st.st_mode) and path.samestat(orig_st, stat(topfd)))): @@ -524,7 +524,7 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): assert entries is not None name, entry = name orig_st = entry.stat(follow_symlinks=False) - dirfd = open(name, O_RDONLY, dir_fd=topfd) + dirfd = open(name, O_RDONLY | O_NONBLOCK, dir_fd=topfd) except OSError as err: if onerror is not None: onerror(err) diff --git a/Lib/shutil.py b/Lib/shutil.py index f8be82d97616d3..94b09509008b0b 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -681,7 +681,7 @@ def _rmtree_safe_fd(topfd, path, onexc): continue if is_dir: try: - dirfd = os.open(entry.name, os.O_RDONLY, dir_fd=topfd) + dirfd = os.open(entry.name, os.O_RDONLY | os.O_NONBLOCK, dir_fd=topfd) dirfd_closed = False except FileNotFoundError: continue @@ -786,7 +786,7 @@ def onexc(*args): onexc(os.lstat, path, err) return try: - fd = os.open(path, os.O_RDONLY, dir_fd=dir_fd) + fd = os.open(path, os.O_RDONLY | os.O_NONBLOCK, dir_fd=dir_fd) fd_closed = False except OSError as err: onexc(os.open, path, err) diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index 8b2ea8f89f5daf..1fdf2817e236f6 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -344,6 +344,18 @@ def test_glob_non_directory(self): eq(self.rglob('nonexistent', '*'), []) eq(self.rglob('nonexistent', '**'), []) + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_glob_named_pipe(self): + path = os.path.join(self.tempdir, 'mypipe') + os.mkfifo(path) + self.assertEqual(self.rglob('mypipe'), [path]) + self.assertEqual(self.rglob('mypipe*'), [path]) + self.assertEqual(self.rglob('mypipe', ''), []) + self.assertEqual(self.rglob('mypipe', 'sub'), []) + self.assertEqual(self.rglob('mypipe', '*'), []) + def test_glob_many_open_files(self): depth = 30 base = os.path.join(self.tempdir, 'deep') diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index fc886f967c11bf..4c157842d95523 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1298,6 +1298,7 @@ def test_ror_operator(self): class WalkTests(unittest.TestCase): """Tests for os.walk().""" + is_fwalk = False # Wrapper to hide minor differences between os.walk and os.fwalk # to tests both functions with the same code base @@ -1332,14 +1333,14 @@ def setUp(self): self.sub11_path = join(self.sub1_path, "SUB11") sub2_path = join(self.walk_path, "SUB2") sub21_path = join(sub2_path, "SUB21") - tmp1_path = join(self.walk_path, "tmp1") + self.tmp1_path = join(self.walk_path, "tmp1") tmp2_path = join(self.sub1_path, "tmp2") tmp3_path = join(sub2_path, "tmp3") tmp5_path = join(sub21_path, "tmp3") self.link_path = join(sub2_path, "link") t2_path = join(os_helper.TESTFN, "TEST2") tmp4_path = join(os_helper.TESTFN, "TEST2", "tmp4") - broken_link_path = join(sub2_path, "broken_link") + self.broken_link_path = join(sub2_path, "broken_link") broken_link2_path = join(sub2_path, "broken_link2") broken_link3_path = join(sub2_path, "broken_link3") @@ -1349,13 +1350,13 @@ def setUp(self): os.makedirs(sub21_path) os.makedirs(t2_path) - for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path: + for path in self.tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path: with open(path, "x", encoding='utf-8') as f: f.write("I'm " + path + " and proud of it. Blame test_os.\n") if os_helper.can_symlink(): os.symlink(os.path.abspath(t2_path), self.link_path) - os.symlink('broken', broken_link_path, True) + os.symlink('broken', self.broken_link_path, True) os.symlink(join('tmp3', 'broken'), broken_link2_path, True) os.symlink(join('SUB21', 'tmp5'), broken_link3_path, True) self.sub2_tree = (sub2_path, ["SUB21", "link"], @@ -1451,6 +1452,11 @@ def test_walk_symlink(self): else: self.fail("Didn't follow symlink with followlinks=True") + walk_it = self.walk(self.broken_link_path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + def test_walk_bad_dir(self): # Walk top-down. errors = [] @@ -1472,6 +1478,73 @@ def test_walk_bad_dir(self): finally: os.rename(path1new, path1) + def test_walk_bad_dir2(self): + walk_it = self.walk('nonexisting') + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk('nonexisting', follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(self.tmp1_path) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(self.tmp1_path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_walk_named_pipe(self): + path = os_helper.TESTFN + '-pipe' + os.mkfifo(path) + self.addCleanup(os.unlink, path) + + walk_it = self.walk(path) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_walk_named_pipe2(self): + path = os_helper.TESTFN + '-dir' + os.mkdir(path) + self.addCleanup(shutil.rmtree, path) + os.mkfifo(os.path.join(path, 'mypipe')) + + errors = [] + walk_it = self.walk(path, onerror=errors.append) + next(walk_it) + self.assertRaises(StopIteration, next, walk_it) + self.assertEqual(errors, []) + + errors = [] + walk_it = self.walk(path, onerror=errors.append) + root, dirs, files = next(walk_it) + self.assertEqual(root, path) + self.assertEqual(dirs, []) + self.assertEqual(files, ['mypipe']) + dirs.extend(files) + files.clear() + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + if self.is_fwalk: + self.assertEqual(errors, []) + else: + self.assertEqual(len(errors), 1, errors) + self.assertIsInstance(errors[0], NotADirectoryError) + def test_walk_many_open_files(self): depth = 30 base = os.path.join(os_helper.TESTFN, 'deep') @@ -1537,6 +1610,7 @@ def test_walk_above_recursion_limit(self): @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()") class FwalkTests(WalkTests): """Tests for os.fwalk().""" + is_fwalk = True def walk(self, top, **kwargs): for root, dirs, files, root_fd in self.fwalk(top, **kwargs): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index d96dad4eb9475d..60e88d57b2b23d 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -667,6 +667,23 @@ def test_rmtree_on_junction(self): finally: shutil.rmtree(TESTFN, ignore_errors=True) + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_rmtree_on_named_pipe(self): + os.mkfifo(TESTFN) + try: + with self.assertRaises(NotADirectoryError): + shutil.rmtree(TESTFN) + self.assertTrue(os.path.exists(TESTFN)) + finally: + os.unlink(TESTFN) + + os.mkdir(TESTFN) + os.mkfifo(os.path.join(TESTFN, 'mypipe')) + shutil.rmtree(TESTFN) + self.assertFalse(os.path.exists(TESTFN)) + @unittest.skipIf(sys.platform[:6] == 'cygwin', "This test can't be run on Cygwin (issue #1071513).") @os_helper.skip_if_dac_override diff --git a/Misc/NEWS.d/next/Library/2024-03-06-18-30-37.gh-issue-116401.3Wcda2.rst b/Misc/NEWS.d/next/Library/2024-03-06-18-30-37.gh-issue-116401.3Wcda2.rst new file mode 100644 index 00000000000000..121f0065ecca95 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-06-18-30-37.gh-issue-116401.3Wcda2.rst @@ -0,0 +1,2 @@ +Fix blocking :func:`os.fwalk` and :func:`shutil.rmtree` on opening named +pipe. From fcd49b4f47f1edd9a2717f6619da7e7af8ea73cf Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 13 Mar 2024 15:38:03 +0300 Subject: [PATCH 060/158] gh-116714: Handle errors correctly in `PyFloat_GetInfo` (#116715) Co-authored-by: Serhiy Storchaka --- Objects/floatobject.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 37d2d312a6a0b7..96227f2cf7d76f 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -98,10 +98,18 @@ PyFloat_GetInfo(void) return NULL; } -#define SetIntFlag(flag) \ - PyStructSequence_SET_ITEM(floatinfo, pos++, PyLong_FromLong(flag)) -#define SetDblFlag(flag) \ - PyStructSequence_SET_ITEM(floatinfo, pos++, PyFloat_FromDouble(flag)) +#define SetFlag(CALL) \ + do { \ + PyObject *flag = (CALL); \ + if (flag == NULL) { \ + Py_CLEAR(floatinfo); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(floatinfo, pos++, flag); \ + } while (0) + +#define SetIntFlag(FLAG) SetFlag(PyLong_FromLong((FLAG))) +#define SetDblFlag(FLAG) SetFlag(PyFloat_FromDouble((FLAG))) SetDblFlag(DBL_MAX); SetIntFlag(DBL_MAX_EXP); @@ -116,11 +124,8 @@ PyFloat_GetInfo(void) SetIntFlag(FLT_ROUNDS); #undef SetIntFlag #undef SetDblFlag +#undef SetFlag - if (PyErr_Occurred()) { - Py_CLEAR(floatinfo); - return NULL; - } return floatinfo; } From 617aca9e745b3dfb5e6bc8cda07632d2f716426d Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Wed, 13 Mar 2024 20:57:48 +0800 Subject: [PATCH 061/158] gh-115419: Change default sym to not_null (GH-116562) --- Lib/test/test_generated_cases.py | 4 +- Python/optimizer_bytecodes.c | 9 +- Python/optimizer_cases.c.h | 181 ++++++++++--------- Tools/cases_generator/optimizer_generator.py | 4 +- 4 files changed, 104 insertions(+), 94 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 32c2c2fca05c4e..7b9dd36f85454f 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -908,7 +908,7 @@ def test_overridden_abstract_args(self): case OP2: { _Py_UopsSymbol *out; - out = sym_new_unknown(ctx); + out = sym_new_not_null(ctx); if (out == NULL) goto out_of_space; stack_pointer[-1] = out; break; @@ -933,7 +933,7 @@ def test_no_overridden_case(self): output = """ case OP: { _Py_UopsSymbol *out; - out = sym_new_unknown(ctx); + out = sym_new_not_null(ctx); if (out == NULL) goto out_of_space; stack_pointer[-1] = out; break; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index e3f7c9822103e6..54abbcd74d7934 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -449,6 +449,14 @@ dummy_func(void) { } } + op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) { + (void)owner; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + if (oparg & 1) { + OUT_OF_SPACE_IF_NULL(self_or_null = sym_new_unknown(ctx)); + } + } + op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { (void)index; OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); @@ -513,7 +521,6 @@ dummy_func(void) { OUT_OF_SPACE_IF_NULL(self = sym_new_not_null(ctx)); } - op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { if (!sym_set_type(callable, &PyFunction_Type)) { goto hit_bottom; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index fed5730d2e50c1..7e4214cc9acf39 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -80,7 +80,7 @@ case _END_SEND: { _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); + value = sym_new_not_null(ctx); if (value == NULL) goto out_of_space; stack_pointer[-2] = value; stack_pointer += -1; @@ -89,7 +89,7 @@ case _UNARY_NEGATIVE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -97,7 +97,7 @@ case _UNARY_NOT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -205,7 +205,7 @@ case _REPLACE_WITH_TRUE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -213,7 +213,7 @@ case _UNARY_INVERT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -482,7 +482,7 @@ case _BINARY_SUBSCR: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -491,7 +491,7 @@ case _BINARY_SLICE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-3] = res; stack_pointer += -2; @@ -505,7 +505,7 @@ case _BINARY_SUBSCR_LIST_INT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -514,7 +514,7 @@ case _BINARY_SUBSCR_STR_INT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -523,7 +523,7 @@ case _BINARY_SUBSCR_TUPLE_INT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -532,7 +532,7 @@ case _BINARY_SUBSCR_DICT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -573,7 +573,7 @@ case _CALL_INTRINSIC_1: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -581,7 +581,7 @@ case _CALL_INTRINSIC_2: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -608,7 +608,7 @@ case _GET_AITER: { _Py_UopsSymbol *iter; - iter = sym_new_unknown(ctx); + iter = sym_new_not_null(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -616,7 +616,7 @@ case _GET_ANEXT: { _Py_UopsSymbol *awaitable; - awaitable = sym_new_unknown(ctx); + awaitable = sym_new_not_null(ctx); if (awaitable == NULL) goto out_of_space; stack_pointer[0] = awaitable; stack_pointer += 1; @@ -625,7 +625,7 @@ case _GET_AWAITABLE: { _Py_UopsSymbol *iter; - iter = sym_new_unknown(ctx); + iter = sym_new_not_null(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -644,7 +644,7 @@ case _LOAD_ASSERTION_ERROR: { _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); + value = sym_new_not_null(ctx); if (value == NULL) goto out_of_space; stack_pointer[0] = value; stack_pointer += 1; @@ -653,7 +653,7 @@ case _LOAD_BUILD_CLASS: { _Py_UopsSymbol *bc; - bc = sym_new_unknown(ctx); + bc = sym_new_not_null(ctx); if (bc == NULL) goto out_of_space; stack_pointer[0] = bc; stack_pointer += 1; @@ -686,9 +686,9 @@ case _UNPACK_SEQUENCE_TWO_TUPLE: { _Py_UopsSymbol *val1; _Py_UopsSymbol *val0; - val1 = sym_new_unknown(ctx); + val1 = sym_new_not_null(ctx); if (val1 == NULL) goto out_of_space; - val0 = sym_new_unknown(ctx); + val0 = sym_new_not_null(ctx); if (val0 == NULL) goto out_of_space; stack_pointer[-1] = val1; stack_pointer[0] = val0; @@ -700,7 +700,7 @@ _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = sym_new_unknown(ctx); + values[_i] = sym_new_not_null(ctx); if (values[_i] == NULL) goto out_of_space; } stack_pointer += -1 + oparg; @@ -711,7 +711,7 @@ _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = sym_new_unknown(ctx); + values[_i] = sym_new_not_null(ctx); if (values[_i] == NULL) goto out_of_space; } stack_pointer += -1 + oparg; @@ -754,7 +754,7 @@ case _LOAD_LOCALS: { _Py_UopsSymbol *locals; - locals = sym_new_unknown(ctx); + locals = sym_new_not_null(ctx); if (locals == NULL) goto out_of_space; stack_pointer[0] = locals; stack_pointer += 1; @@ -763,7 +763,7 @@ case _LOAD_FROM_DICT_OR_GLOBALS: { _Py_UopsSymbol *v; - v = sym_new_unknown(ctx); + v = sym_new_not_null(ctx); if (v == NULL) goto out_of_space; stack_pointer[-1] = v; break; @@ -771,7 +771,7 @@ case _LOAD_NAME: { _Py_UopsSymbol *v; - v = sym_new_unknown(ctx); + v = sym_new_not_null(ctx); if (v == NULL) goto out_of_space; stack_pointer[0] = v; stack_pointer += 1; @@ -781,7 +781,7 @@ case _LOAD_GLOBAL: { _Py_UopsSymbol *res; _Py_UopsSymbol *null = NULL; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; null = sym_new_null(ctx); if (null == NULL) goto out_of_space; @@ -802,7 +802,7 @@ case _LOAD_GLOBAL_MODULE: { _Py_UopsSymbol *res; _Py_UopsSymbol *null = NULL; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; null = sym_new_null(ctx); if (null == NULL) goto out_of_space; @@ -815,7 +815,7 @@ case _LOAD_GLOBAL_BUILTINS: { _Py_UopsSymbol *res; _Py_UopsSymbol *null = NULL; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; null = sym_new_null(ctx); if (null == NULL) goto out_of_space; @@ -839,7 +839,7 @@ case _LOAD_FROM_DICT_OR_DEREF: { _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); + value = sym_new_not_null(ctx); if (value == NULL) goto out_of_space; stack_pointer[-1] = value; break; @@ -847,7 +847,7 @@ case _LOAD_DEREF: { _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); + value = sym_new_not_null(ctx); if (value == NULL) goto out_of_space; stack_pointer[0] = value; stack_pointer += 1; @@ -865,7 +865,7 @@ case _BUILD_STRING: { _Py_UopsSymbol *str; - str = sym_new_unknown(ctx); + str = sym_new_not_null(ctx); if (str == NULL) goto out_of_space; stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; @@ -874,7 +874,7 @@ case _BUILD_TUPLE: { _Py_UopsSymbol *tup; - tup = sym_new_unknown(ctx); + tup = sym_new_not_null(ctx); if (tup == NULL) goto out_of_space; stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; @@ -883,7 +883,7 @@ case _BUILD_LIST: { _Py_UopsSymbol *list; - list = sym_new_unknown(ctx); + list = sym_new_not_null(ctx); if (list == NULL) goto out_of_space; stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; @@ -902,7 +902,7 @@ case _BUILD_SET: { _Py_UopsSymbol *set; - set = sym_new_unknown(ctx); + set = sym_new_not_null(ctx); if (set == NULL) goto out_of_space; stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; @@ -911,7 +911,7 @@ case _BUILD_MAP: { _Py_UopsSymbol *map; - map = sym_new_unknown(ctx); + map = sym_new_not_null(ctx); if (map == NULL) goto out_of_space; stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; @@ -924,7 +924,7 @@ case _BUILD_CONST_KEY_MAP: { _Py_UopsSymbol *map; - map = sym_new_unknown(ctx); + map = sym_new_not_null(ctx); if (map == NULL) goto out_of_space; stack_pointer[-1 - oparg] = map; stack_pointer += -oparg; @@ -950,7 +950,7 @@ case _LOAD_SUPER_ATTR_ATTR: { _Py_UopsSymbol *attr; - attr = sym_new_unknown(ctx); + attr = sym_new_not_null(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-3] = attr; stack_pointer += -2; @@ -960,9 +960,9 @@ case _LOAD_SUPER_ATTR_METHOD: { _Py_UopsSymbol *attr; _Py_UopsSymbol *self_or_null; - attr = sym_new_unknown(ctx); + attr = sym_new_not_null(ctx); if (attr == NULL) goto out_of_space; - self_or_null = sym_new_unknown(ctx); + self_or_null = sym_new_not_null(ctx); if (self_or_null == NULL) goto out_of_space; stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; @@ -971,12 +971,15 @@ } case _LOAD_ATTR: { + _Py_UopsSymbol *owner; _Py_UopsSymbol *attr; _Py_UopsSymbol *self_or_null = NULL; - attr = sym_new_unknown(ctx); - if (attr == NULL) goto out_of_space; - self_or_null = sym_new_unknown(ctx); - if (self_or_null == NULL) goto out_of_space; + owner = stack_pointer[-1]; + (void)owner; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + if (oparg & 1) { + OUT_OF_SPACE_IF_NULL(self_or_null = sym_new_unknown(ctx)); + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = self_or_null; stack_pointer += (oparg & 1); @@ -1222,7 +1225,7 @@ case _CONTAINS_OP_SET: { _Py_UopsSymbol *b; - b = sym_new_unknown(ctx); + b = sym_new_not_null(ctx); if (b == NULL) goto out_of_space; stack_pointer[-2] = b; stack_pointer += -1; @@ -1231,7 +1234,7 @@ case _CONTAINS_OP_DICT: { _Py_UopsSymbol *b; - b = sym_new_unknown(ctx); + b = sym_new_not_null(ctx); if (b == NULL) goto out_of_space; stack_pointer[-2] = b; stack_pointer += -1; @@ -1241,9 +1244,9 @@ case _CHECK_EG_MATCH: { _Py_UopsSymbol *rest; _Py_UopsSymbol *match; - rest = sym_new_unknown(ctx); + rest = sym_new_not_null(ctx); if (rest == NULL) goto out_of_space; - match = sym_new_unknown(ctx); + match = sym_new_not_null(ctx); if (match == NULL) goto out_of_space; stack_pointer[-2] = rest; stack_pointer[-1] = match; @@ -1252,7 +1255,7 @@ case _CHECK_EXC_MATCH: { _Py_UopsSymbol *b; - b = sym_new_unknown(ctx); + b = sym_new_not_null(ctx); if (b == NULL) goto out_of_space; stack_pointer[-1] = b; break; @@ -1264,7 +1267,7 @@ case _IS_NONE: { _Py_UopsSymbol *b; - b = sym_new_unknown(ctx); + b = sym_new_not_null(ctx); if (b == NULL) goto out_of_space; stack_pointer[-1] = b; break; @@ -1272,7 +1275,7 @@ case _GET_LEN: { _Py_UopsSymbol *len_o; - len_o = sym_new_unknown(ctx); + len_o = sym_new_not_null(ctx); if (len_o == NULL) goto out_of_space; stack_pointer[0] = len_o; stack_pointer += 1; @@ -1281,7 +1284,7 @@ case _MATCH_CLASS: { _Py_UopsSymbol *attrs; - attrs = sym_new_unknown(ctx); + attrs = sym_new_not_null(ctx); if (attrs == NULL) goto out_of_space; stack_pointer[-3] = attrs; stack_pointer += -2; @@ -1290,7 +1293,7 @@ case _MATCH_MAPPING: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; @@ -1299,7 +1302,7 @@ case _MATCH_SEQUENCE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; @@ -1308,7 +1311,7 @@ case _MATCH_KEYS: { _Py_UopsSymbol *values_or_none; - values_or_none = sym_new_unknown(ctx); + values_or_none = sym_new_not_null(ctx); if (values_or_none == NULL) goto out_of_space; stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -1317,7 +1320,7 @@ case _GET_ITER: { _Py_UopsSymbol *iter; - iter = sym_new_unknown(ctx); + iter = sym_new_not_null(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -1325,7 +1328,7 @@ case _GET_YIELD_FROM_ITER: { _Py_UopsSymbol *iter; - iter = sym_new_unknown(ctx); + iter = sym_new_not_null(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -1335,7 +1338,7 @@ case _FOR_ITER_TIER_TWO: { _Py_UopsSymbol *next; - next = sym_new_unknown(ctx); + next = sym_new_not_null(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; @@ -1356,7 +1359,7 @@ case _ITER_NEXT_LIST: { _Py_UopsSymbol *next; - next = sym_new_unknown(ctx); + next = sym_new_not_null(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; @@ -1375,7 +1378,7 @@ case _ITER_NEXT_TUPLE: { _Py_UopsSymbol *next; - next = sym_new_unknown(ctx); + next = sym_new_not_null(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; @@ -1408,9 +1411,9 @@ case _BEFORE_ASYNC_WITH: { _Py_UopsSymbol *exit; _Py_UopsSymbol *res; - exit = sym_new_unknown(ctx); + exit = sym_new_not_null(ctx); if (exit == NULL) goto out_of_space; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = exit; stack_pointer[0] = res; @@ -1421,9 +1424,9 @@ case _BEFORE_WITH: { _Py_UopsSymbol *exit; _Py_UopsSymbol *res; - exit = sym_new_unknown(ctx); + exit = sym_new_not_null(ctx); if (exit == NULL) goto out_of_space; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = exit; stack_pointer[0] = res; @@ -1433,7 +1436,7 @@ case _WITH_EXCEPT_START: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; @@ -1443,9 +1446,9 @@ case _PUSH_EXC_INFO: { _Py_UopsSymbol *prev_exc; _Py_UopsSymbol *new_exc; - prev_exc = sym_new_unknown(ctx); + prev_exc = sym_new_not_null(ctx); if (prev_exc == NULL) goto out_of_space; - new_exc = sym_new_unknown(ctx); + new_exc = sym_new_not_null(ctx); if (new_exc == NULL) goto out_of_space; stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; @@ -1493,7 +1496,7 @@ case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { _Py_UopsSymbol *attr; - attr = sym_new_unknown(ctx); + attr = sym_new_not_null(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; break; @@ -1501,7 +1504,7 @@ case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { _Py_UopsSymbol *attr; - attr = sym_new_unknown(ctx); + attr = sym_new_not_null(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; break; @@ -1632,7 +1635,7 @@ case _CALL_TYPE_1: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-3] = res; stack_pointer += -2; @@ -1641,7 +1644,7 @@ case _CALL_STR_1: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-3] = res; stack_pointer += -2; @@ -1650,7 +1653,7 @@ case _CALL_TUPLE_1: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-3] = res; stack_pointer += -2; @@ -1666,7 +1669,7 @@ case _CALL_BUILTIN_CLASS: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1675,7 +1678,7 @@ case _CALL_BUILTIN_O: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1684,7 +1687,7 @@ case _CALL_BUILTIN_FAST: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1693,7 +1696,7 @@ case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1702,7 +1705,7 @@ case _CALL_LEN: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1711,7 +1714,7 @@ case _CALL_ISINSTANCE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1720,7 +1723,7 @@ case _CALL_METHOD_DESCRIPTOR_O: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1729,7 +1732,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1738,7 +1741,7 @@ case _CALL_METHOD_DESCRIPTOR_NOARGS: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1747,7 +1750,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1764,7 +1767,7 @@ case _MAKE_FUNCTION: { _Py_UopsSymbol *func; - func = sym_new_unknown(ctx); + func = sym_new_not_null(ctx); if (func == NULL) goto out_of_space; stack_pointer[-1] = func; break; @@ -1772,7 +1775,7 @@ case _SET_FUNCTION_ATTRIBUTE: { _Py_UopsSymbol *func; - func = sym_new_unknown(ctx); + func = sym_new_not_null(ctx); if (func == NULL) goto out_of_space; stack_pointer[-2] = func; stack_pointer += -1; @@ -1781,7 +1784,7 @@ case _BUILD_SLICE: { _Py_UopsSymbol *slice; - slice = sym_new_unknown(ctx); + slice = sym_new_not_null(ctx); if (slice == NULL) goto out_of_space; stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); @@ -1790,7 +1793,7 @@ case _CONVERT_VALUE: { _Py_UopsSymbol *result; - result = sym_new_unknown(ctx); + result = sym_new_not_null(ctx); if (result == NULL) goto out_of_space; stack_pointer[-1] = result; break; @@ -1798,7 +1801,7 @@ case _FORMAT_SIMPLE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -1806,7 +1809,7 @@ case _FORMAT_WITH_SPEC: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1826,7 +1829,7 @@ case _BINARY_OP: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1953,7 +1956,7 @@ case _POP_TOP_LOAD_CONST_INLINE_BORROW: { _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); + value = sym_new_not_null(ctx); if (value == NULL) goto out_of_space; stack_pointer[-1] = value; break; diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index a0a2f10aa760b7..1c6b708e82321a 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -83,14 +83,14 @@ def emit_default(out: CWriter, uop: Uop) -> None: if var.name != "unused" and not var.peek: if var.is_array(): out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"{var.name}[_i] = sym_new_unknown(ctx);\n") + out.emit(f"{var.name}[_i] = sym_new_not_null(ctx);\n") out.emit(f"if ({var.name}[_i] == NULL) goto out_of_space;\n") out.emit("}\n") elif var.name == "null": out.emit(f"{var.name} = sym_new_null(ctx);\n") out.emit(f"if ({var.name} == NULL) goto out_of_space;\n") else: - out.emit(f"{var.name} = sym_new_unknown(ctx);\n") + out.emit(f"{var.name} = sym_new_not_null(ctx);\n") out.emit(f"if ({var.name} == NULL) goto out_of_space;\n") From 33662d4e01d73cd4f29a25efc2ef09288129023f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 13 Mar 2024 15:03:13 +0200 Subject: [PATCH 062/158] gh-90300: Fix cmdline.rst (GH-116721) * Fix the description of the "-b" option. * Add references to environment variables for "-s" and "-X dev" options. --- Doc/using/cmdline.rst | 13 ++++++++----- Python/initconfig.c | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 36cddffb9eae34..565d86cb1a0dd3 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -242,12 +242,13 @@ Miscellaneous options .. option:: -b - Issue a warning when comparing :class:`bytes` or :class:`bytearray` with - :class:`str` or :class:`bytes` with :class:`int`. Issue an error when the - option is given twice (:option:`!-bb`). + Issue a warning when converting :class:`bytes` or :class:`bytearray` to + :class:`str` without specifying encoding or comparing :class:`!bytes` or + :class:`!bytearray` with :class:`!str` or :class:`!bytes` with :class:`int`. + Issue an error when the option is given twice (:option:`!-bb`). .. versionchanged:: 3.5 - Affects comparisons of :class:`bytes` with :class:`int`. + Affects also comparisons of :class:`bytes` with :class:`int`. .. option:: -B @@ -386,6 +387,8 @@ Miscellaneous options Don't add the :data:`user site-packages directory ` to :data:`sys.path`. + See also :envvar:`PYTHONNOUSERSITE`. + .. seealso:: :pep:`370` -- Per user site-packages directory @@ -517,7 +520,7 @@ Miscellaneous options asyncio'``. See also :envvar:`PYTHONPROFILEIMPORTTIME`. * ``-X dev``: enable :ref:`Python Development Mode `, introducing additional runtime checks that are too expensive to be enabled by - default. + default. See also :envvar:`PYTHONDEVMODE`. * ``-X utf8`` enables the :ref:`Python UTF-8 Mode `. ``-X utf8=0`` explicitly disables :ref:`Python UTF-8 Mode ` (even when it would otherwise activate automatically). diff --git a/Python/initconfig.c b/Python/initconfig.c index e3a62e53334163..a01a9f56f37832 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -142,8 +142,8 @@ static const char usage_line[] = /* Lines sorted by option name; keep in sync with usage_envvars* below */ static const char usage_help[] = "\ Options (and corresponding environment variables):\n\ --b : issue warnings about str(bytes_instance), str(bytearray_instance)\n\ - and comparing bytes/bytearray with str. (-bb: issue errors)\n\ +-b : issue warnings about converting bytes/bytearray to str and comparing\n\ + bytes/bytearray with str or bytes with int. (-bb: issue errors)\n\ -B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\ -c cmd : program passed in as string (terminates option list)\n\ -d : turn on parser debugging output (for experts only, only works on\n\ From 612f1ec988314bc0bc42a1b908751950331e2ede Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 13 Mar 2024 14:20:33 +0100 Subject: [PATCH 063/158] gh-110918: Fix side effects of regrtest test_match_tests() (#116718) test_match_tests now saves and restores patterns. Add get_match_tests() function to libregrtest.filter. Previously, running test_regrtest multiple times in a row only ran tests once: "./python -m test test_regrtest -R 3:3. --- Lib/test/libregrtest/filter.py | 5 +++++ Lib/test/test_regrtest.py | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/filter.py b/Lib/test/libregrtest/filter.py index 817624d79e9263..41372e427ffd03 100644 --- a/Lib/test/libregrtest/filter.py +++ b/Lib/test/libregrtest/filter.py @@ -27,6 +27,11 @@ def _is_full_match_test(pattern): return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) +def get_match_tests(): + global _test_patterns + return _test_patterns + + def set_match_tests(patterns): global _test_matchers, _test_patterns diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 7e1eaa7d6a515e..903ad50ba088e8 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -27,7 +27,7 @@ from test.libregrtest import main from test.libregrtest import setup from test.libregrtest import utils -from test.libregrtest.filter import set_match_tests, match_test +from test.libregrtest.filter import get_match_tests, set_match_tests, match_test from test.libregrtest.result import TestStats from test.libregrtest.utils import normalize_test_name @@ -2298,6 +2298,10 @@ def __init__(self, test_id): def id(self): return self.test_id + # Restore patterns once the test completes + patterns = get_match_tests() + self.addCleanup(set_match_tests, patterns) + test_access = Test('test.test_os.FileTests.test_access') test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') test_copy = Test('test.test_shutil.TestCopy.test_copy') From 186af3cf21705badec086ec16f231ac390747d3b Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Wed, 13 Mar 2024 13:22:47 +0000 Subject: [PATCH 064/158] [doc]: Update cookbook recipe for Qt6. (GH-116719) --- Doc/howto/logging-cookbook.rst | 49 +++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index f7d885ec88483d..ad3e34d0b33bd2 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -3418,9 +3418,10 @@ The worker thread is implemented using Qt's ``QThread`` class rather than the :mod:`threading` module, as there are circumstances where one has to use ``QThread``, which offers better integration with other ``Qt`` components. -The code should work with recent releases of either ``PySide2`` or ``PyQt5``. -You should be able to adapt the approach to earlier versions of Qt. Please -refer to the comments in the code snippet for more detailed information. +The code should work with recent releases of either ``PySide6``, ``PyQt6``, +``PySide2`` or ``PyQt5``. You should be able to adapt the approach to earlier +versions of Qt. Please refer to the comments in the code snippet for more +detailed information. .. code-block:: python3 @@ -3430,16 +3431,25 @@ refer to the comments in the code snippet for more detailed information. import sys import time - # Deal with minor differences between PySide2 and PyQt5 + # Deal with minor differences between different Qt packages try: - from PySide2 import QtCore, QtGui, QtWidgets + from PySide6 import QtCore, QtGui, QtWidgets Signal = QtCore.Signal Slot = QtCore.Slot except ImportError: - from PyQt5 import QtCore, QtGui, QtWidgets - Signal = QtCore.pyqtSignal - Slot = QtCore.pyqtSlot - + try: + from PyQt6 import QtCore, QtGui, QtWidgets + Signal = QtCore.pyqtSignal + Slot = QtCore.pyqtSlot + except ImportError: + try: + from PySide2 import QtCore, QtGui, QtWidgets + Signal = QtCore.Signal + Slot = QtCore.Slot + except ImportError: + from PyQt5 import QtCore, QtGui, QtWidgets + Signal = QtCore.pyqtSignal + Slot = QtCore.pyqtSlot logger = logging.getLogger(__name__) @@ -3511,8 +3521,14 @@ refer to the comments in the code snippet for more detailed information. while not QtCore.QThread.currentThread().isInterruptionRequested(): delay = 0.5 + random.random() * 2 time.sleep(delay) - level = random.choice(LEVELS) - logger.log(level, 'Message after delay of %3.1f: %d', delay, i, extra=extra) + try: + if random.random() < 0.1: + raise ValueError('Exception raised: %d' % i) + else: + level = random.choice(LEVELS) + logger.log(level, 'Message after delay of %3.1f: %d', delay, i, extra=extra) + except ValueError as e: + logger.exception('Failed: %s', e, extra=extra) i += 1 # @@ -3539,7 +3555,10 @@ refer to the comments in the code snippet for more detailed information. self.textedit = te = QtWidgets.QPlainTextEdit(self) # Set whatever the default monospace font is for the platform f = QtGui.QFont('nosuchfont') - f.setStyleHint(f.Monospace) + if hasattr(f, 'Monospace'): + f.setStyleHint(f.Monospace) + else: + f.setStyleHint(f.StyleHint.Monospace) # for Qt6 te.setFont(f) te.setReadOnly(True) PB = QtWidgets.QPushButton @@ -3626,7 +3645,11 @@ refer to the comments in the code snippet for more detailed information. app = QtWidgets.QApplication(sys.argv) example = Window(app) example.show() - sys.exit(app.exec_()) + if hasattr(app, 'exec'): + rc = app.exec() + else: + rc = app.exec_() + sys.exit(rc) if __name__=='__main__': main() From 8e2aab7ad5e1c8b3360c1e1b80ddadc0845eaa3e Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 13 Mar 2024 09:27:36 -0400 Subject: [PATCH 065/158] gh-116604: Fix test_gc on free-threaded build (#116662) The free-threaded GC only does full collections, so it uses a threshold that is a maximum of a fixed value (default 2000) and proportional to the number of live objects. If there were many live objects after the previous collection, then the threshold may be larger than 10,000 causing `test_indirect_calls_with_gc_disabled` to fail. This manually sets the threshold to `(1000, 0, 0)` for the test. The `0` disables the proportional scaling. --- Lib/test/support/__init__.py | 10 ++++++++++ Lib/test/test_gc.py | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index af43446c26120e..ce693e51aab31c 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -797,6 +797,16 @@ def disable_gc(): if have_gc: gc.enable() +@contextlib.contextmanager +def gc_threshold(*args): + import gc + old_threshold = gc.get_threshold() + gc.set_threshold(*args) + try: + yield + finally: + gc.set_threshold(*old_threshold) + def python_is_optimized(): """Find if Python was built with optimizations.""" diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 2aea025fcc140a..f1a7afac0bcd19 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -5,7 +5,7 @@ from test.support.import_helper import import_module from test.support.os_helper import temp_dir, TESTFN, unlink from test.support.script_helper import assert_python_ok, make_script -from test.support import threading_helper +from test.support import threading_helper, gc_threshold import gc import sys @@ -1330,6 +1330,7 @@ def callback(ignored): # with an empty __dict__. self.assertEqual(x, None) + @gc_threshold(1000, 0, 0) def test_bug1055820d(self): # Corresponds to temp2d.py in the bug report. This is very much like # test_bug1055820c, but uses a __del__ method instead of a weakref @@ -1397,6 +1398,7 @@ def __del__(self): # empty __dict__. self.assertEqual(x, None) + @gc_threshold(1000, 0, 0) def test_indirect_calls_with_gc_disabled(self): junk = [] i = 0 From 7f418fb111dec325b5c9fe6f6e96076049322f02 Mon Sep 17 00:00:00 2001 From: Nir Friedman Date: Wed, 13 Mar 2024 11:58:30 -0400 Subject: [PATCH 066/158] gh-98731: Improvements to the logging documentation (GH-101618) Co-authored-by: Vinay Sajip --- Doc/howto/logging.rst | 118 +++++++++++-------------------- Doc/library/logging.rst | 152 ++++++++++++++++++---------------------- 2 files changed, 108 insertions(+), 162 deletions(-) diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 347330e98dd00c..ab758a885b3556 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -25,10 +25,12 @@ or *severity*. When to use logging ^^^^^^^^^^^^^^^^^^^ -Logging provides a set of convenience functions for simple logging usage. These -are :func:`debug`, :func:`info`, :func:`warning`, :func:`error` and -:func:`critical`. To determine when to use logging, see the table below, which -states, for each of a set of common tasks, the best tool to use for it. +You can access logging functionality by creating a logger via ``logger = +getLogger(__name__)``, and then calling the logger's :meth:`~Logger.debug`, +:meth:`~Logger.info`, :meth:`~Logger.warning`, :meth:`~Logger.error` and +:meth:`~Logger.critical` methods. To determine when to use logging, and to see +which logger methods to use when, see the table below. It states, for each of a +set of common tasks, the best tool to use for that task. +-------------------------------------+--------------------------------------+ | Task you want to perform | The best tool for the task | @@ -37,8 +39,8 @@ states, for each of a set of common tasks, the best tool to use for it. | usage of a command line script or | | | program | | +-------------------------------------+--------------------------------------+ -| Report events that occur during | :func:`logging.info` (or | -| normal operation of a program (e.g. | :func:`logging.debug` for very | +| Report events that occur during | A logger's :meth:`~Logger.info` (or | +| normal operation of a program (e.g. | :meth:`~Logger.debug` method for very| | for status monitoring or fault | detailed output for diagnostic | | investigation) | purposes) | +-------------------------------------+--------------------------------------+ @@ -47,22 +49,23 @@ states, for each of a set of common tasks, the best tool to use for it. | | the client application should be | | | modified to eliminate the warning | | | | -| | :func:`logging.warning` if there is | -| | nothing the client application can do| -| | about the situation, but the event | -| | should still be noted | +| | A logger's :meth:`~Logger.warning` | +| | method if there is nothing the client| +| | application can do about the | +| | situation, but the event should still| +| | be noted | +-------------------------------------+--------------------------------------+ | Report an error regarding a | Raise an exception | | particular runtime event | | +-------------------------------------+--------------------------------------+ -| Report suppression of an error | :func:`logging.error`, | -| without raising an exception (e.g. | :func:`logging.exception` or | -| error handler in a long-running | :func:`logging.critical` as | +| Report suppression of an error | A logger's :meth:`~Logger.error`, | +| without raising an exception (e.g. | :meth:`~Logger.exception` or | +| error handler in a long-running | :meth:`~Logger.critical` method as | | server process) | appropriate for the specific error | | | and application domain | +-------------------------------------+--------------------------------------+ -The logging functions are named after the level or severity of the events +The logger methods are named after the level or severity of the events they are used to track. The standard levels and their applicability are described below (in increasing order of severity): @@ -115,12 +118,18 @@ If you type these lines into a script and run it, you'll see: WARNING:root:Watch out! printed out on the console. The ``INFO`` message doesn't appear because the -default level is ``WARNING``. The printed message includes the indication of -the level and the description of the event provided in the logging call, i.e. -'Watch out!'. Don't worry about the 'root' part for now: it will be explained -later. The actual output can be formatted quite flexibly if you need that; -formatting options will also be explained later. - +default level is ``WARNING``. The printed message includes the indication of the +level and the description of the event provided in the logging call, i.e. +'Watch out!'. The actual output can be formatted quite flexibly if you need +that; formatting options will also be explained later. + +Notice that in this example, we use functions directly on the ``logging`` +module, like ``logging.debug``, rather than creating a logger and calling +functions on it. These functions operation on the root logger, but can be useful +as they will call :func:`~logging.basicConfig` for you if it has not been called yet, like in +this example. In larger programs you'll usually want to control the logging +configuration explicitly however - so for that reason as well as others, it's +better to create loggers and call their methods. Logging to a file ^^^^^^^^^^^^^^^^^ @@ -130,11 +139,12 @@ look at that next. Be sure to try the following in a newly started Python interpreter, and don't just continue from the session described above:: import logging + logger = logging.getLogger(__name__) logging.basicConfig(filename='example.log', encoding='utf-8', level=logging.DEBUG) - logging.debug('This message should go to the log file') - logging.info('So should this') - logging.warning('And this, too') - logging.error('And non-ASCII stuff, too, like Øresund and Malmö') + logger.debug('This message should go to the log file') + logger.info('So should this') + logger.warning('And this, too') + logger.error('And non-ASCII stuff, too, like Øresund and Malmö') .. versionchanged:: 3.9 The *encoding* argument was added. In earlier Python versions, or if not @@ -148,10 +158,10 @@ messages: .. code-block:: none - DEBUG:root:This message should go to the log file - INFO:root:So should this - WARNING:root:And this, too - ERROR:root:And non-ASCII stuff, too, like Øresund and Malmö + DEBUG:__main__:This message should go to the log file + INFO:__main__:So should this + WARNING:__main__:And this, too + ERROR:__main__:And non-ASCII stuff, too, like Øresund and Malmö This example also shows how you can set the logging level which acts as the threshold for tracking. In this case, because we set the threshold to @@ -180,11 +190,9 @@ following example:: raise ValueError('Invalid log level: %s' % loglevel) logging.basicConfig(level=numeric_level, ...) -The call to :func:`basicConfig` should come *before* any calls to -:func:`debug`, :func:`info`, etc. Otherwise, those functions will call -:func:`basicConfig` for you with the default options. As it's intended as a -one-off simple configuration facility, only the first call will actually do -anything: subsequent calls are effectively no-ops. +The call to :func:`basicConfig` should come *before* any calls to a logger's +methods such as :meth:`~Logger.debug`, :meth:`~Logger.info`, etc. Otherwise, +that logging event may not be handled in the desired manner. If you run the above script several times, the messages from successive runs are appended to the file *example.log*. If you want each run to start afresh, @@ -197,50 +205,6 @@ The output will be the same as before, but the log file is no longer appended to, so the messages from earlier runs are lost. -Logging from multiple modules -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If your program consists of multiple modules, here's an example of how you -could organize logging in it:: - - # myapp.py - import logging - import mylib - - def main(): - logging.basicConfig(filename='myapp.log', level=logging.INFO) - logging.info('Started') - mylib.do_something() - logging.info('Finished') - - if __name__ == '__main__': - main() - -:: - - # mylib.py - import logging - - def do_something(): - logging.info('Doing something') - -If you run *myapp.py*, you should see this in *myapp.log*: - -.. code-block:: none - - INFO:root:Started - INFO:root:Doing something - INFO:root:Finished - -which is hopefully what you were expecting to see. You can generalize this to -multiple modules, using the pattern in *mylib.py*. Note that for this simple -usage pattern, you won't know, by looking in the log file, *where* in your -application your messages came from, apart from looking at the event -description. If you want to track the location of your messages, you'll need -to refer to the documentation beyond the tutorial level -- see -:ref:`logging-advanced-tutorial`. - - Logging variable data ^^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index e38b7435498507..4e7d18bda8bf7d 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -30,13 +30,53 @@ is that all Python modules can participate in logging, so your application log can include your own messages integrated with messages from third-party modules. -The simplest example: +Here's a simple example of idiomatic usage: :: + + # myapp.py + import logging + import mylib + logger = logging.getLogger(__name__) + + def main(): + logging.basicConfig(filename='myapp.log', level=logging.INFO) + logger.info('Started') + mylib.do_something() + logger.info('Finished') + + if __name__ == '__main__': + main() + +:: + + # mylib.py + import logging + logger = logging.getLogger(__name__) + + def do_something(): + logger.info('Doing something') + +If you run *myapp.py*, you should see this in *myapp.log*: .. code-block:: none - >>> import logging - >>> logging.warning('Watch out!') - WARNING:root:Watch out! + INFO:__main__:Started + INFO:mylib:Doing something + INFO:__main__:Finished + +The key features of this idiomatic usage is that the majority of code is simply +creating a module level logger with ``getLogger(__name__)``, and using that +logger to do any needed logging. This is concise while allowing downstream code +fine grained control if needed. Logged messages to the module-level logger get +forwarded up to handlers of loggers in higher-level modules, all the way up to +the root logger; for this reason this approach is known as hierarchical logging. + +For logging to be useful, it needs to be configured: setting the levels and +destinations for each logger, potentially changing how specific modules log, +often based on command-line arguments or application configuration. In most +cases, like the one above, only the root logger needs to be so configured, since +all the lower level loggers at module level eventually forward their messages to +its handlers. :func:`~logging.basicConfig` provides a quick way to configure +the root logger that handles many use cases. The module provides a lot of functionality and flexibility. If you are unfamiliar with logging, the best way to get to grips with it is to view the @@ -1151,89 +1191,31 @@ functions. .. function:: debug(msg, *args, **kwargs) - Logs a message with level :const:`DEBUG` on the root logger. The *msg* is the - message format string, and the *args* are the arguments which are merged into - *msg* using the string formatting operator. (Note that this means that you can - use keywords in the format string, together with a single dictionary argument.) - - There are three keyword arguments in *kwargs* which are inspected: *exc_info* - which, if it does not evaluate as false, causes exception information to be - added to the logging message. If an exception tuple (in the format returned by - :func:`sys.exc_info`) or an exception instance is provided, it is used; - otherwise, :func:`sys.exc_info` is called to get the exception information. - - The second optional keyword argument is *stack_info*, which defaults to - ``False``. If true, stack information is added to the logging - message, including the actual logging call. Note that this is not the same - stack information as that displayed through specifying *exc_info*: The - former is stack frames from the bottom of the stack up to the logging call - in the current thread, whereas the latter is information about stack frames - which have been unwound, following an exception, while searching for - exception handlers. - - You can specify *stack_info* independently of *exc_info*, e.g. to just show - how you got to a certain point in your code, even when no exceptions were - raised. The stack frames are printed following a header line which says: - - .. code-block:: none + This is a convenience function that calls :meth:`Logger.debug`, on the root + logger. The handling of the arguments is in every way identical + to what is described in that method. - Stack (most recent call last): + The only difference is that if the root logger has no handlers, then + :func:`basicConfig` is called, prior to calling ``debug`` on the root logger. - This mimics the ``Traceback (most recent call last):`` which is used when - displaying exception frames. + For very short scripts or quick demonstrations of ``logging`` facilities, + ``debug`` and the other module-level functions may be convenient. However, + most programs will want to carefully and explicitly control the logging + configuration, and should therefore prefer creating a module-level logger and + calling :meth:`Logger.debug` (or other level-specific methods) on it, as + described at the beginnning of this documentation. - The third optional keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the LogRecord created for - the logging event with user-defined attributes. These custom attributes can then - be used as you like. For example, they could be incorporated into logged - messages. For example:: - - FORMAT = '%(asctime)s %(clientip)-15s %(user)-8s %(message)s' - logging.basicConfig(format=FORMAT) - d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} - logging.warning('Protocol problem: %s', 'connection reset', extra=d) - - would print something like: - - .. code-block:: none - - 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset - - The keys in the dictionary passed in *extra* should not clash with the keys used - by the logging system. (See the :class:`Formatter` documentation for more - information on which keys are used by the logging system.) - - If you choose to use these attributes in logged messages, you need to exercise - some care. In the above example, for instance, the :class:`Formatter` has been - set up with a format string which expects 'clientip' and 'user' in the attribute - dictionary of the LogRecord. If these are missing, the message will not be - logged because a string formatting exception will occur. So in this case, you - always need to pass the *extra* dictionary with these keys. - - While this might be annoying, this feature is intended for use in specialized - circumstances, such as multi-threaded servers where the same code executes in - many contexts, and interesting conditions which arise are dependent on this - context (such as remote client IP address and authenticated user name, in the - above example). In such circumstances, it is likely that specialized - :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. - - This function (as well as :func:`info`, :func:`warning`, :func:`error` and - :func:`critical`) will call :func:`basicConfig` if the root logger doesn't - have any handler attached. - - .. versionchanged:: 3.2 - The *stack_info* parameter was added. .. function:: info(msg, *args, **kwargs) - Logs a message with level :const:`INFO` on the root logger. The arguments are - interpreted as for :func:`debug`. + Logs a message with level :const:`INFO` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: warning(msg, *args, **kwargs) - Logs a message with level :const:`WARNING` on the root logger. The arguments - are interpreted as for :func:`debug`. + Logs a message with level :const:`WARNING` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. note:: There is an obsolete function ``warn`` which is functionally identical to ``warning``. As ``warn`` is deprecated, please do not use @@ -1246,26 +1228,26 @@ functions. .. function:: error(msg, *args, **kwargs) - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. + Logs a message with level :const:`ERROR` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: critical(msg, *args, **kwargs) - Logs a message with level :const:`CRITICAL` on the root logger. The arguments - are interpreted as for :func:`debug`. + Logs a message with level :const:`CRITICAL` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: exception(msg, *args, **kwargs) - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. Exception info is added to the logging + Logs a message with level :const:`ERROR` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. Exception info is added to the logging message. This function should only be called from an exception handler. .. function:: log(level, msg, *args, **kwargs) - Logs a message with level *level* on the root logger. The other arguments are - interpreted as for :func:`debug`. + Logs a message with level *level* on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: disable(level=CRITICAL) From 25684e71310642ffd20b45eea9b5226a1fa809a5 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 13 Mar 2024 21:12:40 +0300 Subject: [PATCH 067/158] gh-100746: Improve `test_named_expressions.py` (#116713) --- Lib/test/test_named_expressions.py | 70 +++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index f2017bdffcf968..cf44080670dc2e 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -298,6 +298,72 @@ def test_named_expression_invalid_set_comprehension_iterable_expression(self): with self.assertRaisesRegex(SyntaxError, msg): exec(f"lambda: {code}", {}) # Function scope + def test_named_expression_invalid_rebinding_dict_comprehension_iteration_variable(self): + cases = [ + ("Key reuse", 'i', "{(i := 0): 1 for i in range(5)}"), + ("Value reuse", 'i', "{1: (i := 0) for i in range(5)}"), + ("Both reuse", 'i', "{(i := 0): (i := 0) for i in range(5)}"), + ("Nested reuse", 'j', "{{(j := 0): 1 for i in range(5)} for j in range(5)}"), + ("Reuse inner loop target", 'j', "{(j := 0): 1 for i in range(5) for j in range(5)}"), + ("Unpacking key reuse", 'i', "{(i := 0): 1 for i, j in {(0, 1)}}"), + ("Unpacking value reuse", 'i', "{1: (i := 0) for i, j in {(0, 1)}}"), + ("Reuse in loop condition", 'i', "{i+1: 1 for i in range(5) if (i := 0)}"), + ("Unreachable reuse", 'i', "{(False or (i:=0)): 1 for i in range(5)}"), + ("Unreachable nested reuse", 'i', + "{i: j for i in range(5) for j in range(5) if True or (i:=10)}"), + # Regression tests from https://github.com/python/cpython/issues/87447 + ("Complex expression: a", "a", + "{(a := 1): 1 for a, (*b, c[d+e::f(g)], h.i) in j}"), + ("Complex expression: b", "b", + "{(b := 1): 1 for a, (*b, c[d+e::f(g)], h.i) in j}"), + ] + for case, target, code in cases: + msg = f"assignment expression cannot rebind comprehension iteration variable '{target}'" + with self.subTest(case=case): + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}) # Module scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}, {}) # Class scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(f"lambda: {code}", {}) # Function scope + + def test_named_expression_invalid_rebinding_dict_comprehension_inner_loop(self): + cases = [ + ("Inner reuse", 'j', "{i: 1 for i in range(5) if (j := 0) for j in range(5)}"), + ("Inner unpacking reuse", 'j', "{i: 1 for i in range(5) if (j := 0) for j, k in {(0, 1)}}"), + ] + for case, target, code in cases: + msg = f"comprehension inner loop cannot rebind assignment expression target '{target}'" + with self.subTest(case=case): + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}) # Module scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}, {}) # Class scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(f"lambda: {code}", {}) # Function scope + + def test_named_expression_invalid_dict_comprehension_iterable_expression(self): + cases = [ + ("Top level", "{i: 1 for i in (i := range(5))}"), + ("Inside tuple", "{i: 1 for i in (2, 3, i := range(5))}"), + ("Inside list", "{i: 1 for i in [2, 3, i := range(5)]}"), + ("Different name", "{i: 1 for i in (j := range(5))}"), + ("Lambda expression", "{i: 1 for i in (lambda:(j := range(5)))()}"), + ("Inner loop", "{i: 1 for i in range(5) for j in (i := range(5))}"), + ("Nested comprehension", "{i: 1 for i in {j: 2 for j in (k := range(5))}}"), + ("Nested comprehension condition", "{i: 1 for i in {j: 2 for j in range(5) if (j := True)}}"), + ("Nested comprehension body", "{i: 1 for i in {(j := True) for j in range(5)}}"), + ] + msg = "assignment expression cannot be used in a comprehension iterable expression" + for case, code in cases: + with self.subTest(case=case): + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}) # Module scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}, {}) # Class scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(f"lambda: {code}", {}) # Function scope + def test_named_expression_invalid_mangled_class_variables(self): code = """class Foo: def bar(self): @@ -361,7 +427,7 @@ def test_named_expression_assignment_09(self): def test_named_expression_assignment_10(self): if (match := 10) == 10: - pass + self.assertEqual(match, 10) else: self.fail("variable was not assigned using named expression") def test_named_expression_assignment_11(self): @@ -403,7 +469,7 @@ def test_named_expression_assignment_14(self): def test_named_expression_assignment_15(self): while a := False: - pass # This will not run + self.fail("While body executed") # This will not run self.assertEqual(a, False) From 98ab21cce6d4c7bd2b5a0a1521b50b1ce2566a44 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 13 Mar 2024 14:56:28 -0400 Subject: [PATCH 068/158] gh-116631: Fix race condition in `test_shutdown_immediate_put_join` (#116670) The test case had a race condition: if `q.task_done()` was executed after `shutdown(immediate=True)`, then it would raise an exception because the immediate shutdown already emptied the queue. This happened rarely with the GIL (due to the switching interval), but frequently in the free-threaded build. --- Lib/test/test_queue.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 92d670ca6f8f5b..ad31ba1af03b6f 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -567,7 +567,6 @@ def _shutdown_put_join(self, immediate): results = [] go = threading.Event() q.put("Y") - nb = q.qsize() # queue not fulled thrds = ( @@ -578,13 +577,19 @@ def _shutdown_put_join(self, immediate): for func, params in thrds: threads.append(threading.Thread(target=func, args=params)) threads[-1].start() - self.assertEqual(q.unfinished_tasks, nb) - for i in range(nb): - t = threading.Thread(target=q.task_done) - t.start() - threads.append(t) + self.assertEqual(q.unfinished_tasks, 1) + q.shutdown(immediate) go.set() + + if immediate: + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() + else: + result = q.get() + self.assertEqual(result, "Y") + q.task_done() + for t in threads: t.join() From e54bdeab9ce2958a22ba08c1f1f1431c5e8056bd Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 13 Mar 2024 21:15:44 +0200 Subject: [PATCH 069/158] gh-90300: Sort the -X options and some envvars in the Python CLI help (GH-116739) --- Python/initconfig.c | 83 +++++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index a01a9f56f37832..020312a015922c 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -187,19 +187,9 @@ arg ...: arguments passed to program in sys.argv[1:]\n\ static const char usage_xoptions[] = "\ The following implementation-specific options are available:\n\ --X faulthandler: enable faulthandler\n\ --X showrefcount: output the total reference count and number of used\n\ - memory blocks when the program finishes or after each statement in\n\ - the interactive interpreter. This only works on debug builds\n\ --X tracemalloc: start tracing Python memory allocations using the\n\ - tracemalloc module. By default, only the most recent frame is stored\n\ - in a traceback of a trace. Use -X tracemalloc=NFRAME to start\n\ - tracing with a traceback limit of NFRAME frames\n\ --X importtime: show how long each import takes. It shows module name,\n\ - cumulative time (including nested imports) and self time (excluding\n\ - nested imports). Note that its output may be broken in\n\ - multi-threaded application.\n\ - Typical usage is python3 -X importtime -c 'import asyncio'\n\ +-X cpu_count=[n|default]: Override the return value of os.cpu_count(),\n\ + os.process_cpu_count(), and multiprocessing.cpu_count(). This can\n\ + help users who need to limit resources in a container.\n\ -X dev : enable CPython's \"development mode\", introducing additional runtime\n\ checks which are too expensive to be enabled by default. Effect of\n\ the developer mode:\n\ @@ -211,12 +201,18 @@ The following implementation-specific options are available:\n\ * Enable asyncio debug mode\n\ * Set the dev_mode attribute of sys.flags to True\n\ * io.IOBase destructor logs close() exceptions\n\ --X utf8: enable UTF-8 mode for operating system interfaces, overriding the\n\ - default locale-aware mode. -X utf8=0 explicitly disables UTF-8 mode\n\ - (even when it would otherwise activate automatically)\n\ --X pycache_prefix=PATH: enable writing .pyc files to a parallel tree rooted\n\ - at the given directory instead of to the code tree\n\ --X warn_default_encoding: enable opt-in EncodingWarning for 'encoding=None'\n\ +-X faulthandler: enable faulthandler\n\ +-X frozen_modules=[on|off]: whether or not frozen modules should be used.\n\ + The default is \"on\" (or \"off\" if you are running a local build).\n\ +-X importtime: show how long each import takes. It shows module name,\n\ + cumulative time (including nested imports) and self time (excluding\n\ + nested imports). Note that its output may be broken in\n\ + multi-threaded application.\n\ + Typical usage is python3 -X importtime -c 'import asyncio'\n\ +-X int_max_str_digits=number: limit the size of int<->str conversions.\n\ + This helps avoid denial of service attacks when parsing untrusted\n\ + data. The default is sys.int_info.default_max_str_digits.\n\ + 0 disables.\n\ -X no_debug_ranges: disable the inclusion of the tables mapping extra location\n\ information (end line, start column offset and end column offset) to\n\ every instruction in code objects. This is useful when smaller code\n\ @@ -227,23 +223,30 @@ The following implementation-specific options are available:\n\ profiler will be able to report Python calls. This option is only\n\ available on some platforms and will do nothing if is not supported\n\ on the current system. The default value is \"off\".\n\ --X frozen_modules=[on|off]: whether or not frozen modules should be used.\n\ - The default is \"on\" (or \"off\" if you are running a local build).\n\ --X int_max_str_digits=number: limit the size of int<->str conversions.\n\ - This helps avoid denial of service attacks when parsing untrusted\n\ - data. The default is sys.int_info.default_max_str_digits.\n\ - 0 disables.\n\ --X cpu_count=[n|default]: Override the return value of os.cpu_count(),\n\ - os.process_cpu_count(), and multiprocessing.cpu_count(). This can\n\ - help users who need to limit resources in a container." -#ifdef Py_STATS -"\n\ --X pystats: Enable pystats collection at startup." -#endif +" #ifdef Py_DEBUG -"\n\ --X presite=package.module: import this module before site.py is run." +"-X presite=package.module: import this module before site.py is run.\n" #endif +"\ +-X pycache_prefix=PATH: enable writing .pyc files to a parallel tree rooted\n\ + at the given directory instead of to the code tree\n\ +" +#ifdef Py_STATS +"-X pystats: Enable pystats collection at startup.\n" +#endif +"\ +-X showrefcount: output the total reference count and number of used\n\ + memory blocks when the program finishes or after each statement in\n\ + the interactive interpreter. This only works on debug builds\n\ +-X tracemalloc: start tracing Python memory allocations using the\n\ + tracemalloc module. By default, only the most recent frame is stored\n\ + in a traceback of a trace. Use -X tracemalloc=NFRAME to start\n\ + tracing with a traceback limit of NFRAME frames\n\ +-X utf8: enable UTF-8 mode for operating system interfaces, overriding the\n\ + default locale-aware mode. -X utf8=0 explicitly disables UTF-8 mode\n\ + (even when it would otherwise activate automatically)\n\ +-X warn_default_encoding: enable opt-in EncodingWarning for 'encoding=None'\ +" ; /* Envvars that don't have equivalent command-line options are listed first */ @@ -310,16 +313,16 @@ static const char usage_envvars[] = " (-X int_max_str_digits=number)\n" "PYTHONNOUSERSITE: disable user site directory (-s)\n" "PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" +#ifdef Py_DEBUG +"PYTHON_PRESITE=pkg.mod: import this module before site.py is run\n" +#endif "PYTHONSAFEPATH : don't prepend a potentially unsafe path to sys.path.\n" -"PYTHONUNBUFFERED: disable stdout/stderr buffering (-u)\n" -"PYTHONVERBOSE : trace import statements (-v)\n" -"PYTHONWARNINGS=arg: warning control (-W arg)\n" #ifdef Py_STATS "PYTHONSTATS : turns on statistics gathering\n" #endif -#ifdef Py_DEBUG -"PYTHON_PRESITE=pkg.mod: import this module before site.py is run\n" -#endif +"PYTHONUNBUFFERED: disable stdout/stderr buffering (-u)\n" +"PYTHONVERBOSE : trace import statements (-v)\n" +"PYTHONWARNINGS=arg: warning control (-W arg)\n" ; #if defined(MS_WINDOWS) From 5ff012a4495060fe9f53ed034c90033e7eafb780 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 13 Mar 2024 15:02:56 -0500 Subject: [PATCH 070/158] Better presentation order for recipes. (gh-116755) --- Doc/library/itertools.rst | 86 +++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 26d5cdaff8c14f..962d23bb53bf6b 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -785,8 +785,8 @@ well as with the built-in itertools such as ``map()``, ``filter()``, A secondary purpose of the recipes is to serve as an incubator. The ``accumulate()``, ``compress()``, and ``pairwise()`` itertools started out as -recipes. Currently, the ``sliding_window()`` and ``iter_index()`` recipes -are being tested to see whether they prove their worth. +recipes. Currently, the ``sliding_window()``, ``iter_index()``, and ``sieve()`` +recipes are being tested to see whether they prove their worth. Substantially all of these recipes and many, many others can be installed from the `more-itertools project `_ found @@ -795,12 +795,12 @@ on the Python Package Index:: python -m pip install more-itertools Many of the recipes offer the same high performance as the underlying toolset. -Superior memory performance is kept by processing elements one at a time -rather than bringing the whole iterable into memory all at once. Code volume is -kept small by linking the tools together in a functional style which helps -eliminate temporary variables. High speed is retained by preferring -"vectorized" building blocks over the use of for-loops and :term:`generator`\s -which incur interpreter overhead. +Superior memory performance is kept by processing elements one at a time rather +than bringing the whole iterable into memory all at once. Code volume is kept +small by linking the tools together in a `functional style +`_. High speed +is retained by preferring "vectorized" building blocks over the use of for-loops +and :term:`generators ` which incur interpreter overhead. .. testcode:: @@ -873,6 +873,14 @@ which incur interpreter overhead. "Returns True if all the elements are equal to each other." return len(take(2, groupby(iterable, key))) <= 1 + def unique_justseen(iterable, key=None): + "List unique elements, preserving order. Remember only the element just seen." + # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B + # unique_justseen('ABBcCAD', str.casefold) --> A B c A D + if key is None: + return map(operator.itemgetter(0), groupby(iterable)) + return map(next, map(operator.itemgetter(1), groupby(iterable, key))) + def unique_everseen(iterable, key=None): "List unique elements, preserving order. Remember all elements ever seen." # unique_everseen('AAAABBBCCDAABBB') --> A B C D @@ -889,35 +897,6 @@ which incur interpreter overhead. seen.add(k) yield element - def unique_justseen(iterable, key=None): - "List unique elements, preserving order. Remember only the element just seen." - # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B - # unique_justseen('ABBcCAD', str.casefold) --> A B c A D - if key is None: - return map(operator.itemgetter(0), groupby(iterable)) - return map(next, map(operator.itemgetter(1), groupby(iterable, key))) - - def iter_index(iterable, value, start=0, stop=None): - "Return indices where a value occurs in a sequence or iterable." - # iter_index('AABCADEAF', 'A') --> 0 1 4 7 - seq_index = getattr(iterable, 'index', None) - if seq_index is None: - # Path for general iterables - it = islice(iterable, start, stop) - for i, element in enumerate(it, start): - if element is value or element == value: - yield i - else: - # Path for sequences with an index() method - stop = len(iterable) if stop is None else stop - i = start - try: - while True: - yield (i := seq_index(value, i, stop)) - i += 1 - except ValueError: - pass - def sliding_window(iterable, n): "Collect data into overlapping fixed-length chunks or blocks." # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG @@ -967,6 +946,27 @@ which incur interpreter overhead. slices = starmap(slice, combinations(range(len(seq) + 1), 2)) return map(operator.getitem, repeat(seq), slices) + def iter_index(iterable, value, start=0, stop=None): + "Return indices where a value occurs in a sequence or iterable." + # iter_index('AABCADEAF', 'A') --> 0 1 4 7 + seq_index = getattr(iterable, 'index', None) + if seq_index is None: + # Path for general iterables + it = islice(iterable, start, stop) + for i, element in enumerate(it, start): + if element is value or element == value: + yield i + else: + # Path for sequences with an index() method + stop = len(iterable) if stop is None else stop + i = start + try: + while True: + yield (i := seq_index(value, i, stop)) + i += 1 + except ValueError: + pass + def iter_except(func, exception, first=None): """ Call a function repeatedly until an exception is raised. @@ -1047,8 +1047,8 @@ The following recipes have a more mathematical flavor: Computes with better numeric stability than Horner's method. """ - # Evaluate x³ -4x² -17x + 60 at x = 2.5 - # polynomial_eval([1, -4, -17, 60], x=2.5) --> 8.125 + # Evaluate x³ -4x² -17x + 60 at x = 5 + # polynomial_eval([1, -4, -17, 60], x=5) --> 0 n = len(coefficients) if not n: return type(x)(0) @@ -1311,10 +1311,10 @@ The following recipes have a more mathematical flavor: >>> from fractions import Fraction >>> from decimal import Decimal - >>> polynomial_eval([1, -4, -17, 60], x=2) - 18 - >>> x = 2; x**3 - 4*x**2 -17*x + 60 - 18 + >>> polynomial_eval([1, -4, -17, 60], x=5) + 0 + >>> x = 5; x**3 - 4*x**2 -17*x + 60 + 0 >>> polynomial_eval([1, -4, -17, 60], x=2.5) 8.125 >>> x = 2.5; x**3 - 4*x**2 -17*x + 60 From 61733a2fb9dc36d2246d922146a3462a2248832d Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 13 Mar 2024 13:24:28 -0700 Subject: [PATCH 071/158] GH-115979: update test_importlib to work under WASI SDK 21 (GH-116754) --- .../extension/test_case_sensitivity.py | 3 +- .../test_importlib/extension/test_finder.py | 2 +- .../test_importlib/extension/test_loader.py | 6 +-- .../extension/test_path_hook.py | 2 + Lib/test/test_importlib/test_spec.py | 9 ++-- Lib/test/test_importlib/test_util.py | 2 +- Lib/test/test_importlib/util.py | 48 +++++++++++-------- ...-03-13-12-06-49.gh-issue-115979.zsNpQD.rst | 1 + 8 files changed, 45 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2024-03-13-12-06-49.gh-issue-115979.zsNpQD.rst diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py index 0bb74fff5fcf96..40311627a144e8 100644 --- a/Lib/test/test_importlib/extension/test_case_sensitivity.py +++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py @@ -8,7 +8,8 @@ machinery = util.import_importlib('importlib.machinery') -@unittest.skipIf(util.EXTENSIONS.filename is None, f'{util.EXTENSIONS.name} not available') +@unittest.skipIf(util.EXTENSIONS is None or util.EXTENSIONS.filename is None, + 'dynamic loading not supported or test module not available') @util.case_insensitive_tests class ExtensionModuleCaseSensitivityTest(util.CASEOKTestBase): diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py index 1d5b6e7a5de94b..3de120958fd27d 100644 --- a/Lib/test/test_importlib/extension/test_finder.py +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -11,7 +11,7 @@ class FinderTests(abc.FinderTests): """Test the finder for extension modules.""" def setUp(self): - if not self.machinery.EXTENSION_SUFFIXES: + if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") if util.EXTENSIONS.name in sys.builtin_module_names: raise unittest.SkipTest( diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 84a0680e4ec653..f4879e75847d8d 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -17,7 +17,7 @@ class LoaderTests: """Test ExtensionFileLoader.""" def setUp(self): - if not self.machinery.EXTENSION_SUFFIXES: + if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") if util.EXTENSIONS.name in sys.builtin_module_names: raise unittest.SkipTest( @@ -99,7 +99,7 @@ class SinglePhaseExtensionModuleTests(abc.LoaderTests): # Test loading extension modules without multi-phase initialization. def setUp(self): - if not self.machinery.EXTENSION_SUFFIXES: + if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") self.name = '_testsinglephase' if self.name in sys.builtin_module_names: @@ -180,7 +180,7 @@ class MultiPhaseExtensionModuleTests(abc.LoaderTests): # Test loading extension modules with multi-phase initialization (PEP 489). def setUp(self): - if not self.machinery.EXTENSION_SUFFIXES: + if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") self.name = '_testmultiphase' if self.name in sys.builtin_module_names: diff --git a/Lib/test/test_importlib/extension/test_path_hook.py b/Lib/test/test_importlib/extension/test_path_hook.py index ec9644dc520534..314a635c77e082 100644 --- a/Lib/test/test_importlib/extension/test_path_hook.py +++ b/Lib/test/test_importlib/extension/test_path_hook.py @@ -5,6 +5,8 @@ import unittest +@unittest.skipIf(util.EXTENSIONS is None or util.EXTENSIONS.filename is None, + 'dynamic loading not supported or test module not available') class PathHookTests: """Test the path hook for extension modules.""" diff --git a/Lib/test/test_importlib/test_spec.py b/Lib/test/test_importlib/test_spec.py index 80aa3609c6f96e..02318926f35eee 100644 --- a/Lib/test/test_importlib/test_spec.py +++ b/Lib/test/test_importlib/test_spec.py @@ -502,7 +502,8 @@ def test_spec_from_loader_is_package_true_with_fileloader(self): self.assertEqual(spec.loader, self.fileloader) self.assertEqual(spec.origin, self.path) self.assertIs(spec.loader_state, None) - self.assertEqual(spec.submodule_search_locations, [os.getcwd()]) + location = cwd if (cwd := os.getcwd()) != '/' else '' + self.assertEqual(spec.submodule_search_locations, [location]) self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) @@ -601,7 +602,8 @@ def test_spec_from_file_location_smsl_empty(self): self.assertEqual(spec.loader, self.fileloader) self.assertEqual(spec.origin, self.path) self.assertIs(spec.loader_state, None) - self.assertEqual(spec.submodule_search_locations, [os.getcwd()]) + location = cwd if (cwd := os.getcwd()) != '/' else '' + self.assertEqual(spec.submodule_search_locations, [location]) self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) @@ -626,7 +628,8 @@ def test_spec_from_file_location_smsl_default(self): self.assertEqual(spec.loader, self.pkgloader) self.assertEqual(spec.origin, self.path) self.assertIs(spec.loader_state, None) - self.assertEqual(spec.submodule_search_locations, [os.getcwd()]) + location = cwd if (cwd := os.getcwd()) != '/' else '' + self.assertEqual(spec.submodule_search_locations, [location]) self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index fe5e7b31d9c32b..a09286806e5152 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -577,7 +577,7 @@ def test_cache_from_source_respects_pycache_prefix_relative(self): with util.temporary_pycache_prefix(pycache_prefix): self.assertEqual( self.util.cache_from_source(path, optimization=''), - expect) + os.path.normpath(expect)) @unittest.skipIf(sys.implementation.cache_tag is None, 'requires sys.implementation.cache_tag to not be None') diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index c25be096e52874..a900cc1dddf425 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -6,6 +6,7 @@ import marshal import os import os.path +from test import support from test.support import import_helper from test.support import os_helper import unittest @@ -22,25 +23,34 @@ if 'importlib' not in sys.builtin_module_names: BUILTINS.bad_name = 'importlib' -EXTENSIONS = types.SimpleNamespace() -EXTENSIONS.path = None -EXTENSIONS.ext = None -EXTENSIONS.filename = None -EXTENSIONS.file_path = None -EXTENSIONS.name = '_testsinglephase' - -def _extension_details(): - global EXTENSIONS - for path in sys.path: - for ext in machinery.EXTENSION_SUFFIXES: - filename = EXTENSIONS.name + ext - file_path = os.path.join(path, filename) - if os.path.exists(file_path): - EXTENSIONS.path = path - EXTENSIONS.ext = ext - EXTENSIONS.filename = filename - EXTENSIONS.file_path = file_path - return +if support.is_wasi: + # dlopen() is a shim for WASI as of WASI SDK which fails by default. + # We don't provide an implementation, so tests will fail. + # But we also don't want to turn off dynamic loading for those that provide + # a working implementation. + def _extension_details(): + global EXTENSIONS + EXTENSIONS = None +else: + EXTENSIONS = types.SimpleNamespace() + EXTENSIONS.path = None + EXTENSIONS.ext = None + EXTENSIONS.filename = None + EXTENSIONS.file_path = None + EXTENSIONS.name = '_testsinglephase' + + def _extension_details(): + global EXTENSIONS + for path in sys.path: + for ext in machinery.EXTENSION_SUFFIXES: + filename = EXTENSIONS.name + ext + file_path = os.path.join(path, filename) + if os.path.exists(file_path): + EXTENSIONS.path = path + EXTENSIONS.ext = ext + EXTENSIONS.filename = filename + EXTENSIONS.file_path = file_path + return _extension_details() diff --git a/Misc/NEWS.d/next/Tests/2024-03-13-12-06-49.gh-issue-115979.zsNpQD.rst b/Misc/NEWS.d/next/Tests/2024-03-13-12-06-49.gh-issue-115979.zsNpQD.rst new file mode 100644 index 00000000000000..02bc2b88942e4f --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-03-13-12-06-49.gh-issue-115979.zsNpQD.rst @@ -0,0 +1 @@ +Update test_importlib so that it passes under WASI SDK 21. From 8c6db45ce34df7081d7497e638daf3e130303295 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 13 Mar 2024 22:59:16 +0200 Subject: [PATCH 072/158] gh-90300: Document equivalent -X options for envvars in the Python CLI help (GH-116756) --- Python/initconfig.c | 60 +++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index 020312a015922c..f365bee89a5f3b 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -259,18 +259,11 @@ static const char usage_envvars[] = " The default module search path uses %s.\n" "PYTHONPLATLIBDIR: override sys.platlibdir.\n" "PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" -"PYTHONUTF8 : if set to 1, enable the UTF-8 mode.\n" "PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" -"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors.\n" "PYTHONHASHSEED : if this variable is set to 'random', a random value is used\n" " to seed the hashes of str and bytes objects. It can also be\n" " set to an integer in the range [0,4294967295] to get hash\n" " values with a predictable seed.\n" -"PYTHONINTMAXSTRDIGITS: limits the maximum digit characters in an int value\n" -" when converting from a string and when converting an int\n" -" back to a str. A value of 0 disables the limit.\n" -" Conversions to or from bases 2, 4, 8, 16, and 32 are never\n" -" limited.\n" "PYTHONMALLOC : set the Python memory allocators and/or install debug hooks\n" " on Python memory allocators. Use PYTHONMALLOC=debug to\n" " install debug hooks.\n" @@ -281,47 +274,56 @@ static const char usage_envvars[] = "PYTHONBREAKPOINT: if this variable is set to 0, it disables the default\n" " debugger. It can be set to the callable of your debugger of\n" " choice.\n" -"PYTHON_CPU_COUNT: Overrides the return value of os.process_cpu_count(),\n" -" os.cpu_count(), and multiprocessing.cpu_count() if set to\n" -" a positive integer.\n" -#ifdef Py_GIL_DISABLED -"PYTHON_GIL : When set to 0, disables the GIL.\n" -#endif -"PYTHONDEVMODE : enable the development mode.\n" -"PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files.\n" -"PYTHONWARNDEFAULTENCODING: enable opt-in EncodingWarning for 'encoding=None'.\n" -"PYTHONNODEBUGRANGES: if this variable is set, it disables the inclusion of\n" -" the tables mapping extra location information (end line,\n" -" start column offset and end column offset) to every\n" -" instruction in code objects. This is useful when smaller\n" -" code objects and pyc files are desired as well as\n" -" suppressing the extra visual location indicators when the\n" -" interpreter displays tracebacks.\n" -"PYTHON_FROZEN_MODULES: if this variable is set, it determines whether or not\n" -" frozen modules should be used. The default is \"on\" (or\n" -" \"off\" if you are running a local build).\n" "PYTHON_COLORS : if this variable is set to 1, the interpreter will colorize\n" " various kinds of output. Setting it to 0 deactivates\n" " this behavior.\n" "PYTHON_HISTORY : the location of a .python_history file.\n" "\n" "These variables have equivalent command-line options (see --help for details):\n" +"PYTHON_CPU_COUNT: Overrides the return value of os.process_cpu_count(),\n" +" os.cpu_count(), and multiprocessing.cpu_count() if set to\n" +" a positive integer. (-X cpu_count)\n" "PYTHONDEBUG : enable parser debug mode (-d)\n" +"PYTHONDEVMODE : enable the development mode (-X dev)\n" "PYTHONDONTWRITEBYTECODE: don't write .pyc files (-B)\n" +"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors (-X faulthandler)\n" +"PYTHON_FROZEN_MODULES: if this variable is set, it determines whether or not\n" +" frozen modules should be used. The default is \"on\" (or\n" +" \"off\" if you are running a local build).\n" +" (-X frozen_modules)\n" +#ifdef Py_GIL_DISABLED +"PYTHON_GIL : when set to 0, disables the GIL (-X gil)\n" +#endif "PYTHONINSPECT : inspect interactively after running script (-i)\n" -"PYTHONINTMAXSTRDIGITS: limit max digit characters in an int value\n" +"PYTHONINTMAXSTRDIGITS: limits the maximum digit characters in an int value\n" +" when converting from a string and when converting an int\n" +" back to a str. A value of 0 disables the limit.\n" +" Conversions to or from bases 2, 4, 8, 16, and 32 are never\n" +" limited.\n" " (-X int_max_str_digits=number)\n" +"PYTHONNODEBUGRANGES: if this variable is set, it disables the inclusion of\n" +" the tables mapping extra location information (end line,\n" +" start column offset and end column offset) to every\n" +" instruction in code objects. This is useful when smaller\n" +" code objects and pyc files are desired as well as\n" +" suppressing the extra visual location indicators when the\n" +" interpreter displays tracebacks. (-X no_debug_ranges)\n" "PYTHONNOUSERSITE: disable user site directory (-s)\n" "PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" #ifdef Py_DEBUG -"PYTHON_PRESITE=pkg.mod: import this module before site.py is run\n" +"PYTHON_PRESITE=pkg.mod: import this module before site.py is run (-X presite)\n" #endif +"PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files\n" +" (-X pycache_prefix)\n" "PYTHONSAFEPATH : don't prepend a potentially unsafe path to sys.path.\n" #ifdef Py_STATS -"PYTHONSTATS : turns on statistics gathering\n" +"PYTHONSTATS : turns on statistics gathering (-X pystats)\n" #endif "PYTHONUNBUFFERED: disable stdout/stderr buffering (-u)\n" +"PYTHONUTF8 : if set to 1, enable the UTF-8 mode (-X utf8)\n" "PYTHONVERBOSE : trace import statements (-v)\n" +"PYTHONWARNDEFAULTENCODING: enable opt-in EncodingWarning for 'encoding=None'\n" +" (-X warn_default_encoding)\n" "PYTHONWARNINGS=arg: warning control (-W arg)\n" ; From cef0ec1a3ca40db69b56bcd736c1b3bb05a1cf48 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 13 Mar 2024 18:13:33 -0400 Subject: [PATCH 073/158] gh-116760: Fix pystats for trace attempts (GH-116761) There are now at least two bytecodes that may attempt to optimize, JUMP_BACK, and more recently, COLD_EXIT. Only the JUMP_BACK was counting the attempt in the stats. This moves that counter to uop_optimize itself so it should always happen no matter where it is called from. --- Python/bytecodes.c | 1 - Python/generated_cases.c.h | 1 - Python/optimizer.c | 1 + 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ec05e40bd23fcb..af2e2c8f52ee29 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2349,7 +2349,6 @@ dummy_func( // Use '>=' not '>' so that the optimizer/backoff bits do not effect the result. // Double-check that the opcode isn't instrumented or something: if (offset_counter >= threshold && this_instr->op.code == JUMP_BACKWARD) { - OPT_STAT_INC(attempts); _Py_CODEUNIT *start = this_instr; /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ while (oparg > 255) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 72892725fb25d8..7d02e49d040c23 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3371,7 +3371,6 @@ // Use '>=' not '>' so that the optimizer/backoff bits do not effect the result. // Double-check that the opcode isn't instrumented or something: if (offset_counter >= threshold && this_instr->op.code == JUMP_BACKWARD) { - OPT_STAT_INC(attempts); _Py_CODEUNIT *start = this_instr; /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ while (oparg > 255) { diff --git a/Python/optimizer.c b/Python/optimizer.c index aaf75b2339cd2e..88c45f2e73c682 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1003,6 +1003,7 @@ uop_optimize( _PyBloomFilter dependencies; _Py_BloomFilter_Init(&dependencies); _PyUOpInstruction buffer[UOP_MAX_TRACE_LENGTH]; + OPT_STAT_INC(attempts); int err = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies); if (err <= 0) { // Error or nothing translated From 7af4b9f25383793c9504c71f7b414d34528b8600 Mon Sep 17 00:00:00 2001 From: guangwu Date: Thu, 14 Mar 2024 06:53:32 +0800 Subject: [PATCH 074/158] Docs: fix spelling of the word 'transferring' (#116641) --- Python/tier2_engine.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/tier2_engine.md b/Python/tier2_engine.md index df9f6c124509bd..5ceda8e806045d 100644 --- a/Python/tier2_engine.md +++ b/Python/tier2_engine.md @@ -52,7 +52,7 @@ The JIT compiler converts superblocks into machine code executors. These have identical behavior to interpreted executors, except that they consume more memory for the generated machine code and are a lot faster. -## Transfering control +## Transferring control There are three types of control transfer that we need to consider: * Tier 1 to tier 2 @@ -104,7 +104,7 @@ reference to the previous executor in the thread state's #### The interpreter The tier 2 interpreter has a variable `current_executor` which -points to the currently live executor. When transfering from executor +points to the currently live executor. When transferring from executor `A` to executor `B` we do the following: (Initially `current_executor` points to `A`, and the refcount of `A` is elevated by one) @@ -122,7 +122,7 @@ increment the refcount of `B` and set `current_executor` to point to `B`. #### In the JIT -Transfering control from one executor to another is done via tailcalls. +Transferring control from one executor to another is done via tailcalls. The compiled executor should do the same, except that there is no local variable `current_executor`. From e39795f2cbad5375536f4be6b3c3906f457992bf Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Thu, 14 Mar 2024 02:01:13 +0300 Subject: [PATCH 075/158] Docs: PyUnstable_Long_IsCompact() docs now mention PyLong_AsNativeBytes() (#116634) --- Doc/c-api/long.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 582f5c7bf05471..6a7eba7761de1a 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -450,7 +450,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. a “fast path” for small integers. For compact values use :c:func:`PyUnstable_Long_CompactValue`; for others fall back to a :c:func:`PyLong_As* ` function or - :c:func:`calling ` :meth:`int.to_bytes`. + :c:func:`PyLong_AsNativeBytes`. The speedup is expected to be negligible for most users. From 8c094c3095feb4de2efebd00f67fb6cc3b2bc240 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 13 Mar 2024 16:25:50 -0700 Subject: [PATCH 076/158] GH-115983: skip building shared modules for testing under WASI (GH-116528) --- .../Build/2024-03-08-17-05-15.gh-issue-115983.ZQqk0Q.rst | 1 + configure | 6 ++++++ configure.ac | 8 ++++++++ 3 files changed, 15 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2024-03-08-17-05-15.gh-issue-115983.ZQqk0Q.rst diff --git a/Misc/NEWS.d/next/Build/2024-03-08-17-05-15.gh-issue-115983.ZQqk0Q.rst b/Misc/NEWS.d/next/Build/2024-03-08-17-05-15.gh-issue-115983.ZQqk0Q.rst new file mode 100644 index 00000000000000..a8d39921d59092 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-03-08-17-05-15.gh-issue-115983.ZQqk0Q.rst @@ -0,0 +1 @@ +Skip building test modules that must be built as shared under WASI. diff --git a/configure b/configure index 267a5183379ce0..9b40c48979b1be 100755 --- a/configure +++ b/configure @@ -28587,9 +28587,15 @@ case $ac_sys_system in #( py_cv_module__ctypes_test=n/a + py_cv_module__testexternalinspection=n/a + py_cv_module__testimportmultiple=n/a + py_cv_module__testmultiphase=n/a + py_cv_module__testsinglephase=n/a py_cv_module_fcntl=n/a py_cv_module_mmap=n/a py_cv_module_termios=n/a + py_cv_module_xxlimited=n/a + py_cv_module_xxlimited_35=n/a py_cv_module_=n/a diff --git a/configure.ac b/configure.ac index 681d081f571ea5..02913cef06a5f9 100644 --- a/configure.ac +++ b/configure.ac @@ -7398,11 +7398,19 @@ AS_CASE([$ac_sys_system], [Emscripten/node*], [], [WASI/*], [ dnl WASI SDK 15.0 does not support file locking, mmap, and more. + dnl Test modules that must be compiled as shared libraries are not supported + dnl (see Modules/Setup.stdlib.in). PY_STDLIB_MOD_SET_NA( [_ctypes_test], + [_testexternalinspection], + [_testimportmultiple], + [_testmultiphase], + [_testsinglephase], [fcntl], [mmap], [termios], + [xxlimited], + [xxlimited_35], ) ] ) From 3a25d9c5a95d4e57513ea7edd9e184f4609ebe20 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 13 Mar 2024 18:45:33 -0700 Subject: [PATCH 077/158] GH-114736: Use WASI SDK 21 (GH-116771) --- .devcontainer/Dockerfile | 4 ++-- .github/workflows/reusable-wasi.yml | 4 ++-- .../next/Build/2024-03-13-16-16-43.gh-issue-114736.ZhmauG.rst | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-03-13-16-16-43.gh-issue-114736.ZhmauG.rst diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 365756458bba30..fa30aee478cf33 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -2,11 +2,11 @@ FROM docker.io/library/fedora:37 ENV CC=clang -ENV WASI_SDK_VERSION=20 +ENV WASI_SDK_VERSION=21 ENV WASI_SDK_PATH=/opt/wasi-sdk ENV WASMTIME_HOME=/opt/wasmtime -ENV WASMTIME_VERSION=18.0.2 +ENV WASMTIME_VERSION=18.0.3 ENV WASMTIME_CPU_ARCH=x86_64 RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \ diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 995e669c228b5c..60eef7bc478bbf 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -11,8 +11,8 @@ jobs: timeout-minutes: 60 runs-on: ubuntu-20.04 env: - WASMTIME_VERSION: 18.0.2 - WASI_SDK_VERSION: 20 + WASMTIME_VERSION: 18.0.3 + WASI_SDK_VERSION: 21 WASI_SDK_PATH: /opt/wasi-sdk CROSS_BUILD_PYTHON: cross-build/build CROSS_BUILD_WASI: cross-build/wasm32-wasi diff --git a/Misc/NEWS.d/next/Build/2024-03-13-16-16-43.gh-issue-114736.ZhmauG.rst b/Misc/NEWS.d/next/Build/2024-03-13-16-16-43.gh-issue-114736.ZhmauG.rst new file mode 100644 index 00000000000000..cc863c3a3ceb48 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-03-13-16-16-43.gh-issue-114736.ZhmauG.rst @@ -0,0 +1 @@ +Have WASI builds use WASI SDK 21. From a18c9854e8c255981f07c0a1c1503253f85b7540 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 14 Mar 2024 09:07:01 +0100 Subject: [PATCH 078/158] gh-113317, AC: Move warn() and fail() to libclinic.errors (#116770) --- Tools/clinic/clinic.py | 48 +----------------------------- Tools/clinic/libclinic/__init__.py | 4 +++ Tools/clinic/libclinic/errors.py | 46 ++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 47 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 893f4cc12ed084..4c7c4dca37cd0f 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -44,14 +44,13 @@ Protocol, TypeVar, cast, - overload, ) # Local imports. import libclinic import libclinic.cpp -from libclinic import ClinicError +from libclinic import ClinicError, fail, warn # TODO: @@ -94,51 +93,6 @@ def __repr__(self) -> str: TemplateDict = dict[str, str] -@overload -def warn_or_fail( - *args: object, - fail: Literal[True], - filename: str | None = None, - line_number: int | None = None, -) -> NoReturn: ... - -@overload -def warn_or_fail( - *args: object, - fail: Literal[False] = False, - filename: str | None = None, - line_number: int | None = None, -) -> None: ... - -def warn_or_fail( - *args: object, - fail: bool = False, - filename: str | None = None, - line_number: int | None = None, -) -> None: - joined = " ".join([str(a) for a in args]) - error = ClinicError(joined, filename=filename, lineno=line_number) - if fail: - raise error - else: - print(error.report(warn_only=True)) - - -def warn( - *args: object, - filename: str | None = None, - line_number: int | None = None, -) -> None: - return warn_or_fail(*args, filename=filename, line_number=line_number, fail=False) - -def fail( - *args: object, - filename: str | None = None, - line_number: int | None = None, -) -> NoReturn: - warn_or_fail(*args, filename=filename, line_number=line_number, fail=True) - - class CRenderData: def __init__(self) -> None: diff --git a/Tools/clinic/libclinic/__init__.py b/Tools/clinic/libclinic/__init__.py index 738864a48c08d3..8efaad6539d7ae 100644 --- a/Tools/clinic/libclinic/__init__.py +++ b/Tools/clinic/libclinic/__init__.py @@ -2,6 +2,8 @@ from .errors import ( ClinicError, + warn, + fail, ) from .formatting import ( SIG_END_MARKER, @@ -32,6 +34,8 @@ __all__ = [ # Error handling "ClinicError", + "warn", + "fail", # Formatting helpers "SIG_END_MARKER", diff --git a/Tools/clinic/libclinic/errors.py b/Tools/clinic/libclinic/errors.py index afb21b02386fe7..f06bdfbd864b2c 100644 --- a/Tools/clinic/libclinic/errors.py +++ b/Tools/clinic/libclinic/errors.py @@ -1,4 +1,5 @@ import dataclasses as dc +from typing import Literal, NoReturn, overload @dc.dataclass @@ -24,3 +25,48 @@ def report(self, *, warn_only: bool = False) -> str: class ParseError(ClinicError): pass + + +@overload +def warn_or_fail( + *args: object, + fail: Literal[True], + filename: str | None = None, + line_number: int | None = None, +) -> NoReturn: ... + +@overload +def warn_or_fail( + *args: object, + fail: Literal[False] = False, + filename: str | None = None, + line_number: int | None = None, +) -> None: ... + +def warn_or_fail( + *args: object, + fail: bool = False, + filename: str | None = None, + line_number: int | None = None, +) -> None: + joined = " ".join([str(a) for a in args]) + error = ClinicError(joined, filename=filename, lineno=line_number) + if fail: + raise error + else: + print(error.report(warn_only=True)) + + +def warn( + *args: object, + filename: str | None = None, + line_number: int | None = None, +) -> None: + return warn_or_fail(*args, filename=filename, line_number=line_number, fail=False) + +def fail( + *args: object, + filename: str | None = None, + line_number: int | None = None, +) -> NoReturn: + warn_or_fail(*args, filename=filename, line_number=line_number, fail=True) From d4028724f2c8c674202615b772913765423c69fd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 14 Mar 2024 10:28:58 +0100 Subject: [PATCH 079/158] gh-116646: Add limited C API support to AC fildes converter (#116769) Add tests on the "fildes" converter to _testclinic_limited. --- Lib/test/test_clinic.py | 33 ++++++++++++++++++++++++ Modules/_testclinic_limited.c | 18 +++++++++++++ Modules/clinic/_testclinic_limited.c.h | 35 +++++++++++++++++++++++++- Tools/clinic/clinic.py | 25 +++++++++--------- 4 files changed, 98 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index cf3eeaadf0010c..a60f087ef2816e 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -3698,6 +3698,39 @@ def test_my_double_sum(self): with self.assertRaises(TypeError): func(1., "2") + def test_get_file_descriptor(self): + # test 'file descriptor' converter: call PyObject_AsFileDescriptor() + get_fd = _testclinic_limited.get_file_descriptor + + class MyInt(int): + pass + + class MyFile: + def __init__(self, fd): + self._fd = fd + def fileno(self): + return self._fd + + for fd in (0, 1, 2, 5, 123_456): + self.assertEqual(get_fd(fd), fd) + + myint = MyInt(fd) + self.assertEqual(get_fd(myint), fd) + + myfile = MyFile(fd) + self.assertEqual(get_fd(myfile), fd) + + with self.assertRaises(OverflowError): + get_fd(2**256) + with self.assertWarnsRegex(RuntimeWarning, + "bool is used as a file descriptor"): + get_fd(True) + with self.assertRaises(TypeError): + get_fd(1.0) + with self.assertRaises(TypeError): + get_fd("abc") + with self.assertRaises(TypeError): + get_fd(None) class PermutationTests(unittest.TestCase): diff --git a/Modules/_testclinic_limited.c b/Modules/_testclinic_limited.c index 1a73c04aecb8af..29f1b7c13e4c50 100644 --- a/Modules/_testclinic_limited.c +++ b/Modules/_testclinic_limited.c @@ -105,12 +105,30 @@ my_double_sum_impl(PyObject *module, double x, double y) } +/*[clinic input] +get_file_descriptor -> int + + file as fd: fildes + / + +Get a file descriptor. +[clinic start generated code]*/ + +static int +get_file_descriptor_impl(PyObject *module, int fd) +/*[clinic end generated code: output=80051ebad54db8a8 input=82e2a1418848cd5b]*/ +{ + return fd; +} + + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF MY_INT_FUNC_METHODDEF MY_INT_SUM_METHODDEF MY_FLOAT_SUM_METHODDEF MY_DOUBLE_SUM_METHODDEF + GET_FILE_DESCRIPTOR_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_testclinic_limited.c.h b/Modules/clinic/_testclinic_limited.c.h index 690e782b839cb5..94897f4c6dc427 100644 --- a/Modules/clinic/_testclinic_limited.c.h +++ b/Modules/clinic/_testclinic_limited.c.h @@ -173,4 +173,37 @@ my_double_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=bb9f6b8c5d9e6a79 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(get_file_descriptor__doc__, +"get_file_descriptor($module, file, /)\n" +"--\n" +"\n" +"Get a file descriptor."); + +#define GET_FILE_DESCRIPTOR_METHODDEF \ + {"get_file_descriptor", (PyCFunction)get_file_descriptor, METH_O, get_file_descriptor__doc__}, + +static int +get_file_descriptor_impl(PyObject *module, int fd); + +static PyObject * +get_file_descriptor(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + int _return_value; + + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { + goto exit; + } + _return_value = get_file_descriptor_impl(module, fd); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} +/*[clinic end generated code: output=03fd7811c056dc74 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 4c7c4dca37cd0f..c81af5e696e924 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -3800,18 +3800,19 @@ class fildes_converter(CConverter): type = 'int' converter = '_PyLong_FileDescriptor_Converter' - def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: - self.add_include('pycore_fileutils.h', - '_PyLong_FileDescriptor_Converter()') - - def _parse_arg(self, argname: str, displayname: str) -> str | None: - return self.format_code(""" - {paramname} = PyObject_AsFileDescriptor({argname}); - if ({paramname} == -1) {{{{ - goto exit; - }}}} - """, - argname=argname) + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if limited_capi: + return self.format_code(""" + {paramname} = PyObject_AsFileDescriptor({argname}); + if ({paramname} < 0) {{{{ + goto exit; + }}}} + """, + argname=argname) + else: + self.add_include('pycore_fileutils.h', + '_PyLong_FileDescriptor_Converter()') + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class float_converter(CConverter): From 66fb613d90fe3dea32130a5937963a9362c8a59e Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 14 Mar 2024 12:55:54 +0300 Subject: [PATCH 080/158] gh-116785: Fix direct invocation of `test_inspect` (#116787) --- Lib/test/test_inspect/test_inspect.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 52cf68b93b85fa..9a9d34df236084 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -40,11 +40,11 @@ from test.support import has_subprocess_support, SuppressCrashReport from test import support -from . import inspect_fodder as mod -from . import inspect_fodder2 as mod2 -from . import inspect_stock_annotations -from . import inspect_stringized_annotations -from . import inspect_stringized_annotations_2 +from test.test_inspect import inspect_fodder as mod +from test.test_inspect import inspect_fodder2 as mod2 +from test.test_inspect import inspect_stock_annotations +from test.test_inspect import inspect_stringized_annotations +from test.test_inspect import inspect_stringized_annotations_2 # Functions tested in this suite: From 61f576a5eff7cc1a7c163fbed491c1e75e0bd025 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 14 Mar 2024 13:01:41 +0300 Subject: [PATCH 081/158] gh-113308: Remove some internal parts of `uuid` module (#115934) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Lib/test/test_uuid.py | 16 ++++++++++------ Lib/uuid.py | 16 ---------------- ...024-02-26-10-06-50.gh-issue-113308.MbvOFt.rst | 4 ++++ 3 files changed, 14 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-02-26-10-06-50.gh-issue-113308.MbvOFt.rst diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 9cec1e87fd3c2d..e177464c00f7a6 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -530,7 +530,14 @@ def test_uuid1(self): @support.requires_mac_ver(10, 5) @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') def test_uuid1_safe(self): - if not self.uuid._has_uuid_generate_time_safe: + try: + import _uuid + except ImportError: + has_uuid_generate_time_safe = False + else: + has_uuid_generate_time_safe = _uuid.has_uuid_generate_time_safe + + if not has_uuid_generate_time_safe or not self.uuid._generate_time_safe: self.skipTest('requires uuid_generate_time_safe(3)') u = self.uuid.uuid1() @@ -546,7 +553,6 @@ def mock_generate_time_safe(self, safe_value): """ if os.name != 'posix': self.skipTest('POSIX-only test') - self.uuid._load_system_functions() f = self.uuid._generate_time_safe if f is None: self.skipTest('need uuid._generate_time_safe') @@ -581,8 +587,7 @@ def test_uuid1_bogus_return_value(self): self.assertEqual(u.is_safe, self.uuid.SafeUUID.unknown) def test_uuid1_time(self): - with mock.patch.object(self.uuid, '_has_uuid_generate_time_safe', False), \ - mock.patch.object(self.uuid, '_generate_time_safe', None), \ + with mock.patch.object(self.uuid, '_generate_time_safe', None), \ mock.patch.object(self.uuid, '_last_timestamp', None), \ mock.patch.object(self.uuid, 'getnode', return_value=93328246233727), \ mock.patch('time.time_ns', return_value=1545052026752910643), \ @@ -590,8 +595,7 @@ def test_uuid1_time(self): u = self.uuid.uuid1() self.assertEqual(u, self.uuid.UUID('a7a55b92-01fc-11e9-94c5-54e1acf6da7f')) - with mock.patch.object(self.uuid, '_has_uuid_generate_time_safe', False), \ - mock.patch.object(self.uuid, '_generate_time_safe', None), \ + with mock.patch.object(self.uuid, '_generate_time_safe', None), \ mock.patch.object(self.uuid, '_last_timestamp', None), \ mock.patch('time.time_ns', return_value=1545052026752910643): u = self.uuid.uuid1(node=93328246233727, clock_seq=5317) diff --git a/Lib/uuid.py b/Lib/uuid.py index da2f1d50563095..c286eac38e1ef4 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -567,32 +567,16 @@ def _netstat_getnode(): # This works on AIX and might work on Tru64 UNIX. return _find_mac_under_heading('netstat', '-ian', b'Address') -def _ipconfig_getnode(): - """[DEPRECATED] Get the hardware address on Windows.""" - # bpo-40501: UuidCreateSequential() is now the only supported approach - return _windll_getnode() - -def _netbios_getnode(): - """[DEPRECATED] Get the hardware address on Windows.""" - # bpo-40501: UuidCreateSequential() is now the only supported approach - return _windll_getnode() - # Import optional C extension at toplevel, to help disabling it when testing try: import _uuid _generate_time_safe = getattr(_uuid, "generate_time_safe", None) _UuidCreate = getattr(_uuid, "UuidCreate", None) - _has_uuid_generate_time_safe = _uuid.has_uuid_generate_time_safe except ImportError: _uuid = None _generate_time_safe = None _UuidCreate = None - _has_uuid_generate_time_safe = None - - -def _load_system_functions(): - """[DEPRECATED] Platform-specific functions loaded at import time""" def _unix_getnode(): diff --git a/Misc/NEWS.d/next/Library/2024-02-26-10-06-50.gh-issue-113308.MbvOFt.rst b/Misc/NEWS.d/next/Library/2024-02-26-10-06-50.gh-issue-113308.MbvOFt.rst new file mode 100644 index 00000000000000..c4c242fe3d578f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-26-10-06-50.gh-issue-113308.MbvOFt.rst @@ -0,0 +1,4 @@ +Remove some internal protected parts from :mod:`uuid`: +``_has_uuid_generate_time_safe``, ``_netbios_getnode``, +``_ipconfig_getnode``, and ``_load_system_functions``. +They were unused. From 19ac28bd08fdb16795e6f82ea7bfac73e8f3791b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 14 Mar 2024 12:09:19 +0200 Subject: [PATCH 082/158] gh-90300: Fix undocumented envvars in the Python CLI help (GH-116765) --- Python/initconfig.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Python/initconfig.c b/Python/initconfig.c index f365bee89a5f3b..bbd611f7f7a48c 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -310,15 +310,18 @@ static const char usage_envvars[] = " interpreter displays tracebacks. (-X no_debug_ranges)\n" "PYTHONNOUSERSITE: disable user site directory (-s)\n" "PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" +"PYTHONPERFSUPPORT: support the Linux \"perf\" profiler (-X perf)\n" #ifdef Py_DEBUG "PYTHON_PRESITE=pkg.mod: import this module before site.py is run (-X presite)\n" #endif +"PYTHONPROFILEIMPORTTIME: show how long each import takes (-X importtime)\n" "PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files\n" " (-X pycache_prefix)\n" "PYTHONSAFEPATH : don't prepend a potentially unsafe path to sys.path.\n" #ifdef Py_STATS "PYTHONSTATS : turns on statistics gathering (-X pystats)\n" #endif +"PYTHONTRACEMALLOC: trace Python memory allocations (-X tracemalloc)\n" "PYTHONUNBUFFERED: disable stdout/stderr buffering (-u)\n" "PYTHONUTF8 : if set to 1, enable the UTF-8 mode (-X utf8)\n" "PYTHONVERBOSE : trace import statements (-v)\n" From f20dfb7569a769da50cd75f4932b9abe78e55d75 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 14 Mar 2024 13:16:18 +0300 Subject: [PATCH 083/158] gh-116780: Fix `test_inspect` in `-OO` mode (#116788) --- Lib/test/test_inspect/test_inspect.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 9a9d34df236084..c3a9dc998e38d0 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -991,7 +991,11 @@ def test_class_decorator(self): self.assertSourceEqual(mod2.cls196, 194, 201) self.assertSourceEqual(mod2.cls196.cls200, 198, 201) + @support.requires_docstrings def test_class_inside_conditional(self): + # We skip this test when docstrings are not present, + # because docstrings are one of the main factors of + # finding the correct class in the source code. self.assertSourceEqual(mod2.cls238.cls239, 239, 240) def test_multiple_children_classes(self): @@ -5284,6 +5288,7 @@ def func(*args, **kwargs): with self.assertRaises(ValueError): inspect.signature(func) + @support.requires_docstrings def test_base_class_have_text_signature(self): # see issue 43118 from test.typinganndata.ann_module7 import BufferedReader From 97b80af897cd3376a5be66c8d5d63310723a1590 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 14 Mar 2024 13:01:13 +0100 Subject: [PATCH 084/158] gh-85283: Build fcntl extension with the limited C API (#116791) --- Doc/whatsnew/3.13.rst | 4 +- ...4-03-14-10-33-58.gh-issue-85283.LOgmdU.rst | 2 + Modules/clinic/fcntlmodule.c.h | 49 +++++++++++++------ Modules/fcntlmodule.c | 19 ++++--- 4 files changed, 48 insertions(+), 26 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index d78f219ed3a746..aec02954b9fea9 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1460,8 +1460,8 @@ Build Changes * Building CPython now requires a compiler with support for the C11 atomic library, GCC built-in atomic functions, or MSVC interlocked intrinsics. -* The ``errno``, ``md5``, ``resource``, ``winsound``, ``_ctypes_test``, - ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, +* The ``errno``, ``fcntl``, ``grp``, ``md5``, ``resource``, ``winsound``, + ``_ctypes_test``, ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, ``_testimportmultiple`` and ``_uuid`` C extensions are now built with the :ref:`limited C API `. (Contributed by Victor Stinner in :gh:`85283`.) diff --git a/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst b/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst new file mode 100644 index 00000000000000..a02b8a8210f9bd --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst @@ -0,0 +1,2 @@ +The ``fcntl`` and ``grp`` C extensions are now built with the :ref:`limited +C API `. (Contributed by Victor Stinner in :gh:`85283`.) diff --git a/Modules/clinic/fcntlmodule.c.h b/Modules/clinic/fcntlmodule.c.h index 5dc2fc068d0f7e..d4846ddf8df7e4 100644 --- a/Modules/clinic/fcntlmodule.c.h +++ b/Modules/clinic/fcntlmodule.c.h @@ -2,9 +2,6 @@ preserve [clinic start generated code]*/ -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() -#include "pycore_modsupport.h" // _PyArg_CheckPositional() - PyDoc_STRVAR(fcntl_fcntl__doc__, "fcntl($module, fd, cmd, arg=0, /)\n" "--\n" @@ -22,7 +19,7 @@ PyDoc_STRVAR(fcntl_fcntl__doc__, "corresponding to the return value of the fcntl call in the C code."); #define FCNTL_FCNTL_METHODDEF \ - {"fcntl", _PyCFunction_CAST(fcntl_fcntl), METH_FASTCALL, fcntl_fcntl__doc__}, + {"fcntl", (PyCFunction)(void(*)(void))fcntl_fcntl, METH_FASTCALL, fcntl_fcntl__doc__}, static PyObject * fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg); @@ -35,10 +32,16 @@ fcntl_fcntl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int code; PyObject *arg = NULL; - if (!_PyArg_CheckPositional("fcntl", nargs, 2, 3)) { + if (nargs < 2) { + PyErr_Format(PyExc_TypeError, "fcntl expected at least 2 arguments, got %zd", nargs); + goto exit; + } + if (nargs > 3) { + PyErr_Format(PyExc_TypeError, "fcntl expected at most 3 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } code = PyLong_AsInt(args[1]); @@ -90,7 +93,7 @@ PyDoc_STRVAR(fcntl_ioctl__doc__, "code."); #define FCNTL_IOCTL_METHODDEF \ - {"ioctl", _PyCFunction_CAST(fcntl_ioctl), METH_FASTCALL, fcntl_ioctl__doc__}, + {"ioctl", (PyCFunction)(void(*)(void))fcntl_ioctl, METH_FASTCALL, fcntl_ioctl__doc__}, static PyObject * fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, @@ -105,10 +108,16 @@ fcntl_ioctl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *ob_arg = NULL; int mutate_arg = 1; - if (!_PyArg_CheckPositional("ioctl", nargs, 2, 4)) { + if (nargs < 2) { + PyErr_Format(PyExc_TypeError, "ioctl expected at least 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + if (nargs > 4) { + PyErr_Format(PyExc_TypeError, "ioctl expected at most 4 arguments, got %zd", nargs); + goto exit; + } + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } code = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); @@ -143,7 +152,7 @@ PyDoc_STRVAR(fcntl_flock__doc__, "function is emulated using fcntl())."); #define FCNTL_FLOCK_METHODDEF \ - {"flock", _PyCFunction_CAST(fcntl_flock), METH_FASTCALL, fcntl_flock__doc__}, + {"flock", (PyCFunction)(void(*)(void))fcntl_flock, METH_FASTCALL, fcntl_flock__doc__}, static PyObject * fcntl_flock_impl(PyObject *module, int fd, int code); @@ -155,10 +164,12 @@ fcntl_flock(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int code; - if (!_PyArg_CheckPositional("flock", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "flock expected 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } code = PyLong_AsInt(args[1]); @@ -199,7 +210,7 @@ PyDoc_STRVAR(fcntl_lockf__doc__, " 2 - relative to the end of the file (SEEK_END)"); #define FCNTL_LOCKF_METHODDEF \ - {"lockf", _PyCFunction_CAST(fcntl_lockf), METH_FASTCALL, fcntl_lockf__doc__}, + {"lockf", (PyCFunction)(void(*)(void))fcntl_lockf, METH_FASTCALL, fcntl_lockf__doc__}, static PyObject * fcntl_lockf_impl(PyObject *module, int fd, int code, PyObject *lenobj, @@ -215,10 +226,16 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *startobj = NULL; int whence = 0; - if (!_PyArg_CheckPositional("lockf", nargs, 2, 5)) { + if (nargs < 2) { + PyErr_Format(PyExc_TypeError, "lockf expected at least 2 arguments, got %zd", nargs); + goto exit; + } + if (nargs > 5) { + PyErr_Format(PyExc_TypeError, "lockf expected at most 5 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } code = PyLong_AsInt(args[1]); @@ -246,4 +263,4 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=732e33ba92042031 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=26793691ab1c75ba input=a9049054013a1b77]*/ diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 0d16602692b62d..e24e5f98f4bc4d 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -1,22 +1,25 @@ /* fcntl module */ -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 +// Need limited C API version 3.13 for PyLong_AsInt() +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" +#include // EINTR +#include // fcntl() +#include // memcpy() +#include // ioctl() #ifdef HAVE_SYS_FILE_H -#include +# include // flock() #endif #ifdef HAVE_LINUX_FS_H -#include +# include #endif - -#include -#include #ifdef HAVE_STROPTS_H -#include +# include // I_FLUSHBAND #endif /*[clinic input] From 2a54c4b25e05e30c50b915302fa846dd0d4e018b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 14 Mar 2024 13:57:02 +0100 Subject: [PATCH 085/158] gh-116646, AC: Add CConverter.use_converter() method (#116793) Only add includes when the converter is effectively used. --- Modules/clinic/_ssl.c.h | 3 +-- Tools/clinic/clinic.py | 41 +++++++++++++++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 2940f16a2cb7f6..e8d1342ed35e66 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -6,7 +6,6 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_ssl__SSLSocket_do_handshake__doc__, @@ -1664,4 +1663,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=fd1c3378fbba5240 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=28a22f2b09d631cb input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index c81af5e696e924..21ec6cf7650a37 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -884,6 +884,7 @@ def parser_body( displayname = parameters[0].get_displayname(0) parsearg = converters[0].parse_arg(argname, displayname, limited_capi=limited_capi) if parsearg is None: + converters[0].use_converter() parsearg = """ if (!PyArg_Parse(%s, "{format_units}:{name}", {parse_arguments})) {{ goto exit; @@ -1016,6 +1017,9 @@ def parser_body( if has_optional: parser_code.append("skip_optional:") else: + for parameter in parameters: + parameter.converter.use_converter() + if limited_capi: fastcall = False if fastcall: @@ -1184,6 +1188,9 @@ def parser_body( if add_label: parser_code.append("%s:" % add_label) else: + for parameter in parameters: + parameter.converter.use_converter() + declarations = declare_parser(f, clinic=clinic, hasformat=True, limited_capi=limited_capi) @@ -3155,8 +3162,13 @@ def format_code(self, fmt: str, *, fmt = fmt.replace('{bad_argument2}', bad_argument2) return fmt.format(argname=argname, paramname=self.parser_name, **kwargs) + def use_converter(self) -> None: + """Method called when self.converter is used to parse an argument.""" + pass + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'O&': + self.use_converter() return self.format_code(""" if (!{converter}({argname}, &{paramname})) {{{{ goto exit; @@ -3435,6 +3447,9 @@ def converter_init(self, *, bitwise: bool = False) -> None: self.format_unit = 'H' else: self.converter = '_PyLong_UnsignedShort_Converter' + + def use_converter(self) -> None: + if self.converter == '_PyLong_UnsignedShort_Converter': self.add_include('pycore_long.h', '_PyLong_UnsignedShort_Converter()') @@ -3519,6 +3534,9 @@ def converter_init(self, *, bitwise: bool = False) -> None: self.format_unit = 'I' else: self.converter = '_PyLong_UnsignedInt_Converter' + + def use_converter(self) -> None: + if self.converter == '_PyLong_UnsignedInt_Converter': self.add_include('pycore_long.h', '_PyLong_UnsignedInt_Converter()') @@ -3577,6 +3595,9 @@ def converter_init(self, *, bitwise: bool = False) -> None: self.format_unit = 'k' else: self.converter = '_PyLong_UnsignedLong_Converter' + + def use_converter(self) -> None: + if self.converter == '_PyLong_UnsignedLong_Converter': self.add_include('pycore_long.h', '_PyLong_UnsignedLong_Converter()') @@ -3630,6 +3651,9 @@ def converter_init(self, *, bitwise: bool = False) -> None: self.format_unit = 'K' else: self.converter = '_PyLong_UnsignedLongLong_Converter' + + def use_converter(self) -> None: + if self.converter == '_PyLong_UnsignedLongLong_Converter': self.add_include('pycore_long.h', '_PyLong_UnsignedLongLong_Converter()') @@ -3664,20 +3688,23 @@ def converter_init(self, *, accept: TypeSet = {int}) -> None: if accept == {int}: self.format_unit = 'n' self.default_type = int - self.add_include('pycore_abstract.h', '_PyNumber_Index()') elif accept == {int, NoneType}: self.converter = '_Py_convert_optional_to_ssize_t' - self.add_include('pycore_abstract.h', - '_Py_convert_optional_to_ssize_t()') else: fail(f"Py_ssize_t_converter: illegal 'accept' argument {accept!r}") + def use_converter(self) -> None: + if self.converter == '_Py_convert_optional_to_ssize_t': + self.add_include('pycore_abstract.h', + '_Py_convert_optional_to_ssize_t()') + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'n': if limited_capi: PyNumber_Index = 'PyNumber_Index' else: PyNumber_Index = '_PyNumber_Index' + self.add_include('pycore_abstract.h', '_PyNumber_Index()') return self.format_code(""" {{{{ Py_ssize_t ival = -1; @@ -3771,7 +3798,7 @@ class size_t_converter(CConverter): converter = '_PyLong_Size_t_Converter' c_ignored_default = "0" - def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: + def use_converter(self) -> None: self.add_include('pycore_long.h', '_PyLong_Size_t_Converter()') @@ -3800,6 +3827,10 @@ class fildes_converter(CConverter): type = 'int' converter = '_PyLong_FileDescriptor_Converter' + def use_converter(self) -> None: + self.add_include('pycore_fileutils.h', + '_PyLong_FileDescriptor_Converter()') + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if limited_capi: return self.format_code(""" @@ -3810,8 +3841,6 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st """, argname=argname) else: - self.add_include('pycore_fileutils.h', - '_PyLong_FileDescriptor_Converter()') return super().parse_arg(argname, displayname, limited_capi=limited_capi) From a76288ad9ba715216a29c89e046846b3eaf0fab7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 14 Mar 2024 14:58:07 +0100 Subject: [PATCH 086/158] gh-116646, AC: Always use PyObject_AsFileDescriptor() in fildes (#116806) The fildes converter of Argument Clinic now always call PyObject_AsFileDescriptor(), not only for the limited C API. The _PyLong_FileDescriptor_Converter() converter stays as a fallback when PyObject_AsFileDescriptor() cannot be used. --- Modules/clinic/posixmodule.c.h | 48 +++++++++++++++++++++------------ Modules/clinic/selectmodule.c.h | 30 +++++++++++++-------- Modules/clinic/termios.c.h | 27 ++++++++++++------- Tools/clinic/clinic.py | 17 +++++------- 4 files changed, 74 insertions(+), 48 deletions(-) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index b7338d138e91ce..0398629e3c10ce 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -7,7 +7,6 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() #include "pycore_long.h" // _PyLong_UnsignedInt_Converter() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() @@ -481,7 +480,8 @@ os_fchdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = os_fchdir_impl(module, fd); @@ -1024,7 +1024,8 @@ os_fsync(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = os_fsync_impl(module, fd); @@ -1107,7 +1108,8 @@ os_fdatasync(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = os_fdatasync_impl(module, fd); @@ -4531,7 +4533,8 @@ os_grantpt(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_grantpt_impl(module, fd); @@ -4567,7 +4570,8 @@ os_unlockpt(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_unlockpt_impl(module, fd); @@ -4604,7 +4608,8 @@ os_ptsname(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_ptsname_impl(module, fd); @@ -4664,7 +4669,8 @@ os_login_tty(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_login_tty_impl(module, fd); @@ -5881,7 +5887,8 @@ os_setns(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!noptargs) { @@ -6322,7 +6329,8 @@ os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!noptargs) { @@ -6435,7 +6443,8 @@ os_timerfd_settime_ns(PyObject *module, PyObject *const *args, Py_ssize_t nargs, if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!noptargs) { @@ -6495,7 +6504,8 @@ os_timerfd_gettime(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_timerfd_gettime_impl(module, fd); @@ -6529,7 +6539,8 @@ os_timerfd_gettime_ns(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_timerfd_gettime_ns_impl(module, fd); @@ -9691,7 +9702,8 @@ os_fpathconf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("fpathconf", nargs, 2, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!conv_path_confname(args[1], &name)) { @@ -10834,7 +10846,8 @@ os_eventfd_read(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = os_eventfd_read_impl(module, fd); @@ -10896,7 +10909,8 @@ os_eventfd_write(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!_PyLong_UnsignedLongLong_Converter(args[1], &value)) { @@ -12588,4 +12602,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=2965306970f31c5d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=511f0788a6b90db0 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index 67e76d4c89f59a..dc7d3fb814396d 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -6,7 +6,6 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() @@ -100,7 +99,8 @@ select_poll_register(pollObject *self, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("register", nargs, 1, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (nargs < 2) { @@ -148,7 +148,8 @@ select_poll_modify(pollObject *self, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("modify", nargs, 2, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!_PyLong_UnsignedShort_Converter(args[1], &eventmask)) { @@ -182,7 +183,8 @@ select_poll_unregister(pollObject *self, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = select_poll_unregister_impl(self, fd); @@ -268,7 +270,8 @@ select_devpoll_register(devpollObject *self, PyObject *const *args, Py_ssize_t n if (!_PyArg_CheckPositional("register", nargs, 1, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (nargs < 2) { @@ -318,7 +321,8 @@ select_devpoll_modify(devpollObject *self, PyObject *const *args, Py_ssize_t nar if (!_PyArg_CheckPositional("modify", nargs, 1, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (nargs < 2) { @@ -356,7 +360,8 @@ select_devpoll_unregister(devpollObject *self, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = select_devpoll_unregister_impl(self, fd); @@ -730,7 +735,8 @@ select_epoll_register(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t na if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!noptargs) { @@ -806,7 +812,8 @@ select_epoll_modify(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t narg if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } eventmask = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); @@ -874,7 +881,8 @@ select_epoll_unregister(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = select_epoll_unregister_impl(self, fd); @@ -1311,4 +1319,4 @@ select_kqueue_control(kqueue_queue_Object *self, PyObject *const *args, Py_ssize #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=4c2dcb31cb17c2c6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4fc17ae9b6cfdc86 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/termios.c.h b/Modules/clinic/termios.c.h index 1c340681e5862f..2813e5e4700352 100644 --- a/Modules/clinic/termios.c.h +++ b/Modules/clinic/termios.c.h @@ -2,7 +2,6 @@ preserve [clinic start generated code]*/ -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(termios_tcgetattr__doc__, @@ -30,7 +29,8 @@ termios_tcgetattr(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = termios_tcgetattr_impl(module, fd); @@ -69,7 +69,8 @@ termios_tcsetattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("tcsetattr", nargs, 3, 3)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } when = PyLong_AsInt(args[1]); @@ -108,7 +109,8 @@ termios_tcsendbreak(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("tcsendbreak", nargs, 2, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } duration = PyLong_AsInt(args[1]); @@ -139,7 +141,8 @@ termios_tcdrain(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = termios_tcdrain_impl(module, fd); @@ -174,7 +177,8 @@ termios_tcflush(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("tcflush", nargs, 2, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } queue = PyLong_AsInt(args[1]); @@ -213,7 +217,8 @@ termios_tcflow(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("tcflow", nargs, 2, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } action = PyLong_AsInt(args[1]); @@ -246,7 +251,8 @@ termios_tcgetwinsize(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = termios_tcgetwinsize_impl(module, fd); @@ -280,7 +286,8 @@ termios_tcsetwinsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("tcsetwinsize", nargs, 2, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } winsz = args[1]; @@ -289,4 +296,4 @@ termios_tcsetwinsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=f31382658135c774 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7327a2085972bf59 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 21ec6cf7650a37..fb56bd203749f9 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -3832,16 +3832,13 @@ def use_converter(self) -> None: '_PyLong_FileDescriptor_Converter()') def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if limited_capi: - return self.format_code(""" - {paramname} = PyObject_AsFileDescriptor({argname}); - if ({paramname} < 0) {{{{ - goto exit; - }}}} - """, - argname=argname) - else: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) + return self.format_code(""" + {paramname} = PyObject_AsFileDescriptor({argname}); + if ({paramname} < 0) {{{{ + goto exit; + }}}} + """, + argname=argname) class float_converter(CConverter): From b1236a44101410e28a69ec6df5f29e7a5f2eb0b2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 14 Mar 2024 15:37:22 +0100 Subject: [PATCH 087/158] gh-113317, AC: Add libclinic.function (#116807) Move Module, Class, Function and Parameter classes to a new libclinic.function module. Move VersionTuple and Sentinels to libclinic.utils. --- Tools/clinic/clinic.py | 248 +---------------------------- Tools/clinic/libclinic/__init__.py | 8 + Tools/clinic/libclinic/function.py | 237 +++++++++++++++++++++++++++ Tools/clinic/libclinic/utils.py | 18 ++- 4 files changed, 270 insertions(+), 241 deletions(-) create mode 100644 Tools/clinic/libclinic/function.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index fb56bd203749f9..6488d913168319 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -12,7 +12,6 @@ import builtins as bltns import collections import contextlib -import copy import dataclasses as dc import enum import functools @@ -50,7 +49,14 @@ # Local imports. import libclinic import libclinic.cpp -from libclinic import ClinicError, fail, warn +from libclinic import ( + ClinicError, Sentinels, VersionTuple, + fail, warn, unspecified, unknown) +from libclinic.function import ( + Module, Class, Function, Parameter, + ClassDict, ModuleDict, FunctionKind, + CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW, + GETTER, SETTER) # TODO: @@ -70,18 +76,6 @@ LIMITED_CAPI_REGEX = re.compile(r'# *define +Py_LIMITED_API') -class Sentinels(enum.Enum): - unspecified = "unspecified" - unknown = "unknown" - - def __repr__(self) -> str: - return f"<{self.value.capitalize()}>" - - -unspecified: Final = Sentinels.unspecified -unknown: Final = Sentinels.unknown - - # This one needs to be a distinct class, unlike the other two class Null: def __repr__(self) -> str: @@ -2096,9 +2090,7 @@ def dump(self) -> str: extensions['py'] = PythonLanguage -ClassDict = dict[str, "Class"] DestinationDict = dict[str, Destination] -ModuleDict = dict[str, "Module"] class Parser(Protocol): @@ -2418,38 +2410,6 @@ def parse(self, block: Block) -> None: block.output = s.getvalue() -@dc.dataclass(repr=False) -class Module: - name: str - module: Module | Clinic - - def __post_init__(self) -> None: - self.parent = self.module - self.modules: ModuleDict = {} - self.classes: ClassDict = {} - self.functions: list[Function] = [] - - def __repr__(self) -> str: - return "" - - -@dc.dataclass(repr=False) -class Class: - name: str - module: Module | Clinic - cls: Class | None - typedef: str - type_object: str - - def __post_init__(self) -> None: - self.parent = self.cls or self.module - self.classes: ClassDict = {} - self.functions: list[Function] = [] - - def __repr__(self) -> str: - return "" - - unsupported_special_methods: set[str] = set(""" __abs__ @@ -2522,201 +2482,9 @@ def __repr__(self) -> str: """.strip().split()) -class FunctionKind(enum.Enum): - INVALID = enum.auto() - CALLABLE = enum.auto() - STATIC_METHOD = enum.auto() - CLASS_METHOD = enum.auto() - METHOD_INIT = enum.auto() - METHOD_NEW = enum.auto() - GETTER = enum.auto() - SETTER = enum.auto() - - @functools.cached_property - def new_or_init(self) -> bool: - return self in {FunctionKind.METHOD_INIT, FunctionKind.METHOD_NEW} - - def __repr__(self) -> str: - return f"" - - -INVALID: Final = FunctionKind.INVALID -CALLABLE: Final = FunctionKind.CALLABLE -STATIC_METHOD: Final = FunctionKind.STATIC_METHOD -CLASS_METHOD: Final = FunctionKind.CLASS_METHOD -METHOD_INIT: Final = FunctionKind.METHOD_INIT -METHOD_NEW: Final = FunctionKind.METHOD_NEW -GETTER: Final = FunctionKind.GETTER -SETTER: Final = FunctionKind.SETTER - -ParamDict = dict[str, "Parameter"] ReturnConverterType = Callable[..., "CReturnConverter"] -@dc.dataclass(repr=False) -class Function: - """ - Mutable duck type for inspect.Function. - - docstring - a str containing - * embedded line breaks - * text outdented to the left margin - * no trailing whitespace. - It will always be true that - (not docstring) or ((not docstring[0].isspace()) and (docstring.rstrip() == docstring)) - """ - parameters: ParamDict = dc.field(default_factory=dict) - _: dc.KW_ONLY - name: str - module: Module | Clinic - cls: Class | None - c_basename: str - full_name: str - return_converter: CReturnConverter - kind: FunctionKind - coexist: bool - return_annotation: object = inspect.Signature.empty - docstring: str = '' - # docstring_only means "don't generate a machine-readable - # signature, just a normal docstring". it's True for - # functions with optional groups because we can't represent - # those accurately with inspect.Signature in 3.4. - docstring_only: bool = False - critical_section: bool = False - target_critical_section: list[str] = dc.field(default_factory=list) - - def __post_init__(self) -> None: - self.parent = self.cls or self.module - self.self_converter: self_converter | None = None - self.__render_parameters__: list[Parameter] | None = None - - @functools.cached_property - def displayname(self) -> str: - """Pretty-printable name.""" - if self.kind.new_or_init: - assert isinstance(self.cls, Class) - return self.cls.name - else: - return self.name - - @functools.cached_property - def fulldisplayname(self) -> str: - parent: Class | Module | Clinic | None - if self.kind.new_or_init: - parent = getattr(self.cls, "parent", None) - else: - parent = self.parent - name = self.displayname - while isinstance(parent, (Module, Class)): - name = f"{parent.name}.{name}" - parent = parent.parent - return name - - @property - def render_parameters(self) -> list[Parameter]: - if not self.__render_parameters__: - l: list[Parameter] = [] - self.__render_parameters__ = l - for p in self.parameters.values(): - p = p.copy() - p.converter.pre_render() - l.append(p) - return self.__render_parameters__ - - @property - def methoddef_flags(self) -> str | None: - if self.kind.new_or_init: - return None - flags = [] - match self.kind: - case FunctionKind.CLASS_METHOD: - flags.append('METH_CLASS') - case FunctionKind.STATIC_METHOD: - flags.append('METH_STATIC') - case _ as kind: - acceptable_kinds = {FunctionKind.CALLABLE, FunctionKind.GETTER, FunctionKind.SETTER} - assert kind in acceptable_kinds, f"unknown kind: {kind!r}" - if self.coexist: - flags.append('METH_COEXIST') - return '|'.join(flags) - - def __repr__(self) -> str: - return f'' - - def copy(self, **overrides: Any) -> Function: - f = dc.replace(self, **overrides) - f.parameters = { - name: value.copy(function=f) - for name, value in f.parameters.items() - } - return f - - -VersionTuple = tuple[int, int] - - -@dc.dataclass(repr=False, slots=True) -class Parameter: - """ - Mutable duck type of inspect.Parameter. - """ - name: str - kind: inspect._ParameterKind - _: dc.KW_ONLY - default: object = inspect.Parameter.empty - function: Function - converter: CConverter - annotation: object = inspect.Parameter.empty - docstring: str = '' - group: int = 0 - # (`None` signifies that there is no deprecation) - deprecated_positional: VersionTuple | None = None - deprecated_keyword: VersionTuple | None = None - right_bracket_count: int = dc.field(init=False, default=0) - - def __repr__(self) -> str: - return f'' - - def is_keyword_only(self) -> bool: - return self.kind == inspect.Parameter.KEYWORD_ONLY - - def is_positional_only(self) -> bool: - return self.kind == inspect.Parameter.POSITIONAL_ONLY - - def is_vararg(self) -> bool: - return self.kind == inspect.Parameter.VAR_POSITIONAL - - def is_optional(self) -> bool: - return not self.is_vararg() and (self.default is not unspecified) - - def copy( - self, - /, - *, - converter: CConverter | None = None, - function: Function | None = None, - **overrides: Any - ) -> Parameter: - function = function or self.function - if not converter: - converter = copy.copy(self.converter) - converter.function = function - return dc.replace(self, **overrides, function=function, converter=converter) - - def get_displayname(self, i: int) -> str: - if i == 0: - return 'argument' - if not self.is_positional_only(): - return f'argument {self.name!r}' - else: - return f'argument {i}' - - def render_docstring(self) -> str: - lines = [f" {self.name}"] - lines.extend(f" {line}" for line in self.docstring.split("\n")) - return "\n".join(lines).rstrip() - - CConverterClassT = TypeVar("CConverterClassT", bound=type["CConverter"]) def add_c_converter( diff --git a/Tools/clinic/libclinic/__init__.py b/Tools/clinic/libclinic/__init__.py index 8efaad6539d7ae..32231b82bfdc07 100644 --- a/Tools/clinic/libclinic/__init__.py +++ b/Tools/clinic/libclinic/__init__.py @@ -28,6 +28,10 @@ compute_checksum, create_regex, write_file, + VersionTuple, + Sentinels, + unspecified, + unknown, ) @@ -60,6 +64,10 @@ "compute_checksum", "create_regex", "write_file", + "VersionTuple", + "Sentinels", + "unspecified", + "unknown", ] diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py new file mode 100644 index 00000000000000..48cb7d05a7caef --- /dev/null +++ b/Tools/clinic/libclinic/function.py @@ -0,0 +1,237 @@ +from __future__ import annotations +import dataclasses as dc +import copy +import enum +import functools +import inspect +from typing import Final, Any, TYPE_CHECKING +if TYPE_CHECKING: + from clinic import Clinic, CConverter, CReturnConverter, self_converter + +from libclinic import VersionTuple, unspecified + + +ClassDict = dict[str, "Class"] +ModuleDict = dict[str, "Module"] +ParamDict = dict[str, "Parameter"] + + +@dc.dataclass(repr=False) +class Module: + name: str + module: Module | Clinic + + def __post_init__(self) -> None: + self.parent = self.module + self.modules: ModuleDict = {} + self.classes: ClassDict = {} + self.functions: list[Function] = [] + + def __repr__(self) -> str: + return "" + + +@dc.dataclass(repr=False) +class Class: + name: str + module: Module | Clinic + cls: Class | None + typedef: str + type_object: str + + def __post_init__(self) -> None: + self.parent = self.cls or self.module + self.classes: ClassDict = {} + self.functions: list[Function] = [] + + def __repr__(self) -> str: + return "" + + +class FunctionKind(enum.Enum): + INVALID = enum.auto() + CALLABLE = enum.auto() + STATIC_METHOD = enum.auto() + CLASS_METHOD = enum.auto() + METHOD_INIT = enum.auto() + METHOD_NEW = enum.auto() + GETTER = enum.auto() + SETTER = enum.auto() + + @functools.cached_property + def new_or_init(self) -> bool: + return self in {FunctionKind.METHOD_INIT, FunctionKind.METHOD_NEW} + + def __repr__(self) -> str: + return f"" + + +INVALID: Final = FunctionKind.INVALID +CALLABLE: Final = FunctionKind.CALLABLE +STATIC_METHOD: Final = FunctionKind.STATIC_METHOD +CLASS_METHOD: Final = FunctionKind.CLASS_METHOD +METHOD_INIT: Final = FunctionKind.METHOD_INIT +METHOD_NEW: Final = FunctionKind.METHOD_NEW +GETTER: Final = FunctionKind.GETTER +SETTER: Final = FunctionKind.SETTER + + +@dc.dataclass(repr=False) +class Function: + """ + Mutable duck type for inspect.Function. + + docstring - a str containing + * embedded line breaks + * text outdented to the left margin + * no trailing whitespace. + It will always be true that + (not docstring) or ((not docstring[0].isspace()) and (docstring.rstrip() == docstring)) + """ + parameters: ParamDict = dc.field(default_factory=dict) + _: dc.KW_ONLY + name: str + module: Module | Clinic + cls: Class | None + c_basename: str + full_name: str + return_converter: CReturnConverter + kind: FunctionKind + coexist: bool + return_annotation: object = inspect.Signature.empty + docstring: str = '' + # docstring_only means "don't generate a machine-readable + # signature, just a normal docstring". it's True for + # functions with optional groups because we can't represent + # those accurately with inspect.Signature in 3.4. + docstring_only: bool = False + critical_section: bool = False + target_critical_section: list[str] = dc.field(default_factory=list) + + def __post_init__(self) -> None: + self.parent = self.cls or self.module + self.self_converter: self_converter | None = None + self.__render_parameters__: list[Parameter] | None = None + + @functools.cached_property + def displayname(self) -> str: + """Pretty-printable name.""" + if self.kind.new_or_init: + assert isinstance(self.cls, Class) + return self.cls.name + else: + return self.name + + @functools.cached_property + def fulldisplayname(self) -> str: + parent: Class | Module | Clinic | None + if self.kind.new_or_init: + parent = getattr(self.cls, "parent", None) + else: + parent = self.parent + name = self.displayname + while isinstance(parent, (Module, Class)): + name = f"{parent.name}.{name}" + parent = parent.parent + return name + + @property + def render_parameters(self) -> list[Parameter]: + if not self.__render_parameters__: + l: list[Parameter] = [] + self.__render_parameters__ = l + for p in self.parameters.values(): + p = p.copy() + p.converter.pre_render() + l.append(p) + return self.__render_parameters__ + + @property + def methoddef_flags(self) -> str | None: + if self.kind.new_or_init: + return None + flags = [] + match self.kind: + case FunctionKind.CLASS_METHOD: + flags.append('METH_CLASS') + case FunctionKind.STATIC_METHOD: + flags.append('METH_STATIC') + case _ as kind: + acceptable_kinds = {FunctionKind.CALLABLE, FunctionKind.GETTER, FunctionKind.SETTER} + assert kind in acceptable_kinds, f"unknown kind: {kind!r}" + if self.coexist: + flags.append('METH_COEXIST') + return '|'.join(flags) + + def __repr__(self) -> str: + return f'' + + def copy(self, **overrides: Any) -> Function: + f = dc.replace(self, **overrides) + f.parameters = { + name: value.copy(function=f) + for name, value in f.parameters.items() + } + return f + + +@dc.dataclass(repr=False, slots=True) +class Parameter: + """ + Mutable duck type of inspect.Parameter. + """ + name: str + kind: inspect._ParameterKind + _: dc.KW_ONLY + default: object = inspect.Parameter.empty + function: Function + converter: CConverter + annotation: object = inspect.Parameter.empty + docstring: str = '' + group: int = 0 + # (`None` signifies that there is no deprecation) + deprecated_positional: VersionTuple | None = None + deprecated_keyword: VersionTuple | None = None + right_bracket_count: int = dc.field(init=False, default=0) + + def __repr__(self) -> str: + return f'' + + def is_keyword_only(self) -> bool: + return self.kind == inspect.Parameter.KEYWORD_ONLY + + def is_positional_only(self) -> bool: + return self.kind == inspect.Parameter.POSITIONAL_ONLY + + def is_vararg(self) -> bool: + return self.kind == inspect.Parameter.VAR_POSITIONAL + + def is_optional(self) -> bool: + return not self.is_vararg() and (self.default is not unspecified) + + def copy( + self, + /, + *, + converter: CConverter | None = None, + function: Function | None = None, + **overrides: Any + ) -> Parameter: + function = function or self.function + if not converter: + converter = copy.copy(self.converter) + converter.function = function + return dc.replace(self, **overrides, function=function, converter=converter) + + def get_displayname(self, i: int) -> str: + if i == 0: + return 'argument' + if not self.is_positional_only(): + return f'argument {self.name!r}' + else: + return f'argument {i}' + + def render_docstring(self) -> str: + lines = [f" {self.name}"] + lines.extend(f" {line}" for line in self.docstring.split("\n")) + return "\n".join(lines).rstrip() diff --git a/Tools/clinic/libclinic/utils.py b/Tools/clinic/libclinic/utils.py index d2d09387a73d1e..95a69f70c5499d 100644 --- a/Tools/clinic/libclinic/utils.py +++ b/Tools/clinic/libclinic/utils.py @@ -1,9 +1,10 @@ import collections +import enum import hashlib import os import re import string -from typing import Literal +from typing import Literal, Final def write_file(filename: str, new_contents: str) -> None: @@ -66,3 +67,18 @@ def get_value( ) -> Literal[""]: self.counts[key] += 1 return "" + + +VersionTuple = tuple[int, int] + + +class Sentinels(enum.Enum): + unspecified = "unspecified" + unknown = "unknown" + + def __repr__(self) -> str: + return f"<{self.value.capitalize()}>" + + +unspecified: Final = Sentinels.unspecified +unknown: Final = Sentinels.unknown From 415cd06d724762f23b42f1ab36867b8114714684 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 14 Mar 2024 17:10:56 +0200 Subject: [PATCH 088/158] CI: Only test free-threading with faster macOS M1 (#116814) Only test free-threading with faster macOS M1 --- .github/workflows/build.yml | 4 ++++ .github/workflows/reusable-macos.yml | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ae14046935b97b..d43b83e830e1fb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -206,6 +206,8 @@ jobs: uses: ./.github/workflows/reusable-macos.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} + # macos-14 is M1, macos-13 is Intel + os-matrix: '["macos-14", "macos-13"]' build_macos_free_threading: name: 'macOS (free-threading)' @@ -215,6 +217,8 @@ jobs: with: config_hash: ${{ needs.check_source.outputs.config_hash }} free-threading: true + # macos-14 is M1 + os-matrix: '["macos-14"]' build_ubuntu: name: 'Ubuntu' diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 65b73cd791c75c..dabeca8c81ece1 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -8,6 +8,9 @@ on: required: false type: boolean default: false + os-matrix: + required: false + type: string jobs: build_macos: @@ -22,10 +25,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ - "macos-14", # M1 - "macos-13", # Intel - ] + os: ${{fromJson(inputs.os-matrix)}} runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 From 846ad5a26ac0ff988a3fceec8f8e830f68bdf48a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 14 Mar 2024 16:42:41 +0100 Subject: [PATCH 089/158] gh-88494: Use QueryPerformanceCounter() for time.monotonic() (#116781) On Windows, time.monotonic() now uses the QueryPerformanceCounter() clock to have a resolution better than 1 us, instead of the gGetTickCount64() clock which has a resolution of 15.6 ms. --- Doc/library/time.rst | 24 ++ Doc/whatsnew/3.13.rst | 9 + ...4-03-14-09-14-21.gh-issue-88494.Bwfmp7.rst | 4 + Python/pytime.c | 211 ++++++------------ 4 files changed, 110 insertions(+), 138 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-03-14-09-14-21.gh-issue-88494.Bwfmp7.rst diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 029663e0801a0d..d79ca6e1208107 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -287,6 +287,15 @@ Functions The reference point of the returned value is undefined, so that only the difference between the results of two calls is valid. + Clock: + + * On Windows, call ``QueryPerformanceCounter()`` and + ``QueryPerformanceFrequency()``. + * On macOS, call ``mach_absolute_time()`` and ``mach_timebase_info()``. + * On HP-UX, call ``gethrtime()``. + * Call ``clock_gettime(CLOCK_HIGHRES)`` if available. + * Otherwise, call ``clock_gettime(CLOCK_MONOTONIC)``. + Use :func:`monotonic_ns` to avoid the precision loss caused by the :class:`float` type. @@ -316,6 +325,11 @@ Functions point of the returned value is undefined, so that only the difference between the results of two calls is valid. + .. impl-detail:: + + On CPython, use the same clock than :func:`time.monotonic()` and is a + monotonic clock, i.e. a clock that cannot go backwards. + Use :func:`perf_counter_ns` to avoid the precision loss caused by the :class:`float` type. @@ -324,6 +338,10 @@ Functions .. versionchanged:: 3.10 On Windows, the function is now system-wide. + .. versionchanged:: 3.13 + Use the same clock than :func:`time.monotonic()`. + + .. function:: perf_counter_ns() -> int Similar to :func:`perf_counter`, but return time as nanoseconds. @@ -666,6 +684,12 @@ Functions :class:`struct_time` object is returned, from which the components of the calendar date may be accessed as attributes. + Clock: + + * On Windows, call ``GetSystemTimeAsFileTime()``. + * Call ``clock_gettime(CLOCK_REALTIME)`` if available. + * Otherwise, call ``gettimeofday()``. + Use :func:`time_ns` to avoid the precision loss caused by the :class:`float` type. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index aec02954b9fea9..ea45fa7c825dfe 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -552,6 +552,15 @@ sys This function is not guaranteed to exist in all implementations of Python. (Contributed by Serhiy Storchaka in :gh:`78573`.) +time +---- + +* On Windows, :func:`time.monotonic()` now uses the + ``QueryPerformanceCounter()`` clock to have a resolution better than 1 us, + instead of the ``GetTickCount64()`` clock which has a resolution of 15.6 ms. + (Contributed by Victor Stinner in :gh:`88494`.) + + tkinter ------- diff --git a/Misc/NEWS.d/next/Windows/2024-03-14-09-14-21.gh-issue-88494.Bwfmp7.rst b/Misc/NEWS.d/next/Windows/2024-03-14-09-14-21.gh-issue-88494.Bwfmp7.rst new file mode 100644 index 00000000000000..5a96af0231918f --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-03-14-09-14-21.gh-issue-88494.Bwfmp7.rst @@ -0,0 +1,4 @@ +On Windows, :func:`time.monotonic()` now uses the ``QueryPerformanceCounter()`` +clock to have a resolution better than 1 us, instead of the +``GetTickCount64()`` clock which has a resolution of 15.6 ms. Patch by Victor +Stinner. diff --git a/Python/pytime.c b/Python/pytime.c index 70d92ca00ee28e..45be6a3dbd3341 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -1027,9 +1027,76 @@ _PyTime_TimeWithInfo(PyTime_t *t, _Py_clock_info_t *info) } +#ifdef MS_WINDOWS +static int +py_win_perf_counter_frequency(_PyTimeFraction *base, int raise_exc) +{ + LARGE_INTEGER freq; + // Since Windows XP, the function cannot fail. + (void)QueryPerformanceFrequency(&freq); + LONGLONG frequency = freq.QuadPart; + + // Since Windows XP, frequency cannot be zero. + assert(frequency >= 1); + + Py_BUILD_ASSERT(sizeof(PyTime_t) == sizeof(frequency)); + PyTime_t denom = (PyTime_t)frequency; + + // Known QueryPerformanceFrequency() values: + // + // * 10,000,000 (10 MHz): 100 ns resolution + // * 3,579,545 Hz (3.6 MHz): 279 ns resolution + if (_PyTimeFraction_Set(base, SEC_TO_NS, denom) < 0) { + if (raise_exc) { + PyErr_SetString(PyExc_RuntimeError, + "invalid QueryPerformanceFrequency"); + } + return -1; + } + return 0; +} + + +// N.B. If raise_exc=0, this may be called without the GIL. +static int +py_get_win_perf_counter(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) +{ + assert(info == NULL || raise_exc); + + static _PyTimeFraction base = {0, 0}; + if (base.denom == 0) { + if (py_win_perf_counter_frequency(&base, raise_exc) < 0) { + return -1; + } + } + + if (info) { + info->implementation = "QueryPerformanceCounter()"; + info->resolution = _PyTimeFraction_Resolution(&base); + info->monotonic = 1; + info->adjustable = 0; + } + + LARGE_INTEGER now; + QueryPerformanceCounter(&now); + LONGLONG ticksll = now.QuadPart; + + /* Make sure that casting LONGLONG to PyTime_t cannot overflow, + both types are signed */ + PyTime_t ticks; + static_assert(sizeof(ticksll) <= sizeof(ticks), + "LONGLONG is larger than PyTime_t"); + ticks = (PyTime_t)ticksll; + + *tp = _PyTimeFraction_Mul(ticks, &base); + return 0; +} +#endif // MS_WINDOWS + + #ifdef __APPLE__ static int -py_mach_timebase_info(_PyTimeFraction *base, int raise) +py_mach_timebase_info(_PyTimeFraction *base, int raise_exc) { mach_timebase_info_data_t timebase; // According to the Technical Q&A QA1398, mach_timebase_info() cannot @@ -1051,7 +1118,7 @@ py_mach_timebase_info(_PyTimeFraction *base, int raise) // * (1000000000, 33333335) on PowerPC: ~30 ns // * (1000000000, 25000000) on PowerPC: 40 ns if (_PyTimeFraction_Set(base, numer, denom) < 0) { - if (raise) { + if (raise_exc) { PyErr_SetString(PyExc_RuntimeError, "invalid mach_timebase_info"); } @@ -1069,42 +1136,9 @@ py_get_monotonic_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) assert(info == NULL || raise_exc); #if defined(MS_WINDOWS) - ULONGLONG ticks = GetTickCount64(); - static_assert(sizeof(ticks) <= sizeof(PyTime_t), - "ULONGLONG is larger than PyTime_t"); - PyTime_t t; - if (ticks <= (ULONGLONG)PyTime_MAX) { - t = (PyTime_t)ticks; - } - else { - // GetTickCount64() maximum is larger than PyTime_t maximum: - // ULONGLONG is unsigned, whereas PyTime_t is signed. - t = PyTime_MAX; - } - - int res = pytime_mul(&t, MS_TO_NS); - *tp = t; - - if (raise_exc && res < 0) { - pytime_overflow(); + if (py_get_win_perf_counter(tp, info, raise_exc) < 0) { return -1; } - - if (info) { - DWORD timeAdjustment, timeIncrement; - BOOL isTimeAdjustmentDisabled, ok; - info->implementation = "GetTickCount64()"; - info->monotonic = 1; - ok = GetSystemTimeAdjustment(&timeAdjustment, &timeIncrement, - &isTimeAdjustmentDisabled); - if (!ok) { - PyErr_SetFromWindowsErr(0); - return -1; - } - info->resolution = timeIncrement * 1e-7; - info->adjustable = 0; - } - #elif defined(__APPLE__) static _PyTimeFraction base = {0, 0}; if (base.denom == 0) { @@ -1190,8 +1224,7 @@ _PyTime_MonotonicUnchecked(void) { PyTime_t t; if (py_get_monotonic_clock(&t, NULL, 0) < 0) { - // If mach_timebase_info(), clock_gettime() or gethrtime() fails: - // silently ignore the failure and return 0. + // Ignore silently the error and return 0. t = 0; } return t; @@ -1216,122 +1249,24 @@ _PyTime_MonotonicWithInfo(PyTime_t *tp, _Py_clock_info_t *info) } -#ifdef MS_WINDOWS -static int -py_win_perf_counter_frequency(_PyTimeFraction *base, int raise) -{ - LONGLONG frequency; - - LARGE_INTEGER freq; - // Since Windows XP, the function cannot fail. - (void)QueryPerformanceFrequency(&freq); - frequency = freq.QuadPart; - - // Since Windows XP, frequency cannot be zero. - assert(frequency >= 1); - - Py_BUILD_ASSERT(sizeof(PyTime_t) == sizeof(frequency)); - PyTime_t denom = (PyTime_t)frequency; - - // Known QueryPerformanceFrequency() values: - // - // * 10,000,000 (10 MHz): 100 ns resolution - // * 3,579,545 Hz (3.6 MHz): 279 ns resolution - if (_PyTimeFraction_Set(base, SEC_TO_NS, denom) < 0) { - if (raise) { - PyErr_SetString(PyExc_RuntimeError, - "invalid QueryPerformanceFrequency"); - } - return -1; - } - return 0; -} - - -// N.B. If raise_exc=0, this may be called without the GIL. -static int -py_get_win_perf_counter(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) -{ - assert(info == NULL || raise_exc); - - static _PyTimeFraction base = {0, 0}; - if (base.denom == 0) { - if (py_win_perf_counter_frequency(&base, raise_exc) < 0) { - return -1; - } - } - - if (info) { - info->implementation = "QueryPerformanceCounter()"; - info->resolution = _PyTimeFraction_Resolution(&base); - info->monotonic = 1; - info->adjustable = 0; - } - - LARGE_INTEGER now; - QueryPerformanceCounter(&now); - LONGLONG ticksll = now.QuadPart; - - /* Make sure that casting LONGLONG to PyTime_t cannot overflow, - both types are signed */ - PyTime_t ticks; - static_assert(sizeof(ticksll) <= sizeof(ticks), - "LONGLONG is larger than PyTime_t"); - ticks = (PyTime_t)ticksll; - - PyTime_t ns = _PyTimeFraction_Mul(ticks, &base); - *tp = ns; - return 0; -} -#endif // MS_WINDOWS - - int _PyTime_PerfCounterWithInfo(PyTime_t *t, _Py_clock_info_t *info) { -#ifdef MS_WINDOWS - return py_get_win_perf_counter(t, info, 1); -#else return _PyTime_MonotonicWithInfo(t, info); -#endif } PyTime_t _PyTime_PerfCounterUnchecked(void) { - PyTime_t t; - int res; -#ifdef MS_WINDOWS - res = py_get_win_perf_counter(&t, NULL, 0); -#else - res = py_get_monotonic_clock(&t, NULL, 0); -#endif - if (res < 0) { - // If py_win_perf_counter_frequency() or py_get_monotonic_clock() - // fails: silently ignore the failure and return 0. - t = 0; - } - return t; + return _PyTime_MonotonicUnchecked(); } int PyTime_PerfCounter(PyTime_t *result) { - int res; -#ifdef MS_WINDOWS - res = py_get_win_perf_counter(result, NULL, 1); -#else - res = py_get_monotonic_clock(result, NULL, 1); -#endif - if (res < 0) { - // If py_win_perf_counter_frequency() or py_get_monotonic_clock() - // fails: silently ignore the failure and return 0. - *result = 0; - return -1; - } - return 0; + return PyTime_Monotonic(result); } From bae6579b46df50dee4dbb77ea242270d27cd0c9d Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 14 Mar 2024 16:47:12 +0100 Subject: [PATCH 090/158] gh-116731: libregrtest: Clear inspect & importlib.metadata caches in clear_caches (GH-116805) Co-authored-by: Jason R. Coombs --- Lib/test/libregrtest/utils.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 7765ae8a933be4..837f73b28b4018 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -276,6 +276,15 @@ def clear_caches(): pass else: inspect._shadowed_dict_from_mro_tuple.cache_clear() + inspect._filesbymodname.clear() + inspect.modulesbyfile.clear() + + try: + importlib_metadata = sys.modules['importlib.metadata'] + except KeyError: + pass + else: + importlib_metadata.FastPath.__new__.cache_clear() def get_build_info(): From b54d7c87aaf23fbd67171d0dd3e4f4ab736e6a48 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 14 Mar 2024 17:11:39 +0100 Subject: [PATCH 091/158] gh-113317, AC: Add libclinic.block_parser module (#116819) * Move Block and BlockParser classes to a new libclinic.block_parser module. * Move Language and PythonLanguage classes to a new libclinic.language module. --- Tools/clinic/clinic.py | 338 +------------------------ Tools/clinic/libclinic/block_parser.py | 256 +++++++++++++++++++ Tools/clinic/libclinic/language.py | 103 ++++++++ 3 files changed, 361 insertions(+), 336 deletions(-) create mode 100644 Tools/clinic/libclinic/block_parser.py create mode 100644 Tools/clinic/libclinic/language.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 6488d913168319..ac205866f9d291 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -6,11 +6,9 @@ # from __future__ import annotations -import abc import argparse import ast import builtins as bltns -import collections import contextlib import dataclasses as dc import enum @@ -57,6 +55,8 @@ ClassDict, ModuleDict, FunctionKind, CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW, GETTER, SETTER) +from libclinic.language import Language, PythonLanguage +from libclinic.block_parser import Block, BlockParser # TODO: @@ -144,96 +144,6 @@ def __init__(self) -> None: self.unlock: list[str] = [] -class Language(metaclass=abc.ABCMeta): - - start_line = "" - body_prefix = "" - stop_line = "" - checksum_line = "" - - def __init__(self, filename: str) -> None: - self.filename = filename - - @abc.abstractmethod - def render( - self, - clinic: Clinic, - signatures: Iterable[Module | Class | Function] - ) -> str: - ... - - def parse_line(self, line: str) -> None: - ... - - def validate(self) -> None: - def assert_only_one( - attr: str, - *additional_fields: str - ) -> None: - """ - Ensures that the string found at getattr(self, attr) - contains exactly one formatter replacement string for - each valid field. The list of valid fields is - ['dsl_name'] extended by additional_fields. - - e.g. - self.fmt = "{dsl_name} {a} {b}" - - # this passes - self.assert_only_one('fmt', 'a', 'b') - - # this fails, the format string has a {b} in it - self.assert_only_one('fmt', 'a') - - # this fails, the format string doesn't have a {c} in it - self.assert_only_one('fmt', 'a', 'b', 'c') - - # this fails, the format string has two {a}s in it, - # it must contain exactly one - self.fmt2 = '{dsl_name} {a} {a}' - self.assert_only_one('fmt2', 'a') - - """ - fields = ['dsl_name'] - fields.extend(additional_fields) - line: str = getattr(self, attr) - fcf = libclinic.FormatCounterFormatter() - fcf.format(line) - def local_fail(should_be_there_but_isnt: bool) -> None: - if should_be_there_but_isnt: - fail("{} {} must contain {{{}}} exactly once!".format( - self.__class__.__name__, attr, name)) - else: - fail("{} {} must not contain {{{}}}!".format( - self.__class__.__name__, attr, name)) - - for name, count in fcf.counts.items(): - if name in fields: - if count > 1: - local_fail(True) - else: - local_fail(False) - for name in fields: - if fcf.counts.get(name) != 1: - local_fail(True) - - assert_only_one('start_line') - assert_only_one('stop_line') - - field = "arguments" if "{arguments}" in self.checksum_line else "checksum" - assert_only_one('checksum_line', field) - - - -class PythonLanguage(Language): - - language = 'Python' - start_line = "#/*[{dsl_name} input]" - body_prefix = "#" - stop_line = "#[{dsl_name} start generated code]*/" - checksum_line = "#/*[{dsl_name} end generated code: {arguments}]*/" - - ParamTuple = tuple["Parameter", ...] @@ -1646,250 +1556,6 @@ def render_function( return clinic.get_destination('block').dump() -@dc.dataclass(slots=True, repr=False) -class Block: - r""" - Represents a single block of text embedded in - another file. If dsl_name is None, the block represents - verbatim text, raw original text from the file, in - which case "input" will be the only non-false member. - If dsl_name is not None, the block represents a Clinic - block. - - input is always str, with embedded \n characters. - input represents the original text from the file; - if it's a Clinic block, it is the original text with - the body_prefix and redundant leading whitespace removed. - - dsl_name is either str or None. If str, it's the text - found on the start line of the block between the square - brackets. - - signatures is a list. - It may only contain clinic.Module, clinic.Class, and - clinic.Function objects. At the moment it should - contain at most one of each. - - output is either str or None. If str, it's the output - from this block, with embedded '\n' characters. - - indent is a str. It's the leading whitespace - that was found on every line of input. (If body_prefix is - not empty, this is the indent *after* removing the - body_prefix.) - - "indent" is different from the concept of "preindent" - (which is not stored as state on Block objects). - "preindent" is the whitespace that - was found in front of every line of input *before* the - "body_prefix" (see the Language object). If body_prefix - is empty, preindent must always be empty too. - - To illustrate the difference between "indent" and "preindent": - - Assume that '_' represents whitespace. - If the block processed was in a Python file, and looked like this: - ____#/*[python] - ____#__for a in range(20): - ____#____print(a) - ____#[python]*/ - "preindent" would be "____" and "indent" would be "__". - - """ - input: str - dsl_name: str | None = None - signatures: list[Module | Class | Function] = dc.field(default_factory=list) - output: Any = None # TODO: Very dynamic; probably untypeable in its current form? - indent: str = '' - - def __repr__(self) -> str: - dsl_name = self.dsl_name or "text" - def summarize(s: object) -> str: - s = repr(s) - if len(s) > 30: - return s[:26] + "..." + s[0] - return s - parts = ( - repr(dsl_name), - f"input={summarize(self.input)}", - f"output={summarize(self.output)}" - ) - return f"" - - -class BlockParser: - """ - Block-oriented parser for Argument Clinic. - Iterator, yields Block objects. - """ - - def __init__( - self, - input: str, - language: Language, - *, - verify: bool = True - ) -> None: - """ - "input" should be a str object - with embedded \n characters. - - "language" should be a Language object. - """ - language.validate() - - self.input = collections.deque(reversed(input.splitlines(keepends=True))) - self.block_start_line_number = self.line_number = 0 - - self.language = language - before, _, after = language.start_line.partition('{dsl_name}') - assert _ == '{dsl_name}' - self.find_start_re = libclinic.create_regex(before, after, - whole_line=False) - self.start_re = libclinic.create_regex(before, after) - self.verify = verify - self.last_checksum_re: re.Pattern[str] | None = None - self.last_dsl_name: str | None = None - self.dsl_name: str | None = None - self.first_block = True - - def __iter__(self) -> BlockParser: - return self - - def __next__(self) -> Block: - while True: - if not self.input: - raise StopIteration - - if self.dsl_name: - try: - return_value = self.parse_clinic_block(self.dsl_name) - except ClinicError as exc: - exc.filename = self.language.filename - exc.lineno = self.line_number - raise - self.dsl_name = None - self.first_block = False - return return_value - block = self.parse_verbatim_block() - if self.first_block and not block.input: - continue - self.first_block = False - return block - - - def is_start_line(self, line: str) -> str | None: - match = self.start_re.match(line.lstrip()) - return match.group(1) if match else None - - def _line(self, lookahead: bool = False) -> str: - self.line_number += 1 - line = self.input.pop() - if not lookahead: - self.language.parse_line(line) - return line - - def parse_verbatim_block(self) -> Block: - lines = [] - self.block_start_line_number = self.line_number - - while self.input: - line = self._line() - dsl_name = self.is_start_line(line) - if dsl_name: - self.dsl_name = dsl_name - break - lines.append(line) - - return Block("".join(lines)) - - def parse_clinic_block(self, dsl_name: str) -> Block: - in_lines = [] - self.block_start_line_number = self.line_number + 1 - stop_line = self.language.stop_line.format(dsl_name=dsl_name) - body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) - - def is_stop_line(line: str) -> bool: - # make sure to recognize stop line even if it - # doesn't end with EOL (it could be the very end of the file) - if line.startswith(stop_line): - remainder = line.removeprefix(stop_line) - if remainder and not remainder.isspace(): - fail(f"Garbage after stop line: {remainder!r}") - return True - else: - # gh-92256: don't allow incorrectly formatted stop lines - if line.lstrip().startswith(stop_line): - fail(f"Whitespace is not allowed before the stop line: {line!r}") - return False - - # consume body of program - while self.input: - line = self._line() - if is_stop_line(line) or self.is_start_line(line): - break - if body_prefix: - line = line.lstrip() - assert line.startswith(body_prefix) - line = line.removeprefix(body_prefix) - in_lines.append(line) - - # consume output and checksum line, if present. - if self.last_dsl_name == dsl_name: - checksum_re = self.last_checksum_re - else: - before, _, after = self.language.checksum_line.format(dsl_name=dsl_name, arguments='{arguments}').partition('{arguments}') - assert _ == '{arguments}' - checksum_re = libclinic.create_regex(before, after, word=False) - self.last_dsl_name = dsl_name - self.last_checksum_re = checksum_re - assert checksum_re is not None - - # scan forward for checksum line - out_lines = [] - arguments = None - while self.input: - line = self._line(lookahead=True) - match = checksum_re.match(line.lstrip()) - arguments = match.group(1) if match else None - if arguments: - break - out_lines.append(line) - if self.is_start_line(line): - break - - output: str | None - output = "".join(out_lines) - if arguments: - d = {} - for field in shlex.split(arguments): - name, equals, value = field.partition('=') - if not equals: - fail(f"Mangled Argument Clinic marker line: {line!r}") - d[name.strip()] = value.strip() - - if self.verify: - if 'input' in d: - checksum = d['output'] - else: - checksum = d['checksum'] - - computed = libclinic.compute_checksum(output, len(checksum)) - if checksum != computed: - fail("Checksum mismatch! " - f"Expected {checksum!r}, computed {computed!r}. " - "Suggested fix: remove all generated code including " - "the end marker, or use the '-f' option.") - else: - # put back output - output_lines = output.splitlines(keepends=True) - self.line_number -= len(output_lines) - self.input.extend(reversed(output_lines)) - output = None - - return Block("".join(in_lines), dsl_name, output=output) - - @dc.dataclass(slots=True, frozen=True) class Include: """ diff --git a/Tools/clinic/libclinic/block_parser.py b/Tools/clinic/libclinic/block_parser.py new file mode 100644 index 00000000000000..4c0198b53592a9 --- /dev/null +++ b/Tools/clinic/libclinic/block_parser.py @@ -0,0 +1,256 @@ +from __future__ import annotations +import collections +import dataclasses as dc +import re +import shlex +from typing import Any + +import libclinic +from libclinic import fail, ClinicError +from libclinic.language import Language +from libclinic.function import ( + Module, Class, Function) + + +@dc.dataclass(slots=True, repr=False) +class Block: + r""" + Represents a single block of text embedded in + another file. If dsl_name is None, the block represents + verbatim text, raw original text from the file, in + which case "input" will be the only non-false member. + If dsl_name is not None, the block represents a Clinic + block. + + input is always str, with embedded \n characters. + input represents the original text from the file; + if it's a Clinic block, it is the original text with + the body_prefix and redundant leading whitespace removed. + + dsl_name is either str or None. If str, it's the text + found on the start line of the block between the square + brackets. + + signatures is a list. + It may only contain clinic.Module, clinic.Class, and + clinic.Function objects. At the moment it should + contain at most one of each. + + output is either str or None. If str, it's the output + from this block, with embedded '\n' characters. + + indent is a str. It's the leading whitespace + that was found on every line of input. (If body_prefix is + not empty, this is the indent *after* removing the + body_prefix.) + + "indent" is different from the concept of "preindent" + (which is not stored as state on Block objects). + "preindent" is the whitespace that + was found in front of every line of input *before* the + "body_prefix" (see the Language object). If body_prefix + is empty, preindent must always be empty too. + + To illustrate the difference between "indent" and "preindent": + + Assume that '_' represents whitespace. + If the block processed was in a Python file, and looked like this: + ____#/*[python] + ____#__for a in range(20): + ____#____print(a) + ____#[python]*/ + "preindent" would be "____" and "indent" would be "__". + + """ + input: str + dsl_name: str | None = None + signatures: list[Module | Class | Function] = dc.field(default_factory=list) + output: Any = None # TODO: Very dynamic; probably untypeable in its current form? + indent: str = '' + + def __repr__(self) -> str: + dsl_name = self.dsl_name or "text" + def summarize(s: object) -> str: + s = repr(s) + if len(s) > 30: + return s[:26] + "..." + s[0] + return s + parts = ( + repr(dsl_name), + f"input={summarize(self.input)}", + f"output={summarize(self.output)}" + ) + return f"" + + +class BlockParser: + """ + Block-oriented parser for Argument Clinic. + Iterator, yields Block objects. + """ + + def __init__( + self, + input: str, + language: Language, + *, + verify: bool = True + ) -> None: + """ + "input" should be a str object + with embedded \n characters. + + "language" should be a Language object. + """ + language.validate() + + self.input = collections.deque(reversed(input.splitlines(keepends=True))) + self.block_start_line_number = self.line_number = 0 + + self.language = language + before, _, after = language.start_line.partition('{dsl_name}') + assert _ == '{dsl_name}' + self.find_start_re = libclinic.create_regex(before, after, + whole_line=False) + self.start_re = libclinic.create_regex(before, after) + self.verify = verify + self.last_checksum_re: re.Pattern[str] | None = None + self.last_dsl_name: str | None = None + self.dsl_name: str | None = None + self.first_block = True + + def __iter__(self) -> BlockParser: + return self + + def __next__(self) -> Block: + while True: + if not self.input: + raise StopIteration + + if self.dsl_name: + try: + return_value = self.parse_clinic_block(self.dsl_name) + except ClinicError as exc: + exc.filename = self.language.filename + exc.lineno = self.line_number + raise + self.dsl_name = None + self.first_block = False + return return_value + block = self.parse_verbatim_block() + if self.first_block and not block.input: + continue + self.first_block = False + return block + + + def is_start_line(self, line: str) -> str | None: + match = self.start_re.match(line.lstrip()) + return match.group(1) if match else None + + def _line(self, lookahead: bool = False) -> str: + self.line_number += 1 + line = self.input.pop() + if not lookahead: + self.language.parse_line(line) + return line + + def parse_verbatim_block(self) -> Block: + lines = [] + self.block_start_line_number = self.line_number + + while self.input: + line = self._line() + dsl_name = self.is_start_line(line) + if dsl_name: + self.dsl_name = dsl_name + break + lines.append(line) + + return Block("".join(lines)) + + def parse_clinic_block(self, dsl_name: str) -> Block: + in_lines = [] + self.block_start_line_number = self.line_number + 1 + stop_line = self.language.stop_line.format(dsl_name=dsl_name) + body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) + + def is_stop_line(line: str) -> bool: + # make sure to recognize stop line even if it + # doesn't end with EOL (it could be the very end of the file) + if line.startswith(stop_line): + remainder = line.removeprefix(stop_line) + if remainder and not remainder.isspace(): + fail(f"Garbage after stop line: {remainder!r}") + return True + else: + # gh-92256: don't allow incorrectly formatted stop lines + if line.lstrip().startswith(stop_line): + fail(f"Whitespace is not allowed before the stop line: {line!r}") + return False + + # consume body of program + while self.input: + line = self._line() + if is_stop_line(line) or self.is_start_line(line): + break + if body_prefix: + line = line.lstrip() + assert line.startswith(body_prefix) + line = line.removeprefix(body_prefix) + in_lines.append(line) + + # consume output and checksum line, if present. + if self.last_dsl_name == dsl_name: + checksum_re = self.last_checksum_re + else: + before, _, after = self.language.checksum_line.format(dsl_name=dsl_name, arguments='{arguments}').partition('{arguments}') + assert _ == '{arguments}' + checksum_re = libclinic.create_regex(before, after, word=False) + self.last_dsl_name = dsl_name + self.last_checksum_re = checksum_re + assert checksum_re is not None + + # scan forward for checksum line + out_lines = [] + arguments = None + while self.input: + line = self._line(lookahead=True) + match = checksum_re.match(line.lstrip()) + arguments = match.group(1) if match else None + if arguments: + break + out_lines.append(line) + if self.is_start_line(line): + break + + output: str | None + output = "".join(out_lines) + if arguments: + d = {} + for field in shlex.split(arguments): + name, equals, value = field.partition('=') + if not equals: + fail(f"Mangled Argument Clinic marker line: {line!r}") + d[name.strip()] = value.strip() + + if self.verify: + if 'input' in d: + checksum = d['output'] + else: + checksum = d['checksum'] + + computed = libclinic.compute_checksum(output, len(checksum)) + if checksum != computed: + fail("Checksum mismatch! " + f"Expected {checksum!r}, computed {computed!r}. " + "Suggested fix: remove all generated code including " + "the end marker, or use the '-f' option.") + else: + # put back output + output_lines = output.splitlines(keepends=True) + self.line_number -= len(output_lines) + self.input.extend(reversed(output_lines)) + output = None + + return Block("".join(in_lines), dsl_name, output=output) diff --git a/Tools/clinic/libclinic/language.py b/Tools/clinic/libclinic/language.py new file mode 100644 index 00000000000000..a90a9bb24e2201 --- /dev/null +++ b/Tools/clinic/libclinic/language.py @@ -0,0 +1,103 @@ +from __future__ import annotations +import abc +import typing +from collections.abc import ( + Iterable, +) + +import libclinic +from libclinic import fail +from libclinic.function import ( + Module, Class, Function) + +if typing.TYPE_CHECKING: + from clinic import Clinic + + +class Language(metaclass=abc.ABCMeta): + + start_line = "" + body_prefix = "" + stop_line = "" + checksum_line = "" + + def __init__(self, filename: str) -> None: + self.filename = filename + + @abc.abstractmethod + def render( + self, + clinic: Clinic, + signatures: Iterable[Module | Class | Function] + ) -> str: + ... + + def parse_line(self, line: str) -> None: + ... + + def validate(self) -> None: + def assert_only_one( + attr: str, + *additional_fields: str + ) -> None: + """ + Ensures that the string found at getattr(self, attr) + contains exactly one formatter replacement string for + each valid field. The list of valid fields is + ['dsl_name'] extended by additional_fields. + + e.g. + self.fmt = "{dsl_name} {a} {b}" + + # this passes + self.assert_only_one('fmt', 'a', 'b') + + # this fails, the format string has a {b} in it + self.assert_only_one('fmt', 'a') + + # this fails, the format string doesn't have a {c} in it + self.assert_only_one('fmt', 'a', 'b', 'c') + + # this fails, the format string has two {a}s in it, + # it must contain exactly one + self.fmt2 = '{dsl_name} {a} {a}' + self.assert_only_one('fmt2', 'a') + + """ + fields = ['dsl_name'] + fields.extend(additional_fields) + line: str = getattr(self, attr) + fcf = libclinic.FormatCounterFormatter() + fcf.format(line) + def local_fail(should_be_there_but_isnt: bool) -> None: + if should_be_there_but_isnt: + fail("{} {} must contain {{{}}} exactly once!".format( + self.__class__.__name__, attr, name)) + else: + fail("{} {} must not contain {{{}}}!".format( + self.__class__.__name__, attr, name)) + + for name, count in fcf.counts.items(): + if name in fields: + if count > 1: + local_fail(True) + else: + local_fail(False) + for name in fields: + if fcf.counts.get(name) != 1: + local_fail(True) + + assert_only_one('start_line') + assert_only_one('stop_line') + + field = "arguments" if "{arguments}" in self.checksum_line else "checksum" + assert_only_one('checksum_line', field) + + +class PythonLanguage(Language): + + language = 'Python' + start_line = "#/*[{dsl_name} input]" + body_prefix = "#" + stop_line = "#[{dsl_name} start generated code]*/" + checksum_line = "#/*[{dsl_name} end generated code: {arguments}]*/" From 19c3a2ff91ccf7444efadbc8f7e67269060050a2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 14 Mar 2024 17:19:36 +0100 Subject: [PATCH 092/158] gh-111696, PEP 737: Add PyType_GetFullyQualifiedName() function (#116815) Rewrite tests on type names in Python, they were written in C. --- Doc/c-api/type.rst | 8 ++ Doc/data/stable_abi.dat | 1 + Doc/whatsnew/3.13.rst | 6 ++ Include/object.h | 3 + Lib/test/test_capi/test_misc.py | 76 ++++++++++++++--- Lib/test/test_stable_abi_ctypes.py | 1 + ...-03-14-15-17-11.gh-issue-111696.YmnvAi.rst | 4 + Misc/stable_abi.toml | 2 + Modules/_testcapimodule.c | 85 ++++--------------- Objects/typeobject.c | 62 +++++++++++--- PC/python3dll.c | 1 + 11 files changed, 158 insertions(+), 91 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-03-14-15-17-11.gh-issue-111696.YmnvAi.rst diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 5aaa8147dd3176..c5234233ba7124 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -185,6 +185,14 @@ Type Objects .. versionadded:: 3.11 +.. c:function:: PyObject* PyType_GetFullyQualifiedName(PyTypeObject *type) + + Return the type's fully qualified name. Equivalent to + ``f"{type.__module__}.{type.__qualname__}"``, or ``type.__qualname__`` if + ``type.__module__`` is not a string or is equal to ``"builtins"``. + + .. versionadded:: 3.13 + .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 25629b4da053da..03fe3cef3843b6 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -677,6 +677,7 @@ function,PyType_FromSpecWithBases,3.3,, function,PyType_GenericAlloc,3.2,, function,PyType_GenericNew,3.2,, function,PyType_GetFlags,3.2,, +function,PyType_GetFullyQualifiedName,3.13,, function,PyType_GetModule,3.10,, function,PyType_GetModuleState,3.10,, function,PyType_GetName,3.11,, diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index ea45fa7c825dfe..cbb5e02aef1ce3 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1658,6 +1658,12 @@ New Features between native integer types and Python :class:`int` objects. (Contributed by Steve Dower in :gh:`111140`.) +* Add :c:func:`PyType_GetFullyQualifiedName` function to get the type's fully + qualified name. Equivalent to ``f"{type.__module__}.{type.__qualname__}"``, + or ``type.__qualname__`` if ``type.__module__`` is not a string or is equal + to ``"builtins"``. + (Contributed by Victor Stinner in :gh:`111696`.) + Porting to Python 3.13 ---------------------- diff --git a/Include/object.h b/Include/object.h index 05187fe5dc4f20..3f6f1ab1e68cc6 100644 --- a/Include/object.h +++ b/Include/object.h @@ -522,6 +522,9 @@ PyAPI_FUNC(void *) PyType_GetModuleState(PyTypeObject *); PyAPI_FUNC(PyObject *) PyType_GetName(PyTypeObject *); PyAPI_FUNC(PyObject *) PyType_GetQualName(PyTypeObject *); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030D0000 +PyAPI_FUNC(PyObject *) PyType_GetFullyQualifiedName(PyTypeObject *); +#endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030C0000 PyAPI_FUNC(PyObject *) PyType_FromMetaclass(PyTypeObject*, PyObject*, PyType_Spec*, PyObject*); PyAPI_FUNC(void *) PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls); diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index c1395ab00077cb..6b4f535cc6550a 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1100,21 +1100,75 @@ class Data(_testcapi.ObjExtraData): del d.extra self.assertIsNone(d.extra) - def test_get_type_module_name(self): + def test_get_type_name(self): + class MyType: + pass + + from _testcapi import get_type_name, get_type_qualname, get_type_fullyqualname + from _testinternalcapi import get_type_module_name + from collections import OrderedDict ht = _testcapi.get_heaptype_for_name() - for cls, expected in { - int: 'builtins', - OrderedDict: 'collections', - ht: '_testcapi', - }.items(): - with self.subTest(repr(cls)): - modname = _testinternalcapi.get_type_module_name(cls) - self.assertEqual(modname, expected) + for cls, fullname, modname, qualname, name in ( + (int, + 'int', + 'builtins', + 'int', + 'int'), + (OrderedDict, + 'collections.OrderedDict', + 'collections', + 'OrderedDict', + 'OrderedDict'), + (ht, + '_testcapi.HeapTypeNameType', + '_testcapi', + 'HeapTypeNameType', + 'HeapTypeNameType'), + (MyType, + f'{__name__}.CAPITest.test_get_type_name..MyType', + __name__, + 'CAPITest.test_get_type_name..MyType', + 'MyType'), + ): + with self.subTest(cls=repr(cls)): + self.assertEqual(get_type_fullyqualname(cls), fullname) + self.assertEqual(get_type_module_name(cls), modname) + self.assertEqual(get_type_qualname(cls), qualname) + self.assertEqual(get_type_name(cls), name) + # override __module__ ht.__module__ = 'test_module' - modname = _testinternalcapi.get_type_module_name(ht) - self.assertEqual(modname, 'test_module') + self.assertEqual(get_type_fullyqualname(ht), 'test_module.HeapTypeNameType') + self.assertEqual(get_type_module_name(ht), 'test_module') + self.assertEqual(get_type_qualname(ht), 'HeapTypeNameType') + self.assertEqual(get_type_name(ht), 'HeapTypeNameType') + + # override __name__ and __qualname__ + MyType.__name__ = 'my_name' + MyType.__qualname__ = 'my_qualname' + self.assertEqual(get_type_fullyqualname(MyType), f'{__name__}.my_qualname') + self.assertEqual(get_type_module_name(MyType), __name__) + self.assertEqual(get_type_qualname(MyType), 'my_qualname') + self.assertEqual(get_type_name(MyType), 'my_name') + + # override also __module__ + MyType.__module__ = 'my_module' + self.assertEqual(get_type_fullyqualname(MyType), 'my_module.my_qualname') + self.assertEqual(get_type_module_name(MyType), 'my_module') + self.assertEqual(get_type_qualname(MyType), 'my_qualname') + self.assertEqual(get_type_name(MyType), 'my_name') + + # PyType_GetFullyQualifiedName() ignores the module if it's "builtins" + # or "__main__" of it is not a string + MyType.__module__ = 'builtins' + self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') + MyType.__module__ = '__main__' + self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') + MyType.__module__ = 123 + self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') + + @requires_limited_api class TestHeapTypeRelative(unittest.TestCase): diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 8bd373976426ef..f0b449ac1708a1 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -706,6 +706,7 @@ def test_windows_feature_macros(self): "PyType_GenericAlloc", "PyType_GenericNew", "PyType_GetFlags", + "PyType_GetFullyQualifiedName", "PyType_GetModule", "PyType_GetModuleState", "PyType_GetName", diff --git a/Misc/NEWS.d/next/C API/2024-03-14-15-17-11.gh-issue-111696.YmnvAi.rst b/Misc/NEWS.d/next/C API/2024-03-14-15-17-11.gh-issue-111696.YmnvAi.rst new file mode 100644 index 00000000000000..3d87c56bf2493a --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-03-14-15-17-11.gh-issue-111696.YmnvAi.rst @@ -0,0 +1,4 @@ +Add :c:func:`PyType_GetFullyQualifiedName` function to get the type's fully +qualified name. Equivalent to ``f"{type.__module__}.{type.__qualname__}"``, or +``type.__qualname__`` if ``type.__module__`` is not a string or is equal to +``"builtins"``. Patch by Victor Stinner. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index ca7cf02961571e..c76a3cea4da3f7 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2496,3 +2496,5 @@ [typedef.PyCFunctionFastWithKeywords] added = '3.13' # "abi-only" since 3.10. (Same story as PyCFunctionFast.) +[function.PyType_GetFullyQualifiedName] + added = '3.13' diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b5e646f904b2d1..07f96466abdfc9 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -597,83 +597,31 @@ get_heaptype_for_name(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyType_FromSpec(&HeapTypeNameType_Spec); } + static PyObject * -test_get_type_name(PyObject *self, PyObject *Py_UNUSED(ignored)) +get_type_name(PyObject *self, PyObject *type) { - PyObject *tp_name = PyType_GetName(&PyLong_Type); - assert(strcmp(PyUnicode_AsUTF8(tp_name), "int") == 0); - Py_DECREF(tp_name); - - tp_name = PyType_GetName(&PyModule_Type); - assert(strcmp(PyUnicode_AsUTF8(tp_name), "module") == 0); - Py_DECREF(tp_name); - - PyObject *HeapTypeNameType = PyType_FromSpec(&HeapTypeNameType_Spec); - if (HeapTypeNameType == NULL) { - Py_RETURN_NONE; - } - tp_name = PyType_GetName((PyTypeObject *)HeapTypeNameType); - assert(strcmp(PyUnicode_AsUTF8(tp_name), "HeapTypeNameType") == 0); - Py_DECREF(tp_name); - - PyObject *name = PyUnicode_FromString("test_name"); - if (name == NULL) { - goto done; - } - if (PyObject_SetAttrString(HeapTypeNameType, "__name__", name) < 0) { - Py_DECREF(name); - goto done; - } - tp_name = PyType_GetName((PyTypeObject *)HeapTypeNameType); - assert(strcmp(PyUnicode_AsUTF8(tp_name), "test_name") == 0); - Py_DECREF(name); - Py_DECREF(tp_name); - - done: - Py_DECREF(HeapTypeNameType); - Py_RETURN_NONE; + assert(PyType_Check(type)); + return PyType_GetName((PyTypeObject *)type); } static PyObject * -test_get_type_qualname(PyObject *self, PyObject *Py_UNUSED(ignored)) +get_type_qualname(PyObject *self, PyObject *type) { - PyObject *tp_qualname = PyType_GetQualName(&PyLong_Type); - assert(strcmp(PyUnicode_AsUTF8(tp_qualname), "int") == 0); - Py_DECREF(tp_qualname); - - tp_qualname = PyType_GetQualName(&PyODict_Type); - assert(strcmp(PyUnicode_AsUTF8(tp_qualname), "OrderedDict") == 0); - Py_DECREF(tp_qualname); - - PyObject *HeapTypeNameType = PyType_FromSpec(&HeapTypeNameType_Spec); - if (HeapTypeNameType == NULL) { - Py_RETURN_NONE; - } - tp_qualname = PyType_GetQualName((PyTypeObject *)HeapTypeNameType); - assert(strcmp(PyUnicode_AsUTF8(tp_qualname), "HeapTypeNameType") == 0); - Py_DECREF(tp_qualname); + assert(PyType_Check(type)); + return PyType_GetQualName((PyTypeObject *)type); +} - PyObject *spec_name = PyUnicode_FromString(HeapTypeNameType_Spec.name); - if (spec_name == NULL) { - goto done; - } - if (PyObject_SetAttrString(HeapTypeNameType, - "__qualname__", spec_name) < 0) { - Py_DECREF(spec_name); - goto done; - } - tp_qualname = PyType_GetQualName((PyTypeObject *)HeapTypeNameType); - assert(strcmp(PyUnicode_AsUTF8(tp_qualname), - "_testcapi.HeapTypeNameType") == 0); - Py_DECREF(spec_name); - Py_DECREF(tp_qualname); - done: - Py_DECREF(HeapTypeNameType); - Py_RETURN_NONE; +static PyObject * +get_type_fullyqualname(PyObject *self, PyObject *type) +{ + assert(PyType_Check(type)); + return PyType_GetFullyQualifiedName((PyTypeObject *)type); } + static PyObject * test_get_type_dict(PyObject *self, PyObject *Py_UNUSED(ignored)) { @@ -3317,8 +3265,9 @@ static PyMethodDef TestMethods[] = { {"test_buildvalue_N", test_buildvalue_N, METH_NOARGS}, {"test_get_statictype_slots", test_get_statictype_slots, METH_NOARGS}, {"get_heaptype_for_name", get_heaptype_for_name, METH_NOARGS}, - {"test_get_type_name", test_get_type_name, METH_NOARGS}, - {"test_get_type_qualname", test_get_type_qualname, METH_NOARGS}, + {"get_type_name", get_type_name, METH_O}, + {"get_type_qualname", get_type_qualname, METH_O}, + {"get_type_fullyqualname", get_type_fullyqualname, METH_O}, {"test_get_type_dict", test_get_type_dict, METH_NOARGS}, {"_test_thread_state", test_thread_state, METH_VARARGS}, #ifndef MS_WINDOWS diff --git a/Objects/typeobject.c b/Objects/typeobject.c index d8c3e920106bc3..e51adac7e9d636 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1201,6 +1201,41 @@ type_set_module(PyTypeObject *type, PyObject *value, void *context) return PyDict_SetItem(dict, &_Py_ID(__module__), value); } + +PyObject * +PyType_GetFullyQualifiedName(PyTypeObject *type) +{ + if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + return PyUnicode_FromString(type->tp_name); + } + + PyObject *qualname = type_qualname(type, NULL); + if (qualname == NULL) { + return NULL; + } + + PyObject *module = type_module(type, NULL); + if (module == NULL) { + Py_DECREF(qualname); + return NULL; + } + + PyObject *result; + if (PyUnicode_Check(module) + && !_PyUnicode_Equal(module, &_Py_ID(builtins)) + && !_PyUnicode_Equal(module, &_Py_ID(__main__))) + { + result = PyUnicode_FromFormat("%U.%U", module, qualname); + } + else { + result = Py_NewRef(qualname); + } + Py_DECREF(module); + Py_DECREF(qualname); + return result; +} + + static PyObject * type_abstractmethods(PyTypeObject *type, void *context) { @@ -1708,28 +1743,31 @@ type_repr(PyObject *self) return PyUnicode_FromFormat("", type); } - PyObject *mod, *name, *rtn; - - mod = type_module(type, NULL); - if (mod == NULL) + PyObject *mod = type_module(type, NULL); + if (mod == NULL) { PyErr_Clear(); + } else if (!PyUnicode_Check(mod)) { - Py_SETREF(mod, NULL); + Py_CLEAR(mod); } - name = type_qualname(type, NULL); + + PyObject *name = type_qualname(type, NULL); if (name == NULL) { Py_XDECREF(mod); return NULL; } - if (mod != NULL && !_PyUnicode_Equal(mod, &_Py_ID(builtins))) - rtn = PyUnicode_FromFormat("", mod, name); - else - rtn = PyUnicode_FromFormat("", type->tp_name); - + PyObject *result; + if (mod != NULL && !_PyUnicode_Equal(mod, &_Py_ID(builtins))) { + result = PyUnicode_FromFormat("", mod, name); + } + else { + result = PyUnicode_FromFormat("", type->tp_name); + } Py_XDECREF(mod); Py_DECREF(name); - return rtn; + + return result; } static PyObject * diff --git a/PC/python3dll.c b/PC/python3dll.c index aa6bfe2c4022db..81d55af7074383 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -637,6 +637,7 @@ EXPORT_FUNC(PyType_FromSpecWithBases) EXPORT_FUNC(PyType_GenericAlloc) EXPORT_FUNC(PyType_GenericNew) EXPORT_FUNC(PyType_GetFlags) +EXPORT_FUNC(PyType_GetFullyQualifiedName) EXPORT_FUNC(PyType_GetModule) EXPORT_FUNC(PyType_GetModuleState) EXPORT_FUNC(PyType_GetName) From 61e54bfcee9f08a8e09aa1b2fd6cea69ca6a26e0 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 14 Mar 2024 16:31:47 +0000 Subject: [PATCH 093/158] GH-116422: Factor out eval breaker checks at end of calls into its own micro-op. (GH-116817) --- Include/internal/pycore_opcode_metadata.h | 20 +- Include/internal/pycore_uop_ids.h | 249 ++++++------- Include/internal/pycore_uop_metadata.h | 22 +- Python/bytecodes.c | 100 +++-- Python/executor_cases.c.h | 20 +- Python/generated_cases.c.h | 430 ++++++++++++---------- Python/optimizer_cases.c.h | 4 + 7 files changed, 474 insertions(+), 371 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index de93d4ef14de2a..f754de3706c812 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1212,21 +1212,21 @@ _PyOpcode_macro_expansion[256] = { [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } }, [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_BUILTIN_CLASS] = { .nuops = 1, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 } } }, - [CALL_BUILTIN_FAST] = { .nuops = 1, .uops = { { _CALL_BUILTIN_FAST, 0, 0 } } }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 } } }, - [CALL_BUILTIN_O] = { .nuops = 1, .uops = { { _CALL_BUILTIN_O, 0, 0 } } }, + [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_BUILTIN_O] = { .nuops = 2, .uops = { { _CALL_BUILTIN_O, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, 0, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, 0, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { _CALL_ISINSTANCE, 0, 0 } } }, [CALL_LEN] = { .nuops = 1, .uops = { { _CALL_LEN, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_O, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_STR_1] = { .nuops = 1, .uops = { { _CALL_STR_1, 0, 0 } } }, - [CALL_TUPLE_1] = { .nuops = 1, .uops = { { _CALL_TUPLE_1, 0, 0 } } }, + [CALL_STR_1] = { .nuops = 2, .uops = { { _CALL_STR_1, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_TUPLE_1] = { .nuops = 2, .uops = { { _CALL_TUPLE_1, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_TYPE_1] = { .nuops = 1, .uops = { { _CALL_TYPE_1, 0, 0 } } }, [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { _CHECK_EG_MATCH, 0, 0 } } }, [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { _CHECK_EXC_MATCH, 0, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 1209d736abe696..b569b80c5f110a 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -37,44 +37,45 @@ extern "C" { #define _BUILD_TUPLE BUILD_TUPLE #define _CALL 311 #define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT -#define _CALL_BUILTIN_CLASS CALL_BUILTIN_CLASS -#define _CALL_BUILTIN_FAST CALL_BUILTIN_FAST -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS CALL_BUILTIN_FAST_WITH_KEYWORDS -#define _CALL_BUILTIN_O CALL_BUILTIN_O +#define _CALL_BUILTIN_CLASS 312 +#define _CALL_BUILTIN_FAST 313 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 314 +#define _CALL_BUILTIN_O 315 #define _CALL_FUNCTION_EX CALL_FUNCTION_EX #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 #define _CALL_ISINSTANCE CALL_ISINSTANCE #define _CALL_KW CALL_KW #define _CALL_LEN CALL_LEN -#define _CALL_METHOD_DESCRIPTOR_FAST CALL_METHOD_DESCRIPTOR_FAST -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS -#define _CALL_METHOD_DESCRIPTOR_NOARGS CALL_METHOD_DESCRIPTOR_NOARGS -#define _CALL_METHOD_DESCRIPTOR_O CALL_METHOD_DESCRIPTOR_O +#define _CALL_METHOD_DESCRIPTOR_FAST 316 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 317 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 318 +#define _CALL_METHOD_DESCRIPTOR_O 319 #define _CALL_PY_WITH_DEFAULTS CALL_PY_WITH_DEFAULTS -#define _CALL_STR_1 CALL_STR_1 -#define _CALL_TUPLE_1 CALL_TUPLE_1 +#define _CALL_STR_1 320 +#define _CALL_TUPLE_1 321 #define _CALL_TYPE_1 CALL_TYPE_1 -#define _CHECK_ATTR_CLASS 312 -#define _CHECK_ATTR_METHOD_LAZY_DICT 313 -#define _CHECK_ATTR_MODULE 314 -#define _CHECK_ATTR_WITH_HINT 315 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 316 +#define _CHECK_ATTR_CLASS 322 +#define _CHECK_ATTR_METHOD_LAZY_DICT 323 +#define _CHECK_ATTR_MODULE 324 +#define _CHECK_ATTR_WITH_HINT 325 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 326 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 317 -#define _CHECK_FUNCTION_EXACT_ARGS 318 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 319 -#define _CHECK_PEP_523 320 -#define _CHECK_STACK_SPACE 321 -#define _CHECK_VALIDITY 322 -#define _CHECK_VALIDITY_AND_SET_IP 323 -#define _COLD_EXIT 324 -#define _COMPARE_OP 325 -#define _COMPARE_OP_FLOAT 326 -#define _COMPARE_OP_INT 327 -#define _COMPARE_OP_STR 328 -#define _CONTAINS_OP 329 +#define _CHECK_FUNCTION 327 +#define _CHECK_FUNCTION_EXACT_ARGS 328 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 329 +#define _CHECK_PEP_523 330 +#define _CHECK_PERIODIC 331 +#define _CHECK_STACK_SPACE 332 +#define _CHECK_VALIDITY 333 +#define _CHECK_VALIDITY_AND_SET_IP 334 +#define _COLD_EXIT 335 +#define _COMPARE_OP 336 +#define _COMPARE_OP_FLOAT 337 +#define _COMPARE_OP_INT 338 +#define _COMPARE_OP_STR 339 +#define _CONTAINS_OP 340 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT #define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE @@ -90,41 +91,41 @@ extern "C" { #define _DICT_UPDATE DICT_UPDATE #define _END_SEND END_SEND #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _FATAL_ERROR 330 +#define _FATAL_ERROR 341 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 331 +#define _FOR_ITER 342 #define _FOR_ITER_GEN FOR_ITER_GEN -#define _FOR_ITER_TIER_TWO 332 +#define _FOR_ITER_TIER_TWO 343 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 333 -#define _GUARD_BOTH_INT 334 -#define _GUARD_BOTH_UNICODE 335 -#define _GUARD_BUILTINS_VERSION 336 -#define _GUARD_DORV_VALUES 337 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 338 -#define _GUARD_GLOBALS_VERSION 339 -#define _GUARD_IS_FALSE_POP 340 -#define _GUARD_IS_NONE_POP 341 -#define _GUARD_IS_NOT_NONE_POP 342 -#define _GUARD_IS_TRUE_POP 343 -#define _GUARD_KEYS_VERSION 344 -#define _GUARD_NOT_EXHAUSTED_LIST 345 -#define _GUARD_NOT_EXHAUSTED_RANGE 346 -#define _GUARD_NOT_EXHAUSTED_TUPLE 347 -#define _GUARD_TYPE_VERSION 348 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 349 -#define _INIT_CALL_PY_EXACT_ARGS 350 -#define _INIT_CALL_PY_EXACT_ARGS_0 351 -#define _INIT_CALL_PY_EXACT_ARGS_1 352 -#define _INIT_CALL_PY_EXACT_ARGS_2 353 -#define _INIT_CALL_PY_EXACT_ARGS_3 354 -#define _INIT_CALL_PY_EXACT_ARGS_4 355 +#define _GUARD_BOTH_FLOAT 344 +#define _GUARD_BOTH_INT 345 +#define _GUARD_BOTH_UNICODE 346 +#define _GUARD_BUILTINS_VERSION 347 +#define _GUARD_DORV_VALUES 348 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 349 +#define _GUARD_GLOBALS_VERSION 350 +#define _GUARD_IS_FALSE_POP 351 +#define _GUARD_IS_NONE_POP 352 +#define _GUARD_IS_NOT_NONE_POP 353 +#define _GUARD_IS_TRUE_POP 354 +#define _GUARD_KEYS_VERSION 355 +#define _GUARD_NOT_EXHAUSTED_LIST 356 +#define _GUARD_NOT_EXHAUSTED_RANGE 357 +#define _GUARD_NOT_EXHAUSTED_TUPLE 358 +#define _GUARD_TYPE_VERSION 359 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 360 +#define _INIT_CALL_PY_EXACT_ARGS 361 +#define _INIT_CALL_PY_EXACT_ARGS_0 362 +#define _INIT_CALL_PY_EXACT_ARGS_1 363 +#define _INIT_CALL_PY_EXACT_ARGS_2 364 +#define _INIT_CALL_PY_EXACT_ARGS_3 365 +#define _INIT_CALL_PY_EXACT_ARGS_4 366 #define _INSTRUMENTED_CALL INSTRUMENTED_CALL #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW @@ -141,65 +142,65 @@ extern "C" { #define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST #define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 356 -#define _IS_NONE 357 +#define _INTERNAL_INCREMENT_OPT_COUNTER 367 +#define _IS_NONE 368 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 358 -#define _ITER_CHECK_RANGE 359 -#define _ITER_CHECK_TUPLE 360 -#define _ITER_JUMP_LIST 361 -#define _ITER_JUMP_RANGE 362 -#define _ITER_JUMP_TUPLE 363 -#define _ITER_NEXT_LIST 364 -#define _ITER_NEXT_RANGE 365 -#define _ITER_NEXT_TUPLE 366 -#define _JUMP_TO_TOP 367 +#define _ITER_CHECK_LIST 369 +#define _ITER_CHECK_RANGE 370 +#define _ITER_CHECK_TUPLE 371 +#define _ITER_JUMP_LIST 372 +#define _ITER_JUMP_RANGE 373 +#define _ITER_JUMP_TUPLE 374 +#define _ITER_NEXT_LIST 375 +#define _ITER_NEXT_RANGE 376 +#define _ITER_NEXT_TUPLE 377 +#define _JUMP_TO_TOP 378 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND #define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR -#define _LOAD_ATTR 368 -#define _LOAD_ATTR_CLASS 369 -#define _LOAD_ATTR_CLASS_0 370 -#define _LOAD_ATTR_CLASS_1 371 +#define _LOAD_ATTR 379 +#define _LOAD_ATTR_CLASS 380 +#define _LOAD_ATTR_CLASS_0 381 +#define _LOAD_ATTR_CLASS_1 382 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 372 -#define _LOAD_ATTR_INSTANCE_VALUE_0 373 -#define _LOAD_ATTR_INSTANCE_VALUE_1 374 -#define _LOAD_ATTR_METHOD_LAZY_DICT 375 -#define _LOAD_ATTR_METHOD_NO_DICT 376 -#define _LOAD_ATTR_METHOD_WITH_VALUES 377 -#define _LOAD_ATTR_MODULE 378 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 379 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 380 +#define _LOAD_ATTR_INSTANCE_VALUE 383 +#define _LOAD_ATTR_INSTANCE_VALUE_0 384 +#define _LOAD_ATTR_INSTANCE_VALUE_1 385 +#define _LOAD_ATTR_METHOD_LAZY_DICT 386 +#define _LOAD_ATTR_METHOD_NO_DICT 387 +#define _LOAD_ATTR_METHOD_WITH_VALUES 388 +#define _LOAD_ATTR_MODULE 389 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 390 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 391 #define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_SLOT 381 -#define _LOAD_ATTR_SLOT_0 382 -#define _LOAD_ATTR_SLOT_1 383 -#define _LOAD_ATTR_WITH_HINT 384 +#define _LOAD_ATTR_SLOT 392 +#define _LOAD_ATTR_SLOT_0 393 +#define _LOAD_ATTR_SLOT_1 394 +#define _LOAD_ATTR_WITH_HINT 395 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 385 -#define _LOAD_CONST_INLINE_BORROW 386 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 387 -#define _LOAD_CONST_INLINE_WITH_NULL 388 +#define _LOAD_CONST_INLINE 396 +#define _LOAD_CONST_INLINE_BORROW 397 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 398 +#define _LOAD_CONST_INLINE_WITH_NULL 399 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 389 -#define _LOAD_FAST_0 390 -#define _LOAD_FAST_1 391 -#define _LOAD_FAST_2 392 -#define _LOAD_FAST_3 393 -#define _LOAD_FAST_4 394 -#define _LOAD_FAST_5 395 -#define _LOAD_FAST_6 396 -#define _LOAD_FAST_7 397 +#define _LOAD_FAST 400 +#define _LOAD_FAST_0 401 +#define _LOAD_FAST_1 402 +#define _LOAD_FAST_2 403 +#define _LOAD_FAST_3 404 +#define _LOAD_FAST_4 405 +#define _LOAD_FAST_5 406 +#define _LOAD_FAST_6 407 +#define _LOAD_FAST_7 408 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 398 -#define _LOAD_GLOBAL_BUILTINS 399 -#define _LOAD_GLOBAL_MODULE 400 +#define _LOAD_GLOBAL 409 +#define _LOAD_GLOBAL_BUILTINS 410 +#define _LOAD_GLOBAL_MODULE 411 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR @@ -213,48 +214,48 @@ extern "C" { #define _MATCH_SEQUENCE MATCH_SEQUENCE #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_FRAME 401 -#define _POP_JUMP_IF_FALSE 402 -#define _POP_JUMP_IF_TRUE 403 +#define _POP_FRAME 412 +#define _POP_JUMP_IF_FALSE 413 +#define _POP_JUMP_IF_TRUE 414 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 404 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 415 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 405 +#define _PUSH_FRAME 416 #define _PUSH_NULL PUSH_NULL -#define _REPLACE_WITH_TRUE 406 +#define _REPLACE_WITH_TRUE 417 #define _RESUME_CHECK RESUME_CHECK -#define _SAVE_RETURN_OFFSET 407 -#define _SEND 408 +#define _SAVE_RETURN_OFFSET 418 +#define _SEND 419 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 409 -#define _STORE_ATTR 410 -#define _STORE_ATTR_INSTANCE_VALUE 411 -#define _STORE_ATTR_SLOT 412 +#define _START_EXECUTOR 420 +#define _STORE_ATTR 421 +#define _STORE_ATTR_INSTANCE_VALUE 422 +#define _STORE_ATTR_SLOT 423 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 413 -#define _STORE_FAST_0 414 -#define _STORE_FAST_1 415 -#define _STORE_FAST_2 416 -#define _STORE_FAST_3 417 -#define _STORE_FAST_4 418 -#define _STORE_FAST_5 419 -#define _STORE_FAST_6 420 -#define _STORE_FAST_7 421 +#define _STORE_FAST 424 +#define _STORE_FAST_0 425 +#define _STORE_FAST_1 426 +#define _STORE_FAST_2 427 +#define _STORE_FAST_3 428 +#define _STORE_FAST_4 429 +#define _STORE_FAST_5 430 +#define _STORE_FAST_6 431 +#define _STORE_FAST_7 432 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 422 +#define _STORE_SUBSCR 433 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TO_BOOL 423 +#define _TO_BOOL 434 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -264,12 +265,12 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 424 +#define _UNPACK_SEQUENCE 435 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define MAX_UOP_ID 424 +#define MAX_UOP_ID 435 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 62405a362fd7ab..507bd27c01c553 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -188,6 +188,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG, [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, + [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG, [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG, [_CHECK_PEP_523] = HAS_DEOPT_FLAG, @@ -201,19 +202,19 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_PUSH_FRAME] = 0, [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_CALL_STR_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, - [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, + [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, @@ -301,6 +302,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", [_CHECK_PEP_523] = "_CHECK_PEP_523", + [_CHECK_PERIODIC] = "_CHECK_PERIODIC", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", [_CHECK_VALIDITY_AND_SET_IP] = "_CHECK_VALIDITY_AND_SET_IP", diff --git a/Python/bytecodes.c b/Python/bytecodes.c index af2e2c8f52ee29..e8fcb6c6d0067c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3108,10 +3108,13 @@ dummy_func( Py_DECREF(args[i]); } ERROR_IF(res == NULL, error); + } + + op(_CHECK_PERIODIC, (--)) { CHECK_EVAL_BREAKER(); } - macro(CALL) = _SPECIALIZE_CALL + unused/2 + _CALL; + macro(CALL) = _SPECIALIZE_CALL + unused/2 + _CALL + _CHECK_PERIODIC; op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { DEOPT_IF(null != NULL); @@ -3246,7 +3249,7 @@ dummy_func( Py_DECREF(arg); } - inst(CALL_STR_1, (unused/1, unused/2, callable, null, arg -- res)) { + op(_CALL_STR_1, (callable, null, arg -- res)) { assert(oparg == 1); DEOPT_IF(null != NULL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type); @@ -3254,10 +3257,15 @@ dummy_func( res = PyObject_Str(arg); Py_DECREF(arg); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } - inst(CALL_TUPLE_1, (unused/1, unused/2, callable, null, arg -- res)) { + macro(CALL_STR_1) = + unused/1 + + unused/2 + + _CALL_STR_1 + + _CHECK_PERIODIC; + + op(_CALL_TUPLE_1, (callable, null, arg -- res)) { assert(oparg == 1); DEOPT_IF(null != NULL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type); @@ -3265,9 +3273,14 @@ dummy_func( res = PySequence_Tuple(arg); Py_DECREF(arg); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } + macro(CALL_TUPLE_1) = + unused/1 + + unused/2 + + _CALL_TUPLE_1 + + _CHECK_PERIODIC; + inst(CALL_ALLOC_AND_ENTER_INIT, (unused/1, unused/2, callable, null, args[oparg] -- unused)) { /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) @@ -3328,7 +3341,7 @@ dummy_func( } } - inst(CALL_BUILTIN_CLASS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg] -- res)) { int total_args = oparg; if (self_or_null != NULL) { args--; @@ -3345,10 +3358,15 @@ dummy_func( } Py_DECREF(tp); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } - inst(CALL_BUILTIN_O, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_BUILTIN_CLASS) = + unused/1 + + unused/2 + + _CALL_BUILTIN_CLASS + + _CHECK_PERIODIC; + + op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_O functions */ int total_args = oparg; if (self_or_null != NULL) { @@ -3373,10 +3391,15 @@ dummy_func( Py_DECREF(arg); Py_DECREF(callable); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } - inst(CALL_BUILTIN_FAST, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_BUILTIN_O) = + unused/1 + + unused/2 + + _CALL_BUILTIN_O + + _CHECK_PERIODIC; + + op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL functions, without keywords */ int total_args = oparg; if (self_or_null != NULL) { @@ -3400,15 +3423,15 @@ dummy_func( } Py_DECREF(callable); ERROR_IF(res == NULL, error); - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ - CHECK_EVAL_BREAKER(); } - inst(CALL_BUILTIN_FAST_WITH_KEYWORDS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_BUILTIN_FAST) = + unused/1 + + unused/2 + + _CALL_BUILTIN_FAST + + _CHECK_PERIODIC; + + op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int total_args = oparg; if (self_or_null != NULL) { @@ -3431,9 +3454,14 @@ dummy_func( } Py_DECREF(callable); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } + macro(CALL_BUILTIN_FAST_WITH_KEYWORDS) = + unused/1 + + unused/2 + + _CALL_BUILTIN_FAST_WITH_KEYWORDS + + _CHECK_PERIODIC; + inst(CALL_LEN, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { /* len(o) */ int total_args = oparg; @@ -3504,7 +3532,7 @@ dummy_func( DISPATCH(); } - inst(CALL_METHOD_DESCRIPTOR_O, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res)) { int total_args = oparg; if (self_or_null != NULL) { args--; @@ -3532,10 +3560,15 @@ dummy_func( Py_DECREF(arg); Py_DECREF(callable); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } - inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_METHOD_DESCRIPTOR_O) = + unused/1 + + unused/2 + + _CALL_METHOD_DESCRIPTOR_O + + _CHECK_PERIODIC; + + op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { int total_args = oparg; if (self_or_null != NULL) { args--; @@ -3561,10 +3594,15 @@ dummy_func( } Py_DECREF(callable); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } - inst(CALL_METHOD_DESCRIPTOR_NOARGS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) = + unused/1 + + unused/2 + + _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + + _CHECK_PERIODIC; + + op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res)) { assert(oparg == 0 || oparg == 1); int total_args = oparg; if (self_or_null != NULL) { @@ -3591,10 +3629,15 @@ dummy_func( Py_DECREF(self); Py_DECREF(callable); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } - inst(CALL_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_METHOD_DESCRIPTOR_NOARGS) = + unused/1 + + unused/2 + + _CALL_METHOD_DESCRIPTOR_NOARGS + + _CHECK_PERIODIC; + + op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- res)) { int total_args = oparg; if (self_or_null != NULL) { args--; @@ -3619,9 +3662,14 @@ dummy_func( } Py_DECREF(callable); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } + macro(CALL_METHOD_DESCRIPTOR_FAST) = + unused/1 + + unused/2 + + _CALL_METHOD_DESCRIPTOR_FAST + + _CHECK_PERIODIC; + inst(INSTRUMENTED_CALL_KW, ( -- )) { int is_meth = PEEK(oparg + 2) != NULL; int total_args = oparg + is_meth; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 42e884c20ba04f..077499ebc01687 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2819,6 +2819,11 @@ /* _CALL is not a viable micro-op for tier 2 */ + case _CHECK_PERIODIC: { + CHECK_EVAL_BREAKER(); + break; + } + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { PyObject *null; PyObject *callable; @@ -3096,7 +3101,6 @@ if (res == NULL) goto pop_3_error_tier_two; stack_pointer[-3] = res; stack_pointer += -2; - CHECK_EVAL_BREAKER(); break; } @@ -3118,7 +3122,6 @@ if (res == NULL) goto pop_3_error_tier_two; stack_pointer[-3] = res; stack_pointer += -2; - CHECK_EVAL_BREAKER(); break; } @@ -3165,7 +3168,6 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); break; } @@ -3203,7 +3205,6 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); break; } @@ -3238,14 +3239,8 @@ } Py_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); break; } @@ -3281,7 +3276,6 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); break; } @@ -3393,7 +3387,6 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); break; } @@ -3432,7 +3425,6 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); break; } @@ -3473,7 +3465,6 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); break; } @@ -3512,7 +3503,6 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7d02e49d040c23..645d0fb510e6fb 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -842,6 +842,9 @@ } if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } + // _CHECK_PERIODIC + { + } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1020,25 +1023,31 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_BUILTIN_CLASS args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + { + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyType_Check(callable), CALL); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + STAT_INC(CALL, hit); + res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(tp); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); - STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + // _CHECK_PERIODIC + { } - Py_DECREF(tp); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1056,36 +1065,37 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL functions, without keywords */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + { + /* Builtin METH_FASTCALL functions, without keywords */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + /* res = func(self, args, nargs) */ + res = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable), + args, + total_args); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - /* res = func(self, args, nargs) */ - res = ((PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, - total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + // _CHECK_PERIODIC + { } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1103,30 +1113,36 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST_WITH_KEYWORDS args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + { + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); + STAT_INC(CALL, hit); + /* res = func(self, args, nargs, kwnames) */ + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); + res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); - STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + // _CHECK_PERIODIC + { } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1144,32 +1160,38 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_BUILTIN_O args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + { + /* Builtin METH_O functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + PyObject *arg = args[0]; + res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + // _CHECK_PERIODIC + { } - PyObject *arg = args[0]; - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1509,33 +1531,39 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + { + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + /* Builtin METH_FASTCALL methods, without keywords */ + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; + int nargs = total_args - 1; + res = cfunc(self, args + 1, nargs); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - PyCFunctionFast cfunc = - (PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + // _CHECK_PERIODIC + { } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1553,33 +1581,39 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + { + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + res = cfunc(self, args + 1, nargs, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + // _CHECK_PERIODIC + { } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1597,35 +1631,41 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_NOARGS args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + { + assert(oparg == 0 || oparg == 1); + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + // _CHECK_PERIODIC + { } - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1643,36 +1683,42 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_O args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + { + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_O, CALL); + PyObject *arg = args[1]; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + res = _PyCFunction_TrampolineCall(cfunc, self, arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); - PyObject *arg = args[1]; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + // _CHECK_PERIODIC + { } - res = _PyCFunction_TrampolineCall(cfunc, self, arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1816,16 +1862,22 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_STR_1 arg = stack_pointer[-1]; null = stack_pointer[-2]; callable = stack_pointer[-3]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); - STAT_INC(CALL, hit); - res = PyObject_Str(arg); - Py_DECREF(arg); - if (res == NULL) goto pop_3_error; + { + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); + STAT_INC(CALL, hit); + res = PyObject_Str(arg); + Py_DECREF(arg); + if (res == NULL) goto pop_3_error; + } + // _CHECK_PERIODIC + { + } stack_pointer[-3] = res; stack_pointer += -2; CHECK_EVAL_BREAKER(); @@ -1843,16 +1895,22 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_TUPLE_1 arg = stack_pointer[-1]; null = stack_pointer[-2]; callable = stack_pointer[-3]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); - STAT_INC(CALL, hit); - res = PySequence_Tuple(arg); - Py_DECREF(arg); - if (res == NULL) goto pop_3_error; + { + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); + STAT_INC(CALL, hit); + res = PySequence_Tuple(arg); + Py_DECREF(arg); + if (res == NULL) goto pop_3_error; + } + // _CHECK_PERIODIC + { + } stack_pointer[-3] = res; stack_pointer += -2; CHECK_EVAL_BREAKER(); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 7e4214cc9acf39..cf36f1ba792775 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1533,6 +1533,10 @@ /* _CALL is not a viable micro-op for tier 2 */ + case _CHECK_PERIODIC: { + break; + } + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { _Py_UopsSymbol *null; _Py_UopsSymbol *callable; From fd8e30eb62d0ecfb75786df1ac25593b0143cc98 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 14 Mar 2024 13:50:24 -0400 Subject: [PATCH 094/158] gh-112795: Move the test for ZipFile into the core tests for zipfile. (#116823) Move the test for ZipFile into the core tests for zipfile. --- Lib/test/test_zipfile/_path/test_path.py | 12 ------------ Lib/test/test_zipfile/test_core.py | 11 +++++++++++ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index 171ab6fdb5fc28..c66cb3cba69ebd 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -577,15 +577,3 @@ def test_getinfo_missing(self, alpharep): zipfile.Path(alpharep) with self.assertRaises(KeyError): alpharep.getinfo('does-not-exist') - - def test_root_folder_in_zipfile(self): - """ - gh-112795: Some tools or self constructed codes will add '/' folder to - the zip file, this is a strange behavior, but we should support it. - """ - in_memory_file = io.BytesIO() - zf = zipfile.ZipFile(in_memory_file, "w") - zf.mkdir('/') - zf.writestr('./a.txt', 'aaa') - tmpdir = pathlib.Path(self.fixtures.enter_context(temp_dir())) - zf.extractall(tmpdir) diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 7d89f753448dd7..368e60a37b0555 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -3033,6 +3033,17 @@ def test_create_directory_with_write(self): self.assertEqual(set(os.listdir(target)), {"directory", "directory2"}) + def test_root_folder_in_zipfile(self): + """ + gh-112795: Some tools or self constructed codes will add '/' folder to + the zip file, this is a strange behavior, but we should support it. + """ + in_memory_file = io.BytesIO() + zf = zipfile.ZipFile(in_memory_file, "w") + zf.mkdir('/') + zf.writestr('./a.txt', 'aaa') + zf.extractall(TESTFN2) + def tearDown(self): rmtree(TESTFN2) if os.path.exists(TESTFN): From 25cd8730aa39c18e767f63586dfd1da2fbb307c9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 14 Mar 2024 18:59:43 +0100 Subject: [PATCH 095/158] gh-113317, AC: Add libclinic.converter module (#116821) * Move CConverter class to a new libclinic.converter module. * Move CRenderData and Include classes to a new libclinic.crenderdata module. --- Tools/clinic/clinic.py | 604 +------------------------- Tools/clinic/libclinic/converter.py | 534 +++++++++++++++++++++++ Tools/clinic/libclinic/crenderdata.py | 81 ++++ Tools/clinic/libclinic/function.py | 3 +- 4 files changed, 621 insertions(+), 601 deletions(-) create mode 100644 Tools/clinic/libclinic/converter.py create mode 100644 Tools/clinic/libclinic/crenderdata.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index ac205866f9d291..c9641cb9c82bf7 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -32,15 +32,12 @@ from operator import attrgetter from types import FunctionType, NoneType from typing import ( - TYPE_CHECKING, Any, Final, Literal, NamedTuple, NoReturn, Protocol, - TypeVar, - cast, ) @@ -57,6 +54,10 @@ GETTER, SETTER) from libclinic.language import Language, PythonLanguage from libclinic.block_parser import Block, BlockParser +from libclinic.crenderdata import CRenderData, Include, TemplateDict +from libclinic.converter import ( + CConverter, CConverterClassT, + converters, legacy_converters) # TODO: @@ -84,65 +85,6 @@ def __repr__(self) -> str: NULL = Null() -TemplateDict = dict[str, str] - - -class CRenderData: - def __init__(self) -> None: - - # The C statements to declare variables. - # Should be full lines with \n eol characters. - self.declarations: list[str] = [] - - # The C statements required to initialize the variables before the parse call. - # Should be full lines with \n eol characters. - self.initializers: list[str] = [] - - # The C statements needed to dynamically modify the values - # parsed by the parse call, before calling the impl. - self.modifications: list[str] = [] - - # The entries for the "keywords" array for PyArg_ParseTuple. - # Should be individual strings representing the names. - self.keywords: list[str] = [] - - # The "format units" for PyArg_ParseTuple. - # Should be individual strings that will get - self.format_units: list[str] = [] - - # The varargs arguments for PyArg_ParseTuple. - self.parse_arguments: list[str] = [] - - # The parameter declarations for the impl function. - self.impl_parameters: list[str] = [] - - # The arguments to the impl function at the time it's called. - self.impl_arguments: list[str] = [] - - # For return converters: the name of the variable that - # should receive the value returned by the impl. - self.return_value = "return_value" - - # For return converters: the code to convert the return - # value from the parse function. This is also where - # you should check the _return_value for errors, and - # "goto exit" if there are any. - self.return_conversion: list[str] = [] - self.converter_retval = "_return_value" - - # The C statements required to do some operations - # after the end of parsing but before cleaning up. - # These operations may be, for example, memory deallocations which - # can only be done without any error happening during argument parsing. - self.post_parsing: list[str] = [] - - # The C statements required to clean up after the impl call. - self.cleanup: list[str] = [] - - # The C statements to generate critical sections (per-object locking). - self.lock: list[str] = [] - self.unlock: list[str] = [] - ParamTuple = tuple["Parameter", ...] @@ -1556,26 +1498,6 @@ def render_function( return clinic.get_destination('block').dump() -@dc.dataclass(slots=True, frozen=True) -class Include: - """ - An include like: #include "pycore_long.h" // _Py_ID() - """ - # Example: "pycore_long.h". - filename: str - - # Example: "_Py_ID()". - reason: str - - # None means unconditional include. - # Example: "#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)". - condition: str | None - - def sort_key(self) -> tuple[str, str]: - # order: '#if' comes before 'NO_CONDITION' - return (self.condition or 'NO_CONDITION', self.filename) - - @dc.dataclass(slots=True) class BlockPrinter: language: Language @@ -2151,29 +2073,6 @@ def parse(self, block: Block) -> None: ReturnConverterType = Callable[..., "CReturnConverter"] -CConverterClassT = TypeVar("CConverterClassT", bound=type["CConverter"]) - -def add_c_converter( - f: CConverterClassT, - name: str | None = None -) -> CConverterClassT: - if not name: - name = f.__name__ - if not name.endswith('_converter'): - return f - name = name.removesuffix('_converter') - converters[name] = f - return f - -def add_default_legacy_c_converter(cls: CConverterClassT) -> CConverterClassT: - # automatically add converter for default format unit - # (but without stomping on the existing one if it's already - # set, in case you subclass) - if ((cls.format_unit not in ('O&', '')) and - (cls.format_unit not in legacy_converters)): - legacy_converters[cls.format_unit] = cls - return cls - def add_legacy_c_converter( format_unit: str, **kwargs: Any @@ -2192,501 +2091,6 @@ def closure(f: CConverterClassT) -> CConverterClassT: return f return closure -class CConverterAutoRegister(type): - def __init__( - cls, name: str, bases: tuple[type[object], ...], classdict: dict[str, Any] - ) -> None: - converter_cls = cast(type["CConverter"], cls) - add_c_converter(converter_cls) - add_default_legacy_c_converter(converter_cls) - -class CConverter(metaclass=CConverterAutoRegister): - """ - For the init function, self, name, function, and default - must be keyword-or-positional parameters. All other - parameters must be keyword-only. - """ - - # The C name to use for this variable. - name: str - - # The Python name to use for this variable. - py_name: str - - # The C type to use for this variable. - # 'type' should be a Python string specifying the type, e.g. "int". - # If this is a pointer type, the type string should end with ' *'. - type: str | None = None - - # The Python default value for this parameter, as a Python value. - # Or the magic value "unspecified" if there is no default. - # Or the magic value "unknown" if this value is a cannot be evaluated - # at Argument-Clinic-preprocessing time (but is presumed to be valid - # at runtime). - default: object = unspecified - - # If not None, default must be isinstance() of this type. - # (You can also specify a tuple of types.) - default_type: bltns.type[object] | tuple[bltns.type[object], ...] | None = None - - # "default" converted into a C value, as a string. - # Or None if there is no default. - c_default: str | None = None - - # "default" converted into a Python value, as a string. - # Or None if there is no default. - py_default: str | None = None - - # The default value used to initialize the C variable when - # there is no default, but not specifying a default may - # result in an "uninitialized variable" warning. This can - # easily happen when using option groups--although - # properly-written code won't actually use the variable, - # the variable does get passed in to the _impl. (Ah, if - # only dataflow analysis could inline the static function!) - # - # This value is specified as a string. - # Every non-abstract subclass should supply a valid value. - c_ignored_default: str = 'NULL' - - # If true, wrap with Py_UNUSED. - unused = False - - # The C converter *function* to be used, if any. - # (If this is not None, format_unit must be 'O&'.) - converter: str | None = None - - # Should Argument Clinic add a '&' before the name of - # the variable when passing it into the _impl function? - impl_by_reference = False - - # Should Argument Clinic add a '&' before the name of - # the variable when passing it into PyArg_ParseTuple (AndKeywords)? - parse_by_reference = True - - ############################################################# - ############################################################# - ## You shouldn't need to read anything below this point to ## - ## write your own converter functions. ## - ############################################################# - ############################################################# - - # The "format unit" to specify for this variable when - # parsing arguments using PyArg_ParseTuple (AndKeywords). - # Custom converters should always use the default value of 'O&'. - format_unit = 'O&' - - # What encoding do we want for this variable? Only used - # by format units starting with 'e'. - encoding: str | None = None - - # Should this object be required to be a subclass of a specific type? - # If not None, should be a string representing a pointer to a - # PyTypeObject (e.g. "&PyUnicode_Type"). - # Only used by the 'O!' format unit (and the "object" converter). - subclass_of: str | None = None - - # See also the 'length_name' property. - # Only used by format units ending with '#'. - length = False - - # Should we show this parameter in the generated - # __text_signature__? This is *almost* always True. - # (It's only False for __new__, __init__, and METH_STATIC functions.) - show_in_signature = True - - # Overrides the name used in a text signature. - # The name used for a "self" parameter must be one of - # self, type, or module; however users can set their own. - # This lets the self_converter overrule the user-settable - # name, *just* for the text signature. - # Only set by self_converter. - signature_name: str | None = None - - broken_limited_capi: bool = False - - # keep in sync with self_converter.__init__! - def __init__(self, - # Positional args: - name: str, - py_name: str, - function: Function, - default: object = unspecified, - *, # Keyword only args: - c_default: str | None = None, - py_default: str | None = None, - annotation: str | Literal[Sentinels.unspecified] = unspecified, - unused: bool = False, - **kwargs: Any - ) -> None: - self.name = libclinic.ensure_legal_c_identifier(name) - self.py_name = py_name - self.unused = unused - self.includes: list[Include] = [] - - if default is not unspecified: - if (self.default_type - and default is not unknown - and not isinstance(default, self.default_type) - ): - if isinstance(self.default_type, type): - types_str = self.default_type.__name__ - else: - names = [cls.__name__ for cls in self.default_type] - types_str = ', '.join(names) - cls_name = self.__class__.__name__ - fail(f"{cls_name}: default value {default!r} for field " - f"{name!r} is not of type {types_str!r}") - self.default = default - - if c_default: - self.c_default = c_default - if py_default: - self.py_default = py_default - - if annotation is not unspecified: - fail("The 'annotation' parameter is not currently permitted.") - - # Make sure not to set self.function until after converter_init() has been called. - # This prevents you from caching information - # about the function in converter_init(). - # (That breaks if we get cloned.) - self.converter_init(**kwargs) - self.function = function - - # Add a custom __getattr__ method to improve the error message - # if somebody tries to access self.function in converter_init(). - # - # mypy will assume arbitrary access is okay for a class with a __getattr__ method, - # and that's not what we want, - # so put it inside an `if not TYPE_CHECKING` block - if not TYPE_CHECKING: - def __getattr__(self, attr): - if attr == "function": - fail( - f"{self.__class__.__name__!r} object has no attribute 'function'.\n" - f"Note: accessing self.function inside converter_init is disallowed!" - ) - return super().__getattr__(attr) - # this branch is just here for coverage reporting - else: # pragma: no cover - pass - - def converter_init(self) -> None: - pass - - def is_optional(self) -> bool: - return (self.default is not unspecified) - - def _render_self(self, parameter: Parameter, data: CRenderData) -> None: - self.parameter = parameter - name = self.parser_name - - # impl_arguments - s = ("&" if self.impl_by_reference else "") + name - data.impl_arguments.append(s) - if self.length: - data.impl_arguments.append(self.length_name) - - # impl_parameters - data.impl_parameters.append(self.simple_declaration(by_reference=self.impl_by_reference)) - if self.length: - data.impl_parameters.append(f"Py_ssize_t {self.length_name}") - - def _render_non_self( - self, - parameter: Parameter, - data: CRenderData - ) -> None: - self.parameter = parameter - name = self.name - - # declarations - d = self.declaration(in_parser=True) - data.declarations.append(d) - - # initializers - initializers = self.initialize() - if initializers: - data.initializers.append('/* initializers for ' + name + ' */\n' + initializers.rstrip()) - - # modifications - modifications = self.modify() - if modifications: - data.modifications.append('/* modifications for ' + name + ' */\n' + modifications.rstrip()) - - # keywords - if parameter.is_vararg(): - pass - elif parameter.is_positional_only(): - data.keywords.append('') - else: - data.keywords.append(parameter.name) - - # format_units - if self.is_optional() and '|' not in data.format_units: - data.format_units.append('|') - if parameter.is_keyword_only() and '$' not in data.format_units: - data.format_units.append('$') - data.format_units.append(self.format_unit) - - # parse_arguments - self.parse_argument(data.parse_arguments) - - # post_parsing - if post_parsing := self.post_parsing(): - data.post_parsing.append('/* Post parse cleanup for ' + name + ' */\n' + post_parsing.rstrip() + '\n') - - # cleanup - cleanup = self.cleanup() - if cleanup: - data.cleanup.append('/* Cleanup for ' + name + ' */\n' + cleanup.rstrip() + "\n") - - def render(self, parameter: Parameter, data: CRenderData) -> None: - """ - parameter is a clinic.Parameter instance. - data is a CRenderData instance. - """ - self._render_self(parameter, data) - self._render_non_self(parameter, data) - - @functools.cached_property - def length_name(self) -> str: - """Computes the name of the associated "length" variable.""" - assert self.length is not None - return self.parser_name + "_length" - - # Why is this one broken out separately? - # For "positional-only" function parsing, - # which generates a bunch of PyArg_ParseTuple calls. - def parse_argument(self, args: list[str]) -> None: - assert not (self.converter and self.encoding) - if self.format_unit == 'O&': - assert self.converter - args.append(self.converter) - - if self.encoding: - args.append(libclinic.c_repr(self.encoding)) - elif self.subclass_of: - args.append(self.subclass_of) - - s = ("&" if self.parse_by_reference else "") + self.parser_name - args.append(s) - - if self.length: - args.append(f"&{self.length_name}") - - # - # All the functions after here are intended as extension points. - # - - def simple_declaration( - self, - by_reference: bool = False, - *, - in_parser: bool = False - ) -> str: - """ - Computes the basic declaration of the variable. - Used in computing the prototype declaration and the - variable declaration. - """ - assert isinstance(self.type, str) - prototype = [self.type] - if by_reference or not self.type.endswith('*'): - prototype.append(" ") - if by_reference: - prototype.append('*') - if in_parser: - name = self.parser_name - else: - name = self.name - if self.unused: - name = f"Py_UNUSED({name})" - prototype.append(name) - return "".join(prototype) - - def declaration(self, *, in_parser: bool = False) -> str: - """ - The C statement to declare this variable. - """ - declaration = [self.simple_declaration(in_parser=True)] - default = self.c_default - if not default and self.parameter.group: - default = self.c_ignored_default - if default: - declaration.append(" = ") - declaration.append(default) - declaration.append(";") - if self.length: - declaration.append('\n') - declaration.append(f"Py_ssize_t {self.length_name};") - return "".join(declaration) - - def initialize(self) -> str: - """ - The C statements required to set up this variable before parsing. - Returns a string containing this code indented at column 0. - If no initialization is necessary, returns an empty string. - """ - return "" - - def modify(self) -> str: - """ - The C statements required to modify this variable after parsing. - Returns a string containing this code indented at column 0. - If no modification is necessary, returns an empty string. - """ - return "" - - def post_parsing(self) -> str: - """ - The C statements required to do some operations after the end of parsing but before cleaning up. - Return a string containing this code indented at column 0. - If no operation is necessary, return an empty string. - """ - return "" - - def cleanup(self) -> str: - """ - The C statements required to clean up after this variable. - Returns a string containing this code indented at column 0. - If no cleanup is necessary, returns an empty string. - """ - return "" - - def pre_render(self) -> None: - """ - A second initialization function, like converter_init, - called just before rendering. - You are permitted to examine self.function here. - """ - pass - - def bad_argument(self, displayname: str, expected: str, *, limited_capi: bool, expected_literal: bool = True) -> str: - assert '"' not in expected - if limited_capi: - if expected_literal: - return (f'PyErr_Format(PyExc_TypeError, ' - f'"{{{{name}}}}() {displayname} must be {expected}, not %.50s", ' - f'{{argname}} == Py_None ? "None" : Py_TYPE({{argname}})->tp_name);') - else: - return (f'PyErr_Format(PyExc_TypeError, ' - f'"{{{{name}}}}() {displayname} must be %.50s, not %.50s", ' - f'"{expected}", ' - f'{{argname}} == Py_None ? "None" : Py_TYPE({{argname}})->tp_name);') - else: - if expected_literal: - expected = f'"{expected}"' - self.add_include('pycore_modsupport.h', '_PyArg_BadArgument()') - return f'_PyArg_BadArgument("{{{{name}}}}", "{displayname}", {expected}, {{argname}});' - - def format_code(self, fmt: str, *, - argname: str, - bad_argument: str | None = None, - bad_argument2: str | None = None, - **kwargs: Any) -> str: - if '{bad_argument}' in fmt: - if not bad_argument: - raise TypeError("required 'bad_argument' argument") - fmt = fmt.replace('{bad_argument}', bad_argument) - if '{bad_argument2}' in fmt: - if not bad_argument2: - raise TypeError("required 'bad_argument2' argument") - fmt = fmt.replace('{bad_argument2}', bad_argument2) - return fmt.format(argname=argname, paramname=self.parser_name, **kwargs) - - def use_converter(self) -> None: - """Method called when self.converter is used to parse an argument.""" - pass - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'O&': - self.use_converter() - return self.format_code(""" - if (!{converter}({argname}, &{paramname})) {{{{ - goto exit; - }}}} - """, - argname=argname, - converter=self.converter) - if self.format_unit == 'O!': - cast = '(%s)' % self.type if self.type != 'PyObject *' else '' - if self.subclass_of in type_checks: - typecheck, typename = type_checks[self.subclass_of] - return self.format_code(""" - if (!{typecheck}({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = {cast}{argname}; - """, - argname=argname, - bad_argument=self.bad_argument(displayname, typename, limited_capi=limited_capi), - typecheck=typecheck, typename=typename, cast=cast) - return self.format_code(""" - if (!PyObject_TypeCheck({argname}, {subclass_of})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = {cast}{argname}; - """, - argname=argname, - bad_argument=self.bad_argument(displayname, '({subclass_of})->tp_name', - expected_literal=False, limited_capi=limited_capi), - subclass_of=self.subclass_of, cast=cast) - if self.format_unit == 'O': - cast = '(%s)' % self.type if self.type != 'PyObject *' else '' - return self.format_code(""" - {paramname} = {cast}{argname}; - """, - argname=argname, cast=cast) - return None - - def set_template_dict(self, template_dict: TemplateDict) -> None: - pass - - @property - def parser_name(self) -> str: - if self.name in libclinic.CLINIC_PREFIXED_ARGS: # bpo-39741 - return libclinic.CLINIC_PREFIX + self.name - else: - return self.name - - def add_include(self, name: str, reason: str, - *, condition: str | None = None) -> None: - include = Include(name, reason, condition) - self.includes.append(include) - -type_checks = { - '&PyLong_Type': ('PyLong_Check', 'int'), - '&PyTuple_Type': ('PyTuple_Check', 'tuple'), - '&PyList_Type': ('PyList_Check', 'list'), - '&PySet_Type': ('PySet_Check', 'set'), - '&PyFrozenSet_Type': ('PyFrozenSet_Check', 'frozenset'), - '&PyDict_Type': ('PyDict_Check', 'dict'), - '&PyUnicode_Type': ('PyUnicode_Check', 'str'), - '&PyBytes_Type': ('PyBytes_Check', 'bytes'), - '&PyByteArray_Type': ('PyByteArray_Check', 'bytearray'), -} - - -ConverterType = Callable[..., CConverter] -ConverterDict = dict[str, ConverterType] - -# maps strings to callables. -# these callables must be of the form: -# def foo(name, default, *, ...) -# The callable may have any number of keyword-only parameters. -# The callable must return a CConverter object. -# The callable should not call builtins.print. -converters: ConverterDict = {} - -# maps strings to callables. -# these callables follow the same rules as those for "converters" above. -# note however that they will never be called with keyword-only parameters. -legacy_converters: ConverterDict = {} - # maps strings to callables. # these callables must be of the form: # def foo(*, ...) diff --git a/Tools/clinic/libclinic/converter.py b/Tools/clinic/libclinic/converter.py new file mode 100644 index 00000000000000..da28ba56a346a1 --- /dev/null +++ b/Tools/clinic/libclinic/converter.py @@ -0,0 +1,534 @@ +from __future__ import annotations +import builtins as bltns +import functools +from typing import Any, TypeVar, Literal, TYPE_CHECKING, cast +from collections.abc import Callable + +import libclinic +from libclinic import fail +from libclinic import Sentinels, unspecified, unknown +from libclinic.crenderdata import CRenderData, Include, TemplateDict +from libclinic.function import Function, Parameter + + +CConverterClassT = TypeVar("CConverterClassT", bound=type["CConverter"]) + + +type_checks = { + '&PyLong_Type': ('PyLong_Check', 'int'), + '&PyTuple_Type': ('PyTuple_Check', 'tuple'), + '&PyList_Type': ('PyList_Check', 'list'), + '&PySet_Type': ('PySet_Check', 'set'), + '&PyFrozenSet_Type': ('PyFrozenSet_Check', 'frozenset'), + '&PyDict_Type': ('PyDict_Check', 'dict'), + '&PyUnicode_Type': ('PyUnicode_Check', 'str'), + '&PyBytes_Type': ('PyBytes_Check', 'bytes'), + '&PyByteArray_Type': ('PyByteArray_Check', 'bytearray'), +} + + +def add_c_converter( + f: CConverterClassT, + name: str | None = None +) -> CConverterClassT: + if not name: + name = f.__name__ + if not name.endswith('_converter'): + return f + name = name.removesuffix('_converter') + converters[name] = f + return f + + +def add_default_legacy_c_converter(cls: CConverterClassT) -> CConverterClassT: + # automatically add converter for default format unit + # (but without stomping on the existing one if it's already + # set, in case you subclass) + if ((cls.format_unit not in ('O&', '')) and + (cls.format_unit not in legacy_converters)): + legacy_converters[cls.format_unit] = cls + return cls + + +class CConverterAutoRegister(type): + def __init__( + cls, name: str, bases: tuple[type[object], ...], classdict: dict[str, Any] + ) -> None: + converter_cls = cast(type["CConverter"], cls) + add_c_converter(converter_cls) + add_default_legacy_c_converter(converter_cls) + +class CConverter(metaclass=CConverterAutoRegister): + """ + For the init function, self, name, function, and default + must be keyword-or-positional parameters. All other + parameters must be keyword-only. + """ + + # The C name to use for this variable. + name: str + + # The Python name to use for this variable. + py_name: str + + # The C type to use for this variable. + # 'type' should be a Python string specifying the type, e.g. "int". + # If this is a pointer type, the type string should end with ' *'. + type: str | None = None + + # The Python default value for this parameter, as a Python value. + # Or the magic value "unspecified" if there is no default. + # Or the magic value "unknown" if this value is a cannot be evaluated + # at Argument-Clinic-preprocessing time (but is presumed to be valid + # at runtime). + default: object = unspecified + + # If not None, default must be isinstance() of this type. + # (You can also specify a tuple of types.) + default_type: bltns.type[object] | tuple[bltns.type[object], ...] | None = None + + # "default" converted into a C value, as a string. + # Or None if there is no default. + c_default: str | None = None + + # "default" converted into a Python value, as a string. + # Or None if there is no default. + py_default: str | None = None + + # The default value used to initialize the C variable when + # there is no default, but not specifying a default may + # result in an "uninitialized variable" warning. This can + # easily happen when using option groups--although + # properly-written code won't actually use the variable, + # the variable does get passed in to the _impl. (Ah, if + # only dataflow analysis could inline the static function!) + # + # This value is specified as a string. + # Every non-abstract subclass should supply a valid value. + c_ignored_default: str = 'NULL' + + # If true, wrap with Py_UNUSED. + unused = False + + # The C converter *function* to be used, if any. + # (If this is not None, format_unit must be 'O&'.) + converter: str | None = None + + # Should Argument Clinic add a '&' before the name of + # the variable when passing it into the _impl function? + impl_by_reference = False + + # Should Argument Clinic add a '&' before the name of + # the variable when passing it into PyArg_ParseTuple (AndKeywords)? + parse_by_reference = True + + ############################################################# + ############################################################# + ## You shouldn't need to read anything below this point to ## + ## write your own converter functions. ## + ############################################################# + ############################################################# + + # The "format unit" to specify for this variable when + # parsing arguments using PyArg_ParseTuple (AndKeywords). + # Custom converters should always use the default value of 'O&'. + format_unit = 'O&' + + # What encoding do we want for this variable? Only used + # by format units starting with 'e'. + encoding: str | None = None + + # Should this object be required to be a subclass of a specific type? + # If not None, should be a string representing a pointer to a + # PyTypeObject (e.g. "&PyUnicode_Type"). + # Only used by the 'O!' format unit (and the "object" converter). + subclass_of: str | None = None + + # See also the 'length_name' property. + # Only used by format units ending with '#'. + length = False + + # Should we show this parameter in the generated + # __text_signature__? This is *almost* always True. + # (It's only False for __new__, __init__, and METH_STATIC functions.) + show_in_signature = True + + # Overrides the name used in a text signature. + # The name used for a "self" parameter must be one of + # self, type, or module; however users can set their own. + # This lets the self_converter overrule the user-settable + # name, *just* for the text signature. + # Only set by self_converter. + signature_name: str | None = None + + broken_limited_capi: bool = False + + # keep in sync with self_converter.__init__! + def __init__(self, + # Positional args: + name: str, + py_name: str, + function: Function, + default: object = unspecified, + *, # Keyword only args: + c_default: str | None = None, + py_default: str | None = None, + annotation: str | Literal[Sentinels.unspecified] = unspecified, + unused: bool = False, + **kwargs: Any + ) -> None: + self.name = libclinic.ensure_legal_c_identifier(name) + self.py_name = py_name + self.unused = unused + self.includes: list[Include] = [] + + if default is not unspecified: + if (self.default_type + and default is not unknown + and not isinstance(default, self.default_type) + ): + if isinstance(self.default_type, type): + types_str = self.default_type.__name__ + else: + names = [cls.__name__ for cls in self.default_type] + types_str = ', '.join(names) + cls_name = self.__class__.__name__ + fail(f"{cls_name}: default value {default!r} for field " + f"{name!r} is not of type {types_str!r}") + self.default = default + + if c_default: + self.c_default = c_default + if py_default: + self.py_default = py_default + + if annotation is not unspecified: + fail("The 'annotation' parameter is not currently permitted.") + + # Make sure not to set self.function until after converter_init() has been called. + # This prevents you from caching information + # about the function in converter_init(). + # (That breaks if we get cloned.) + self.converter_init(**kwargs) + self.function = function + + # Add a custom __getattr__ method to improve the error message + # if somebody tries to access self.function in converter_init(). + # + # mypy will assume arbitrary access is okay for a class with a __getattr__ method, + # and that's not what we want, + # so put it inside an `if not TYPE_CHECKING` block + if not TYPE_CHECKING: + def __getattr__(self, attr): + if attr == "function": + fail( + f"{self.__class__.__name__!r} object has no attribute 'function'.\n" + f"Note: accessing self.function inside converter_init is disallowed!" + ) + return super().__getattr__(attr) + # this branch is just here for coverage reporting + else: # pragma: no cover + pass + + def converter_init(self) -> None: + pass + + def is_optional(self) -> bool: + return (self.default is not unspecified) + + def _render_self(self, parameter: Parameter, data: CRenderData) -> None: + self.parameter = parameter + name = self.parser_name + + # impl_arguments + s = ("&" if self.impl_by_reference else "") + name + data.impl_arguments.append(s) + if self.length: + data.impl_arguments.append(self.length_name) + + # impl_parameters + data.impl_parameters.append(self.simple_declaration(by_reference=self.impl_by_reference)) + if self.length: + data.impl_parameters.append(f"Py_ssize_t {self.length_name}") + + def _render_non_self( + self, + parameter: Parameter, + data: CRenderData + ) -> None: + self.parameter = parameter + name = self.name + + # declarations + d = self.declaration(in_parser=True) + data.declarations.append(d) + + # initializers + initializers = self.initialize() + if initializers: + data.initializers.append('/* initializers for ' + name + ' */\n' + initializers.rstrip()) + + # modifications + modifications = self.modify() + if modifications: + data.modifications.append('/* modifications for ' + name + ' */\n' + modifications.rstrip()) + + # keywords + if parameter.is_vararg(): + pass + elif parameter.is_positional_only(): + data.keywords.append('') + else: + data.keywords.append(parameter.name) + + # format_units + if self.is_optional() and '|' not in data.format_units: + data.format_units.append('|') + if parameter.is_keyword_only() and '$' not in data.format_units: + data.format_units.append('$') + data.format_units.append(self.format_unit) + + # parse_arguments + self.parse_argument(data.parse_arguments) + + # post_parsing + if post_parsing := self.post_parsing(): + data.post_parsing.append('/* Post parse cleanup for ' + name + ' */\n' + post_parsing.rstrip() + '\n') + + # cleanup + cleanup = self.cleanup() + if cleanup: + data.cleanup.append('/* Cleanup for ' + name + ' */\n' + cleanup.rstrip() + "\n") + + def render(self, parameter: Parameter, data: CRenderData) -> None: + """ + parameter is a clinic.Parameter instance. + data is a CRenderData instance. + """ + self._render_self(parameter, data) + self._render_non_self(parameter, data) + + @functools.cached_property + def length_name(self) -> str: + """Computes the name of the associated "length" variable.""" + assert self.length is not None + return self.parser_name + "_length" + + # Why is this one broken out separately? + # For "positional-only" function parsing, + # which generates a bunch of PyArg_ParseTuple calls. + def parse_argument(self, args: list[str]) -> None: + assert not (self.converter and self.encoding) + if self.format_unit == 'O&': + assert self.converter + args.append(self.converter) + + if self.encoding: + args.append(libclinic.c_repr(self.encoding)) + elif self.subclass_of: + args.append(self.subclass_of) + + s = ("&" if self.parse_by_reference else "") + self.parser_name + args.append(s) + + if self.length: + args.append(f"&{self.length_name}") + + # + # All the functions after here are intended as extension points. + # + + def simple_declaration( + self, + by_reference: bool = False, + *, + in_parser: bool = False + ) -> str: + """ + Computes the basic declaration of the variable. + Used in computing the prototype declaration and the + variable declaration. + """ + assert isinstance(self.type, str) + prototype = [self.type] + if by_reference or not self.type.endswith('*'): + prototype.append(" ") + if by_reference: + prototype.append('*') + if in_parser: + name = self.parser_name + else: + name = self.name + if self.unused: + name = f"Py_UNUSED({name})" + prototype.append(name) + return "".join(prototype) + + def declaration(self, *, in_parser: bool = False) -> str: + """ + The C statement to declare this variable. + """ + declaration = [self.simple_declaration(in_parser=True)] + default = self.c_default + if not default and self.parameter.group: + default = self.c_ignored_default + if default: + declaration.append(" = ") + declaration.append(default) + declaration.append(";") + if self.length: + declaration.append('\n') + declaration.append(f"Py_ssize_t {self.length_name};") + return "".join(declaration) + + def initialize(self) -> str: + """ + The C statements required to set up this variable before parsing. + Returns a string containing this code indented at column 0. + If no initialization is necessary, returns an empty string. + """ + return "" + + def modify(self) -> str: + """ + The C statements required to modify this variable after parsing. + Returns a string containing this code indented at column 0. + If no modification is necessary, returns an empty string. + """ + return "" + + def post_parsing(self) -> str: + """ + The C statements required to do some operations after the end of parsing but before cleaning up. + Return a string containing this code indented at column 0. + If no operation is necessary, return an empty string. + """ + return "" + + def cleanup(self) -> str: + """ + The C statements required to clean up after this variable. + Returns a string containing this code indented at column 0. + If no cleanup is necessary, returns an empty string. + """ + return "" + + def pre_render(self) -> None: + """ + A second initialization function, like converter_init, + called just before rendering. + You are permitted to examine self.function here. + """ + pass + + def bad_argument(self, displayname: str, expected: str, *, limited_capi: bool, expected_literal: bool = True) -> str: + assert '"' not in expected + if limited_capi: + if expected_literal: + return (f'PyErr_Format(PyExc_TypeError, ' + f'"{{{{name}}}}() {displayname} must be {expected}, not %.50s", ' + f'{{argname}} == Py_None ? "None" : Py_TYPE({{argname}})->tp_name);') + else: + return (f'PyErr_Format(PyExc_TypeError, ' + f'"{{{{name}}}}() {displayname} must be %.50s, not %.50s", ' + f'"{expected}", ' + f'{{argname}} == Py_None ? "None" : Py_TYPE({{argname}})->tp_name);') + else: + if expected_literal: + expected = f'"{expected}"' + self.add_include('pycore_modsupport.h', '_PyArg_BadArgument()') + return f'_PyArg_BadArgument("{{{{name}}}}", "{displayname}", {expected}, {{argname}});' + + def format_code(self, fmt: str, *, + argname: str, + bad_argument: str | None = None, + bad_argument2: str | None = None, + **kwargs: Any) -> str: + if '{bad_argument}' in fmt: + if not bad_argument: + raise TypeError("required 'bad_argument' argument") + fmt = fmt.replace('{bad_argument}', bad_argument) + if '{bad_argument2}' in fmt: + if not bad_argument2: + raise TypeError("required 'bad_argument2' argument") + fmt = fmt.replace('{bad_argument2}', bad_argument2) + return fmt.format(argname=argname, paramname=self.parser_name, **kwargs) + + def use_converter(self) -> None: + """Method called when self.converter is used to parse an argument.""" + pass + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'O&': + self.use_converter() + return self.format_code(""" + if (!{converter}({argname}, &{paramname})) {{{{ + goto exit; + }}}} + """, + argname=argname, + converter=self.converter) + if self.format_unit == 'O!': + cast = '(%s)' % self.type if self.type != 'PyObject *' else '' + if self.subclass_of in type_checks: + typecheck, typename = type_checks[self.subclass_of] + return self.format_code(""" + if (!{typecheck}({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = {cast}{argname}; + """, + argname=argname, + bad_argument=self.bad_argument(displayname, typename, limited_capi=limited_capi), + typecheck=typecheck, typename=typename, cast=cast) + return self.format_code(""" + if (!PyObject_TypeCheck({argname}, {subclass_of})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = {cast}{argname}; + """, + argname=argname, + bad_argument=self.bad_argument(displayname, '({subclass_of})->tp_name', + expected_literal=False, limited_capi=limited_capi), + subclass_of=self.subclass_of, cast=cast) + if self.format_unit == 'O': + cast = '(%s)' % self.type if self.type != 'PyObject *' else '' + return self.format_code(""" + {paramname} = {cast}{argname}; + """, + argname=argname, cast=cast) + return None + + def set_template_dict(self, template_dict: TemplateDict) -> None: + pass + + @property + def parser_name(self) -> str: + if self.name in libclinic.CLINIC_PREFIXED_ARGS: # bpo-39741 + return libclinic.CLINIC_PREFIX + self.name + else: + return self.name + + def add_include(self, name: str, reason: str, + *, condition: str | None = None) -> None: + include = Include(name, reason, condition) + self.includes.append(include) + + +ConverterType = Callable[..., CConverter] +ConverterDict = dict[str, ConverterType] + +# maps strings to callables. +# these callables must be of the form: +# def foo(name, default, *, ...) +# The callable may have any number of keyword-only parameters. +# The callable must return a CConverter object. +# The callable should not call builtins.print. +converters: ConverterDict = {} + +# maps strings to callables. +# these callables follow the same rules as those for "converters" above. +# note however that they will never be called with keyword-only parameters. +legacy_converters: ConverterDict = {} diff --git a/Tools/clinic/libclinic/crenderdata.py b/Tools/clinic/libclinic/crenderdata.py new file mode 100644 index 00000000000000..58976b8185ebae --- /dev/null +++ b/Tools/clinic/libclinic/crenderdata.py @@ -0,0 +1,81 @@ +import dataclasses as dc + + +TemplateDict = dict[str, str] + + +class CRenderData: + def __init__(self) -> None: + + # The C statements to declare variables. + # Should be full lines with \n eol characters. + self.declarations: list[str] = [] + + # The C statements required to initialize the variables before the parse call. + # Should be full lines with \n eol characters. + self.initializers: list[str] = [] + + # The C statements needed to dynamically modify the values + # parsed by the parse call, before calling the impl. + self.modifications: list[str] = [] + + # The entries for the "keywords" array for PyArg_ParseTuple. + # Should be individual strings representing the names. + self.keywords: list[str] = [] + + # The "format units" for PyArg_ParseTuple. + # Should be individual strings that will get + self.format_units: list[str] = [] + + # The varargs arguments for PyArg_ParseTuple. + self.parse_arguments: list[str] = [] + + # The parameter declarations for the impl function. + self.impl_parameters: list[str] = [] + + # The arguments to the impl function at the time it's called. + self.impl_arguments: list[str] = [] + + # For return converters: the name of the variable that + # should receive the value returned by the impl. + self.return_value = "return_value" + + # For return converters: the code to convert the return + # value from the parse function. This is also where + # you should check the _return_value for errors, and + # "goto exit" if there are any. + self.return_conversion: list[str] = [] + self.converter_retval = "_return_value" + + # The C statements required to do some operations + # after the end of parsing but before cleaning up. + # These operations may be, for example, memory deallocations which + # can only be done without any error happening during argument parsing. + self.post_parsing: list[str] = [] + + # The C statements required to clean up after the impl call. + self.cleanup: list[str] = [] + + # The C statements to generate critical sections (per-object locking). + self.lock: list[str] = [] + self.unlock: list[str] = [] + + +@dc.dataclass(slots=True, frozen=True) +class Include: + """ + An include like: #include "pycore_long.h" // _Py_ID() + """ + # Example: "pycore_long.h". + filename: str + + # Example: "_Py_ID()". + reason: str + + # None means unconditional include. + # Example: "#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)". + condition: str | None + + def sort_key(self) -> tuple[str, str]: + # order: '#if' comes before 'NO_CONDITION' + return (self.condition or 'NO_CONDITION', self.filename) diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py index 48cb7d05a7caef..4fafedb617115c 100644 --- a/Tools/clinic/libclinic/function.py +++ b/Tools/clinic/libclinic/function.py @@ -6,7 +6,8 @@ import inspect from typing import Final, Any, TYPE_CHECKING if TYPE_CHECKING: - from clinic import Clinic, CConverter, CReturnConverter, self_converter + from clinic import Clinic, CReturnConverter, self_converter + from libclinic.converter import CConverter from libclinic import VersionTuple, unspecified From c432df6d56f3e02530132321b47dcc7b914a3660 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 14 Mar 2024 19:17:43 +0100 Subject: [PATCH 096/158] gh-111696, PEP 737: Add PyType_GetModuleName() function (#116824) Co-authored-by: Eric Snow --- Doc/c-api/type.rst | 7 +++++++ Doc/data/stable_abi.dat | 1 + Doc/whatsnew/3.13.rst | 4 ++++ Include/internal/pycore_typeobject.h | 4 ---- Include/object.h | 3 ++- Lib/test/test_capi/test_misc.py | 5 +++-- Lib/test/test_stable_abi_ctypes.py | 1 + ...-03-14-18-00-32.gh-issue-111696.L6oIPq.rst | 3 +++ Misc/stable_abi.toml | 2 ++ Modules/_functoolsmodule.c | 2 +- Modules/_testcapimodule.c | 9 ++++++++ Modules/_testinternalcapi.c | 9 -------- Objects/typeobject.c | 21 ++++++++++++------- PC/python3dll.c | 1 + Python/crossinterp.c | 3 +-- 15 files changed, 48 insertions(+), 27 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-03-14-18-00-32.gh-issue-111696.L6oIPq.rst diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index c5234233ba7124..0cae5c09505ebe 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -193,6 +193,13 @@ Type Objects .. versionadded:: 3.13 +.. c:function:: PyObject* PyType_GetModuleName(PyTypeObject *type) + + Return the type's module name. Equivalent to getting the ``type.__module__`` + attribute. + + .. versionadded:: 3.13 + .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 03fe3cef3843b6..9d0ad3d036dac3 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -679,6 +679,7 @@ function,PyType_GenericNew,3.2,, function,PyType_GetFlags,3.2,, function,PyType_GetFullyQualifiedName,3.13,, function,PyType_GetModule,3.10,, +function,PyType_GetModuleName,3.13,, function,PyType_GetModuleState,3.10,, function,PyType_GetName,3.11,, function,PyType_GetQualName,3.11,, diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index cbb5e02aef1ce3..f42197c001f18f 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1664,6 +1664,10 @@ New Features to ``"builtins"``. (Contributed by Victor Stinner in :gh:`111696`.) +* Add :c:func:`PyType_GetModuleName` function to get the type's module name. + Equivalent to getting the ``type.__module__`` attribute. + (Contributed by Eric Snow and Victor Stinner in :gh:`111696`.) + Porting to Python 3.13 ---------------------- diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index c214111fed6f97..5c32d49e85c97b 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -151,10 +151,6 @@ PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found); -// This is exported for the _testinternalcapi module. -PyAPI_FUNC(PyObject *) _PyType_GetModuleName(PyTypeObject *); - - #ifdef __cplusplus } #endif diff --git a/Include/object.h b/Include/object.h index 3f6f1ab1e68cc6..34141af7b7f7ef 100644 --- a/Include/object.h +++ b/Include/object.h @@ -523,7 +523,8 @@ PyAPI_FUNC(PyObject *) PyType_GetName(PyTypeObject *); PyAPI_FUNC(PyObject *) PyType_GetQualName(PyTypeObject *); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030D0000 -PyAPI_FUNC(PyObject *) PyType_GetFullyQualifiedName(PyTypeObject *); +PyAPI_FUNC(PyObject *) PyType_GetFullyQualifiedName(PyTypeObject *type); +PyAPI_FUNC(PyObject *) PyType_GetModuleName(PyTypeObject *type); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030C0000 PyAPI_FUNC(PyObject *) PyType_FromMetaclass(PyTypeObject*, PyObject*, PyType_Spec*, PyObject*); diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 6b4f535cc6550a..eb0bc13911701a 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1104,8 +1104,9 @@ def test_get_type_name(self): class MyType: pass - from _testcapi import get_type_name, get_type_qualname, get_type_fullyqualname - from _testinternalcapi import get_type_module_name + from _testcapi import ( + get_type_name, get_type_qualname, + get_type_fullyqualname, get_type_module_name) from collections import OrderedDict ht = _testcapi.get_heaptype_for_name() diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index f0b449ac1708a1..117c27d27b38dc 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -708,6 +708,7 @@ def test_windows_feature_macros(self): "PyType_GetFlags", "PyType_GetFullyQualifiedName", "PyType_GetModule", + "PyType_GetModuleName", "PyType_GetModuleState", "PyType_GetName", "PyType_GetQualName", diff --git a/Misc/NEWS.d/next/C API/2024-03-14-18-00-32.gh-issue-111696.L6oIPq.rst b/Misc/NEWS.d/next/C API/2024-03-14-18-00-32.gh-issue-111696.L6oIPq.rst new file mode 100644 index 00000000000000..7973d7b16e5826 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-03-14-18-00-32.gh-issue-111696.L6oIPq.rst @@ -0,0 +1,3 @@ +Add :c:func:`PyType_GetModuleName` function to get the type's module name. +Equivalent to getting the ``type.__module__`` attribute. Patch by Eric Snow +and Victor Stinner. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index c76a3cea4da3f7..c68adf8db079f9 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2498,3 +2498,5 @@ # "abi-only" since 3.10. (Same story as PyCFunctionFast.) [function.PyType_GetFullyQualifiedName] added = '3.13' +[function.PyType_GetModuleName] + added = '3.13' diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index d2212d40550a3c..f23b6e0d62bfb1 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -402,7 +402,7 @@ partial_repr(partialobject *pto) goto done; } - mod = _PyType_GetModuleName(Py_TYPE(pto)); + mod = PyType_GetModuleName(Py_TYPE(pto)); if (mod == NULL) { goto error; } diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 07f96466abdfc9..7928cd7d6fe1ae 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -622,6 +622,14 @@ get_type_fullyqualname(PyObject *self, PyObject *type) } +static PyObject * +get_type_module_name(PyObject *self, PyObject *type) +{ + assert(PyType_Check(type)); + return PyType_GetModuleName((PyTypeObject *)type); +} + + static PyObject * test_get_type_dict(PyObject *self, PyObject *Py_UNUSED(ignored)) { @@ -3268,6 +3276,7 @@ static PyMethodDef TestMethods[] = { {"get_type_name", get_type_name, METH_O}, {"get_type_qualname", get_type_qualname, METH_O}, {"get_type_fullyqualname", get_type_fullyqualname, METH_O}, + {"get_type_module_name", get_type_module_name, METH_O}, {"test_get_type_dict", test_get_type_dict, METH_NOARGS}, {"_test_thread_state", test_thread_state, METH_VARARGS}, #ifndef MS_WINDOWS diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index db8817418950b9..b3076a8f548b62 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -28,7 +28,6 @@ #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_typeobject.h" // _PyType_GetModuleName() #include "interpreteridobject.h" // PyInterpreterID_LookUp() @@ -1631,13 +1630,6 @@ perf_trampoline_set_persist_after_fork(PyObject *self, PyObject *args) } -static PyObject * -get_type_module_name(PyObject *self, PyObject *type) -{ - assert(PyType_Check(type)); - return _PyType_GetModuleName((PyTypeObject *)type); -} - static PyObject * get_rare_event_counters(PyObject *self, PyObject *type) { @@ -1741,7 +1733,6 @@ static PyMethodDef module_functions[] = { {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, {"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS}, _TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF - {"get_type_module_name", get_type_module_name, METH_O}, {"get_rare_event_counters", get_rare_event_counters, METH_NOARGS}, {"reset_rare_event_counters", reset_rare_event_counters, METH_NOARGS}, #ifdef Py_GIL_DISABLED diff --git a/Objects/typeobject.c b/Objects/typeobject.c index e51adac7e9d636..1c5729c589da93 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1164,10 +1164,9 @@ type_set_qualname(PyTypeObject *type, PyObject *value, void *context) } static PyObject * -type_module(PyTypeObject *type, void *context) +type_module(PyTypeObject *type) { PyObject *mod; - if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) { PyObject *dict = lookup_tp_dict(type); if (PyDict_GetItemRef(dict, &_Py_ID(__module__), &mod) == 0) { @@ -1189,6 +1188,12 @@ type_module(PyTypeObject *type, void *context) return mod; } +static PyObject * +type_get_module(PyTypeObject *type, void *context) +{ + return type_module(type); +} + static int type_set_module(PyTypeObject *type, PyObject *value, void *context) { @@ -1214,7 +1219,7 @@ PyType_GetFullyQualifiedName(PyTypeObject *type) return NULL; } - PyObject *module = type_module(type, NULL); + PyObject *module = type_module(type); if (module == NULL) { Py_DECREF(qualname); return NULL; @@ -1722,7 +1727,7 @@ static PyGetSetDef type_getsets[] = { {"__qualname__", (getter)type_qualname, (setter)type_set_qualname, NULL}, {"__bases__", (getter)type_get_bases, (setter)type_set_bases, NULL}, {"__mro__", (getter)type_get_mro, NULL, NULL}, - {"__module__", (getter)type_module, (setter)type_set_module, NULL}, + {"__module__", (getter)type_get_module, (setter)type_set_module, NULL}, {"__abstractmethods__", (getter)type_abstractmethods, (setter)type_set_abstractmethods, NULL}, {"__dict__", (getter)type_dict, NULL, NULL}, @@ -1743,7 +1748,7 @@ type_repr(PyObject *self) return PyUnicode_FromFormat("", type); } - PyObject *mod = type_module(type, NULL); + PyObject *mod = type_module(type); if (mod == NULL) { PyErr_Clear(); } @@ -4734,9 +4739,9 @@ PyType_GetQualName(PyTypeObject *type) } PyObject * -_PyType_GetModuleName(PyTypeObject *type) +PyType_GetModuleName(PyTypeObject *type) { - return type_module(type, NULL); + return type_module(type); } void * @@ -5850,7 +5855,7 @@ object_repr(PyObject *self) PyObject *mod, *name, *rtn; type = Py_TYPE(self); - mod = type_module(type, NULL); + mod = type_module(type); if (mod == NULL) PyErr_Clear(); else if (!PyUnicode_Check(mod)) { diff --git a/PC/python3dll.c b/PC/python3dll.c index 81d55af7074383..dbfa3f23bb586d 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -639,6 +639,7 @@ EXPORT_FUNC(PyType_GenericNew) EXPORT_FUNC(PyType_GetFlags) EXPORT_FUNC(PyType_GetFullyQualifiedName) EXPORT_FUNC(PyType_GetModule) +EXPORT_FUNC(PyType_GetModuleName) EXPORT_FUNC(PyType_GetModuleState) EXPORT_FUNC(PyType_GetName) EXPORT_FUNC(PyType_GetQualName) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 143b261f9a5396..18dec4dd959044 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -7,7 +7,6 @@ #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_namespace.h" //_PyNamespace_New() #include "pycore_pyerrors.h" // _PyErr_Clear() -#include "pycore_typeobject.h" // _PyType_GetModuleName() #include "pycore_weakref.h" // _PyWeakref_GET_REF() @@ -510,7 +509,7 @@ _excinfo_init_type(struct _excinfo_type *info, PyObject *exc) } // __module__ - strobj = _PyType_GetModuleName(type); + strobj = PyType_GetModuleName(type); if (strobj == NULL) { return -1; } From ab9e322ae1da1068b615d0375859920c710458e4 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 14 Mar 2024 16:39:50 -0500 Subject: [PATCH 097/158] Minor improvements to the itertools documentation (gh-116833) --- Doc/library/itertools.rst | 178 ++++++++++++++++++++------------------ 1 file changed, 94 insertions(+), 84 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 962d23bb53bf6b..7df8e804507ef5 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -41,9 +41,9 @@ operator can be mapped across two vectors to form an efficient dot-product: ================== ================= ================================================= ========================================= Iterator Arguments Results Example ================== ================= ================================================= ========================================= -:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) --> 10 11 12 13 14 ...`` -:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') --> A B C D A B C D ...`` -:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) --> 10 10 10`` +:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) → 10 11 12 13 14 ...`` +:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') → A B C D A B C D ...`` +:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) → 10 10 10`` ================== ================= ================================================= ========================================= **Iterators terminating on the shortest input sequence:** @@ -51,20 +51,20 @@ Iterator Arguments Results ============================ ============================ ================================================= ============================================================= Iterator Arguments Results Example ============================ ============================ ================================================= ============================================================= -:func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) --> 1 3 6 10 15`` -:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=3) --> ABC DEF G`` -:func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F`` -:func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) --> A B C D E F`` -:func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` -:func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1`` -:func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` +:func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) → 1 3 6 10 15`` +:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=3) → ABC DEF G`` +:func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') → A B C D E F`` +:func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) → A B C D E F`` +:func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) → A C E F`` +:func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) → 6 4 1`` +:func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x%2, range(10)) → 0 2 4 6 8`` :func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) -:func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) --> C D E F G`` -:func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') --> AB BC CD DE EF FG`` -:func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000`` -:func:`takewhile` predicate, seq seq[0], seq[1], until predicate fails ``takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4`` +:func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) → C D E F G`` +:func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') → AB BC CD DE EF FG`` +:func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000`` +:func:`takewhile` predicate, seq seq[0], seq[1], until predicate fails ``takewhile(lambda x: x<5, [1,4,6,4,1]) → 1 4`` :func:`tee` it, n it1, it2, ... itn splits one iterator into n -:func:`zip_longest` p, q, ... (p[0], q[0]), (p[1], q[1]), ... ``zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D-`` +:func:`zip_longest` p, q, ... (p[0], q[0]), (p[1], q[1]), ... ``zip_longest('ABCD', 'xy', fillvalue='-') → Ax By C- D-`` ============================ ============================ ================================================= ============================================================= **Combinatoric iterators:** @@ -84,7 +84,7 @@ Examples Results ``product('ABCD', repeat=2)`` ``AA AB AC AD BA BB BC BD CA CB CC CD DA DB DC DD`` ``permutations('ABCD', 2)`` ``AB AC AD BA BC BD CA CB CD DA DB DC`` ``combinations('ABCD', 2)`` ``AB AC AD BC BD CD`` -``combinations_with_replacement('ABCD', 2)`` ``AA AB AC AD BB BC BD CC CD DD`` +``combinations_with_replacement('ABCD', 2)`` ``AA AB AC AD BB BC BD CC CD DD`` ============================================== ============================================================= @@ -119,9 +119,9 @@ loops that truncate the stream. def accumulate(iterable, func=operator.add, *, initial=None): 'Return running totals' - # accumulate([1,2,3,4,5]) --> 1 3 6 10 15 - # accumulate([1,2,3,4,5], initial=100) --> 100 101 103 106 110 115 - # accumulate([1,2,3,4,5], operator.mul) --> 1 2 6 24 120 + # accumulate([1,2,3,4,5]) → 1 3 6 10 15 + # accumulate([1,2,3,4,5], initial=100) → 100 101 103 106 110 115 + # accumulate([1,2,3,4,5], operator.mul) → 1 2 6 24 120 it = iter(iterable) total = initial if initial is None: @@ -194,7 +194,7 @@ loops that truncate the stream. Roughly equivalent to:: def batched(iterable, n, *, strict=False): - # batched('ABCDEFG', 3) --> ABC DEF G + # batched('ABCDEFG', 3) → ABC DEF G if n < 1: raise ValueError('n must be at least one') it = iter(iterable) @@ -217,7 +217,7 @@ loops that truncate the stream. Roughly equivalent to:: def chain(*iterables): - # chain('ABC', 'DEF') --> A B C D E F + # chain('ABC', 'DEF') → A B C D E F for it in iterables: for element in it: yield element @@ -229,7 +229,7 @@ loops that truncate the stream. single iterable argument that is evaluated lazily. Roughly equivalent to:: def from_iterable(iterables): - # chain.from_iterable(['ABC', 'DEF']) --> A B C D E F + # chain.from_iterable(['ABC', 'DEF']) → A B C D E F for it in iterables: for element in it: yield element @@ -250,8 +250,8 @@ loops that truncate the stream. Roughly equivalent to:: def combinations(iterable, r): - # combinations('ABCD', 2) --> AB AC AD BC BD CD - # combinations(range(4), 3) --> 012 013 023 123 + # combinations('ABCD', 2) → AB AC AD BC BD CD + # combinations(range(4), 3) → 012 013 023 123 pool = tuple(iterable) n = len(pool) if r > n: @@ -299,7 +299,7 @@ loops that truncate the stream. Roughly equivalent to:: def combinations_with_replacement(iterable, r): - # combinations_with_replacement('ABC', 2) --> AA AB AC BB BC CC + # combinations_with_replacement('ABC', 2) → AA AB AC BB BC CC pool = tuple(iterable) n = len(pool) if not n and r: @@ -339,7 +339,7 @@ loops that truncate the stream. Roughly equivalent to:: def compress(data, selectors): - # compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F + # compress('ABCDEF', [1,0,1,0,1,1]) → A C E F return (d for d, s in zip(data, selectors) if s) .. versionadded:: 3.1 @@ -352,8 +352,8 @@ loops that truncate the stream. Also, used with :func:`zip` to add sequence numbers. Roughly equivalent to:: def count(start=0, step=1): - # count(10) --> 10 11 12 13 14 ... - # count(2.5, 0.5) --> 2.5 3.0 3.5 ... + # count(10) → 10 11 12 13 14 ... + # count(2.5, 0.5) → 2.5 3.0 3.5 ... n = start while True: yield n @@ -373,7 +373,7 @@ loops that truncate the stream. indefinitely. Roughly equivalent to:: def cycle(iterable): - # cycle('ABCD') --> A B C D A B C D A B C D ... + # cycle('ABCD') → A B C D A B C D A B C D ... saved = [] for element in iterable: yield element @@ -394,7 +394,7 @@ loops that truncate the stream. start-up time. Roughly equivalent to:: def dropwhile(predicate, iterable): - # dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1 + # dropwhile(lambda x: x<5, [1,4,6,4,1]) → 6 4 1 iterable = iter(iterable) for x in iterable: if not predicate(x): @@ -410,7 +410,7 @@ loops that truncate the stream. that are false. Roughly equivalent to:: def filterfalse(predicate, iterable): - # filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8 + # filterfalse(lambda x: x%2, range(10)) → 0 2 4 6 8 if predicate is None: predicate = bool for x in iterable: @@ -447,8 +447,8 @@ loops that truncate the stream. :func:`groupby` is roughly equivalent to:: class groupby: - # [k for k, g in groupby('AAAABBBCCDAABBB')] --> A B C D A B - # [list(g) for k, g in groupby('AAAABBBCCD')] --> AAAA BBB CC D + # [k for k, g in groupby('AAAABBBCCDAABBB')] → A B C D A B + # [list(g) for k, g in groupby('AAAABBBCCD')] → AAAA BBB CC D def __init__(self, iterable, key=None): if key is None: @@ -499,10 +499,10 @@ loops that truncate the stream. Roughly equivalent to:: def islice(iterable, *args): - # islice('ABCDEFG', 2) --> A B - # islice('ABCDEFG', 2, 4) --> C D - # islice('ABCDEFG', 2, None) --> C D E F G - # islice('ABCDEFG', 0, None, 2) --> A C E G + # islice('ABCDEFG', 2) → A B + # islice('ABCDEFG', 2, 4) → C D + # islice('ABCDEFG', 2, None) → C D E F G + # islice('ABCDEFG', 0, None, 2) → A C E G s = slice(*args) start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 it = iter(range(start, stop, step)) @@ -535,10 +535,12 @@ loops that truncate the stream. Roughly equivalent to:: def pairwise(iterable): - # pairwise('ABCDEFG') --> AB BC CD DE EF FG - a, b = tee(iterable) - next(b, None) - return zip(a, b) + # pairwise('ABCDEFG') → AB BC CD DE EF FG + iterator = iter(iterable) + a = next(iterator, None) + for b in iterator: + yield a, b + a = b .. versionadded:: 3.10 @@ -562,8 +564,8 @@ loops that truncate the stream. Roughly equivalent to:: def permutations(iterable, r=None): - # permutations('ABCD', 2) --> AB AC AD BA BC BD CA CB CD DA DB DC - # permutations(range(3)) --> 012 021 102 120 201 210 + # permutations('ABCD', 2) → AB AC AD BA BC BD CA CB CD DA DB DC + # permutations(range(3)) → 012 021 102 120 201 210 pool = tuple(iterable) n = len(pool) r = n if r is None else r @@ -621,8 +623,8 @@ loops that truncate the stream. actual implementation does not build up intermediate results in memory:: def product(*args, repeat=1): - # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy - # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111 + # product('ABCD', 'xy') → Ax Ay Bx By Cx Cy Dx Dy + # product(range(2), repeat=3) → 000 001 010 011 100 101 110 111 pools = [tuple(pool) for pool in args] * repeat result = [[]] for pool in pools: @@ -642,7 +644,7 @@ loops that truncate the stream. Roughly equivalent to:: def repeat(object, times=None): - # repeat(10, 3) --> 10 10 10 + # repeat(10, 3) → 10 10 10 if times is None: while True: yield object @@ -670,7 +672,7 @@ loops that truncate the stream. equivalent to:: def starmap(function, iterable): - # starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000 + # starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000 for args in iterable: yield function(*args) @@ -681,7 +683,7 @@ loops that truncate the stream. predicate is true. Roughly equivalent to:: def takewhile(predicate, iterable): - # takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4 + # takewhile(lambda x: x<5, [1,4,6,4,1]) → 1 4 for x in iterable: if predicate(x): yield x @@ -741,7 +743,7 @@ loops that truncate the stream. Iteration continues until the longest iterable is exhausted. Roughly equivalent to:: def zip_longest(*args, fillvalue=None): - # zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D- + # zip_longest('ABCD', 'xy', fillvalue='-') → Ax By C- D- iterators = [iter(it) for it in args] num_active = len(iterators) if not num_active: @@ -816,7 +818,7 @@ and :term:`generators ` which incur interpreter overhead. def prepend(value, iterable): "Prepend a single value in front of an iterable." - # prepend(1, [2, 3, 4]) --> 1 2 3 4 + # prepend(1, [2, 3, 4]) → 1 2 3 4 return chain([value], iterable) def tabulate(function, start=0): @@ -842,7 +844,7 @@ and :term:`generators ` which incur interpreter overhead. def tail(n, iterable): "Return an iterator over the last n items." - # tail(3, 'ABCDEFG') --> E F G + # tail(3, 'ABCDEFG') → E F G return iter(collections.deque(iterable, maxlen=n)) def consume(iterator, n=None): @@ -865,26 +867,27 @@ and :term:`generators ` which incur interpreter overhead. def first_true(iterable, default=False, predicate=None): "Returns the first true value or the *default* if there is no true value." - # first_true([a,b,c], x) --> a or b or c or x - # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x + # first_true([a,b,c], x) → a or b or c or x + # first_true([a,b], x, f) → a if f(a) else b if f(b) else x return next(filter(predicate, iterable), default) def all_equal(iterable, key=None): "Returns True if all the elements are equal to each other." + # all_equal('4٤໔４৪', key=int) → True return len(take(2, groupby(iterable, key))) <= 1 def unique_justseen(iterable, key=None): "List unique elements, preserving order. Remember only the element just seen." - # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B - # unique_justseen('ABBcCAD', str.casefold) --> A B c A D + # unique_justseen('AAAABBBCCDAABBB') → A B C D A B + # unique_justseen('ABBcCAD', str.casefold) → A B c A D if key is None: return map(operator.itemgetter(0), groupby(iterable)) return map(next, map(operator.itemgetter(1), groupby(iterable, key))) def unique_everseen(iterable, key=None): "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBcCAD', str.casefold) --> A B c D + # unique_everseen('AAAABBBCCDAABBB') → A B C D + # unique_everseen('ABBcCAD', str.casefold) → A B c D seen = set() if key is None: for element in filterfalse(seen.__contains__, iterable): @@ -899,7 +902,7 @@ and :term:`generators ` which incur interpreter overhead. def sliding_window(iterable, n): "Collect data into overlapping fixed-length chunks or blocks." - # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG + # sliding_window('ABCDEFG', 4) → ABCD BCDE CDEF DEFG it = iter(iterable) window = collections.deque(islice(it, n-1), maxlen=n) for x in it: @@ -908,9 +911,9 @@ and :term:`generators ` which incur interpreter overhead. def grouper(iterable, n, *, incomplete='fill', fillvalue=None): "Collect data into non-overlapping fixed-length chunks or blocks." - # grouper('ABCDEFG', 3, fillvalue='x') --> ABC DEF Gxx - # grouper('ABCDEFG', 3, incomplete='strict') --> ABC DEF ValueError - # grouper('ABCDEFG', 3, incomplete='ignore') --> ABC DEF + # grouper('ABCDEFG', 3, fillvalue='x') → ABC DEF Gxx + # grouper('ABCDEFG', 3, incomplete='strict') → ABC DEF ValueError + # grouper('ABCDEFG', 3, incomplete='ignore') → ABC DEF iterators = [iter(iterable)] * n match incomplete: case 'fill': @@ -924,7 +927,7 @@ and :term:`generators ` which incur interpreter overhead. def roundrobin(*iterables): "Visit input iterables in a cycle until each is exhausted." - # roundrobin('ABC', 'D', 'EF') --> A D E B F C + # roundrobin('ABC', 'D', 'EF') → A D E B F C # Algorithm credited to George Sakkis iterators = map(iter, iterables) for num_active in range(len(iterables), 0, -1): @@ -936,19 +939,19 @@ and :term:`generators ` which incur interpreter overhead. If *predicate* is slow, consider wrapping it with functools.lru_cache(). """ - # partition(is_odd, range(10)) --> 0 2 4 6 8 and 1 3 5 7 9 + # partition(is_odd, range(10)) → 0 2 4 6 8 and 1 3 5 7 9 t1, t2 = tee(iterable) return filterfalse(predicate, t1), filter(predicate, t2) def subslices(seq): "Return all contiguous non-empty subslices of a sequence." - # subslices('ABCD') --> A AB ABC ABCD B BC BCD C CD D + # subslices('ABCD') → A AB ABC ABCD B BC BCD C CD D slices = starmap(slice, combinations(range(len(seq) + 1), 2)) return map(operator.getitem, repeat(seq), slices) def iter_index(iterable, value, start=0, stop=None): "Return indices where a value occurs in a sequence or iterable." - # iter_index('AABCADEAF', 'A') --> 0 1 4 7 + # iter_index('AABCADEAF', 'A') → 0 1 4 7 seq_index = getattr(iterable, 'index', None) if seq_index is None: # Path for general iterables @@ -972,7 +975,7 @@ and :term:`generators ` which incur interpreter overhead. Converts a call-until-exception interface to an iterator interface. """ - # iter_except(d.popitem, KeyError) --> non-blocking dictionary iterator + # iter_except(d.popitem, KeyError) → non-blocking dictionary iterator try: if first is not None: yield first() @@ -987,28 +990,28 @@ The following recipes have a more mathematical flavor: .. testcode:: def powerset(iterable): - "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" + "powerset([1,2,3]) → () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" s = list(iterable) return chain.from_iterable(combinations(s, r) for r in range(len(s)+1)) def sum_of_squares(iterable): "Add up the squares of the input values." - # sum_of_squares([10, 20, 30]) --> 1400 + # sum_of_squares([10, 20, 30]) → 1400 return math.sumprod(*tee(iterable)) def reshape(matrix, cols): "Reshape a 2-D matrix to have a given number of columns." - # reshape([(0, 1), (2, 3), (4, 5)], 3) --> (0, 1, 2), (3, 4, 5) + # reshape([(0, 1), (2, 3), (4, 5)], 3) → (0, 1, 2), (3, 4, 5) return batched(chain.from_iterable(matrix), cols, strict=True) def transpose(matrix): "Swap the rows and columns of a 2-D matrix." - # transpose([(1, 2, 3), (11, 22, 33)]) --> (1, 11) (2, 22) (3, 33) + # transpose([(1, 2, 3), (11, 22, 33)]) → (1, 11) (2, 22) (3, 33) return zip(*matrix, strict=True) def matmul(m1, m2): "Multiply two matrices." - # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) --> (49, 80), (41, 60) + # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) → (49, 80), (41, 60) n = len(m2[0]) return batched(starmap(math.sumprod, product(m1, transpose(m2))), n) @@ -1023,10 +1026,10 @@ The following recipes have a more mathematical flavor: Article: https://betterexplained.com/articles/intuitive-convolution/ Video: https://www.youtube.com/watch?v=KuXjwB4LzSA """ - # convolve([1, -1, -20], [1, -3]) --> 1 -4 -17 60 - # convolve(data, [0.25, 0.25, 0.25, 0.25]) --> Moving average (blur) - # convolve(data, [1/2, 0, -1/2]) --> 1st derivative estimate - # convolve(data, [1, -2, 1]) --> 2nd derivative estimate + # convolve([1, -1, -20], [1, -3]) → 1 -4 -17 60 + # convolve(data, [0.25, 0.25, 0.25, 0.25]) → Moving average (blur) + # convolve(data, [1/2, 0, -1/2]) → 1st derivative estimate + # convolve(data, [1, -2, 1]) → 2nd derivative estimate kernel = tuple(kernel)[::-1] n = len(kernel) padded_signal = chain(repeat(0, n-1), signal, repeat(0, n-1)) @@ -1038,7 +1041,7 @@ The following recipes have a more mathematical flavor: (x - 5) (x + 4) (x - 3) expands to: x³ -4x² -17x + 60 """ - # polynomial_from_roots([5, -4, 3]) --> [1, -4, -17, 60] + # polynomial_from_roots([5, -4, 3]) → [1, -4, -17, 60] factors = zip(repeat(1), map(operator.neg, roots)) return list(functools.reduce(convolve, factors, [1])) @@ -1048,7 +1051,7 @@ The following recipes have a more mathematical flavor: Computes with better numeric stability than Horner's method. """ # Evaluate x³ -4x² -17x + 60 at x = 5 - # polynomial_eval([1, -4, -17, 60], x=5) --> 0 + # polynomial_eval([1, -4, -17, 60], x=5) → 0 n = len(coefficients) if not n: return type(x)(0) @@ -1061,14 +1064,14 @@ The following recipes have a more mathematical flavor: f(x) = x³ -4x² -17x + 60 f'(x) = 3x² -8x -17 """ - # polynomial_derivative([1, -4, -17, 60]) --> [3, -8, -17] + # polynomial_derivative([1, -4, -17, 60]) → [3, -8, -17] n = len(coefficients) powers = reversed(range(1, n)) return list(map(operator.mul, coefficients, powers)) def sieve(n): "Primes less than n." - # sieve(30) --> 2 3 5 7 11 13 17 19 23 29 + # sieve(30) → 2 3 5 7 11 13 17 19 23 29 if n > 2: yield 2 start = 3 @@ -1082,9 +1085,9 @@ The following recipes have a more mathematical flavor: def factor(n): "Prime factors of n." - # factor(99) --> 3 3 11 - # factor(1_000_000_000_000_007) --> 47 59 360620266859 - # factor(1_000_000_000_000_403) --> 1000000000000403 + # factor(99) → 3 3 11 + # factor(1_000_000_000_000_007) → 47 59 360620266859 + # factor(1_000_000_000_000_403) → 1000000000000403 for prime in sieve(math.isqrt(n) + 1): while not n % prime: yield prime @@ -1097,7 +1100,7 @@ The following recipes have a more mathematical flavor: def totient(n): "Count of natural numbers up to n that are coprime to n." # https://mathworld.wolfram.com/TotientFunction.html - # totient(12) --> 4 because len([1, 5, 7, 11]) == 4 + # totient(12) → 4 because len([1, 5, 7, 11]) == 4 for p in unique_justseen(factor(n)): n -= n // p return n @@ -1567,6 +1570,13 @@ The following recipes have a more mathematical flavor: >>> ranges = [range(5, 1000), range(4, 3000), range(0), range(3, 2000), range(2, 5000), range(1, 3500)] >>> collections.Counter(roundrobin(ranges)) == collections.Counter(ranges) True + >>> # Verify that the inputs are consumed lazily + >>> input_iterators = list(map(iter, ['abcd', 'ef', '', 'ghijk', 'l', 'mnopqr'])) + >>> output_iterator = roundrobin(*input_iterators) + >>> ''.join(islice(output_iterator, 10)) + 'aeglmbfhnc' + >>> ''.join(chain(*input_iterators)) + 'dijkopqr' >>> def is_odd(x): ... return x % 2 == 1 @@ -1676,7 +1686,7 @@ The following recipes have a more mathematical flavor: def triplewise(iterable): "Return overlapping triplets from an iterable" - # triplewise('ABCDEFG') --> ABC BCD CDE DEF EFG + # triplewise('ABCDEFG') → ABC BCD CDE DEF EFG for (a, _), (b, c) in pairwise(pairwise(iterable)): yield a, b, c From be59aaf3abec37b27bdb31fadf433665e5471a46 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 14 Mar 2024 17:53:50 -0400 Subject: [PATCH 098/158] gh-106531: Refresh zipfile._path with zipp 3.18. (#116835) * gh-106531: Refresh zipfile._path with zipp 3.18. * Add blurb --- .../test_zipfile/_path/test_complexity.py | 8 +- Lib/test/test_zipfile/_path/test_path.py | 23 ++-- Lib/zipfile/_path/__init__.py | 65 +++++++--- Lib/zipfile/_path/glob.py | 112 ++++++++++++++---- ...-03-14-17-24-59.gh-issue-106531.9ehywi.rst | 5 + 5 files changed, 159 insertions(+), 54 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-14-17-24-59.gh-issue-106531.9ehywi.rst diff --git a/Lib/test/test_zipfile/_path/test_complexity.py b/Lib/test/test_zipfile/_path/test_complexity.py index 7050937738af18..fd7ce57551b7a5 100644 --- a/Lib/test/test_zipfile/_path/test_complexity.py +++ b/Lib/test/test_zipfile/_path/test_complexity.py @@ -43,13 +43,17 @@ def make_zip_path(self, depth=1, width=1) -> zipfile.Path: @classmethod def make_names(cls, width, letters=string.ascii_lowercase): """ + >>> list(TestComplexity.make_names(1)) + ['a'] >>> list(TestComplexity.make_names(2)) ['a', 'b'] >>> list(TestComplexity.make_names(30)) ['aa', 'ab', ..., 'bd'] + >>> list(TestComplexity.make_names(17124)) + ['aaa', 'aab', ..., 'zip'] """ # determine how many products are needed to produce width - n_products = math.ceil(math.log(width, len(letters))) + n_products = max(1, math.ceil(math.log(width, len(letters)))) inputs = (letters,) * n_products combinations = itertools.product(*inputs) names = map(''.join, combinations) @@ -80,7 +84,7 @@ def test_glob_depth(self): max_n=100, min_n=1, ) - assert best <= big_o.complexities.Quadratic + assert best <= big_o.complexities.Linear @pytest.mark.flaky def test_glob_width(self): diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index c66cb3cba69ebd..df5b8c9d8fea40 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -6,6 +6,7 @@ import sys import unittest import zipfile +import zipfile._path from ._functools import compose from ._itertools import Counter @@ -20,16 +21,6 @@ class itertools: Counter = Counter -def add_dirs(zf): - """ - Given a writable zip file zf, inject directory entries for - any directories implied by the presence of children. - """ - for name in zipfile.CompleteDirs._implied_dirs(zf.namelist()): - zf.writestr(name, b"") - return zf - - def build_alpharep_fixture(): """ Create a zip file with this structure: @@ -76,7 +67,7 @@ def build_alpharep_fixture(): alpharep_generators = [ Invoked.wrap(build_alpharep_fixture), - Invoked.wrap(compose(add_dirs, build_alpharep_fixture)), + Invoked.wrap(compose(zipfile._path.CompleteDirs.inject, build_alpharep_fixture)), ] pass_alpharep = parameterize(['alpharep'], alpharep_generators) @@ -210,11 +201,12 @@ def test_open_write(self): with zf.joinpath('file.txt').open('w', encoding="utf-8") as strm: strm.write('text file') - def test_open_extant_directory(self): + @pass_alpharep + def test_open_extant_directory(self, alpharep): """ Attempting to open a directory raises IsADirectoryError. """ - zf = zipfile.Path(add_dirs(build_alpharep_fixture())) + zf = zipfile.Path(alpharep) with self.assertRaises(IsADirectoryError): zf.joinpath('b').open() @@ -226,11 +218,12 @@ def test_open_binary_invalid_args(self, alpharep): with self.assertRaises(ValueError): root.joinpath('a.txt').open('rb', 'utf-8') - def test_open_missing_directory(self): + @pass_alpharep + def test_open_missing_directory(self, alpharep): """ Attempting to open a missing directory raises FileNotFoundError. """ - zf = zipfile.Path(add_dirs(build_alpharep_fixture())) + zf = zipfile.Path(alpharep) with self.assertRaises(FileNotFoundError): zf.joinpath('z').open() diff --git a/Lib/zipfile/_path/__init__.py b/Lib/zipfile/_path/__init__.py index 78c413563bb2b1..4c167563b6b762 100644 --- a/Lib/zipfile/_path/__init__.py +++ b/Lib/zipfile/_path/__init__.py @@ -5,8 +5,9 @@ import contextlib import pathlib import re +import sys -from .glob import translate +from .glob import Translator __all__ = ['Path'] @@ -147,6 +148,16 @@ def make(cls, source): source.__class__ = cls return source + @classmethod + def inject(cls, zf: zipfile.ZipFile) -> zipfile.ZipFile: + """ + Given a writable zip file zf, inject directory entries for + any directories implied by the presence of children. + """ + for name in cls._implied_dirs(zf.namelist()): + zf.writestr(name, b"") + return zf + class FastLookup(CompleteDirs): """ @@ -168,8 +179,10 @@ def _name_set(self): def _extract_text_encoding(encoding=None, *args, **kwargs): - # stacklevel=3 so that the caller of the caller see any warning. - return io.text_encoding(encoding, 3), args, kwargs + # compute stack level so that the caller of the caller sees any warning. + is_pypy = sys.implementation.name == 'pypy' + stack_level = 3 + is_pypy + return io.text_encoding(encoding, stack_level), args, kwargs class Path: @@ -194,13 +207,13 @@ class Path: Path accepts the zipfile object itself or a filename - >>> root = Path(zf) + >>> path = Path(zf) From there, several path operations are available. Directory iteration (including the zip file itself): - >>> a, b = root.iterdir() + >>> a, b = path.iterdir() >>> a Path('mem/abcde.zip', 'a.txt') >>> b @@ -238,16 +251,38 @@ class Path: 'mem/abcde.zip/b/c.txt' At the root, ``name``, ``filename``, and ``parent`` - resolve to the zipfile. Note these attributes are not - valid and will raise a ``ValueError`` if the zipfile - has no filename. + resolve to the zipfile. - >>> root.name + >>> str(path) + 'mem/abcde.zip/' + >>> path.name 'abcde.zip' - >>> str(root.filename).replace(os.sep, posixpath.sep) - 'mem/abcde.zip' - >>> str(root.parent) + >>> path.filename == pathlib.Path('mem/abcde.zip') + True + >>> str(path.parent) 'mem' + + If the zipfile has no filename, such attribtues are not + valid and accessing them will raise an Exception. + + >>> zf.filename = None + >>> path.name + Traceback (most recent call last): + ... + TypeError: ... + + >>> path.filename + Traceback (most recent call last): + ... + TypeError: ... + + >>> path.parent + Traceback (most recent call last): + ... + TypeError: ... + + # workaround python/cpython#106763 + >>> pass """ __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" @@ -364,8 +399,10 @@ def glob(self, pattern): raise ValueError(f"Unacceptable pattern: {pattern!r}") prefix = re.escape(self.at) - matches = re.compile(prefix + translate(pattern)).fullmatch - return map(self._next, filter(matches, self.root.namelist())) + tr = Translator(seps='/') + matches = re.compile(prefix + tr.translate(pattern)).fullmatch + names = (data.filename for data in self.root.filelist) + return map(self._next, filter(matches, names)) def rglob(self, pattern): return self.glob(f'**/{pattern}') diff --git a/Lib/zipfile/_path/glob.py b/Lib/zipfile/_path/glob.py index 4a2e665e27078a..69c41d77c3f654 100644 --- a/Lib/zipfile/_path/glob.py +++ b/Lib/zipfile/_path/glob.py @@ -1,18 +1,97 @@ +import os import re -def translate(pattern): - r""" - Given a glob pattern, produce a regex that matches it. +_default_seps = os.sep + str(os.altsep) * bool(os.altsep) - >>> translate('*.txt') - '[^/]*\\.txt' - >>> translate('a?txt') - 'a.txt' - >>> translate('**/*') - '.*/[^/]*' + +class Translator: + """ + >>> Translator('xyz') + Traceback (most recent call last): + ... + AssertionError: Invalid separators + + >>> Translator('') + Traceback (most recent call last): + ... + AssertionError: Invalid separators """ - return ''.join(map(replace, separate(pattern))) + + seps: str + + def __init__(self, seps: str = _default_seps): + assert seps and set(seps) <= set(_default_seps), "Invalid separators" + self.seps = seps + + def translate(self, pattern): + """ + Given a glob pattern, produce a regex that matches it. + """ + return self.extend(self.translate_core(pattern)) + + def extend(self, pattern): + r""" + Extend regex for pattern-wide concerns. + + Apply '(?s:)' to create a non-matching group that + matches newlines (valid on Unix). + + Append '\Z' to imply fullmatch even when match is used. + """ + return rf'(?s:{pattern})\Z' + + def translate_core(self, pattern): + r""" + Given a glob pattern, produce a regex that matches it. + + >>> t = Translator() + >>> t.translate_core('*.txt').replace('\\\\', '') + '[^/]*\\.txt' + >>> t.translate_core('a?txt') + 'a[^/]txt' + >>> t.translate_core('**/*').replace('\\\\', '') + '.*/[^/][^/]*' + """ + self.restrict_rglob(pattern) + return ''.join(map(self.replace, separate(self.star_not_empty(pattern)))) + + def replace(self, match): + """ + Perform the replacements for a match from :func:`separate`. + """ + return match.group('set') or ( + re.escape(match.group(0)) + .replace('\\*\\*', r'.*') + .replace('\\*', rf'[^{re.escape(self.seps)}]*') + .replace('\\?', r'[^/]') + ) + + def restrict_rglob(self, pattern): + """ + Raise ValueError if ** appears in anything but a full path segment. + + >>> Translator().translate('**foo') + Traceback (most recent call last): + ... + ValueError: ** must appear alone in a path segment + """ + seps_pattern = rf'[{re.escape(self.seps)}]+' + segments = re.split(seps_pattern, pattern) + if any('**' in segment and segment != '**' for segment in segments): + raise ValueError("** must appear alone in a path segment") + + def star_not_empty(self, pattern): + """ + Ensure that * will not match an empty segment. + """ + + def handle_segment(match): + segment = match.group(0) + return '?*' if segment == '*' else segment + + not_seps_pattern = rf'[^{re.escape(self.seps)}]+' + return re.sub(not_seps_pattern, handle_segment, pattern) def separate(pattern): @@ -25,16 +104,3 @@ def separate(pattern): ['a', '[?]', 'txt'] """ return re.finditer(r'([^\[]+)|(?P[\[].*?[\]])|([\[][^\]]*$)', pattern) - - -def replace(match): - """ - Perform the replacements for a match from :func:`separate`. - """ - - return match.group('set') or ( - re.escape(match.group(0)) - .replace('\\*\\*', r'.*') - .replace('\\*', r'[^/]*') - .replace('\\?', r'.') - ) diff --git a/Misc/NEWS.d/next/Library/2024-03-14-17-24-59.gh-issue-106531.9ehywi.rst b/Misc/NEWS.d/next/Library/2024-03-14-17-24-59.gh-issue-106531.9ehywi.rst new file mode 100644 index 00000000000000..e2720d333783c0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-14-17-24-59.gh-issue-106531.9ehywi.rst @@ -0,0 +1,5 @@ +Refreshed zipfile._path from `zipp 3.18 +`_, providing +better compatibility for PyPy, better glob performance for deeply nested +zipfiles, and providing internal access to ``CompleteDirs.inject`` for use +in other tests (like importlib.resources). From 5f52d20a93908196f74271db8437cc1ba7e1e262 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 14 Mar 2024 17:59:00 -0400 Subject: [PATCH 099/158] gh-116811: Ensure MetadataPathFinder.invalidate_caches is reachable when delegated through PathFinder. (#116812) * Make MetadataPathFinder a proper classmethod. * In PathFinder.invalidate_caches, also invoke MetadataPathFinder.invalidate_caches. * Add blurb --- Lib/importlib/_bootstrap_external.py | 3 +++ Lib/importlib/metadata/__init__.py | 1 + .../Library/2024-03-14-10-01-23.gh-issue-116811._h5iKP.rst | 2 ++ 3 files changed, 6 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-03-14-10-01-23.gh-issue-116811._h5iKP.rst diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 46ddceed07b0d4..b26be8583d0f81 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -1470,6 +1470,9 @@ def invalidate_caches(): # https://bugs.python.org/issue45703 _NamespacePath._epoch += 1 + from importlib.metadata import MetadataPathFinder + MetadataPathFinder.invalidate_caches() + @staticmethod def _path_hooks(path): """Search sys.path_hooks for a finder for 'path'.""" diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index c612fbefee2e80..41c2a4a6088b5d 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -797,6 +797,7 @@ def _search_paths(cls, name, paths): path.search(prepared) for path in map(FastPath, paths) ) + @classmethod def invalidate_caches(cls) -> None: FastPath.__new__.cache_clear() diff --git a/Misc/NEWS.d/next/Library/2024-03-14-10-01-23.gh-issue-116811._h5iKP.rst b/Misc/NEWS.d/next/Library/2024-03-14-10-01-23.gh-issue-116811._h5iKP.rst new file mode 100644 index 00000000000000..00168632429996 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-14-10-01-23.gh-issue-116811._h5iKP.rst @@ -0,0 +1,2 @@ +In ``PathFinder.invalidate_caches``, delegate to +``MetadataPathFinder.invalidate_caches``. From 7bbb9b57e67057d5ca3b7e3a434527fb3fcf5a2b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 14 Mar 2024 23:23:00 +0100 Subject: [PATCH 100/158] gh-111696, PEP 737: Add %T and %N to PyUnicode_FromFormat() (#116839) --- Doc/c-api/unicode.rst | 23 ++++++++ Doc/whatsnew/3.13.rst | 6 ++ Include/internal/pycore_typeobject.h | 2 + Lib/test/test_capi/test_unicode.py | 34 +++++++++++ ...-03-14-22-30-07.gh-issue-111696.76UMKi.rst | 4 ++ Objects/typeobject.c | 10 +++- Objects/unicodeobject.c | 58 +++++++++++++++++++ 7 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-03-14-22-30-07.gh-issue-111696.76UMKi.rst diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 666ffe89605c56..78eec14e3a24d6 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -518,6 +518,26 @@ APIs: - :c:expr:`PyObject*` - The result of calling :c:func:`PyObject_Repr`. + * - ``T`` + - :c:expr:`PyObject*` + - Get the fully qualified name of an object type; + call :c:func:`PyType_GetFullyQualifiedName`. + + * - ``T#`` + - :c:expr:`PyObject*` + - Similar to ``T`` format, but use a colon (``:``) as separator between + the module name and the qualified name. + + * - ``N`` + - :c:expr:`PyTypeObject*` + - Get the fully qualified name of a type; + call :c:func:`PyType_GetFullyQualifiedName`. + + * - ``N#`` + - :c:expr:`PyTypeObject*` + - Similar to ``N`` format, but use a colon (``:``) as separator between + the module name and the qualified name. + .. note:: The width formatter unit is number of characters rather than bytes. The precision formatter unit is number of bytes or :c:type:`wchar_t` @@ -553,6 +573,9 @@ APIs: In previous versions it caused all the rest of the format string to be copied as-is to the result string, and any extra arguments discarded. + .. versionchanged:: 3.13 + Support for ``%T``, ``%T#``, ``%N`` and ``%N#`` formats added. + .. c:function:: PyObject* PyUnicode_FromFormatV(const char *format, va_list vargs) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index f42197c001f18f..856c6ee1d6e3f0 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1668,6 +1668,12 @@ New Features Equivalent to getting the ``type.__module__`` attribute. (Contributed by Eric Snow and Victor Stinner in :gh:`111696`.) +* Add support for ``%T``, ``%T#``, ``%N`` and ``%N#`` formats to + :c:func:`PyUnicode_FromFormat`: format the fully qualified name of an object + type and of a type: call :c:func:`PyType_GetModuleName`. See :pep:`737` for + more information. + (Contributed by Victor Stinner in :gh:`111696`.) + Porting to Python 3.13 ---------------------- diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 5c32d49e85c97b..8a25935f308178 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -150,6 +150,8 @@ extern PyTypeObject _PyBufferWrapper_Type; PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found); +extern PyObject* _PyType_GetFullyQualifiedName(PyTypeObject *type, char sep); + #ifdef __cplusplus } diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index bb6161abf4da81..91c425e483f0ff 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -609,6 +609,40 @@ def check_format(expected, format, *args): check_format('xyz', b'%V', None, b'xyz') + # test %T + check_format('type: str', + b'type: %T', py_object("abc")) + check_format(f'type: st', + b'type: %.2T', py_object("abc")) + check_format(f'type: str', + b'type: %10T', py_object("abc")) + + class LocalType: + pass + obj = LocalType() + fullname = f'{__name__}.{LocalType.__qualname__}' + check_format(f'type: {fullname}', + b'type: %T', py_object(obj)) + fullname_alt = f'{__name__}:{LocalType.__qualname__}' + check_format(f'type: {fullname_alt}', + b'type: %T#', py_object(obj)) + + # test %N + check_format('type: str', + b'type: %N', py_object(str)) + check_format(f'type: st', + b'type: %.2N', py_object(str)) + check_format(f'type: str', + b'type: %10N', py_object(str)) + + check_format(f'type: {fullname}', + b'type: %N', py_object(type(obj))) + check_format(f'type: {fullname_alt}', + b'type: %N#', py_object(type(obj))) + with self.assertRaisesRegex(TypeError, "%N argument must be a type"): + check_format('type: str', + b'type: %N', py_object("abc")) + # test %ls check_format('abc', b'%ls', c_wchar_p('abc')) check_format('\u4eba\u6c11', b'%ls', c_wchar_p('\u4eba\u6c11')) diff --git a/Misc/NEWS.d/next/C API/2024-03-14-22-30-07.gh-issue-111696.76UMKi.rst b/Misc/NEWS.d/next/C API/2024-03-14-22-30-07.gh-issue-111696.76UMKi.rst new file mode 100644 index 00000000000000..44c15e4e6a8256 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-03-14-22-30-07.gh-issue-111696.76UMKi.rst @@ -0,0 +1,4 @@ +Add support for ``%T``, ``%T#``, ``%N`` and ``%N#`` formats to +:c:func:`PyUnicode_FromFormat`: format the fully qualified name of an object +type and of a type: call :c:func:`PyType_GetModuleName`. See :pep:`737` for +more information. Patch by Victor Stinner. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 1c5729c589da93..b73dfba37529a3 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1208,7 +1208,7 @@ type_set_module(PyTypeObject *type, PyObject *value, void *context) PyObject * -PyType_GetFullyQualifiedName(PyTypeObject *type) +_PyType_GetFullyQualifiedName(PyTypeObject *type, char sep) { if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { return PyUnicode_FromString(type->tp_name); @@ -1230,7 +1230,7 @@ PyType_GetFullyQualifiedName(PyTypeObject *type) && !_PyUnicode_Equal(module, &_Py_ID(builtins)) && !_PyUnicode_Equal(module, &_Py_ID(__main__))) { - result = PyUnicode_FromFormat("%U.%U", module, qualname); + result = PyUnicode_FromFormat("%U%c%U", module, sep, qualname); } else { result = Py_NewRef(qualname); @@ -1240,6 +1240,12 @@ PyType_GetFullyQualifiedName(PyTypeObject *type) return result; } +PyObject * +PyType_GetFullyQualifiedName(PyTypeObject *type) +{ + return _PyType_GetFullyQualifiedName(type, '.'); +} + static PyObject * type_abstractmethods(PyTypeObject *type, void *context) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 0a569a950e88e2..c8f647a7a71135 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -2791,6 +2791,64 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, break; } + case 'T': + { + PyObject *obj = va_arg(*vargs, PyObject *); + PyTypeObject *type = (PyTypeObject *)Py_NewRef(Py_TYPE(obj)); + + PyObject *type_name; + if (f[1] == '#') { + type_name = _PyType_GetFullyQualifiedName(type, ':'); + f++; + } + else { + type_name = PyType_GetFullyQualifiedName(type); + } + Py_DECREF(type); + if (!type_name) { + return NULL; + } + + if (unicode_fromformat_write_str(writer, type_name, + width, precision, flags) == -1) { + Py_DECREF(type_name); + return NULL; + } + Py_DECREF(type_name); + break; + } + + case 'N': + { + PyObject *type_raw = va_arg(*vargs, PyObject *); + assert(type_raw != NULL); + + if (!PyType_Check(type_raw)) { + PyErr_SetString(PyExc_TypeError, "%N argument must be a type"); + return NULL; + } + PyTypeObject *type = (PyTypeObject*)type_raw; + + PyObject *type_name; + if (f[1] == '#') { + type_name = _PyType_GetFullyQualifiedName(type, ':'); + f++; + } + else { + type_name = PyType_GetFullyQualifiedName(type); + } + if (!type_name) { + return NULL; + } + if (unicode_fromformat_write_str(writer, type_name, + width, precision, flags) == -1) { + Py_DECREF(type_name); + return NULL; + } + Py_DECREF(type_name); + break; + } + default: invalid_format: PyErr_Format(PyExc_SystemError, "invalid format string: %s", p); From be1c808fcad201adc4d5d6cca52ddb24aeb5e367 Mon Sep 17 00:00:00 2001 From: vxiiduu <73044267+vxiiduu@users.noreply.github.com> Date: Fri, 15 Mar 2024 09:09:36 +1000 Subject: [PATCH 101/158] gh-116195: Implements a fast path for nt.getppid (GH-116205) Use the NtQueryInformationProcess system call to efficiently retrieve the parent process ID in a single step, rather than using the process snapshots API which retrieves large amounts of unnecessary information and is more prone to failure (since it makes heap allocations). Includes a fallback to the original win32_getppid implementation in case the unstable API appears to return strange results. --- ...-03-14-20-46-23.gh-issue-116195.Cu_rYs.rst | 1 + Modules/posixmodule.c | 86 ++++++++++++++++++- 2 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-03-14-20-46-23.gh-issue-116195.Cu_rYs.rst diff --git a/Misc/NEWS.d/next/Windows/2024-03-14-20-46-23.gh-issue-116195.Cu_rYs.rst b/Misc/NEWS.d/next/Windows/2024-03-14-20-46-23.gh-issue-116195.Cu_rYs.rst new file mode 100644 index 00000000000000..32122d764e870a --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-03-14-20-46-23.gh-issue-116195.Cu_rYs.rst @@ -0,0 +1 @@ +Improves performance of :func:`os.getppid` by using an alternate system API when available. Contributed by vxiiduu. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 19e925730a5110..7b2d3661ee5546 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -9115,7 +9115,81 @@ os_setpgrp_impl(PyObject *module) #ifdef HAVE_GETPPID #ifdef MS_WINDOWS -#include +#include +#include + +// The structure definition in winternl.h may be incomplete. +// This structure is the full version from the MSDN documentation. +typedef struct _PROCESS_BASIC_INFORMATION_FULL { + NTSTATUS ExitStatus; + PVOID PebBaseAddress; + ULONG_PTR AffinityMask; + LONG BasePriority; + ULONG_PTR UniqueProcessId; + ULONG_PTR InheritedFromUniqueProcessId; +} PROCESS_BASIC_INFORMATION_FULL; + +typedef NTSTATUS (NTAPI *PNT_QUERY_INFORMATION_PROCESS) ( + IN HANDLE ProcessHandle, + IN PROCESSINFOCLASS ProcessInformationClass, + OUT PVOID ProcessInformation, + IN ULONG ProcessInformationLength, + OUT PULONG ReturnLength OPTIONAL); + +// This function returns the process ID of the parent process. +// Returns 0 on failure. +static ULONG +win32_getppid_fast(void) +{ + NTSTATUS status; + HMODULE ntdll; + PNT_QUERY_INFORMATION_PROCESS pNtQueryInformationProcess; + PROCESS_BASIC_INFORMATION_FULL basic_information; + static ULONG cached_ppid = 0; + + if (cached_ppid) { + // No need to query the kernel again. + return cached_ppid; + } + + ntdll = GetModuleHandleW(L"ntdll.dll"); + if (!ntdll) { + return 0; + } + + pNtQueryInformationProcess = (PNT_QUERY_INFORMATION_PROCESS) GetProcAddress(ntdll, "NtQueryInformationProcess"); + if (!pNtQueryInformationProcess) { + return 0; + } + + status = pNtQueryInformationProcess(GetCurrentProcess(), + ProcessBasicInformation, + &basic_information, + sizeof(basic_information), + NULL); + + if (!NT_SUCCESS(status)) { + return 0; + } + + // Perform sanity check on the parent process ID we received from NtQueryInformationProcess. + // The check covers values which exceed the 32-bit range (if running on x64) as well as + // zero and (ULONG) -1. + + if (basic_information.InheritedFromUniqueProcessId == 0 || + basic_information.InheritedFromUniqueProcessId >= ULONG_MAX) + { + return 0; + } + + // Now that we have reached this point, the BasicInformation.InheritedFromUniqueProcessId + // structure member contains a ULONG_PTR which represents the process ID of our parent + // process. This process ID will be correctly returned even if the parent process has + // exited or been terminated. + + cached_ppid = (ULONG) basic_information.InheritedFromUniqueProcessId; + return cached_ppid; +} static PyObject* win32_getppid(void) @@ -9123,8 +9197,16 @@ win32_getppid(void) DWORD error; PyObject* result = NULL; HANDLE process = GetCurrentProcess(); - HPSS snapshot = NULL; + ULONG pid; + + pid = win32_getppid_fast(); + if (pid != 0) { + return PyLong_FromUnsignedLong(pid); + } + + // If failure occurs in win32_getppid_fast(), fall back to using the PSS API. + error = PssCaptureSnapshot(process, PSS_CAPTURE_NONE, 0, &snapshot); if (error != ERROR_SUCCESS) { return PyErr_SetFromWindowsErr(error); From 1904f0a2245f500aa85fba347b260620350efc78 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 15 Mar 2024 00:11:49 +0000 Subject: [PATCH 102/158] GH-113838: Add "Comparison to os.path" section to pathlib docs (#115926) --- Doc/library/pathlib.rst | 63 ++++++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 19 deletions(-) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 9041f37bd668fa..3ff2631d73c0b2 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1664,7 +1664,7 @@ the pattern to match only directories. Comparison to the :mod:`glob` module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------------ The patterns accepted and results generated by :meth:`Path.glob` and :meth:`Path.rglob` differ slightly from those by the :mod:`glob` module: @@ -1682,27 +1682,57 @@ The patterns accepted and results generated by :meth:`Path.glob` and 5. The values returned from pathlib's ``path.glob()`` and ``path.rglob()`` include the *path* as a prefix, unlike the results of ``glob.glob(root_dir=path)``. -6. ``bytes``-based paths and :ref:`paths relative to directory descriptors - ` are not supported by pathlib. -Correspondence to tools in the :mod:`os` module ------------------------------------------------ +Comparison to the :mod:`os` and :mod:`os.path` modules +------------------------------------------------------ -Below is a table mapping various :mod:`os` functions to their corresponding -:class:`PurePath`/:class:`Path` equivalent. +pathlib implements path operations using :class:`PurePath` and :class:`Path` +objects, and so it's said to be *object-oriented*. On the other hand, the +:mod:`os` and :mod:`os.path` modules supply functions that work with low-level +``str`` and ``bytes`` objects, which is a more *procedural* approach. Some +users consider the object-oriented style to be more readable. -.. note:: +Many functions in :mod:`os` and :mod:`os.path` support ``bytes`` paths and +:ref:`paths relative to directory descriptors `. These features aren't +available in pathlib. + +Python's ``str`` and ``bytes`` types, and portions of the :mod:`os` and +:mod:`os.path` modules, are written in C and are very speedy. pathlib is +written in pure Python and is often slower, but rarely slow enough to matter. + +pathlib's path normalization is slightly more opinionated and consistent than +:mod:`os.path`. For example, whereas :func:`os.path.abspath` eliminates +"``..``" segments from a path, which may change its meaning if symlinks are +involved, :meth:`Path.absolute` preserves these segments for greater safety. + +pathlib's path normalization may render it unsuitable for some applications: + +1. pathlib normalizes ``Path("my_folder/")`` to ``Path("my_folder")``, which + changes a path's meaning when supplied to various operating system APIs and + command-line utilities. Specifically, the absence of a trailing separator + may allow the path to be resolved as either a file or directory, rather + than a directory only. +2. pathlib normalizes ``Path("./my_program")`` to ``Path("my_program")``, + which changes a path's meaning when used as an executable search path, such + as in a shell or when spawning a child process. Specifically, the absence + of a separator in the path may force it to be looked up in :envvar:`PATH` + rather than the current directory. - Not all pairs of functions/methods below are equivalent. Some of them, - despite having some overlapping use-cases, have different semantics. They - include :func:`os.path.abspath` and :meth:`Path.absolute`, - :func:`os.path.relpath` and :meth:`PurePath.relative_to`. +As a consequence of these differences, pathlib is not a drop-in replacement +for :mod:`os.path`. + + +Corresponding tools +^^^^^^^^^^^^^^^^^^^ + +Below is a table mapping various :mod:`os` functions to their corresponding +:class:`PurePath`/:class:`Path` equivalent. ==================================== ============================== :mod:`os` and :mod:`os.path` :mod:`pathlib` ==================================== ============================== -:func:`os.path.abspath` :meth:`Path.absolute` [#]_ +:func:`os.path.abspath` :meth:`Path.absolute` :func:`os.path.realpath` :meth:`Path.resolve` :func:`os.chmod` :meth:`Path.chmod` :func:`os.mkdir` :meth:`Path.mkdir` @@ -1723,7 +1753,7 @@ Below is a table mapping various :mod:`os` functions to their corresponding :func:`os.link` :meth:`Path.hardlink_to` :func:`os.symlink` :meth:`Path.symlink_to` :func:`os.readlink` :meth:`Path.readlink` -:func:`os.path.relpath` :meth:`PurePath.relative_to` [#]_ +:func:`os.path.relpath` :meth:`PurePath.relative_to` :func:`os.stat` :meth:`Path.stat`, :meth:`Path.owner`, :meth:`Path.group` @@ -1735,8 +1765,3 @@ Below is a table mapping various :mod:`os` functions to their corresponding :func:`os.path.splitext` :attr:`PurePath.stem` and :attr:`PurePath.suffix` ==================================== ============================== - -.. rubric:: Footnotes - -.. [#] :func:`os.path.abspath` normalizes the resulting path, which may change its meaning in the presence of symlinks, while :meth:`Path.absolute` does not. -.. [#] :meth:`PurePath.relative_to` requires ``self`` to be the subpath of the argument, but :func:`os.path.relpath` does not. From 41e844a4acbd5070f675e034e31c988b4849dec9 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 14 Mar 2024 21:40:36 -0500 Subject: [PATCH 103/158] gh-116842: Improve test comment and fix a doctest (gh-116846) --- Doc/library/itertools.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 7df8e804507ef5..b092efe0dc362b 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1417,7 +1417,7 @@ The following recipes have a more mathematical flavor: [1, 4] >>> # Verify faithfulness to type specific index() method behaviors. - >>> # For example, bytes and str perform subsequence searches + >>> # For example, bytes and str perform continuous-subsequence searches >>> # that do not match the general behavior specified >>> # in collections.abc.Sequence.index(). >>> seq = 'abracadabra' @@ -1568,7 +1568,7 @@ The following recipes have a more mathematical flavor: >>> list(roundrobin('abc', 'd', 'ef')) ['a', 'd', 'e', 'b', 'f', 'c'] >>> ranges = [range(5, 1000), range(4, 3000), range(0), range(3, 2000), range(2, 5000), range(1, 3500)] - >>> collections.Counter(roundrobin(ranges)) == collections.Counter(ranges) + >>> collections.Counter(roundrobin(*ranges)) == collections.Counter(chain(*ranges)) True >>> # Verify that the inputs are consumed lazily >>> input_iterators = list(map(iter, ['abcd', 'ef', '', 'ghijk', 'l', 'mnopqr'])) From 8fc8fbb43a8bb46c04ab55f96049039de243afb0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 15 Mar 2024 08:49:58 +0100 Subject: [PATCH 104/158] gh-85283: Build pwd extension with the limited C API (#116841) Argument Clinic now uses the PEP 737 "%T" format to format type name for the limited C API. --- Doc/whatsnew/3.13.rst | 2 +- .../C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst | 2 +- Modules/clinic/pwdmodule.c.h | 6 ++---- Modules/pwdmodule.c | 9 ++++++++- Tools/clinic/libclinic/converter.py | 9 ++++----- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 856c6ee1d6e3f0..03ae6018905380 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1469,7 +1469,7 @@ Build Changes * Building CPython now requires a compiler with support for the C11 atomic library, GCC built-in atomic functions, or MSVC interlocked intrinsics. -* The ``errno``, ``fcntl``, ``grp``, ``md5``, ``resource``, ``winsound``, +* The ``errno``, ``fcntl``, ``grp``, ``md5``, ``pwd``, ``resource``, ``winsound``, ``_ctypes_test``, ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, ``_testimportmultiple`` and ``_uuid`` C extensions are now built with the :ref:`limited C API `. diff --git a/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst b/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst index a02b8a8210f9bd..8a3185a8649d7d 100644 --- a/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst +++ b/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst @@ -1,2 +1,2 @@ -The ``fcntl`` and ``grp`` C extensions are now built with the :ref:`limited +The ``fcntl``, ``grp`` and ``pwd`` C extensions are now built with the :ref:`limited C API `. (Contributed by Victor Stinner in :gh:`85283`.) diff --git a/Modules/clinic/pwdmodule.c.h b/Modules/clinic/pwdmodule.c.h index 43d4825031c7e6..365d99aab1dd22 100644 --- a/Modules/clinic/pwdmodule.c.h +++ b/Modules/clinic/pwdmodule.c.h @@ -2,8 +2,6 @@ preserve [clinic start generated code]*/ -#include "pycore_modsupport.h" // _PyArg_BadArgument() - PyDoc_STRVAR(pwd_getpwuid__doc__, "getpwuid($module, uidobj, /)\n" "--\n" @@ -36,7 +34,7 @@ pwd_getpwnam(PyObject *module, PyObject *arg) PyObject *name; if (!PyUnicode_Check(arg)) { - _PyArg_BadArgument("getpwnam", "argument", "str", arg); + PyErr_Format(PyExc_TypeError, "getpwnam() argument must be str, not %T", arg); goto exit; } name = arg; @@ -73,4 +71,4 @@ pwd_getpwall(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef PWD_GETPWALL_METHODDEF #define PWD_GETPWALL_METHODDEF #endif /* !defined(PWD_GETPWALL_METHODDEF) */ -/*[clinic end generated code: output=5a8fb12939ff4ea3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dac88d500f6d6f49 input=a9049054013a1b77]*/ diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index c59a8e41aa292a..f58735aff99799 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -1,9 +1,16 @@ /* UNIX password file access module */ +// Need limited C API version 3.13 for PyMem_RawRealloc() +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif + #include "Python.h" #include "posixmodule.h" +#include // ERANGE #include // getpwuid() #include // sysconf() @@ -83,7 +90,7 @@ mkpwent(PyObject *module, struct passwd *p) if (item == NULL) { \ goto error; \ } \ - PyStructSequence_SET_ITEM(v, setIndex++, item); \ + PyStructSequence_SetItem(v, setIndex++, item); \ } while(0) SET_STRING(p->pw_name); diff --git a/Tools/clinic/libclinic/converter.py b/Tools/clinic/libclinic/converter.py index da28ba56a346a1..744d03c2c1fea4 100644 --- a/Tools/clinic/libclinic/converter.py +++ b/Tools/clinic/libclinic/converter.py @@ -426,13 +426,12 @@ def bad_argument(self, displayname: str, expected: str, *, limited_capi: bool, e if limited_capi: if expected_literal: return (f'PyErr_Format(PyExc_TypeError, ' - f'"{{{{name}}}}() {displayname} must be {expected}, not %.50s", ' - f'{{argname}} == Py_None ? "None" : Py_TYPE({{argname}})->tp_name);') + f'"{{{{name}}}}() {displayname} must be {expected}, not %T", ' + f'{{argname}});') else: return (f'PyErr_Format(PyExc_TypeError, ' - f'"{{{{name}}}}() {displayname} must be %.50s, not %.50s", ' - f'"{expected}", ' - f'{{argname}} == Py_None ? "None" : Py_TYPE({{argname}})->tp_name);') + f'"{{{{name}}}}() {displayname} must be %s, not %T", ' + f'"{expected}", {{argname}});') else: if expected_literal: expected = f'"{expected}"' From a50cf6c3d76b34e2ee9f92a248f1b0df24e407f6 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 15 Mar 2024 02:36:04 -0700 Subject: [PATCH 105/158] gh-90095: Ignore empty lines and comments in `.pdbrc` (#116834) --- Doc/library/pdb.rst | 3 ++- Lib/pdb.py | 5 ++++- Lib/test/test_pdb.py | 19 +++++++++++++++++++ ...4-03-14-20-59-28.gh-issue-90095.7UaJ1U.rst | 1 + 4 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-14-20-59-28.gh-issue-90095.7UaJ1U.rst diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 1cfca0cf68a946..ac3007f70c3534 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -288,7 +288,8 @@ There are three preset *convenience variables*: If a file :file:`.pdbrc` exists in the user's home directory or in the current directory, it is read with ``'utf-8'`` encoding and executed as if it had been -typed at the debugger prompt. This is particularly useful for aliases. If both +typed at the debugger prompt, with the exception that empty lines and lines +starting with ``#`` are ignored. This is particularly useful for aliases. If both files exist, the one in the home directory is read first and aliases defined there can be overridden by the local file. diff --git a/Lib/pdb.py b/Lib/pdb.py index f4d19386703de0..88ea900e63f42b 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -364,7 +364,10 @@ def setup(self, f, tb): ) if self.rcLines: - self.cmdqueue = self.rcLines + self.cmdqueue = [ + line for line in self.rcLines + if line.strip() and not line.strip().startswith("#") + ] self.rcLines = [] # Override Bdb methods diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 44728542787423..69691e930562bc 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2933,8 +2933,27 @@ def test_pdbrc_basic(self): """) stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertNotIn("SyntaxError", stdout) self.assertIn("a+8=9", stdout) + def test_pdbrc_empty_line(self): + """Test that empty lines in .pdbrc are ignored.""" + + script = textwrap.dedent(""" + a = 1 + b = 2 + c = 3 + """) + + pdbrc = textwrap.dedent(""" + n + + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("b = 2", stdout) + self.assertNotIn("c = 3", stdout) + def test_pdbrc_alias(self): script = textwrap.dedent(""" class A: diff --git a/Misc/NEWS.d/next/Library/2024-03-14-20-59-28.gh-issue-90095.7UaJ1U.rst b/Misc/NEWS.d/next/Library/2024-03-14-20-59-28.gh-issue-90095.7UaJ1U.rst new file mode 100644 index 00000000000000..b7024c74f7aa7d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-14-20-59-28.gh-issue-90095.7UaJ1U.rst @@ -0,0 +1 @@ +Ignore empty lines and comments in ``.pdbrc`` From 2cf18a44303b6d84faa8ecffaecc427b53ae121e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Mar 2024 10:48:00 +0000 Subject: [PATCH 106/158] GH-116422: Modify a few uops so that they can be supported by tier 2 with hot/cold splitting (GH-116832) --- Include/internal/pycore_ceval.h | 9 +++++++-- Python/bytecodes.c | 34 +++++++++++++++------------------ Python/executor_cases.c.h | 32 +++++++++++++++---------------- Python/generated_cases.c.h | 32 +++++++++++++++---------------- 4 files changed, 52 insertions(+), 55 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 6eab2ba1daedf8..946f82ae3c20e3 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -138,12 +138,12 @@ extern void _PyEval_DeactivateOpCache(void); /* With USE_STACKCHECK macro defined, trigger stack checks in _Py_CheckRecursiveCall() on every 64th call to _Py_EnterRecursiveCall. */ static inline int _Py_MakeRecCheck(PyThreadState *tstate) { - return (tstate->c_recursion_remaining-- <= 0 + return (tstate->c_recursion_remaining-- < 0 || (tstate->c_recursion_remaining & 63) == 0); } #else static inline int _Py_MakeRecCheck(PyThreadState *tstate) { - return tstate->c_recursion_remaining-- <= 0; + return tstate->c_recursion_remaining-- < 0; } #endif @@ -161,6 +161,11 @@ static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate, return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where)); } +static inline void _Py_EnterRecursiveCallTstateUnchecked(PyThreadState *tstate) { + assert(tstate->c_recursion_remaining > 0); + tstate->c_recursion_remaining--; +} + static inline int _Py_EnterRecursiveCall(const char *where) { PyThreadState *tstate = _PyThreadState_GET(); return _Py_EnterRecursiveCallTstate(tstate, where); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e8fcb6c6d0067c..bf7782a65bc5ce 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3376,14 +3376,12 @@ dummy_func( DEOPT_IF(total_args != 1); DEOPT_IF(!PyCFunction_CheckExact(callable)); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } PyObject *arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3480,10 +3478,11 @@ dummy_func( } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - + if (res == NULL) { + GOTO_ERROR(error); + } Py_DECREF(callable); Py_DECREF(arg); - ERROR_IF(res == NULL, error); } inst(CALL_ISINSTANCE, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { @@ -3505,11 +3504,12 @@ dummy_func( } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - + if (res == NULL) { + GOTO_ERROR(error); + } Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - ERROR_IF(res == NULL, error); } // This is secretly a super-instruction @@ -3543,16 +3543,14 @@ dummy_func( DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_O); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0); PyObject *arg = args[1]; PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, self, arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3616,13 +3614,11 @@ dummy_func( PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); DEOPT_IF(meth->ml_flags != METH_NOARGS); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 077499ebc01687..bdc9c0b1501c91 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3189,14 +3189,12 @@ if (total_args != 1) goto deoptimize; if (!PyCFunction_CheckExact(callable)) goto deoptimize; if (PyCFunction_GET_FLAGS(callable) != METH_O) goto deoptimize; + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) goto deoptimize; STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } PyObject *arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3305,9 +3303,11 @@ } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res == NULL) { + GOTO_ERROR(error); + } Py_DECREF(callable); Py_DECREF(arg); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3340,10 +3340,12 @@ } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res == NULL) { + GOTO_ERROR(error); + } Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3368,16 +3370,14 @@ if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_O) goto deoptimize; + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) goto deoptimize; PyObject *arg = args[1]; PyObject *self = args[0]; if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, self, arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3450,13 +3450,11 @@ PyObject *self = args[0]; if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; if (meth->ml_flags != METH_NOARGS) goto deoptimize; + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) goto deoptimize; STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 645d0fb510e6fb..2fbd9ed403f5d1 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1174,14 +1174,12 @@ DEOPT_IF(total_args != 1, CALL); DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } PyObject *arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -1350,10 +1348,12 @@ } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res == NULL) { + GOTO_ERROR(error); + } Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; DISPATCH(); @@ -1481,9 +1481,11 @@ } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res == NULL) { + GOTO_ERROR(error); + } Py_DECREF(callable); Py_DECREF(arg); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; DISPATCH(); @@ -1649,13 +1651,11 @@ PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -1698,16 +1698,14 @@ DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); PyObject *arg = args[1]; PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, self, arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); From 001b21d1c500857fb3721b019eeaf014b5ad76e8 Mon Sep 17 00:00:00 2001 From: mpage Date: Fri, 15 Mar 2024 06:56:13 -0700 Subject: [PATCH 107/158] gh-111926: Simplify weakref creation logic (#116843) Since 3.12, allocating a GC object cannot immediately trigger GC. This allows us to simplify the logic for creating the canonical callback-less weakref. --- Objects/weakrefobject.c | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index df74be6aba7244..6264b1825eda73 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -801,24 +801,14 @@ PyWeakref_NewRef(PyObject *ob, PyObject *callback) if (result != NULL) Py_INCREF(result); else { - /* Note: new_weakref() can trigger cyclic GC, so the weakref - list on ob can be mutated. This means that the ref and - proxy pointers we got back earlier may have been collected, - so we need to compute these values again before we use - them. */ + /* We do not need to recompute ref/proxy; new_weakref() cannot + trigger GC. + */ result = new_weakref(ob, callback); if (result != NULL) { - get_basic_refs(*list, &ref, &proxy); if (callback == NULL) { - if (ref == NULL) - insert_head(result, list); - else { - /* Someone else added a ref without a callback - during GC. Return that one instead of this one - to avoid violating the invariants of the list - of weakrefs for ob. */ - Py_SETREF(result, (PyWeakReference*)Py_NewRef(ref)); - } + assert(ref == NULL); + insert_head(result, list); } else { PyWeakReference *prev; From ce2c996b2f645cd886d05f6ef8f1ba60ced7d4b7 Mon Sep 17 00:00:00 2001 From: mpage Date: Fri, 15 Mar 2024 06:58:40 -0700 Subject: [PATCH 108/158] gh-111926: Simplify proxy creation logic (#116844) Since 3.12, allocating a GC-able object cannot trigger GC. This allows us to simplify the logic for creating the canonical callback-less proxy object. --- Objects/weakrefobject.c | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index 6264b1825eda73..b7b29064151609 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -848,11 +848,9 @@ PyWeakref_NewProxy(PyObject *ob, PyObject *callback) if (result != NULL) Py_INCREF(result); else { - /* Note: new_weakref() can trigger cyclic GC, so the weakref - list on ob can be mutated. This means that the ref and - proxy pointers we got back earlier may have been collected, - so we need to compute these values again before we use - them. */ + /* We do not need to recompute ref/proxy; new_weakref cannot + trigger GC. + */ result = new_weakref(ob, callback); if (result != NULL) { PyWeakReference *prev; @@ -863,16 +861,7 @@ PyWeakref_NewProxy(PyObject *ob, PyObject *callback) else { Py_SET_TYPE(result, &_PyWeakref_ProxyType); } - get_basic_refs(*list, &ref, &proxy); if (callback == NULL) { - if (proxy != NULL) { - /* Someone else added a proxy without a callback - during GC. Return that one instead of this one - to avoid violating the invariants of the list - of weakrefs for ob. */ - Py_SETREF(result, (PyWeakReference*)Py_NewRef(proxy)); - goto skip_insert; - } prev = ref; } else @@ -882,8 +871,6 @@ PyWeakref_NewProxy(PyObject *ob, PyObject *callback) insert_head(result, list); else insert_after(result, prev); - skip_insert: - ; } } return (PyObject *) result; From d180b507c4929be399395bfd7946948f98ffc4f7 Mon Sep 17 00:00:00 2001 From: Zackery Spytz Date: Fri, 15 Mar 2024 07:38:13 -0700 Subject: [PATCH 109/158] gh-63283: IDNA prefix should be case insensitive (GH-17726) Any capitalization of "xn--" should be acceptable for the ACE prefix (see https://tools.ietf.org/html/rfc3490#section-5). Co-authored-by: Pepijn de Vos Co-authored-by: Erlend E. Aasland Co-authored-by: Petr Viktorin --- Lib/encodings/idna.py | 6 +++--- Lib/test/test_codecs.py | 7 +++++++ .../Library/2024-03-13-15-45-54.gh-issue-63283.OToJnG.rst | 2 ++ 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-13-15-45-54.gh-issue-63283.OToJnG.rst diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py index 5396047a7fb0b8..d0f70c00f0ab66 100644 --- a/Lib/encodings/idna.py +++ b/Lib/encodings/idna.py @@ -86,7 +86,7 @@ def ToASCII(label): raise UnicodeError("label empty or too long") # Step 5: Check ACE prefix - if label.startswith(sace_prefix): + if label[:4].lower() == sace_prefix: raise UnicodeError("Label starts with ACE prefix") # Step 6: Encode with PUNYCODE @@ -129,7 +129,7 @@ def ToUnicode(label): except UnicodeError: raise UnicodeError("Invalid character in IDN label") # Step 3: Check for ACE prefix - if not label.startswith(ace_prefix): + if not label[:4].lower() == ace_prefix: return str(label, "ascii") # Step 4: Remove ACE prefix @@ -202,7 +202,7 @@ def decode(self, input, errors='strict'): # XXX obviously wrong, see #3232 input = bytes(input) - if ace_prefix not in input: + if ace_prefix not in input.lower(): # Fast path try: return input.decode('ascii'), len(input) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index ff511a625a0194..9585f947877142 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1547,6 +1547,13 @@ def test_builtin_decode(self): self.assertEqual(str(b"python.org.", "idna"), "python.org.") self.assertEqual(str(b"xn--pythn-mua.org", "idna"), "pyth\xf6n.org") self.assertEqual(str(b"xn--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"XN--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"xN--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"Xn--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"bugs.xn--pythn-mua.org.", "idna"), + "bugs.pyth\xf6n.org.") + self.assertEqual(str(b"bugs.XN--pythn-mua.org.", "idna"), + "bugs.pyth\xf6n.org.") def test_builtin_encode(self): self.assertEqual("python.org".encode("idna"), b"python.org") diff --git a/Misc/NEWS.d/next/Library/2024-03-13-15-45-54.gh-issue-63283.OToJnG.rst b/Misc/NEWS.d/next/Library/2024-03-13-15-45-54.gh-issue-63283.OToJnG.rst new file mode 100644 index 00000000000000..bb4c3a4a8d741b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-13-15-45-54.gh-issue-63283.OToJnG.rst @@ -0,0 +1,2 @@ +In :mod:`encodings.idna`, any capitalization of the the ACE prefix +(``xn--``) is now acceptable. Patch by Pepijn de Vos and Zackery Spytz. From 59e30f41ed6f2388a99ac0a8aebf0a12f7460a4a Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 15 Mar 2024 07:46:18 -0700 Subject: [PATCH 110/158] gh-116735: Use `MISSING` for `CALL` event if argument is absent (GH-116737) --- Lib/test/test_monitoring.py | 5 ++++- .../2024-03-13-16-55-25.gh-issue-116735.o3w6y8.rst | 1 + Python/bytecodes.c | 2 +- Python/generated_cases.c.h | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-13-16-55-25.gh-issue-116735.o3w6y8.rst diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 11fb2d87f6c995..58441ef8b82fd0 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1808,9 +1808,10 @@ def test_gh108976(self): sys.monitoring.set_events(0, 0) def test_call_function_ex(self): - def f(a, b): + def f(a=1, b=2): return a + b args = (1, 2) + empty_args = [] call_data = [] sys.monitoring.use_tool_id(0, "test") @@ -1819,8 +1820,10 @@ def f(a, b): sys.monitoring.register_callback(0, E.CALL, lambda code, offset, callable, arg0: call_data.append((callable, arg0))) sys.monitoring.set_events(0, E.CALL) f(*args) + f(*empty_args) sys.monitoring.set_events(0, 0) self.assertEqual(call_data[0], (f, 1)) + self.assertEqual(call_data[1], (f, sys.monitoring.MISSING)) class TestOptimizer(MonitoringTestBase, unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-13-16-55-25.gh-issue-116735.o3w6y8.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-13-16-55-25.gh-issue-116735.o3w6y8.rst new file mode 100644 index 00000000000000..ca15d484e345db --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-13-16-55-25.gh-issue-116735.o3w6y8.rst @@ -0,0 +1 @@ +For ``INSTRUMENTED_CALL_FUNCTION_EX``, set ``arg0`` to ``sys.monitoring.MISSING`` instead of ``None`` for :monitoring-event:`CALL` event. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index bf7782a65bc5ce..fb66ae583130db 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3774,7 +3774,7 @@ dummy_func( EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : Py_None; + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2fbd9ed403f5d1..82d7b7621f8b25 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1227,7 +1227,7 @@ EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : Py_None; + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); From 8da83f3386da603605358dc9ec68796daa5ef455 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 15 Mar 2024 23:48:34 +0900 Subject: [PATCH 111/158] gh-116621: Specialize list.extend for dict keys/values (gh-116816) --- Objects/listobject.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/Objects/listobject.c b/Objects/listobject.c index 7179d5929e2148..6f919ce02b3ce2 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_dict.h" // _PyDictViewObject #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_interp.h" // PyInterpreterState.list #include "pycore_list.h" // struct _Py_list_freelist, _PyListIterObject @@ -1295,6 +1296,30 @@ list_extend_set(PyListObject *self, PySetObject *other) return 0; } +static int +list_extend_dict(PyListObject *self, PyDictObject *dict, int which_item) +{ + // which_item: 0 for keys and 1 for values + Py_ssize_t m = Py_SIZE(self); + Py_ssize_t n = PyDict_GET_SIZE(dict); + if (list_resize(self, m + n) < 0) { + return -1; + } + + PyObject **dest = self->ob_item + m; + Py_ssize_t pos = 0; + PyObject *keyvalue[2]; + while (_PyDict_Next((PyObject *)dict, &pos, &keyvalue[0], &keyvalue[1], NULL)) { + PyObject *obj = keyvalue[which_item]; + Py_INCREF(obj); + *dest = obj; + dest++; + } + + Py_SET_SIZE(self, m + n); + return 0; +} + static int _list_extend(PyListObject *self, PyObject *iterable) { @@ -1322,6 +1347,18 @@ _list_extend(PyListObject *self, PyObject *iterable) res = list_extend_set(self, (PySetObject *)iterable); Py_END_CRITICAL_SECTION2(); } + else if (Py_IS_TYPE(iterable, &PyDictKeys_Type)) { + PyDictObject *dict = ((_PyDictViewObject *)iterable)->dv_dict; + Py_BEGIN_CRITICAL_SECTION2(self, dict); + res = list_extend_dict(self, dict, 0 /*keys*/); + Py_END_CRITICAL_SECTION2(); + } + else if (Py_IS_TYPE(iterable, &PyDictValues_Type)) { + PyDictObject *dict = ((_PyDictViewObject *)iterable)->dv_dict; + Py_BEGIN_CRITICAL_SECTION2(self, dict); + res = list_extend_dict(self, dict, 1 /*values*/); + Py_END_CRITICAL_SECTION2(); + } else { Py_BEGIN_CRITICAL_SECTION(self); res = list_extend_iter_lock_held(self, iterable); From 16349868d396cc1bff5188de3638321e87fe0293 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 15 Mar 2024 18:47:46 +0300 Subject: [PATCH 112/158] gh-116782: Mention `__type_params__` in `inspect.getmembers` docs (#116783) Co-authored-by: Jelle Zijlstra --- Doc/library/inspect.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index ed8d705da3b0b5..4a0a090facb8bb 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -55,6 +55,11 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | __module__ | name of module in which | | | | this class was defined | +-----------+-------------------+---------------------------+ +| | __type_params__ | A tuple containing the | +| | | :ref:`type parameters | +| | | ` of | +| | | a generic class | ++-----------+-------------------+---------------------------+ | method | __doc__ | documentation string | +-----------+-------------------+---------------------------+ | | __name__ | name with which this | @@ -103,6 +108,11 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | | reserved for return | | | | annotations. | +-----------+-------------------+---------------------------+ +| | __type_params__ | A tuple containing the | +| | | :ref:`type parameters | +| | | ` of | +| | | a generic function | ++-----------+-------------------+---------------------------+ | | __module__ | name of module in which | | | | this function was defined | +-----------+-------------------+---------------------------+ From ebf29b3a02d5b42a747e271e9cfc4dd73c01ebe6 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sat, 16 Mar 2024 01:07:16 +0900 Subject: [PATCH 113/158] gh-112536: Add --tsan test for reasonable TSAN execution times. (gh-116601) --- Lib/test/libregrtest/cmdline.py | 3 +++ Lib/test/libregrtest/main.py | 5 ++++ Lib/test/libregrtest/tsan.py | 24 +++++++++++++++++++ ...-03-11-23-20-28.gh-issue-112536.Qv1RrX.rst | 2 ++ Tools/tsan/supressions.txt | 5 ++++ 5 files changed, 39 insertions(+) create mode 100644 Lib/test/libregrtest/tsan.py create mode 100644 Misc/NEWS.d/next/Tests/2024-03-11-23-20-28.gh-issue-112536.Qv1RrX.rst create mode 100644 Tools/tsan/supressions.txt diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 608b12bb6f2a38..876b1bcd2ca406 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -164,6 +164,7 @@ def __init__(self, **kwargs) -> None: self.match_tests: TestFilter = [] self.pgo = False self.pgo_extended = False + self.tsan = False self.worker_json = None self.start = None self.timeout = None @@ -333,6 +334,8 @@ def _create_parser(): help='enable Profile Guided Optimization (PGO) training') group.add_argument('--pgo-extended', action='store_true', help='enable extended PGO training (slower training)') + group.add_argument('--tsan', dest='tsan', action='store_true', + help='run a subset of test cases that are proper for the TSAN test') group.add_argument('--fail-env-changed', action='store_true', help='if a test file alters the environment, mark ' 'the test as failed') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 126daca388fd7f..70f723a92eb44a 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -18,6 +18,7 @@ from .runtests import RunTests, HuntRefleak from .setup import setup_process, setup_test_dir from .single import run_single_test, PROGRESS_MIN_TIME +from .tsan import setup_tsan_tests from .utils import ( StrPath, StrJSON, TestName, TestList, TestTuple, TestFilter, strip_py_suffix, count, format_duration, @@ -56,6 +57,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.quiet: bool = ns.quiet self.pgo: bool = ns.pgo self.pgo_extended: bool = ns.pgo_extended + self.tsan: bool = ns.tsan # Test results self.results: TestResults = TestResults() @@ -183,6 +185,9 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList # add default PGO tests if no tests are specified setup_pgo_tests(self.cmdline_args, self.pgo_extended) + if self.tsan: + setup_tsan_tests(self.cmdline_args) + exclude_tests = set() if self.exclude: for arg in self.cmdline_args: diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py new file mode 100644 index 00000000000000..150fcec8816179 --- /dev/null +++ b/Lib/test/libregrtest/tsan.py @@ -0,0 +1,24 @@ +# Set of tests run by default if --tsan is specified. The tests below were +# chosen because they use threads and run in a reasonable amount of time. + +TSAN_TESTS = [ + 'test_code', + 'test_enum', + 'test_functools', + 'test_httpservers', + 'test_imaplib', + 'test_importlib', + 'test_io', + 'test_logging', + 'test_ssl', + 'test_syslog', + 'test_thread', + 'test_threadedtempfile', + 'test_threading_local', + 'test_threadsignals', +] + + +def setup_tsan_tests(cmdline_args): + if not cmdline_args: + cmdline_args[:] = TSAN_TESTS[:] diff --git a/Misc/NEWS.d/next/Tests/2024-03-11-23-20-28.gh-issue-112536.Qv1RrX.rst b/Misc/NEWS.d/next/Tests/2024-03-11-23-20-28.gh-issue-112536.Qv1RrX.rst new file mode 100644 index 00000000000000..de9e1c557b093c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-03-11-23-20-28.gh-issue-112536.Qv1RrX.rst @@ -0,0 +1,2 @@ +Add --tsan to test.regrtest for running TSAN tests in reasonable execution +times. Patch by Donghee Na. diff --git a/Tools/tsan/supressions.txt b/Tools/tsan/supressions.txt new file mode 100644 index 00000000000000..448dfac8005c79 --- /dev/null +++ b/Tools/tsan/supressions.txt @@ -0,0 +1,5 @@ +## reference: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions +race:get_allocator_unlocked +race:set_allocator_unlocked +race:mi_heap_visit_pages +race:_mi_heap_delayed_free_partial From 280de3661b42af9b3fe792764d0b09f403df5223 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Fri, 15 Mar 2024 12:35:29 -0400 Subject: [PATCH 114/158] gh-116868: Avoid locking in PyType_IsSubtype (#116829) Make PyType_IsSubType not acquire lock --- Objects/typeobject.c | 80 ++++++++++++++++++++++++++------------------ 1 file changed, 48 insertions(+), 32 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b73dfba37529a3..24f31492985164 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -373,12 +373,23 @@ lookup_tp_mro(PyTypeObject *self) PyObject * _PyType_GetMRO(PyTypeObject *self) { - PyObject *mro; +#ifdef Py_GIL_DISABLED + PyObject *mro = _Py_atomic_load_ptr_relaxed(&self->tp_mro); + if (mro == NULL) { + return NULL; + } + if (_Py_TryIncref(&self->tp_mro, mro)) { + return mro; + } + BEGIN_TYPE_LOCK(); mro = lookup_tp_mro(self); - Py_INCREF(mro); + Py_XINCREF(mro); END_TYPE_LOCK() return mro; +#else + return Py_XNewRef(lookup_tp_mro(self)); +#endif } static inline void @@ -911,7 +922,7 @@ PyType_Modified(PyTypeObject *type) } static int -is_subtype_unlocked(PyTypeObject *a, PyTypeObject *b); +is_subtype_with_mro(PyObject *a_mro, PyTypeObject *a, PyTypeObject *b); static void type_mro_modified(PyTypeObject *type, PyObject *bases) { @@ -957,7 +968,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { PyObject *b = PyTuple_GET_ITEM(bases, i); PyTypeObject *cls = _PyType_CAST(b); - if (!is_subtype_unlocked(type, cls)) { + if (!is_subtype_with_mro(lookup_tp_mro(type), type, cls)) { goto clear; } } @@ -1442,7 +1453,7 @@ type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases, void *context) } PyTypeObject *base = (PyTypeObject*)ob; - if (is_subtype_unlocked(base, type) || + if (is_subtype_with_mro(lookup_tp_mro(base), base, type) || /* In case of reentering here again through a custom mro() the above check is not enough since it relies on base->tp_mro which would gonna be updated inside @@ -2303,37 +2314,41 @@ type_is_subtype_base_chain(PyTypeObject *a, PyTypeObject *b) } static int -is_subtype_unlocked(PyTypeObject *a, PyTypeObject *b) +is_subtype_with_mro(PyObject *a_mro, PyTypeObject *a, PyTypeObject *b) { - PyObject *mro; - - ASSERT_TYPE_LOCK_HELD(); - mro = lookup_tp_mro(a); - if (mro != NULL) { + int res; + if (a_mro != NULL) { /* Deal with multiple inheritance without recursion by walking the MRO tuple */ Py_ssize_t i, n; - assert(PyTuple_Check(mro)); - n = PyTuple_GET_SIZE(mro); + assert(PyTuple_Check(a_mro)); + n = PyTuple_GET_SIZE(a_mro); + res = 0; for (i = 0; i < n; i++) { - if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) - return 1; + if (PyTuple_GET_ITEM(a_mro, i) == (PyObject *)b) { + res = 1; + break; + } } - return 0; } - else + else { /* a is not completely initialized yet; follow tp_base */ - return type_is_subtype_base_chain(a, b); + res = type_is_subtype_base_chain(a, b); + } + return res; } int PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) { - int res; - BEGIN_TYPE_LOCK(); - res = is_subtype_unlocked(a, b); - END_TYPE_LOCK() +#ifdef Py_GIL_DISABLED + PyObject *mro = _PyType_GetMRO(a); + int res = is_subtype_with_mro(mro, a, b); + Py_XDECREF(mro); return res; +#else + return is_subtype_with_mro(lookup_tp_mro(a), a, b); +#endif } /* Routines to do a method lookup in the type without looking in the @@ -2826,7 +2841,7 @@ mro_check(PyTypeObject *type, PyObject *mro) } PyTypeObject *base = (PyTypeObject*)obj; - if (!is_subtype_unlocked(solid, solid_base(base))) { + if (!is_subtype_with_mro(lookup_tp_mro(solid), solid, solid_base(base))) { PyErr_Format( PyExc_TypeError, "mro() returned base with unsuitable layout ('%.500s')", @@ -7082,28 +7097,29 @@ inherit_special(PyTypeObject *type, PyTypeObject *base) #undef COPYVAL /* Setup fast subclass flags */ - if (is_subtype_unlocked(base, (PyTypeObject*)PyExc_BaseException)) { + PyObject *mro = lookup_tp_mro(base); + if (is_subtype_with_mro(mro, base, (PyTypeObject*)PyExc_BaseException)) { type->tp_flags |= Py_TPFLAGS_BASE_EXC_SUBCLASS; } - else if (is_subtype_unlocked(base, &PyType_Type)) { + else if (is_subtype_with_mro(mro, base, &PyType_Type)) { type->tp_flags |= Py_TPFLAGS_TYPE_SUBCLASS; } - else if (is_subtype_unlocked(base, &PyLong_Type)) { + else if (is_subtype_with_mro(mro, base, &PyLong_Type)) { type->tp_flags |= Py_TPFLAGS_LONG_SUBCLASS; } - else if (is_subtype_unlocked(base, &PyBytes_Type)) { + else if (is_subtype_with_mro(mro, base, &PyBytes_Type)) { type->tp_flags |= Py_TPFLAGS_BYTES_SUBCLASS; } - else if (is_subtype_unlocked(base, &PyUnicode_Type)) { + else if (is_subtype_with_mro(mro, base, &PyUnicode_Type)) { type->tp_flags |= Py_TPFLAGS_UNICODE_SUBCLASS; } - else if (is_subtype_unlocked(base, &PyTuple_Type)) { + else if (is_subtype_with_mro(mro, base, &PyTuple_Type)) { type->tp_flags |= Py_TPFLAGS_TUPLE_SUBCLASS; } - else if (is_subtype_unlocked(base, &PyList_Type)) { + else if (is_subtype_with_mro(mro, base, &PyList_Type)) { type->tp_flags |= Py_TPFLAGS_LIST_SUBCLASS; } - else if (is_subtype_unlocked(base, &PyDict_Type)) { + else if (is_subtype_with_mro(mro, base, &PyDict_Type)) { type->tp_flags |= Py_TPFLAGS_DICT_SUBCLASS; } @@ -10204,7 +10220,7 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) d = (PyWrapperDescrObject *)descr; if ((specific == NULL || specific == d->d_wrapped) && d->d_base->wrapper == p->wrapper && - is_subtype_unlocked(type, PyDescr_TYPE(d))) + is_subtype_with_mro(lookup_tp_mro(type), type, PyDescr_TYPE(d))) { specific = d->d_wrapped; } From 950667ed0737144666ea8e1ec1a7e9de2e49a628 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Mar 2024 17:16:30 +0000 Subject: [PATCH 115/158] GH-115802: Reduce the size of _INIT_CALL_PY_EXACT_ARGS. (GH-116856) --- Python/bytecodes.c | 14 +++---- Python/executor_cases.c.h | 84 ++++++++++++++++---------------------- Python/generated_cases.c.h | 28 ++++++------- 3 files changed, 54 insertions(+), 72 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fb66ae583130db..476975d2fbc3c2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3150,16 +3150,14 @@ dummy_func( } replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + int has_self = (self_or_null != NULL); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + PyObject **first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index bdc9c0b1501c91..a55daa2c344944 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2893,16 +2893,14 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + int has_self = (self_or_null != NULL); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + PyObject **first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } stack_pointer[-2 - oparg] = (PyObject *)new_frame; stack_pointer += -1 - oparg; @@ -2919,16 +2917,14 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + int has_self = (self_or_null != NULL); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + PyObject **first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } stack_pointer[-2 - oparg] = (PyObject *)new_frame; stack_pointer += -1 - oparg; @@ -2945,16 +2941,14 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + int has_self = (self_or_null != NULL); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + PyObject **first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } stack_pointer[-2 - oparg] = (PyObject *)new_frame; stack_pointer += -1 - oparg; @@ -2971,16 +2965,14 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + int has_self = (self_or_null != NULL); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + PyObject **first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } stack_pointer[-2 - oparg] = (PyObject *)new_frame; stack_pointer += -1 - oparg; @@ -2997,16 +2989,14 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + int has_self = (self_or_null != NULL); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + PyObject **first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } stack_pointer[-2 - oparg] = (PyObject *)new_frame; stack_pointer += -1 - oparg; @@ -3022,16 +3012,14 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + int has_self = (self_or_null != NULL); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + PyObject **first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } stack_pointer[-2 - oparg] = (PyObject *)new_frame; stack_pointer += -1 - oparg; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 82d7b7621f8b25..2996ee72e7f2c6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -968,16 +968,14 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + int has_self = (self_or_null != NULL); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + PyObject **first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } } // _SAVE_RETURN_OFFSET @@ -1759,16 +1757,14 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + int has_self = (self_or_null != NULL); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + PyObject **first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } } // _SAVE_RETURN_OFFSET From 0c7dc494f2a32494f8971a236ba59c0c35f48d94 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 15 Mar 2024 14:02:10 -0500 Subject: [PATCH 116/158] Minor kde() docstring nit: make presentation order match the function signature (#116876) --- Lib/statistics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/statistics.py b/Lib/statistics.py index 7924123c05b8c3..5d636258fd442b 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -963,7 +963,7 @@ def pdf(x): supported = sample[i : j] return sum(K((x - x_i) / h) for x_i in supported) / (n * h) - pdf.__doc__ = f'PDF estimate with {kernel=!r} and {h=!r}' + pdf.__doc__ = f'PDF estimate with {h=!r} and {kernel=!r}' return pdf From a1c4923d65a3e4cea917745e7f6bc2e377cde5c5 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 16 Mar 2024 11:54:42 +0300 Subject: [PATCH 117/158] gh-116858: Add `@cpython_only` to several tests in `test_cmd_line` (#116859) --- Lib/test/test_cmd_line.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index c633f6493cfab7..fb832aed3152ff 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -38,6 +38,7 @@ def verify_valid_flag(self, cmd_line): self.assertNotIn(b'Traceback', err) return out + @support.cpython_only def test_help(self): self.verify_valid_flag('-h') self.verify_valid_flag('-?') @@ -48,14 +49,17 @@ def test_help(self): self.assertNotIn(b'-X dev', out) self.assertLess(len(lines), 50) + @support.cpython_only def test_help_env(self): out = self.verify_valid_flag('--help-env') self.assertIn(b'PYTHONHOME', out) + @support.cpython_only def test_help_xoptions(self): out = self.verify_valid_flag('--help-xoptions') self.assertIn(b'-X dev', out) + @support.cpython_only def test_help_all(self): out = self.verify_valid_flag('--help-all') lines = out.splitlines() @@ -74,6 +78,7 @@ def test_optimize(self): def test_site_flag(self): self.verify_valid_flag('-S') + @support.cpython_only def test_version(self): version = ('Python %d.%d' % sys.version_info[:2]).encode("ascii") for switch in '-V', '--version', '-VV': @@ -139,6 +144,7 @@ def run_python(*args): else: self.assertEqual(err, b'') + @support.cpython_only def test_xoption_frozen_modules(self): tests = { ('=on', 'FrozenImporter'), @@ -153,6 +159,7 @@ def test_xoption_frozen_modules(self): res = assert_python_ok(*cmd) self.assertRegex(res.out.decode('utf-8'), expected) + @support.cpython_only def test_env_var_frozen_modules(self): tests = { ('on', 'FrozenImporter'), @@ -579,6 +586,7 @@ def test_del___main__(self): print("del sys.modules['__main__']", file=script) assert_python_ok(filename) + @support.cpython_only def test_unknown_options(self): rc, out, err = assert_python_failure('-E', '-z') self.assertIn(b'Unknown option: -z', err) @@ -691,6 +699,7 @@ def run_xdev(self, *args, check_exitcode=True, xdev=True): self.assertEqual(proc.returncode, 0, proc) return proc.stdout.rstrip() + @support.cpython_only def test_xdev(self): # sys.flags.dev_mode code = "import sys; print(sys.flags.dev_mode)" @@ -914,6 +923,7 @@ def test_argv0_normalization(self): self.assertEqual(proc.returncode, 0, proc) self.assertEqual(proc.stdout.strip(), b'0') + @support.cpython_only def test_parsing_error(self): args = [sys.executable, '-I', '--unknown-option'] proc = subprocess.run(args, From 20578a1f68c841a264b72b00591b11ab2fa77b43 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sat, 16 Mar 2024 19:10:37 +0900 Subject: [PATCH 118/158] gh-112536: Add TSAN builds on Github Actions (#116872) --- .github/workflows/build.yml | 22 ++++++++++ .github/workflows/reusable-tsan.yml | 51 ++++++++++++++++++++++++ Lib/test/test_concurrent_futures/util.py | 4 ++ Lib/test/test_logging.py | 4 ++ Lib/test/test_threading.py | 8 ++++ Python/thread_pthread.h | 4 ++ 6 files changed, 93 insertions(+) create mode 100644 .github/workflows/reusable-tsan.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d43b83e830e1fb..e36859e728b67f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -484,6 +484,24 @@ jobs: - name: Tests run: xvfb-run make test + build_tsan: + name: 'Thread sanitizer' + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + uses: ./.github/workflows/reusable-tsan.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + options: ./configure --config-cache --with-thread-sanitizer --with-pydebug + + build_tsan_free_threading: + name: 'Thread sanitizer (free-threading)' + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + uses: ./.github/workflows/reusable-tsan.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + options: ./configure --config-cache --disable-gil --with-thread-sanitizer --with-pydebug + # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: name: CIFuzz @@ -542,6 +560,8 @@ jobs: - build_windows_free_threading - test_hypothesis - build_asan + - build_tsan + - build_tsan_free_threading - cifuzz runs-on: ubuntu-latest @@ -575,6 +595,8 @@ jobs: build_windows, build_windows_free_threading, build_asan, + build_tsan, + build_tsan_free_threading, ' || '' }} diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml new file mode 100644 index 00000000000000..96a9c1b0cda3c3 --- /dev/null +++ b/.github/workflows/reusable-tsan.yml @@ -0,0 +1,51 @@ +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + options: + required: true + type: string + +jobs: + build_tsan_reusable: + name: 'Thread sanitizer' + runs-on: ubuntu-22.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + - name: Install Dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + sudo apt install -y clang + # Reduce ASLR to avoid TSAN crashing + sudo sysctl -w vm.mmap_rnd_bits=28 + - name: TSAN Option Setup + run: | + echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV + echo "CC=clang" >> $GITHUB_ENV + echo "CXX=clang++" >> $GITHUB_ENV + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: Configure CPython + run: ${{ inputs.options }} + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: ./python -m test --tsan -j4 diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py index 3e855031913042..3b8ec3e205d5aa 100644 --- a/Lib/test/test_concurrent_futures/util.py +++ b/Lib/test/test_concurrent_futures/util.py @@ -85,6 +85,8 @@ def get_context(self): self.skipTest("ProcessPoolExecutor unavailable on this system") if sys.platform == "win32": self.skipTest("require unix system") + if support.check_sanitizer(thread=True): + self.skipTest("TSAN doesn't support threads after fork") return super().get_context() @@ -111,6 +113,8 @@ def get_context(self): self.skipTest("ProcessPoolExecutor unavailable on this system") if sys.platform == "win32": self.skipTest("require unix system") + if support.check_sanitizer(thread=True): + self.skipTest("TSAN doesn't support threads after fork") return super().get_context() diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 32bb5171a3f757..c84eca51b52362 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -80,6 +80,9 @@ skip_if_asan_fork = unittest.skipIf( support.HAVE_ASAN_FORK_BUG, "libasan has a pthread_create() dead lock related to thread+fork") +skip_if_tsan_fork = unittest.skipIf( + support.check_sanitizer(thread=True), + "TSAN doesn't support threads after fork") class BaseTest(unittest.TestCase): @@ -731,6 +734,7 @@ def remove_loop(fname, tries): @support.requires_fork() @threading_helper.requires_working_threading() @skip_if_asan_fork + @skip_if_tsan_fork def test_post_fork_child_no_deadlock(self): """Ensure child logging locks are not held; bpo-6721 & bpo-36533.""" class _OurHandler(logging.Handler): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 3b5c37c948c8c3..9769cb41e3e689 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -50,6 +50,11 @@ def skip_unless_reliable_fork(test): return test +skip_if_tsan_fork = unittest.skipIf( + support.check_sanitizer(thread=True), + "TSAN doesn't support threads after fork") + + def requires_subinterpreters(meth): """Decorator to skip a test if subinterpreters are not supported.""" return unittest.skipIf(interpreters is None, @@ -634,6 +639,7 @@ def test_daemon_param(self): self.assertTrue(t.daemon) @skip_unless_reliable_fork + @skip_if_tsan_fork def test_dummy_thread_after_fork(self): # Issue #14308: a dummy thread in the active list doesn't mess up # the after-fork mechanism. @@ -703,6 +709,7 @@ def f(): @skip_unless_reliable_fork @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + @skip_if_tsan_fork def test_main_thread_after_fork(self): code = """if 1: import os, threading @@ -1271,6 +1278,7 @@ def test_2_join_in_forked_process(self): self._run_and_join(script) @skip_unless_reliable_fork + @skip_if_tsan_fork def test_3_join_in_forked_from_thread(self): # Like the test above, but fork() was called from a worker thread # In the forked process, the main Thread object must be marked as stopped. diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 64cc60053e6cf7..65d366e91c322a 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -95,6 +95,10 @@ #endif #endif +/* Thread sanitizer doesn't currently support sem_clockwait */ +#ifdef _Py_THREAD_SANITIZER +#undef HAVE_SEM_CLOCKWAIT +#endif /* Whether or not to use semaphores directly rather than emulating them with * mutexes and condition variables: From 269051d20e65eda30734cbbbdb07d21df61978d6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 16 Mar 2024 12:29:42 +0200 Subject: [PATCH 119/158] gh-90535: Fix support of interval>1 in logging.TimedRotatingFileHandler (GH-116220) Fix support of interval values > 1 in logging.TimedRotatingFileHandler for when='MIDNIGHT' and when='Wx'. --- Lib/logging/handlers.py | 5 +- Lib/test/test_logging.py | 200 +++++++++++++++--- ...4-03-01-20-23-57.gh-issue-90535.wXm-jC.rst | 3 + 3 files changed, 173 insertions(+), 35 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-01-20-23-57.gh-issue-90535.wXm-jC.rst diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 30cfe064478281..410bd9851f366d 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -340,7 +340,10 @@ def computeRollover(self, currentTime): daysToWait = self.dayOfWeek - day else: daysToWait = 6 - day + self.dayOfWeek + 1 - result += daysToWait * (60 * 60 * 24) + result += daysToWait * _MIDNIGHT + result += self.interval - _MIDNIGHT * 7 + else: + result += self.interval - _MIDNIGHT if not self.utc: dstNow = t[-1] dstAtRollover = time.localtime(result)[-1] diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index c84eca51b52362..7b5bc6b6a74180 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -6577,6 +6577,129 @@ def test(current, expected): fh.close() + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_MIDNIGHT_local_interval(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, interval=3) + + test(DT(2012, 3, 8, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 3, 9, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 9, 1, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 13, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 3, 14, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 3, 14, 0, 0)) + + test(DT(2012, 11, 1, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 11, 2, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 11, 2, 1, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 6, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 7, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 7, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, interval=3, + atTime=datetime.time(12, 0, 0)) + + test(DT(2012, 3, 8, 11, 59, 59), DT(2012, 3, 10, 12, 0)) + test(DT(2012, 3, 8, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 8, 13, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 10, 11, 59, 59), DT(2012, 3, 12, 12, 0)) + test(DT(2012, 3, 10, 12, 0), DT(2012, 3, 13, 12, 0)) + test(DT(2012, 3, 10, 13, 0), DT(2012, 3, 13, 12, 0)) + + test(DT(2012, 11, 1, 11, 59, 59), DT(2012, 11, 3, 12, 0)) + test(DT(2012, 11, 1, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 11, 1, 13, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 11, 3, 11, 59, 59), DT(2012, 11, 5, 12, 0)) + test(DT(2012, 11, 3, 12, 0), DT(2012, 11, 6, 12, 0)) + test(DT(2012, 11, 3, 13, 0), DT(2012, 11, 6, 12, 0)) + + fh.close() + + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_W6_local_interval(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, interval=3) + + test(DT(2012, 2, 19, 23, 59, 59), DT(2012, 3, 5, 0, 0)) + test(DT(2012, 2, 20, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 2, 20, 1, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 4, 23, 59, 59), DT(2012, 3, 19, 0, 0)) + test(DT(2012, 3, 5, 0, 0), DT(2012, 3, 26, 0, 0)) + test(DT(2012, 3, 5, 1, 0), DT(2012, 3, 26, 0, 0)) + + test(DT(2012, 10, 14, 23, 59, 59), DT(2012, 10, 29, 0, 0)) + test(DT(2012, 10, 15, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 10, 15, 1, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 10, 28, 23, 59, 59), DT(2012, 11, 12, 0, 0)) + test(DT(2012, 10, 29, 0, 0), DT(2012, 11, 19, 0, 0)) + test(DT(2012, 10, 29, 1, 0), DT(2012, 11, 19, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, interval=3, + atTime=datetime.time(0, 0, 0)) + + test(DT(2012, 2, 25, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 2, 26, 0, 0), DT(2012, 3, 18, 0, 0)) + test(DT(2012, 2, 26, 1, 0), DT(2012, 3, 18, 0, 0)) + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 25, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 4, 1, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 4, 1, 0, 0)) + + test(DT(2012, 10, 20, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 10, 21, 0, 0), DT(2012, 11, 11, 0, 0)) + test(DT(2012, 10, 21, 1, 0), DT(2012, 11, 11, 0, 0)) + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 18, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 25, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 25, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, interval=3, + atTime=datetime.time(12, 0, 0)) + + test(DT(2012, 2, 18, 11, 59, 59), DT(2012, 3, 4, 12, 0)) + test(DT(2012, 2, 19, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 2, 19, 13, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 4, 11, 59, 59), DT(2012, 3, 18, 12, 0)) + test(DT(2012, 3, 4, 12, 0), DT(2012, 3, 25, 12, 0)) + test(DT(2012, 3, 4, 13, 0), DT(2012, 3, 25, 12, 0)) + + test(DT(2012, 10, 14, 11, 59, 59), DT(2012, 10, 28, 12, 0)) + test(DT(2012, 10, 14, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 10, 14, 13, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 10, 28, 11, 59, 59), DT(2012, 11, 11, 12, 0)) + test(DT(2012, 10, 28, 12, 0), DT(2012, 11, 18, 12, 0)) + test(DT(2012, 10, 28, 13, 0), DT(2012, 11, 18, 12, 0)) + + fh.close() + + def secs(**kw): return datetime.timedelta(**kw) // datetime.timedelta(seconds=1) @@ -6588,40 +6711,49 @@ def secs(**kw): # current time (epoch start) is a Thursday, W0 means Monday ('W0', secs(days=4, hours=24)), ): - def test_compute_rollover(self, when=when, exp=exp): - rh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when=when, interval=1, backupCount=0, utc=True) - currentTime = 0.0 - actual = rh.computeRollover(currentTime) - if exp != actual: - # Failures occur on some systems for MIDNIGHT and W0. - # Print detailed calculation for MIDNIGHT so we can try to see - # what's going on - if when == 'MIDNIGHT': - try: - if rh.utc: - t = time.gmtime(currentTime) - else: - t = time.localtime(currentTime) - currentHour = t[3] - currentMinute = t[4] - currentSecond = t[5] - # r is the number of seconds left between now and midnight - r = logging.handlers._MIDNIGHT - ((currentHour * 60 + - currentMinute) * 60 + - currentSecond) - result = currentTime + r - print('t: %s (%s)' % (t, rh.utc), file=sys.stderr) - print('currentHour: %s' % currentHour, file=sys.stderr) - print('currentMinute: %s' % currentMinute, file=sys.stderr) - print('currentSecond: %s' % currentSecond, file=sys.stderr) - print('r: %s' % r, file=sys.stderr) - print('result: %s' % result, file=sys.stderr) - except Exception as e: - print('exception in diagnostic code: %s' % e, file=sys.stderr) - self.assertEqual(exp, actual) - rh.close() - setattr(TimedRotatingFileHandlerTest, "test_compute_rollover_%s" % when, test_compute_rollover) + for interval in 1, 3: + def test_compute_rollover(self, when=when, interval=interval, exp=exp): + rh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when=when, interval=interval, backupCount=0, utc=True) + currentTime = 0.0 + actual = rh.computeRollover(currentTime) + if when.startswith('W'): + exp += secs(days=7*(interval-1)) + else: + exp *= interval + if exp != actual: + # Failures occur on some systems for MIDNIGHT and W0. + # Print detailed calculation for MIDNIGHT so we can try to see + # what's going on + if when == 'MIDNIGHT': + try: + if rh.utc: + t = time.gmtime(currentTime) + else: + t = time.localtime(currentTime) + currentHour = t[3] + currentMinute = t[4] + currentSecond = t[5] + # r is the number of seconds left between now and midnight + r = logging.handlers._MIDNIGHT - ((currentHour * 60 + + currentMinute) * 60 + + currentSecond) + result = currentTime + r + print('t: %s (%s)' % (t, rh.utc), file=sys.stderr) + print('currentHour: %s' % currentHour, file=sys.stderr) + print('currentMinute: %s' % currentMinute, file=sys.stderr) + print('currentSecond: %s' % currentSecond, file=sys.stderr) + print('r: %s' % r, file=sys.stderr) + print('result: %s' % result, file=sys.stderr) + except Exception as e: + print('exception in diagnostic code: %s' % e, file=sys.stderr) + self.assertEqual(exp, actual) + rh.close() + name = "test_compute_rollover_%s" % when + if interval > 1: + name += "_interval" + test_compute_rollover.__name__ = name + setattr(TimedRotatingFileHandlerTest, name, test_compute_rollover) @unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil/pywintypes required for this test.') diff --git a/Misc/NEWS.d/next/Library/2024-03-01-20-23-57.gh-issue-90535.wXm-jC.rst b/Misc/NEWS.d/next/Library/2024-03-01-20-23-57.gh-issue-90535.wXm-jC.rst new file mode 100644 index 00000000000000..9af4efabb6b5b2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-01-20-23-57.gh-issue-90535.wXm-jC.rst @@ -0,0 +1,3 @@ +Fix support of *interval* values > 1 in +:class:`logging.TimedRotatingFileHandler` for ``when='MIDNIGHT'`` and +``when='Wx'``. From 1069a462f611f0b70b6eec0bba603d618a0378f3 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 16 Mar 2024 12:36:05 +0200 Subject: [PATCH 120/158] gh-116764: Fix regressions in urllib.parse.parse_qsl() (GH-116801) * Restore support of None and other false values. * Raise TypeError for non-zero integers and non-empty sequences. The regressions were introduced in gh-74668 (bdba8ef42b15e651dc23374a08143cc2b4c4657d). --- Lib/test/test_urlparse.py | 24 +++++++++++++++++++ Lib/urllib/parse.py | 6 ++++- ...-03-14-14-01-46.gh-issue-116764.moB3Lc.rst | 4 ++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-14-14-01-46.gh-issue-116764.moB3Lc.rst diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 72f028680f4cb5..236b6e4516490a 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -1072,6 +1072,30 @@ def test_parse_qsl_separator(self): result_bytes = urllib.parse.parse_qsl(orig, separator=b';') self.assertEqual(result_bytes, expect, "Error parsing %r" % orig) + def test_parse_qsl_bytes(self): + self.assertEqual(urllib.parse.parse_qsl(b'a=b'), [(b'a', b'b')]) + self.assertEqual(urllib.parse.parse_qsl(bytearray(b'a=b')), [(b'a', b'b')]) + self.assertEqual(urllib.parse.parse_qsl(memoryview(b'a=b')), [(b'a', b'b')]) + + def test_parse_qsl_false_value(self): + kwargs = dict(keep_blank_values=True, strict_parsing=True) + for x in '', b'', None, 0, 0.0, [], {}, memoryview(b''): + self.assertEqual(urllib.parse.parse_qsl(x, **kwargs), []) + self.assertRaises(ValueError, urllib.parse.parse_qsl, x, separator=1) + + def test_parse_qsl_errors(self): + self.assertRaises(TypeError, urllib.parse.parse_qsl, list(b'a=b')) + self.assertRaises(TypeError, urllib.parse.parse_qsl, iter(b'a=b')) + self.assertRaises(TypeError, urllib.parse.parse_qsl, 1) + self.assertRaises(TypeError, urllib.parse.parse_qsl, object()) + + for separator in '', b'', None, 0, 1, 0.0, 1.5: + with self.assertRaises(ValueError): + urllib.parse.parse_qsl('a=b', separator=separator) + with self.assertRaises(UnicodeEncodeError): + urllib.parse.parse_qsl(b'a=b', separator='\xa6') + with self.assertRaises(UnicodeDecodeError): + urllib.parse.parse_qsl('a=b', separator=b'\xa6') def test_urlencode_sequences(self): # Other tests incidentally urlencode things; test non-covered cases: diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index ec52821125005c..fc9e7c99f283be 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -773,7 +773,11 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, def _unquote(s): return unquote_plus(s, encoding=encoding, errors=errors) else: - qs = bytes(qs) + if not qs: + return [] + # Use memoryview() to reject integers and iterables, + # acceptable by the bytes constructor. + qs = bytes(memoryview(qs)) if isinstance(separator, str): separator = bytes(separator, 'ascii') eq = b'=' diff --git a/Misc/NEWS.d/next/Library/2024-03-14-14-01-46.gh-issue-116764.moB3Lc.rst b/Misc/NEWS.d/next/Library/2024-03-14-14-01-46.gh-issue-116764.moB3Lc.rst new file mode 100644 index 00000000000000..e92034b0e8b157 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-14-14-01-46.gh-issue-116764.moB3Lc.rst @@ -0,0 +1,4 @@ +Restore support of ``None`` and other false values in :mod:`urllib.parse` +functions :func:`~urllib.parse.parse_qs` and +:func:`~urllib.parse.parse_qsl`. Also, they now raise a TypeError for +non-zero integers and non-empty sequences. From c61cb507c10c5b597928284e087a9a384ab267d0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 16 Mar 2024 13:31:19 +0200 Subject: [PATCH 121/158] gh-116484: Fix collisions between Checkbutton and ttk.Checkbutton default names (GH-116495) Change automatically generated tkinter.Checkbutton widget names to avoid collisions with automatically generated tkinter.ttk.Checkbutton widget names within the same parent widget. --- Lib/test/test_ttk/test_widgets.py | 22 ++++++++++++++++++- Lib/tkinter/__init__.py | 7 +++++- ...-03-08-11-31-49.gh-issue-116484.VMAsU7.rst | 3 +++ 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-08-11-31-49.gh-issue-116484.VMAsU7.rst diff --git a/Lib/test/test_ttk/test_widgets.py b/Lib/test/test_ttk/test_widgets.py index fd1a748a498ac5..e3e440c45859f7 100644 --- a/Lib/test/test_ttk/test_widgets.py +++ b/Lib/test/test_ttk/test_widgets.py @@ -285,9 +285,29 @@ def test_unique_variables(self): b.pack() buttons.append(b) variables = [str(b['variable']) for b in buttons] - print(variables) self.assertEqual(len(set(variables)), 4, variables) + def test_unique_variables2(self): + buttons = [] + f = ttk.Frame(self.root) + f.pack() + f = ttk.Frame(self.root) + f.pack() + for j in 'AB': + b = tkinter.Checkbutton(f, text=j) + b.pack() + buttons.append(b) + # Should be larger than the number of all previously created + # tkinter.Checkbutton widgets: + for j in range(100): + b = ttk.Checkbutton(f, text=str(j)) + b.pack() + buttons.append(b) + names = [str(b) for b in buttons] + self.assertEqual(len(set(names)), len(buttons), names) + variables = [str(b['variable']) for b in buttons] + self.assertEqual(len(set(variables)), len(buttons), variables) + @add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) class EntryTest(AbstractWidgetTest, unittest.TestCase): diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 175bfbd7d912d2..fd7b48e3519990 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -3166,11 +3166,16 @@ def __init__(self, master=None, cnf={}, **kw): Widget.__init__(self, master, 'checkbutton', cnf, kw) def _setup(self, master, cnf): + # Because Checkbutton defaults to a variable with the same name as + # the widget, Checkbutton default names must be globally unique, + # not just unique within the parent widget. if not cnf.get('name'): global _checkbutton_count name = self.__class__.__name__.lower() _checkbutton_count += 1 - cnf['name'] = f'!{name}{_checkbutton_count}' + # To avoid collisions with ttk.Checkbutton, use the different + # name template. + cnf['name'] = f'!{name}-{_checkbutton_count}' super()._setup(master, cnf) def deselect(self): diff --git a/Misc/NEWS.d/next/Library/2024-03-08-11-31-49.gh-issue-116484.VMAsU7.rst b/Misc/NEWS.d/next/Library/2024-03-08-11-31-49.gh-issue-116484.VMAsU7.rst new file mode 100644 index 00000000000000..265c3810466d39 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-08-11-31-49.gh-issue-116484.VMAsU7.rst @@ -0,0 +1,3 @@ +Change automatically generated :class:`tkinter.Checkbutton` widget names to +avoid collisions with automatically generated +:class:`tkinter.ttk.Checkbutton` widget names within the same parent widget. From 86bc40dd414bceb3f93382cc9f670936de9d68be Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sat, 16 Mar 2024 12:55:46 +0100 Subject: [PATCH 122/158] gh-112536: Add test_threading to TSAN tests (#116898) --- Lib/test/libregrtest/tsan.py | 1 + Lib/test/support/script_helper.py | 4 ++-- Lib/test/test_threading.py | 24 ++++++++++++++++-------- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index 150fcec8816179..fde8ba937c0e4b 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -14,6 +14,7 @@ 'test_syslog', 'test_thread', 'test_threadedtempfile', + 'test_threading', 'test_threading_local', 'test_threadsignals', ] diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py index 759020c33aa700..65e0bc199e7f0b 100644 --- a/Lib/test/support/script_helper.py +++ b/Lib/test/support/script_helper.py @@ -63,8 +63,8 @@ class _PythonRunResult(collections.namedtuple("_PythonRunResult", """Helper for reporting Python subprocess run results""" def fail(self, cmd_line): """Provide helpful details about failed subcommand runs""" - # Limit to 80 lines to ASCII characters - maxlen = 80 * 100 + # Limit to 300 lines of ASCII characters + maxlen = 300 * 100 out, err = self.out, self.err if len(out) > maxlen: out = b'(... truncated stdout ...)' + out[-maxlen:] diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 9769cb41e3e689..886866626dc9f8 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -47,14 +47,11 @@ def skip_unless_reliable_fork(test): return unittest.skip("due to known OS bug related to thread+fork")(test) if support.HAVE_ASAN_FORK_BUG: return unittest.skip("libasan has a pthread_create() dead lock related to thread+fork")(test) + if support.check_sanitizer(thread=True): + return unittest.skip("TSAN doesn't support threads after fork") return test -skip_if_tsan_fork = unittest.skipIf( - support.check_sanitizer(thread=True), - "TSAN doesn't support threads after fork") - - def requires_subinterpreters(meth): """Decorator to skip a test if subinterpreters are not supported.""" return unittest.skipIf(interpreters is None, @@ -428,6 +425,10 @@ def test_finalize_running_thread(self): # Issue 1402: the PyGILState_Ensure / _Release functions may be called # very late on python exit: on deallocation of a running thread for # example. + if support.check_sanitizer(thread=True): + # the thread running `time.sleep(100)` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") import_module("ctypes") rc, out, err = assert_python_failure("-c", """if 1: @@ -460,6 +461,11 @@ def waitingThread(): def test_finalize_with_trace(self): # Issue1733757 # Avoid a deadlock when sys.settrace steps into threading._shutdown + if support.check_sanitizer(thread=True): + # the thread running `time.sleep(2)` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") + assert_python_ok("-c", """if 1: import sys, threading @@ -639,7 +645,6 @@ def test_daemon_param(self): self.assertTrue(t.daemon) @skip_unless_reliable_fork - @skip_if_tsan_fork def test_dummy_thread_after_fork(self): # Issue #14308: a dummy thread in the active list doesn't mess up # the after-fork mechanism. @@ -709,7 +714,6 @@ def f(): @skip_unless_reliable_fork @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") - @skip_if_tsan_fork def test_main_thread_after_fork(self): code = """if 1: import os, threading @@ -1278,7 +1282,6 @@ def test_2_join_in_forked_process(self): self._run_and_join(script) @skip_unless_reliable_fork - @skip_if_tsan_fork def test_3_join_in_forked_from_thread(self): # Like the test above, but fork() was called from a worker thread # In the forked process, the main Thread object must be marked as stopped. @@ -1311,6 +1314,11 @@ def test_4_daemon_threads(self): # Check that a daemon thread cannot crash the interpreter on shutdown # by manipulating internal structures that are being disposed of in # the main thread. + if support.check_sanitizer(thread=True): + # some of the threads running `random_io` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") + script = """if True: import os import random From 33da0e844c922b3dcded75fbb9b7be67cb013a17 Mon Sep 17 00:00:00 2001 From: mpage Date: Sat, 16 Mar 2024 05:56:30 -0700 Subject: [PATCH 123/158] gh-114271: Fix race in `Thread.join()` (#114839) There is a race between when `Thread._tstate_lock` is released[^1] in `Thread._wait_for_tstate_lock()` and when `Thread._stop()` asserts[^2] that it is unlocked. Consider the following execution involving threads A, B, and C: 1. A starts. 2. B joins A, blocking on its `_tstate_lock`. 3. C joins A, blocking on its `_tstate_lock`. 4. A finishes and releases its `_tstate_lock`. 5. B acquires A's `_tstate_lock` in `_wait_for_tstate_lock()`, releases it, but is swapped out before calling `_stop()`. 6. C is scheduled, acquires A's `_tstate_lock` in `_wait_for_tstate_lock()` but is swapped out before releasing it. 7. B is scheduled, calls `_stop()`, which asserts that A's `_tstate_lock` is not held. However, C holds it, so the assertion fails. The race can be reproduced[^3] by inserting sleeps at the appropriate points in the threading code. To do so, run the `repro_join_race.py` from the linked repo. There are two main parts to this PR: 1. `_tstate_lock` is replaced with an event that is attached to `PyThreadState`. The event is set by the runtime prior to the thread being cleared (in the same place that `_tstate_lock` was released). `Thread.join()` blocks waiting for the event to be set. 2. `_PyInterpreterState_WaitForThreads()` provides the ability to wait for all non-daemon threads to exit. To do so, an `is_daemon` predicate was added to `PyThreadState`. This field is set each time a thread is created. `threading._shutdown()` now calls into `_PyInterpreterState_WaitForThreads()` instead of waiting on `_tstate_lock`s. [^1]: https://github.com/python/cpython/blob/441affc9e7f419ef0b68f734505fa2f79fe653c7/Lib/threading.py#L1201 [^2]: https://github.com/python/cpython/blob/441affc9e7f419ef0b68f734505fa2f79fe653c7/Lib/threading.py#L1115 [^3]: https://github.com/mpage/cpython/commit/81946532792f938cd6f6ab4c4ff92a4edf61314f --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Antoine Pitrou --- Include/cpython/pystate.h | 26 - Include/internal/pycore_lock.h | 10 - Include/internal/pycore_pythread.h | 2 +- Lib/test/test_audit.py | 2 +- .../test_process_pool.py | 4 +- Lib/test/test_thread.py | 48 + Lib/test/test_threading.py | 61 +- Lib/threading.py | 218 +--- ...-02-01-03-09-38.gh-issue-114271.raCkt5.rst | 7 + Modules/_threadmodule.c | 979 ++++++++++++------ Python/lock.c | 24 - Python/pystate.c | 25 +- 12 files changed, 767 insertions(+), 639 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-02-01-03-09-38.gh-issue-114271.raCkt5.rst diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index ac7ff83748dbfc..38d0897ea13161 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -161,32 +161,6 @@ struct _ts { */ uintptr_t critical_section; - /* Called when a thread state is deleted normally, but not when it - * is destroyed after fork(). - * Pain: to prevent rare but fatal shutdown errors (issue 18808), - * Thread.join() must wait for the join'ed thread's tstate to be unlinked - * from the tstate chain. That happens at the end of a thread's life, - * in pystate.c. - * The obvious way doesn't quite work: create a lock which the tstate - * unlinking code releases, and have Thread.join() wait to acquire that - * lock. The problem is that we _are_ at the end of the thread's life: - * if the thread holds the last reference to the lock, decref'ing the - * lock will delete the lock, and that may trigger arbitrary Python code - * if there's a weakref, with a callback, to the lock. But by this time - * _PyRuntime.gilstate.tstate_current is already NULL, so only the simplest - * of C code can be allowed to run (in particular it must not be possible to - * release the GIL). - * So instead of holding the lock directly, the tstate holds a weakref to - * the lock: that's the value of on_delete_data below. Decref'ing a - * weakref is harmless. - * on_delete points to _threadmodule.c's static release_sentinel() function. - * After the tstate is unlinked, release_sentinel is called with the - * weakref-to-lock (on_delete_data) argument, and release_sentinel releases - * the indirectly held lock. - */ - void (*on_delete)(void *); - void *on_delete_data; - int coroutine_origin_tracking_depth; PyObject *async_gen_firstiter; diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index 971b4611a87f3b..f993c95ecbf75a 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -153,16 +153,6 @@ PyAPI_FUNC(void) PyEvent_Wait(PyEvent *evt); // and 0 if the timeout expired or thread was interrupted. PyAPI_FUNC(int) PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns); -// A one-time event notification with reference counting. -typedef struct _PyEventRc { - PyEvent event; - Py_ssize_t refcount; -} _PyEventRc; - -_PyEventRc *_PyEventRc_New(void); -void _PyEventRc_Incref(_PyEventRc *erc); -void _PyEventRc_Decref(_PyEventRc *erc); - // _PyRawMutex implements a word-sized mutex that that does not depend on the // parking lot API, and therefore can be used in the parking lot // implementation. diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index d2e7cc2a206ced..f032cb97388657 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -78,7 +78,7 @@ struct _pythread_runtime_state { } stubs; #endif - // Linked list of ThreadHandleObjects + // Linked list of ThreadHandles struct llist_node handles; }; diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index cd0a4e2264865d..c24c8213924196 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -209,7 +209,7 @@ def test_threading(self): expected = [ ("_thread.start_new_thread", "(, (), None)"), ("test.test_func", "()"), - ("_thread.start_joinable_thread", "(,)"), + ("_thread.start_joinable_thread", "(, 1, None)"), ("test.test_func", "()"), ] diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 7fc59a05f3deac..70444bb147fadc 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -201,13 +201,13 @@ def test_python_finalization_error(self): # QueueFeederThread. orig_start_new_thread = threading._start_joinable_thread nthread = 0 - def mock_start_new_thread(func, *args): + def mock_start_new_thread(func, *args, **kwargs): nonlocal nthread if nthread >= 1: raise RuntimeError("can't create new thread at " "interpreter shutdown") nthread += 1 - return orig_start_new_thread(func, *args) + return orig_start_new_thread(func, *args, **kwargs) with support.swap_attr(threading, '_start_joinable_thread', mock_start_new_thread): diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index 83235230d5c112..d94e04250c9307 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -289,6 +289,54 @@ def joiner(): with self.assertRaisesRegex(RuntimeError, "Cannot join current thread"): raise error + def test_join_with_timeout(self): + lock = thread.allocate_lock() + lock.acquire() + + def thr(): + lock.acquire() + + with threading_helper.wait_threads_exit(): + handle = thread.start_joinable_thread(thr) + handle.join(0.1) + self.assertFalse(handle.is_done()) + lock.release() + handle.join() + self.assertTrue(handle.is_done()) + + def test_join_unstarted(self): + handle = thread._ThreadHandle() + with self.assertRaisesRegex(RuntimeError, "thread not started"): + handle.join() + + def test_set_done_unstarted(self): + handle = thread._ThreadHandle() + with self.assertRaisesRegex(RuntimeError, "thread not started"): + handle._set_done() + + def test_start_duplicate_handle(self): + lock = thread.allocate_lock() + lock.acquire() + + def func(): + lock.acquire() + + handle = thread._ThreadHandle() + with threading_helper.wait_threads_exit(): + thread.start_joinable_thread(func, handle=handle) + with self.assertRaisesRegex(RuntimeError, "thread already started"): + thread.start_joinable_thread(func, handle=handle) + lock.release() + handle.join() + + def test_start_with_none_handle(self): + def func(): + pass + + with threading_helper.wait_threads_exit(): + handle = thread.start_joinable_thread(func, handle=None) + handle.join() + class Barrier: def __init__(self, num_threads): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 886866626dc9f8..f1dc12944cb6cc 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -408,7 +408,7 @@ def run(self): def test_limbo_cleanup(self): # Issue 7481: Failure to start thread should cleanup the limbo map. - def fail_new_thread(*args): + def fail_new_thread(*args, **kwargs): raise threading.ThreadError() _start_joinable_thread = threading._start_joinable_thread threading._start_joinable_thread = fail_new_thread @@ -912,41 +912,6 @@ def f(): rc, out, err = assert_python_ok("-c", code) self.assertEqual(err, b"") - def test_tstate_lock(self): - # Test an implementation detail of Thread objects. - started = _thread.allocate_lock() - finish = _thread.allocate_lock() - started.acquire() - finish.acquire() - def f(): - started.release() - finish.acquire() - time.sleep(0.01) - # The tstate lock is None until the thread is started - t = threading.Thread(target=f) - self.assertIs(t._tstate_lock, None) - t.start() - started.acquire() - self.assertTrue(t.is_alive()) - # The tstate lock can't be acquired when the thread is running - # (or suspended). - tstate_lock = t._tstate_lock - self.assertFalse(tstate_lock.acquire(timeout=0), False) - finish.release() - # When the thread ends, the state_lock can be successfully - # acquired. - self.assertTrue(tstate_lock.acquire(timeout=support.SHORT_TIMEOUT), False) - # But is_alive() is still True: we hold _tstate_lock now, which - # prevents is_alive() from knowing the thread's end-of-life C code - # is done. - self.assertTrue(t.is_alive()) - # Let is_alive() find out the C code is done. - tstate_lock.release() - self.assertFalse(t.is_alive()) - # And verify the thread disposed of _tstate_lock. - self.assertIsNone(t._tstate_lock) - t.join() - def test_repr_stopped(self): # Verify that "stopped" shows up in repr(Thread) appropriately. started = _thread.allocate_lock() @@ -1112,30 +1077,6 @@ def checker(): self.assertEqual(threading.getprofile(), old_profile) self.assertEqual(sys.getprofile(), old_profile) - @cpython_only - def test_shutdown_locks(self): - for daemon in (False, True): - with self.subTest(daemon=daemon): - event = threading.Event() - thread = threading.Thread(target=event.wait, daemon=daemon) - - # Thread.start() must add lock to _shutdown_locks, - # but only for non-daemon thread - thread.start() - tstate_lock = thread._tstate_lock - if not daemon: - self.assertIn(tstate_lock, threading._shutdown_locks) - else: - self.assertNotIn(tstate_lock, threading._shutdown_locks) - - # unblock the thread and join it - event.set() - thread.join() - - # Thread._stop() must remove tstate_lock from _shutdown_locks. - # Daemon threads must never add it to _shutdown_locks. - self.assertNotIn(tstate_lock, threading._shutdown_locks) - def test_locals_at_exit(self): # bpo-19466: thread locals must not be deleted before destructors # are called diff --git a/Lib/threading.py b/Lib/threading.py index ec89550d6b022e..31ab77c92b1c20 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -36,8 +36,11 @@ _daemon_threads_allowed = _thread.daemon_threads_allowed _allocate_lock = _thread.allocate_lock _LockType = _thread.LockType -_set_sentinel = _thread._set_sentinel +_thread_shutdown = _thread._shutdown +_make_thread_handle = _thread._make_thread_handle +_ThreadHandle = _thread._ThreadHandle get_ident = _thread.get_ident +_get_main_thread_ident = _thread._get_main_thread_ident _is_main_interpreter = _thread._is_main_interpreter try: get_native_id = _thread.get_native_id @@ -847,25 +850,6 @@ def _newname(name_template): _limbo = {} _dangling = WeakSet() -# Set of Thread._tstate_lock locks of non-daemon threads used by _shutdown() -# to wait until all Python thread states get deleted: -# see Thread._set_tstate_lock(). -_shutdown_locks_lock = _allocate_lock() -_shutdown_locks = set() - -def _maintain_shutdown_locks(): - """ - Drop any shutdown locks that don't correspond to running threads anymore. - - Calling this from time to time avoids an ever-growing _shutdown_locks - set when Thread objects are not joined explicitly. See bpo-37788. - - This must be called with _shutdown_locks_lock acquired. - """ - # If a lock was released, the corresponding thread has exited - to_remove = [lock for lock in _shutdown_locks if not lock.locked()] - _shutdown_locks.difference_update(to_remove) - # Main class for threads @@ -930,10 +914,8 @@ class is implemented. self._ident = None if _HAVE_THREAD_NATIVE_ID: self._native_id = None - self._tstate_lock = None - self._handle = None + self._handle = _ThreadHandle() self._started = Event() - self._is_stopped = False self._initialized = True # Copy of sys.stderr used by self._invoke_excepthook() self._stderr = _sys.stderr @@ -947,28 +929,18 @@ def _after_fork(self, new_ident=None): if new_ident is not None: # This thread is alive. self._ident = new_ident - if self._handle is not None: - assert self._handle.ident == new_ident - # bpo-42350: If the fork happens when the thread is already stopped - # (ex: after threading._shutdown() has been called), _tstate_lock - # is None. Do nothing in this case. - if self._tstate_lock is not None: - self._tstate_lock._at_fork_reinit() - self._tstate_lock.acquire() + assert self._handle.ident == new_ident else: - # This thread isn't alive after fork: it doesn't have a tstate - # anymore. - self._is_stopped = True - self._tstate_lock = None - self._handle = None + # Otherwise, the thread is dead, Jim. _PyThread_AfterFork() + # already marked our handle done. + pass def __repr__(self): assert self._initialized, "Thread.__init__() was not called" status = "initial" if self._started.is_set(): status = "started" - self.is_alive() # easy way to get ._is_stopped set when appropriate - if self._is_stopped: + if self._handle.is_done(): status = "stopped" if self._daemonic: status += " daemon" @@ -996,7 +968,8 @@ def start(self): _limbo[self] = self try: # Start joinable thread - self._handle = _start_joinable_thread(self._bootstrap) + _start_joinable_thread(self._bootstrap, handle=self._handle, + daemon=self.daemon) except Exception: with _active_limbo_lock: del _limbo[self] @@ -1047,23 +1020,9 @@ def _set_ident(self): def _set_native_id(self): self._native_id = get_native_id() - def _set_tstate_lock(self): - """ - Set a lock object which will be released by the interpreter when - the underlying thread state (see pystate.h) gets deleted. - """ - self._tstate_lock = _set_sentinel() - self._tstate_lock.acquire() - - if not self.daemon: - with _shutdown_locks_lock: - _maintain_shutdown_locks() - _shutdown_locks.add(self._tstate_lock) - def _bootstrap_inner(self): try: self._set_ident() - self._set_tstate_lock() if _HAVE_THREAD_NATIVE_ID: self._set_native_id() self._started.set() @@ -1083,33 +1042,6 @@ def _bootstrap_inner(self): finally: self._delete() - def _stop(self): - # After calling ._stop(), .is_alive() returns False and .join() returns - # immediately. ._tstate_lock must be released before calling ._stop(). - # - # Normal case: C code at the end of the thread's life - # (release_sentinel in _threadmodule.c) releases ._tstate_lock, and - # that's detected by our ._wait_for_tstate_lock(), called by .join() - # and .is_alive(). Any number of threads _may_ call ._stop() - # simultaneously (for example, if multiple threads are blocked in - # .join() calls), and they're not serialized. That's harmless - - # they'll just make redundant rebindings of ._is_stopped and - # ._tstate_lock. Obscure: we rebind ._tstate_lock last so that the - # "assert self._is_stopped" in ._wait_for_tstate_lock() always works - # (the assert is executed only if ._tstate_lock is None). - # - # Special case: _main_thread releases ._tstate_lock via this - # module's _shutdown() function. - lock = self._tstate_lock - if lock is not None: - assert not lock.locked() - self._is_stopped = True - self._tstate_lock = None - if not self.daemon: - with _shutdown_locks_lock: - # Remove our lock and other released locks from _shutdown_locks - _maintain_shutdown_locks() - def _delete(self): "Remove current thread from the dict of currently running threads." with _active_limbo_lock: @@ -1150,47 +1082,12 @@ def join(self, timeout=None): if self is current_thread(): raise RuntimeError("cannot join current thread") - if timeout is None: - self._wait_for_tstate_lock() - else: - # the behavior of a negative timeout isn't documented, but - # historically .join(timeout=x) for x<0 has acted as if timeout=0 - self._wait_for_tstate_lock(timeout=max(timeout, 0)) - - if self._is_stopped: - self._join_os_thread() - - def _join_os_thread(self): - # self._handle may be cleared post-fork - if self._handle is not None: - self._handle.join() - - def _wait_for_tstate_lock(self, block=True, timeout=-1): - # Issue #18808: wait for the thread state to be gone. - # At the end of the thread's life, after all knowledge of the thread - # is removed from C data structures, C code releases our _tstate_lock. - # This method passes its arguments to _tstate_lock.acquire(). - # If the lock is acquired, the C code is done, and self._stop() is - # called. That sets ._is_stopped to True, and ._tstate_lock to None. - lock = self._tstate_lock - if lock is None: - # already determined that the C code is done - assert self._is_stopped - return + # the behavior of a negative timeout isn't documented, but + # historically .join(timeout=x) for x<0 has acted as if timeout=0 + if timeout is not None: + timeout = max(timeout, 0) - try: - if lock.acquire(block, timeout): - lock.release() - self._stop() - except: - if lock.locked(): - # bpo-45274: lock.acquire() acquired the lock, but the function - # was interrupted with an exception before reaching the - # lock.release(). It can happen if a signal handler raises an - # exception, like CTRL+C which raises KeyboardInterrupt. - lock.release() - self._stop() - raise + self._handle.join(timeout) @property def name(self): @@ -1241,13 +1138,7 @@ def is_alive(self): """ assert self._initialized, "Thread.__init__() not called" - if self._is_stopped or not self._started.is_set(): - return False - self._wait_for_tstate_lock(False) - if not self._is_stopped: - return True - self._join_os_thread() - return False + return self._started.is_set() and not self._handle.is_done() @property def daemon(self): @@ -1456,18 +1347,14 @@ class _MainThread(Thread): def __init__(self): Thread.__init__(self, name="MainThread", daemon=False) - self._set_tstate_lock() self._started.set() - self._set_ident() + self._ident = _get_main_thread_ident() + self._handle = _make_thread_handle(self._ident) if _HAVE_THREAD_NATIVE_ID: self._set_native_id() with _active_limbo_lock: _active[self._ident] = self - def _join_os_thread(self): - # No ThreadHandle for main thread - pass - # Helper thread-local instance to detect when a _DummyThread # is collected. Not a part of the public API. @@ -1510,17 +1397,15 @@ def __init__(self): daemon=_daemon_threads_allowed()) self._started.set() self._set_ident() + self._handle = _make_thread_handle(self._ident) if _HAVE_THREAD_NATIVE_ID: self._set_native_id() with _active_limbo_lock: _active[self._ident] = self _DeleteDummyThreadOnDel(self) - def _stop(self): - pass - def is_alive(self): - if not self._is_stopped and self._started.is_set(): + if not self._handle.is_done() and self._started.is_set(): return True raise RuntimeError("thread is not alive") @@ -1532,7 +1417,6 @@ def _after_fork(self, new_ident=None): self.__class__ = _MainThread self._name = 'MainThread' self._daemonic = False - self._set_tstate_lock() Thread._after_fork(self, new_ident=new_ident) @@ -1631,12 +1515,11 @@ def _shutdown(): """ Wait until the Python thread state of all non-daemon threads get deleted. """ - # Obscure: other threads may be waiting to join _main_thread. That's - # dubious, but some code does it. We can't wait for C code to release - # the main thread's tstate_lock - that won't happen until the interpreter - # is nearly dead. So we release it here. Note that just calling _stop() - # isn't enough: other threads may already be waiting on _tstate_lock. - if _main_thread._is_stopped and _is_main_interpreter(): + # Obscure: other threads may be waiting to join _main_thread. That's + # dubious, but some code does it. We can't wait for it to be marked as done + # normally - that won't happen until the interpreter is nearly dead. So + # mark it done here. + if _main_thread._handle.is_done() and _is_main_interpreter(): # _shutdown() was already called return @@ -1648,42 +1531,11 @@ def _shutdown(): for atexit_call in reversed(_threading_atexits): atexit_call() - # Main thread - if _main_thread.ident == get_ident(): - tlock = _main_thread._tstate_lock - # The main thread isn't finished yet, so its thread state lock can't - # have been released. - assert tlock is not None - if tlock.locked(): - # It should have been released already by - # _PyInterpreterState_SetNotRunningMain(), but there may be - # embedders that aren't calling that yet. - tlock.release() - _main_thread._stop() - else: - # bpo-1596321: _shutdown() must be called in the main thread. - # If the threading module was not imported by the main thread, - # _main_thread is the thread which imported the threading module. - # In this case, ignore _main_thread, similar behavior than for threads - # spawned by C libraries or using _thread.start_new_thread(). - pass - - # Join all non-deamon threads - while True: - with _shutdown_locks_lock: - locks = list(_shutdown_locks) - _shutdown_locks.clear() - - if not locks: - break - - for lock in locks: - # mimic Thread.join() - lock.acquire() - lock.release() - - # new threads can be spawned while we were waiting for the other - # threads to complete + if _is_main_interpreter(): + _main_thread._handle._set_done() + + # Wait for all non-daemon threads to exit. + _thread_shutdown() def main_thread(): @@ -1703,7 +1555,6 @@ def _after_fork(): # Reset _active_limbo_lock, in case we forked while the lock was held # by another (non-forked) thread. http://bugs.python.org/issue874900 global _active_limbo_lock, _main_thread - global _shutdown_locks_lock, _shutdown_locks _active_limbo_lock = RLock() # fork() only copied the current thread; clear references to others. @@ -1719,10 +1570,6 @@ def _after_fork(): _main_thread = current - # reset _shutdown() locks: threads re-register their _tstate_lock below - _shutdown_locks_lock = _allocate_lock() - _shutdown_locks = set() - with _active_limbo_lock: # Dangling thread instances must still have their locks reset, # because someone may join() them. @@ -1739,7 +1586,6 @@ def _after_fork(): else: # All the others are already stopped. thread._after_fork() - thread._stop() _limbo.clear() _active.clear() diff --git a/Misc/NEWS.d/next/Library/2024-02-01-03-09-38.gh-issue-114271.raCkt5.rst b/Misc/NEWS.d/next/Library/2024-02-01-03-09-38.gh-issue-114271.raCkt5.rst new file mode 100644 index 00000000000000..2cd35eeda830b9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-01-03-09-38.gh-issue-114271.raCkt5.rst @@ -0,0 +1,7 @@ +Fix a race in ``threading.Thread.join()``. + +``threading._MainThread`` now always represents the main thread of the main +interpreter. + +``PyThreadState.on_delete`` and ``PyThreadState.on_delete_data`` have been +removed. diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index cc5396a035018f..6c84b7e08e226b 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -12,7 +12,6 @@ #include "pycore_time.h" // _PyTime_FromSeconds() #include "pycore_weakref.h" // _PyWeakref_GET_REF() -#include #include // offsetof() #ifdef HAVE_SIGNAL_H # include // SIGINT @@ -21,7 +20,6 @@ // ThreadError is just an alias to PyExc_RuntimeError #define ThreadError PyExc_RuntimeError - // Forward declarations static struct PyModuleDef thread_module; @@ -32,6 +30,10 @@ typedef struct { PyTypeObject *local_type; PyTypeObject *local_dummy_type; PyTypeObject *thread_handle_type; + + // Linked list of handles to all non-daemon threads created by the + // threading module. We wait for these to finish at shutdown. + struct llist_node shutdown_handles; } thread_module_state; static inline thread_module_state* @@ -44,76 +46,148 @@ get_thread_state(PyObject *module) // _ThreadHandle type -// Handles transition from RUNNING to one of JOINED, DETACHED, or INVALID (post -// fork). +// Handles state transitions according to the following diagram: +// +// NOT_STARTED -> STARTING -> RUNNING -> DONE +// | ^ +// | | +// +----- error --------+ typedef enum { - THREAD_HANDLE_RUNNING = 1, - THREAD_HANDLE_JOINED = 2, - THREAD_HANDLE_DETACHED = 3, - THREAD_HANDLE_INVALID = 4, + THREAD_HANDLE_NOT_STARTED = 1, + THREAD_HANDLE_STARTING = 2, + THREAD_HANDLE_RUNNING = 3, + THREAD_HANDLE_DONE = 4, } ThreadHandleState; -// A handle around an OS thread. +// A handle to wait for thread completion. +// +// This may be used to wait for threads that were spawned by the threading +// module as well as for the "main" thread of the threading module. In the +// former case an OS thread, identified by the `os_handle` field, will be +// associated with the handle. The handle "owns" this thread and ensures that +// the thread is either joined or detached after the handle is destroyed. // -// The OS thread is either joined or detached after the handle is destroyed. +// Joining the handle is idempotent; the underlying OS thread, if any, is +// joined or detached only once. Concurrent join operations are serialized +// until it is their turn to execute or an earlier operation completes +// successfully. Once a join has completed successfully all future joins +// complete immediately. // -// Joining the handle is idempotent; the underlying OS thread is joined or -// detached only once. Concurrent join operations are serialized until it is -// their turn to execute or an earlier operation completes successfully. Once a -// join has completed successfully all future joins complete immediately. +// This must be separately reference counted because it may be destroyed +// in `thread_run()` after the PyThreadState has been destroyed. typedef struct { - PyObject_HEAD struct llist_node node; // linked list node (see _pythread_runtime_state) - // The `ident` and `handle` fields are immutable once the object is visible - // to threads other than its creator, thus they do not need to be accessed - // atomically. + // linked list node (see thread_module_state) + struct llist_node shutdown_node; + + // The `ident`, `os_handle`, `has_os_handle`, and `state` fields are + // protected by `mutex`. PyThread_ident_t ident; - PyThread_handle_t handle; + PyThread_handle_t os_handle; + int has_os_handle; // Holds a value from the `ThreadHandleState` enum. int state; + PyMutex mutex; + // Set immediately before `thread_run` returns to indicate that the OS // thread is about to exit. This is used to avoid false positives when // detecting self-join attempts. See the comment in `ThreadHandle_join()` // for a more detailed explanation. - _PyEventRc *thread_is_exiting; + PyEvent thread_is_exiting; - // Serializes calls to `join`. + // Serializes calls to `join` and `set_done`. _PyOnceFlag once; -} ThreadHandleObject; + + Py_ssize_t refcount; +} ThreadHandle; static inline int -get_thread_handle_state(ThreadHandleObject *handle) +get_thread_handle_state(ThreadHandle *handle) { - return _Py_atomic_load_int(&handle->state); + PyMutex_Lock(&handle->mutex); + int state = handle->state; + PyMutex_Unlock(&handle->mutex); + return state; } static inline void -set_thread_handle_state(ThreadHandleObject *handle, ThreadHandleState state) +set_thread_handle_state(ThreadHandle *handle, ThreadHandleState state) { - _Py_atomic_store_int(&handle->state, state); + PyMutex_Lock(&handle->mutex); + handle->state = state; + PyMutex_Unlock(&handle->mutex); } -static ThreadHandleObject* -new_thread_handle(thread_module_state* state) +static PyThread_ident_t +ThreadHandle_ident(ThreadHandle *handle) { - _PyEventRc *event = _PyEventRc_New(); - if (event == NULL) { - PyErr_NoMemory(); - return NULL; + PyMutex_Lock(&handle->mutex); + PyThread_ident_t ident = handle->ident; + PyMutex_Unlock(&handle->mutex); + return ident; +} + +static int +ThreadHandle_get_os_handle(ThreadHandle *handle, PyThread_handle_t *os_handle) +{ + PyMutex_Lock(&handle->mutex); + int has_os_handle = handle->has_os_handle; + if (has_os_handle) { + *os_handle = handle->os_handle; + } + PyMutex_Unlock(&handle->mutex); + return has_os_handle; +} + +static void +add_to_shutdown_handles(thread_module_state *state, ThreadHandle *handle) +{ + HEAD_LOCK(&_PyRuntime); + llist_insert_tail(&state->shutdown_handles, &handle->shutdown_node); + HEAD_UNLOCK(&_PyRuntime); +} + +static void +clear_shutdown_handles(thread_module_state *state) +{ + HEAD_LOCK(&_PyRuntime); + struct llist_node *node; + llist_for_each_safe(node, &state->shutdown_handles) { + llist_remove(node); + } + HEAD_UNLOCK(&_PyRuntime); +} + +static void +remove_from_shutdown_handles(ThreadHandle *handle) +{ + HEAD_LOCK(&_PyRuntime); + if (handle->shutdown_node.next != NULL) { + llist_remove(&handle->shutdown_node); } - ThreadHandleObject* self = PyObject_New(ThreadHandleObject, state->thread_handle_type); + HEAD_UNLOCK(&_PyRuntime); +} + +static ThreadHandle * +ThreadHandle_new(void) +{ + ThreadHandle *self = + (ThreadHandle *)PyMem_RawCalloc(1, sizeof(ThreadHandle)); if (self == NULL) { - _PyEventRc_Decref(event); + PyErr_NoMemory(); return NULL; } self->ident = 0; - self->handle = 0; - self->thread_is_exiting = event; + self->os_handle = 0; + self->has_os_handle = 0; + self->thread_is_exiting = (PyEvent){0}; + self->mutex = (PyMutex){_Py_UNLOCKED}; self->once = (_PyOnceFlag){0}; - self->state = THREAD_HANDLE_INVALID; + self->state = THREAD_HANDLE_NOT_STARTED; + self->refcount = 1; HEAD_LOCK(&_PyRuntime); llist_insert_tail(&_PyRuntime.threads.handles, &self->node); @@ -123,9 +197,34 @@ new_thread_handle(thread_module_state* state) } static void -ThreadHandle_dealloc(ThreadHandleObject *self) +ThreadHandle_incref(ThreadHandle *self) { - PyObject *tp = (PyObject *) Py_TYPE(self); + _Py_atomic_add_ssize(&self->refcount, 1); +} + +static int +detach_thread(ThreadHandle *self) +{ + if (!self->has_os_handle) { + return 0; + } + // This is typically short so no need to release the GIL + if (PyThread_detach_thread(self->os_handle)) { + fprintf(stderr, "detach_thread: failed detaching thread\n"); + return -1; + } + return 0; +} + +// NB: This may be called after the PyThreadState in `thread_run` has been +// deleted; it cannot call anything that relies on a valid PyThreadState +// existing. +static void +ThreadHandle_decref(ThreadHandle *self) +{ + if (_Py_atomic_add_ssize(&self->refcount, -1) > 1) { + return; + } // Remove ourself from the global list of handles HEAD_LOCK(&_PyRuntime); @@ -134,23 +233,17 @@ ThreadHandle_dealloc(ThreadHandleObject *self) } HEAD_UNLOCK(&_PyRuntime); + assert(self->shutdown_node.next == NULL); + // It's safe to access state non-atomically: // 1. This is the destructor; nothing else holds a reference. - // 2. The refcount going to zero is a "synchronizes-with" event; - // all changes from other threads are visible. - if (self->state == THREAD_HANDLE_RUNNING) { - // This is typically short so no need to release the GIL - if (PyThread_detach_thread(self->handle)) { - PyErr_SetString(ThreadError, "Failed detaching thread"); - PyErr_WriteUnraisable(tp); - } - else { - self->state = THREAD_HANDLE_DETACHED; - } + // 2. The refcount going to zero is a "synchronizes-with" event; all + // changes from other threads are visible. + if (self->state == THREAD_HANDLE_RUNNING && !detach_thread(self)) { + self->state = THREAD_HANDLE_DONE; } - _PyEventRc_Decref(self->thread_is_exiting); - PyObject_Free(self); - Py_DECREF(tp); + + PyMem_RawFree(self); } void @@ -164,55 +257,229 @@ _PyThread_AfterFork(struct _pythread_runtime_state *state) struct llist_node *node; llist_for_each_safe(node, &state->handles) { - ThreadHandleObject *hobj = llist_data(node, ThreadHandleObject, node); - if (hobj->ident == current) { + ThreadHandle *handle = llist_data(node, ThreadHandle, node); + if (handle->ident == current) { continue; } - // Disallow calls to join() as they could crash. We are the only - // thread; it's safe to set this without an atomic. - hobj->state = THREAD_HANDLE_INVALID; + // Mark all threads as done. Any attempts to join or detach the + // underlying OS thread (if any) could crash. We are the only thread; + // it's safe to set this non-atomically. + handle->state = THREAD_HANDLE_DONE; + handle->once = (_PyOnceFlag){_Py_ONCE_INITIALIZED}; + handle->mutex = (PyMutex){_Py_UNLOCKED}; + _PyEvent_Notify(&handle->thread_is_exiting); llist_remove(node); + remove_from_shutdown_handles(handle); } } -static PyObject * -ThreadHandle_repr(ThreadHandleObject *self) +// bootstate is used to "bootstrap" new threads. Any arguments needed by +// `thread_run()`, which can only take a single argument due to platform +// limitations, are contained in bootstate. +struct bootstate { + PyThreadState *tstate; + PyObject *func; + PyObject *args; + PyObject *kwargs; + ThreadHandle *handle; + PyEvent handle_ready; +}; + +static void +thread_bootstate_free(struct bootstate *boot, int decref) { - return PyUnicode_FromFormat("<%s object: ident=%" PY_FORMAT_THREAD_IDENT_T ">", - Py_TYPE(self)->tp_name, self->ident); + if (decref) { + Py_DECREF(boot->func); + Py_DECREF(boot->args); + Py_XDECREF(boot->kwargs); + } + ThreadHandle_decref(boot->handle); + PyMem_RawFree(boot); } -static PyObject * -ThreadHandle_get_ident(ThreadHandleObject *self, void *ignored) +static void +thread_run(void *boot_raw) { - return PyLong_FromUnsignedLongLong(self->ident); + struct bootstate *boot = (struct bootstate *) boot_raw; + PyThreadState *tstate = boot->tstate; + + // Wait until the handle is marked as running + PyEvent_Wait(&boot->handle_ready); + + // `handle` needs to be manipulated after bootstate has been freed + ThreadHandle *handle = boot->handle; + ThreadHandle_incref(handle); + + // gh-108987: If _thread.start_new_thread() is called before or while + // Python is being finalized, thread_run() can called *after*. + // _PyRuntimeState_SetFinalizing() is called. At this point, all Python + // threads must exit, except of the thread calling Py_Finalize() whch holds + // the GIL and must not exit. + // + // At this stage, tstate can be a dangling pointer (point to freed memory), + // it's ok to call _PyThreadState_MustExit() with a dangling pointer. + if (_PyThreadState_MustExit(tstate)) { + // Don't call PyThreadState_Clear() nor _PyThreadState_DeleteCurrent(). + // These functions are called on tstate indirectly by Py_Finalize() + // which calls _PyInterpreterState_Clear(). + // + // Py_DECREF() cannot be called because the GIL is not held: leak + // references on purpose. Python is being finalized anyway. + thread_bootstate_free(boot, 0); + goto exit; + } + + _PyThreadState_Bind(tstate); + PyEval_AcquireThread(tstate); + _Py_atomic_add_ssize(&tstate->interp->threads.count, 1); + + PyObject *res = PyObject_Call(boot->func, boot->args, boot->kwargs); + if (res == NULL) { + if (PyErr_ExceptionMatches(PyExc_SystemExit)) + /* SystemExit is ignored silently */ + PyErr_Clear(); + else { + PyErr_FormatUnraisable( + "Exception ignored in thread started by %R", boot->func); + } + } + else { + Py_DECREF(res); + } + + thread_bootstate_free(boot, 1); + + _Py_atomic_add_ssize(&tstate->interp->threads.count, -1); + PyThreadState_Clear(tstate); + _PyThreadState_DeleteCurrent(tstate); + +exit: + // Don't need to wait for this thread anymore + remove_from_shutdown_handles(handle); + + _PyEvent_Notify(&handle->thread_is_exiting); + ThreadHandle_decref(handle); + + // bpo-44434: Don't call explicitly PyThread_exit_thread(). On Linux with + // the glibc, pthread_exit() can abort the whole process if dlopen() fails + // to open the libgcc_s.so library (ex: EMFILE error). + return; +} + +static int +force_done(ThreadHandle *handle) +{ + assert(get_thread_handle_state(handle) == THREAD_HANDLE_STARTING); + _PyEvent_Notify(&handle->thread_is_exiting); + set_thread_handle_state(handle, THREAD_HANDLE_DONE); + return 0; +} + +static int +ThreadHandle_start(ThreadHandle *self, PyObject *func, PyObject *args, + PyObject *kwargs) +{ + // Mark the handle as starting to prevent any other threads from doing so + PyMutex_Lock(&self->mutex); + if (self->state != THREAD_HANDLE_NOT_STARTED) { + PyMutex_Unlock(&self->mutex); + PyErr_SetString(ThreadError, "thread already started"); + return -1; + } + self->state = THREAD_HANDLE_STARTING; + PyMutex_Unlock(&self->mutex); + + // Do all the heavy lifting outside of the mutex. All other operations on + // the handle should fail since the handle is in the starting state. + + // gh-109795: Use PyMem_RawMalloc() instead of PyMem_Malloc(), + // because it should be possible to call thread_bootstate_free() + // without holding the GIL. + struct bootstate *boot = PyMem_RawMalloc(sizeof(struct bootstate)); + if (boot == NULL) { + PyErr_NoMemory(); + goto start_failed; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + boot->tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_THREADING); + if (boot->tstate == NULL) { + PyMem_RawFree(boot); + if (!PyErr_Occurred()) { + PyErr_NoMemory(); + } + goto start_failed; + } + boot->func = Py_NewRef(func); + boot->args = Py_NewRef(args); + boot->kwargs = Py_XNewRef(kwargs); + boot->handle = self; + ThreadHandle_incref(self); + boot->handle_ready = (PyEvent){0}; + + PyThread_ident_t ident; + PyThread_handle_t os_handle; + if (PyThread_start_joinable_thread(thread_run, boot, &ident, &os_handle)) { + PyThreadState_Clear(boot->tstate); + thread_bootstate_free(boot, 1); + PyErr_SetString(ThreadError, "can't start new thread"); + goto start_failed; + } + + // Mark the handle running + PyMutex_Lock(&self->mutex); + assert(self->state == THREAD_HANDLE_STARTING); + self->ident = ident; + self->has_os_handle = 1; + self->os_handle = os_handle; + self->state = THREAD_HANDLE_RUNNING; + PyMutex_Unlock(&self->mutex); + + // Unblock the thread + _PyEvent_Notify(&boot->handle_ready); + + return 0; + +start_failed: + _PyOnceFlag_CallOnce(&self->once, (_Py_once_fn_t *)force_done, self); + return -1; } static int -join_thread(ThreadHandleObject *handle) +join_thread(ThreadHandle *handle) { assert(get_thread_handle_state(handle) == THREAD_HANDLE_RUNNING); + PyThread_handle_t os_handle; + if (ThreadHandle_get_os_handle(handle, &os_handle)) { + int err = 0; + Py_BEGIN_ALLOW_THREADS + err = PyThread_join_thread(os_handle); + Py_END_ALLOW_THREADS + if (err) { + PyErr_SetString(ThreadError, "Failed joining thread"); + return -1; + } + } + set_thread_handle_state(handle, THREAD_HANDLE_DONE); + return 0; +} - int err; - Py_BEGIN_ALLOW_THREADS - err = PyThread_join_thread(handle->handle); - Py_END_ALLOW_THREADS - if (err) { - PyErr_SetString(ThreadError, "Failed joining thread"); +static int +check_started(ThreadHandle *self) +{ + ThreadHandleState state = get_thread_handle_state(self); + if (state < THREAD_HANDLE_RUNNING) { + PyErr_SetString(ThreadError, "thread not started"); return -1; } - set_thread_handle_state(handle, THREAD_HANDLE_JOINED); return 0; } -static PyObject * -ThreadHandle_join(ThreadHandleObject *self, void* ignored) +static int +ThreadHandle_join(ThreadHandle *self, PyTime_t timeout_ns) { - if (get_thread_handle_state(self) == THREAD_HANDLE_INVALID) { - PyErr_SetString(PyExc_ValueError, - "the handle is invalid and thus cannot be joined"); - return NULL; + if (check_started(self) < 0) { + return -1; } // We want to perform this check outside of the `_PyOnceFlag` to prevent @@ -225,45 +492,197 @@ ThreadHandle_join(ThreadHandleObject *self, void* ignored) // To work around this, we set `thread_is_exiting` immediately before // `thread_run` returns. We can be sure that we are not attempting to join // ourselves if the handle's thread is about to exit. - if (!_PyEvent_IsSet(&self->thread_is_exiting->event) && - self->ident == PyThread_get_thread_ident_ex()) { + if (!_PyEvent_IsSet(&self->thread_is_exiting) && + ThreadHandle_ident(self) == PyThread_get_thread_ident_ex()) { // PyThread_join_thread() would deadlock or error out. PyErr_SetString(ThreadError, "Cannot join current thread"); - return NULL; + return -1; + } + + // Wait until the deadline for the thread to exit. + PyTime_t deadline = timeout_ns != -1 ? _PyDeadline_Init(timeout_ns) : 0; + while (!PyEvent_WaitTimed(&self->thread_is_exiting, timeout_ns)) { + if (deadline) { + // _PyDeadline_Get will return a negative value if the deadline has + // been exceeded. + timeout_ns = Py_MAX(_PyDeadline_Get(deadline), 0); + } + + if (timeout_ns) { + // Interrupted + if (Py_MakePendingCalls() < 0) { + return -1; + } + } + else { + // Timed out + return 0; + } } if (_PyOnceFlag_CallOnce(&self->once, (_Py_once_fn_t *)join_thread, self) == -1) { + return -1; + } + assert(get_thread_handle_state(self) == THREAD_HANDLE_DONE); + return 0; +} + +static int +set_done(ThreadHandle *handle) +{ + assert(get_thread_handle_state(handle) == THREAD_HANDLE_RUNNING); + if (detach_thread(handle) < 0) { + PyErr_SetString(ThreadError, "failed detaching handle"); + return -1; + } + _PyEvent_Notify(&handle->thread_is_exiting); + set_thread_handle_state(handle, THREAD_HANDLE_DONE); + return 0; +} + +static int +ThreadHandle_set_done(ThreadHandle *self) +{ + if (check_started(self) < 0) { + return -1; + } + + if (_PyOnceFlag_CallOnce(&self->once, (_Py_once_fn_t *)set_done, self) == + -1) { + return -1; + } + assert(get_thread_handle_state(self) == THREAD_HANDLE_DONE); + return 0; +} + +// A wrapper around a ThreadHandle. +typedef struct { + PyObject_HEAD + + ThreadHandle *handle; +} PyThreadHandleObject; + +static PyThreadHandleObject * +PyThreadHandleObject_new(PyTypeObject *type) +{ + ThreadHandle *handle = ThreadHandle_new(); + if (handle == NULL) { + return NULL; + } + + PyThreadHandleObject *self = + (PyThreadHandleObject *)type->tp_alloc(type, 0); + if (self == NULL) { + ThreadHandle_decref(handle); + return NULL; + } + + self->handle = handle; + + return self; +} + +static PyObject * +PyThreadHandleObject_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + return (PyObject *)PyThreadHandleObject_new(type); +} + +static void +PyThreadHandleObject_dealloc(PyThreadHandleObject *self) +{ + PyObject *tp = (PyObject *) Py_TYPE(self); + ThreadHandle_decref(self->handle); + PyObject_Free(self); + Py_DECREF(tp); +} + +static PyObject * +PyThreadHandleObject_repr(PyThreadHandleObject *self) +{ + PyThread_ident_t ident = ThreadHandle_ident(self->handle); + return PyUnicode_FromFormat("<%s object: ident=%" PY_FORMAT_THREAD_IDENT_T ">", + Py_TYPE(self)->tp_name, ident); +} + +static PyObject * +PyThreadHandleObject_get_ident(PyThreadHandleObject *self, + PyObject *Py_UNUSED(ignored)) +{ + return PyLong_FromUnsignedLongLong(ThreadHandle_ident(self->handle)); +} + +static PyObject * +PyThreadHandleObject_join(PyThreadHandleObject *self, PyObject *args) +{ + PyObject *timeout_obj = NULL; + if (!PyArg_ParseTuple(args, "|O:join", &timeout_obj)) { + return NULL; + } + + PyTime_t timeout_ns = -1; + if (timeout_obj != NULL && timeout_obj != Py_None) { + if (_PyTime_FromSecondsObject(&timeout_ns, timeout_obj, + _PyTime_ROUND_TIMEOUT) < 0) { + return NULL; + } + } + + if (ThreadHandle_join(self->handle, timeout_ns) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +PyThreadHandleObject_is_done(PyThreadHandleObject *self, + PyObject *Py_UNUSED(ignored)) +{ + if (_PyEvent_IsSet(&self->handle->thread_is_exiting)) { + Py_RETURN_TRUE; + } + else { + Py_RETURN_FALSE; + } +} + +static PyObject * +PyThreadHandleObject_set_done(PyThreadHandleObject *self, + PyObject *Py_UNUSED(ignored)) +{ + if (ThreadHandle_set_done(self->handle) < 0) { return NULL; } - assert(get_thread_handle_state(self) == THREAD_HANDLE_JOINED); Py_RETURN_NONE; } static PyGetSetDef ThreadHandle_getsetlist[] = { - {"ident", (getter)ThreadHandle_get_ident, NULL, NULL}, + {"ident", (getter)PyThreadHandleObject_get_ident, NULL, NULL}, {0}, }; -static PyMethodDef ThreadHandle_methods[] = -{ - {"join", (PyCFunction)ThreadHandle_join, METH_NOARGS}, +static PyMethodDef ThreadHandle_methods[] = { + {"join", (PyCFunction)PyThreadHandleObject_join, METH_VARARGS, NULL}, + {"_set_done", (PyCFunction)PyThreadHandleObject_set_done, METH_NOARGS, NULL}, + {"is_done", (PyCFunction)PyThreadHandleObject_is_done, METH_NOARGS, NULL}, {0, 0} }; static PyType_Slot ThreadHandle_Type_slots[] = { - {Py_tp_dealloc, (destructor)ThreadHandle_dealloc}, - {Py_tp_repr, (reprfunc)ThreadHandle_repr}, + {Py_tp_dealloc, (destructor)PyThreadHandleObject_dealloc}, + {Py_tp_repr, (reprfunc)PyThreadHandleObject_repr}, {Py_tp_getset, ThreadHandle_getsetlist}, {Py_tp_methods, ThreadHandle_methods}, + {Py_tp_new, PyThreadHandleObject_tp_new}, {0, 0} }; static PyType_Spec ThreadHandle_Type_spec = { "_thread._ThreadHandle", - sizeof(ThreadHandleObject), + sizeof(PyThreadHandleObject), 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, ThreadHandle_Type_slots, }; @@ -1272,98 +1691,6 @@ _localdummy_destroyed(PyObject *localweakref, PyObject *dummyweakref) /* Module functions */ -// bootstate is used to "bootstrap" new threads. Any arguments needed by -// `thread_run()`, which can only take a single argument due to platform -// limitations, are contained in bootstate. -struct bootstate { - PyThreadState *tstate; - PyObject *func; - PyObject *args; - PyObject *kwargs; - _PyEventRc *thread_is_exiting; -}; - - -static void -thread_bootstate_free(struct bootstate *boot, int decref) -{ - if (decref) { - Py_DECREF(boot->func); - Py_DECREF(boot->args); - Py_XDECREF(boot->kwargs); - } - if (boot->thread_is_exiting != NULL) { - _PyEventRc_Decref(boot->thread_is_exiting); - } - PyMem_RawFree(boot); -} - - -static void -thread_run(void *boot_raw) -{ - struct bootstate *boot = (struct bootstate *) boot_raw; - PyThreadState *tstate = boot->tstate; - - // `thread_is_exiting` needs to be set after bootstate has been freed - _PyEventRc *thread_is_exiting = boot->thread_is_exiting; - boot->thread_is_exiting = NULL; - - // gh-108987: If _thread.start_new_thread() is called before or while - // Python is being finalized, thread_run() can called *after*. - // _PyRuntimeState_SetFinalizing() is called. At this point, all Python - // threads must exit, except of the thread calling Py_Finalize() whch holds - // the GIL and must not exit. - // - // At this stage, tstate can be a dangling pointer (point to freed memory), - // it's ok to call _PyThreadState_MustExit() with a dangling pointer. - if (_PyThreadState_MustExit(tstate)) { - // Don't call PyThreadState_Clear() nor _PyThreadState_DeleteCurrent(). - // These functions are called on tstate indirectly by Py_Finalize() - // which calls _PyInterpreterState_Clear(). - // - // Py_DECREF() cannot be called because the GIL is not held: leak - // references on purpose. Python is being finalized anyway. - thread_bootstate_free(boot, 0); - goto exit; - } - - _PyThreadState_Bind(tstate); - PyEval_AcquireThread(tstate); - _Py_atomic_add_ssize(&tstate->interp->threads.count, 1); - - PyObject *res = PyObject_Call(boot->func, boot->args, boot->kwargs); - if (res == NULL) { - if (PyErr_ExceptionMatches(PyExc_SystemExit)) - /* SystemExit is ignored silently */ - PyErr_Clear(); - else { - PyErr_FormatUnraisable( - "Exception ignored in thread started by %R", boot->func); - } - } - else { - Py_DECREF(res); - } - - thread_bootstate_free(boot, 1); - - _Py_atomic_add_ssize(&tstate->interp->threads.count, -1); - PyThreadState_Clear(tstate); - _PyThreadState_DeleteCurrent(tstate); - -exit: - if (thread_is_exiting != NULL) { - _PyEvent_Notify(&thread_is_exiting->event); - _PyEventRc_Decref(thread_is_exiting); - } - - // bpo-44434: Don't call explicitly PyThread_exit_thread(). On Linux with - // the glibc, pthread_exit() can abort the whole process if dlopen() fails - // to open the libgcc_s.so library (ex: EMFILE error). - return; -} - static PyObject * thread_daemon_threads_allowed(PyObject *module, PyObject *Py_UNUSED(ignored)) { @@ -1383,11 +1710,8 @@ Return True if daemon threads are allowed in the current interpreter,\n\ and False otherwise.\n"); static int -do_start_new_thread(thread_module_state* state, - PyObject *func, PyObject* args, PyObject* kwargs, - int joinable, - PyThread_ident_t* ident, PyThread_handle_t* handle, - _PyEventRc *thread_is_exiting) +do_start_new_thread(thread_module_state *state, PyObject *func, PyObject *args, + PyObject *kwargs, ThreadHandle *handle, int daemon) { PyInterpreterState *interp = _PyInterpreterState_GET(); if (!_PyInterpreterState_HasFeature(interp, Py_RTFLAGS_THREADS)) { @@ -1401,44 +1725,20 @@ do_start_new_thread(thread_module_state* state, return -1; } - // gh-109795: Use PyMem_RawMalloc() instead of PyMem_Malloc(), - // because it should be possible to call thread_bootstate_free() - // without holding the GIL. - struct bootstate *boot = PyMem_RawMalloc(sizeof(struct bootstate)); - if (boot == NULL) { - PyErr_NoMemory(); - return -1; + if (!daemon) { + // Add the handle before starting the thread to avoid adding a handle + // to a thread that has already finished (i.e. if the thread finishes + // before the call to `ThreadHandle_start()` below returns). + add_to_shutdown_handles(state, handle); } - boot->tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_THREADING); - if (boot->tstate == NULL) { - PyMem_RawFree(boot); - if (!PyErr_Occurred()) { - PyErr_NoMemory(); + + if (ThreadHandle_start(handle, func, args, kwargs) < 0) { + if (!daemon) { + remove_from_shutdown_handles(handle); } return -1; } - boot->func = Py_NewRef(func); - boot->args = Py_NewRef(args); - boot->kwargs = Py_XNewRef(kwargs); - boot->thread_is_exiting = thread_is_exiting; - if (thread_is_exiting != NULL) { - _PyEventRc_Incref(thread_is_exiting); - } - int err; - if (joinable) { - err = PyThread_start_joinable_thread(thread_run, (void*) boot, ident, handle); - } else { - *handle = 0; - *ident = PyThread_start_new_thread(thread_run, (void*) boot); - err = (*ident == PYTHREAD_INVALID_THREAD_ID); - } - if (err) { - PyErr_SetString(ThreadError, "can't start new thread"); - PyThreadState_Clear(boot->tstate); - thread_bootstate_free(boot, 1); - return -1; - } return 0; } @@ -1472,12 +1772,19 @@ thread_PyThread_start_new_thread(PyObject *module, PyObject *fargs) return NULL; } - PyThread_ident_t ident = 0; - PyThread_handle_t handle; - if (do_start_new_thread(state, func, args, kwargs, /*joinable=*/ 0, - &ident, &handle, NULL)) { + ThreadHandle *handle = ThreadHandle_new(); + if (handle == NULL) { + return NULL; + } + + int st = + do_start_new_thread(state, func, args, kwargs, handle, /*daemon=*/1); + if (st < 0) { + ThreadHandle_decref(handle); return NULL; } + PyThread_ident_t ident = ThreadHandle_ident(handle); + ThreadHandle_decref(handle); return PyLong_FromUnsignedLongLong(ident); } @@ -1495,9 +1802,19 @@ unhandled exception; a stack trace will be printed unless the exception\n\ is SystemExit.\n"); static PyObject * -thread_PyThread_start_joinable_thread(PyObject *module, PyObject *func) +thread_PyThread_start_joinable_thread(PyObject *module, PyObject *fargs, + PyObject *fkwargs) { + static char *keywords[] = {"function", "handle", "daemon", NULL}; + PyObject *func = NULL; + int daemon = 1; thread_module_state *state = get_thread_state(module); + PyObject *hobj = NULL; + if (!PyArg_ParseTupleAndKeywords(fargs, fkwargs, + "O|Op:start_joinable_thread", keywords, + &func, &hobj, &daemon)) { + return NULL; + } if (!PyCallable_Check(func)) { PyErr_SetString(PyExc_TypeError, @@ -1505,32 +1822,45 @@ thread_PyThread_start_joinable_thread(PyObject *module, PyObject *func) return NULL; } - if (PySys_Audit("_thread.start_joinable_thread", "O", func) < 0) { + if (hobj == NULL) { + hobj = Py_None; + } + else if (hobj != Py_None && !Py_IS_TYPE(hobj, state->thread_handle_type)) { + PyErr_SetString(PyExc_TypeError, "'handle' must be a _ThreadHandle"); + return NULL; + } + + if (PySys_Audit("_thread.start_joinable_thread", "OiO", func, daemon, + hobj) < 0) { return NULL; } + if (hobj == Py_None) { + hobj = (PyObject *)PyThreadHandleObject_new(state->thread_handle_type); + if (hobj == NULL) { + return NULL; + } + } + else { + Py_INCREF(hobj); + } + PyObject* args = PyTuple_New(0); if (args == NULL) { return NULL; } - ThreadHandleObject* hobj = new_thread_handle(state); - if (hobj == NULL) { - Py_DECREF(args); - return NULL; - } - if (do_start_new_thread(state, func, args, /*kwargs=*/ NULL, /*joinable=*/ 1, - &hobj->ident, &hobj->handle, hobj->thread_is_exiting)) { - Py_DECREF(args); + int st = do_start_new_thread(state, func, args, + /*kwargs=*/ NULL, ((PyThreadHandleObject*)hobj)->handle, daemon); + Py_DECREF(args); + if (st < 0) { Py_DECREF(hobj); return NULL; } - set_thread_handle_state(hobj, THREAD_HANDLE_RUNNING); - Py_DECREF(args); - return (PyObject*) hobj; + return (PyObject *) hobj; } PyDoc_STRVAR(start_joinable_doc, -"start_joinable_thread(function)\n\ +"start_joinable_thread(function[, daemon=True[, handle=None]])\n\ \n\ *For internal use only*: start a new thread.\n\ \n\ @@ -1538,7 +1868,9 @@ Like start_new_thread(), this starts a new thread calling the given function.\n\ Unlike start_new_thread(), this returns a handle object with methods to join\n\ or detach the given thread.\n\ This function is not for third-party code, please use the\n\ -`threading` module instead.\n"); +`threading` module instead. During finalization the runtime will not wait for\n\ +the thread to exit if daemon is True. If handle is provided it must be a\n\ +newly created thread._ThreadHandle instance."); static PyObject * thread_PyThread_exit_thread(PyObject *self, PyObject *Py_UNUSED(ignored)) @@ -1650,67 +1982,6 @@ yet finished.\n\ This function is meant for internal and specialized purposes only.\n\ In most applications `threading.enumerate()` should be used instead."); -static void -release_sentinel(void *weakref_raw) -{ - PyObject *weakref = _PyObject_CAST(weakref_raw); - - /* Tricky: this function is called when the current thread state - is being deleted. Therefore, only simple C code can safely - execute here. */ - lockobject *lock = (lockobject *)_PyWeakref_GET_REF(weakref); - if (lock != NULL) { - if (lock->locked) { - lock->locked = 0; - PyThread_release_lock(lock->lock_lock); - } - Py_DECREF(lock); - } - - /* Deallocating a weakref with a NULL callback only calls - PyObject_GC_Del(), which can't call any Python code. */ - Py_DECREF(weakref); -} - -static PyObject * -thread__set_sentinel(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - PyObject *wr; - PyThreadState *tstate = _PyThreadState_GET(); - lockobject *lock; - - if (tstate->on_delete_data != NULL) { - /* We must support the re-creation of the lock from a - fork()ed child. */ - assert(tstate->on_delete == &release_sentinel); - wr = (PyObject *) tstate->on_delete_data; - tstate->on_delete = NULL; - tstate->on_delete_data = NULL; - Py_DECREF(wr); - } - lock = newlockobject(module); - if (lock == NULL) - return NULL; - /* The lock is owned by whoever called _set_sentinel(), but the weakref - hangs to the thread state. */ - wr = PyWeakref_NewRef((PyObject *) lock, NULL); - if (wr == NULL) { - Py_DECREF(lock); - return NULL; - } - tstate->on_delete_data = (void *) wr; - tstate->on_delete = &release_sentinel; - return (PyObject *) lock; -} - -PyDoc_STRVAR(_set_sentinel_doc, -"_set_sentinel() -> lock\n\ -\n\ -Set a sentinel lock that will be released when the current thread\n\ -state is finalized (after it is untied from the interpreter).\n\ -\n\ -This is a private API for the threading module."); - static PyObject * thread_stack_size(PyObject *self, PyObject *args) { @@ -1917,13 +2188,101 @@ PyDoc_STRVAR(thread__is_main_interpreter_doc, \n\ Return True if the current interpreter is the main Python interpreter."); +static PyObject * +thread_shutdown(PyObject *self, PyObject *args) +{ + PyThread_ident_t ident = PyThread_get_thread_ident_ex(); + thread_module_state *state = get_thread_state(self); + + for (;;) { + ThreadHandle *handle = NULL; + + // Find a thread that's not yet finished. + HEAD_LOCK(&_PyRuntime); + struct llist_node *node; + llist_for_each_safe(node, &state->shutdown_handles) { + ThreadHandle *cur = llist_data(node, ThreadHandle, shutdown_node); + if (cur->ident != ident) { + ThreadHandle_incref(cur); + handle = cur; + break; + } + } + HEAD_UNLOCK(&_PyRuntime); + + if (!handle) { + // No more threads to wait on! + break; + } + + // Wait for the thread to finish. If we're interrupted, such + // as by a ctrl-c we print the error and exit early. + if (ThreadHandle_join(handle, -1) < 0) { + PyErr_WriteUnraisable(NULL); + ThreadHandle_decref(handle); + Py_RETURN_NONE; + } + + ThreadHandle_decref(handle); + } + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(shutdown_doc, +"_shutdown()\n\ +\n\ +Wait for all non-daemon threads (other than the calling thread) to stop."); + +static PyObject * +thread__make_thread_handle(PyObject *module, PyObject *identobj) +{ + thread_module_state *state = get_thread_state(module); + if (!PyLong_Check(identobj)) { + PyErr_SetString(PyExc_TypeError, "ident must be an integer"); + return NULL; + } + PyThread_ident_t ident = PyLong_AsUnsignedLongLong(identobj); + if (PyErr_Occurred()) { + return NULL; + } + PyThreadHandleObject *hobj = + PyThreadHandleObject_new(state->thread_handle_type); + if (hobj == NULL) { + return NULL; + } + PyMutex_Lock(&hobj->handle->mutex); + hobj->handle->ident = ident; + hobj->handle->state = THREAD_HANDLE_RUNNING; + PyMutex_Unlock(&hobj->handle->mutex); + return (PyObject*) hobj; +} + +PyDoc_STRVAR(thread__make_thread_handle_doc, +"_make_thread_handle(ident)\n\ +\n\ +Internal only. Make a thread handle for threads not spawned\n\ +by the _thread or threading module."); + +static PyObject * +thread__get_main_thread_ident(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return PyLong_FromUnsignedLongLong(_PyRuntime.main_thread); +} + +PyDoc_STRVAR(thread__get_main_thread_ident_doc, +"_get_main_thread_ident()\n\ +\n\ +Internal only. Return a non-zero integer that uniquely identifies the main thread\n\ +of the main interpreter."); + static PyMethodDef thread_methods[] = { {"start_new_thread", (PyCFunction)thread_PyThread_start_new_thread, METH_VARARGS, start_new_doc}, {"start_new", (PyCFunction)thread_PyThread_start_new_thread, METH_VARARGS, start_new_doc}, - {"start_joinable_thread", (PyCFunction)thread_PyThread_start_joinable_thread, - METH_O, start_joinable_doc}, + {"start_joinable_thread", _PyCFunction_CAST(thread_PyThread_start_joinable_thread), + METH_VARARGS | METH_KEYWORDS, start_joinable_doc}, {"daemon_threads_allowed", (PyCFunction)thread_daemon_threads_allowed, METH_NOARGS, daemon_threads_allowed_doc}, {"allocate_lock", thread_PyThread_allocate_lock, @@ -1946,12 +2305,16 @@ static PyMethodDef thread_methods[] = { METH_NOARGS, _count_doc}, {"stack_size", (PyCFunction)thread_stack_size, METH_VARARGS, stack_size_doc}, - {"_set_sentinel", thread__set_sentinel, - METH_NOARGS, _set_sentinel_doc}, {"_excepthook", thread_excepthook, METH_O, excepthook_doc}, {"_is_main_interpreter", thread__is_main_interpreter, METH_NOARGS, thread__is_main_interpreter_doc}, + {"_shutdown", thread_shutdown, + METH_NOARGS, shutdown_doc}, + {"_make_thread_handle", thread__make_thread_handle, + METH_O, thread__make_thread_handle_doc}, + {"_get_main_thread_ident", thread__get_main_thread_ident, + METH_NOARGS, thread__get_main_thread_ident_doc}, {NULL, NULL} /* sentinel */ }; @@ -2041,6 +2404,8 @@ thread_module_exec(PyObject *module) return -1; } + llist_init(&state->shutdown_handles); + return 0; } @@ -2066,6 +2431,10 @@ thread_module_clear(PyObject *module) Py_CLEAR(state->local_type); Py_CLEAR(state->local_dummy_type); Py_CLEAR(state->thread_handle_type); + // Remove any remaining handles (e.g. if shutdown exited early due to + // interrupt) so that attempts to unlink the handle after our module state + // is destroyed do not crash. + clear_shutdown_handles(state); return 0; } diff --git a/Python/lock.c b/Python/lock.c index de25adce385105..7d1ead585dee6c 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -304,30 +304,6 @@ PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns) } } -_PyEventRc * -_PyEventRc_New(void) -{ - _PyEventRc *erc = (_PyEventRc *)PyMem_RawCalloc(1, sizeof(_PyEventRc)); - if (erc != NULL) { - erc->refcount = 1; - } - return erc; -} - -void -_PyEventRc_Incref(_PyEventRc *erc) -{ - _Py_atomic_add_ssize(&erc->refcount, 1); -} - -void -_PyEventRc_Decref(_PyEventRc *erc) -{ - if (_Py_atomic_add_ssize(&erc->refcount, -1) == 1) { - PyMem_RawFree(erc); - } -} - static int unlock_once(_PyOnceFlag *o, int res) { diff --git a/Python/pystate.c b/Python/pystate.c index 635616c5648c18..9f142223aff340 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1032,20 +1032,7 @@ _PyInterpreterState_SetRunningMain(PyInterpreterState *interp) void _PyInterpreterState_SetNotRunningMain(PyInterpreterState *interp) { - PyThreadState *tstate = interp->threads.main; - assert(tstate == current_fast_get()); - - if (tstate->on_delete != NULL) { - // The threading module was imported for the first time in this - // thread, so it was set as threading._main_thread. (See gh-75698.) - // The thread has finished running the Python program so we mark - // the thread object as finished. - assert(tstate->_whence != _PyThreadState_WHENCE_THREADING); - tstate->on_delete(tstate->on_delete_data); - tstate->on_delete = NULL; - tstate->on_delete_data = NULL; - } - + assert(interp->threads.main == current_fast_get()); interp->threads.main = NULL; } @@ -1570,16 +1557,6 @@ PyThreadState_Clear(PyThreadState *tstate) Py_CLEAR(tstate->context); - if (tstate->on_delete != NULL) { - // For the "main" thread of each interpreter, this is meant - // to be done in _PyInterpreterState_SetNotRunningMain(). - // That leaves threads created by the threading module, - // and any threads killed by forking. - // However, we also accommodate "main" threads that still - // don't call _PyInterpreterState_SetNotRunningMain() yet. - tstate->on_delete(tstate->on_delete_data); - } - #ifdef Py_GIL_DISABLED // Each thread should clear own freelists in free-threading builds. struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); From 744c0777952f1e535d1192ee15b286aa67b61533 Mon Sep 17 00:00:00 2001 From: jnchen Date: Sat, 16 Mar 2024 21:01:45 +0800 Subject: [PATCH 124/158] gh-116851: Remove "from ctypes import *" from a ctypes example (GH-116852) It is confusing, because libc is not imported from ctypes, but defined in previous examples, which already contain the import. --- Doc/library/ctypes.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index eed18201e3ede0..36976470b5a468 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -93,7 +93,6 @@ Accessing functions from loaded dlls Functions are accessed as attributes of dll objects:: - >>> from ctypes import * >>> libc.printf <_FuncPtr object at 0x...> >>> print(windll.kernel32.GetModuleHandleA) # doctest: +WINDOWS From bee7e290cdedb17e06f473a2f318c720ba766852 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sat, 16 Mar 2024 23:52:44 +0900 Subject: [PATCH 125/158] gh-112536: Add more TSAN tests (#116896) --------- Co-authored-by: Antoine Pitrou --- Lib/test/libregrtest/tsan.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index fde8ba937c0e4b..c5aed436b829d1 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -10,6 +10,7 @@ 'test_importlib', 'test_io', 'test_logging', + 'test_queue', 'test_ssl', 'test_syslog', 'test_thread', @@ -17,6 +18,7 @@ 'test_threading', 'test_threading_local', 'test_threadsignals', + 'test_weakref', ] From 1a33513f99bf4a9e5122b9cd82945879e73ff44c Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Sat, 16 Mar 2024 11:10:43 -0400 Subject: [PATCH 126/158] gh-116879: Add new optimizer pystats to tables (GH-116880) --- Tools/scripts/summarize_stats.py | 34 ++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 2925e096f4d95e..6af14e1b769b80 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -459,10 +459,7 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: "The number of times a potential trace is identified. Specifically, this " "occurs in the JUMP BACKWARD instruction when the counter reaches a " "threshold.", - ): ( - attempts, - None, - ), + ): (attempts, None), Doc( "Traces created", "The number of traces that were successfully created." ): (created, attempts), @@ -512,6 +509,26 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: ), } + def get_optimizer_stats(self) -> dict[str, tuple[int, int | None]]: + attempts = self._data["Optimization optimizer attempts"] + successes = self._data["Optimization optimizer successes"] + no_memory = self._data["Optimization optimizer failure no memory"] + + return { + Doc( + "Optimizer attempts", + "The number of times the trace optimizer (_Py_uop_analyze_and_optimize) was run.", + ): (attempts, None), + Doc( + "Optimizer successes", + "The number of traces that were successfully optimized.", + ): (successes, attempts), + Doc( + "Optimizer no memory", + "The number of optimizations that failed due to no memory.", + ): (no_memory, attempts), + } + def get_histogram(self, prefix: str) -> list[tuple[int, int]]: rows = [] for k, v in self._data.items(): @@ -1118,6 +1135,14 @@ def calc_optimization_table(stats: Stats) -> Rows: for label, (value, den) in optimization_stats.items() ] + def calc_optimizer_table(stats: Stats) -> Rows: + optimizer_stats = stats.get_optimizer_stats() + + return [ + (label, Count(value), Ratio(value, den)) + for label, (value, den) in optimizer_stats.items() + ] + def calc_histogram_table(key: str, den: str) -> RowCalculator: def calc(stats: Stats) -> Rows: histogram = stats.get_histogram(key) @@ -1159,6 +1184,7 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) return yield Table(("", "Count:", "Ratio:"), calc_optimization_table, JoinMode.CHANGE) + yield Table(("", "Count:", "Ratio:"), calc_optimizer_table, JoinMode.CHANGE) for name, den in [ ("Trace length", "Optimization traces created"), ("Optimized trace length", "Optimization traces created"), From 259dbc448dabc1ad95fcf6f70eff4b6b5355e71c Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sat, 16 Mar 2024 19:59:42 +0200 Subject: [PATCH 127/158] CI: Process stale issues four times per day (#116857) --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 32299d0fc47c01..f97587e68cbbe4 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,7 +2,7 @@ name: Mark stale pull requests on: schedule: - - cron: "0 */12 * * *" + - cron: "0 */6 * * *" permissions: pull-requests: write From 5e0a070dfe33530756fa2811b76bf959b9616590 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 16 Mar 2024 21:37:11 +0100 Subject: [PATCH 128/158] gh-116809: Restore removed _PyErr_ChainExceptions1() function (#116900) --- Include/cpython/pyerrors.h | 4 ++++ Include/internal/pycore_pyerrors.h | 3 --- .../next/C API/2024-03-16-12-21-00.gh-issue-116809.JL786L.rst | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-03-16-12-21-00.gh-issue-116809.JL786L.rst diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 32c5884cd21341..42b4b03b10ca20 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -88,6 +88,10 @@ typedef PyOSErrorObject PyEnvironmentErrorObject; typedef PyOSErrorObject PyWindowsErrorObject; #endif +/* Context manipulation (PEP 3134) */ + +PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *); + /* In exceptions.c */ PyAPI_FUNC(PyObject*) PyUnstable_Exc_PrepReraiseStar( diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 910335fd2cf33b..683d87a0d0b129 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -167,9 +167,6 @@ void _PyErr_FormatNote(const char *format, ...); Py_DEPRECATED(3.12) extern void _PyErr_ChainExceptions(PyObject *, PyObject *, PyObject *); -// Export for '_zoneinfo' shared extension -PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *); - #ifdef __cplusplus } #endif diff --git a/Misc/NEWS.d/next/C API/2024-03-16-12-21-00.gh-issue-116809.JL786L.rst b/Misc/NEWS.d/next/C API/2024-03-16-12-21-00.gh-issue-116809.JL786L.rst new file mode 100644 index 00000000000000..a122e1b45b959a --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-03-16-12-21-00.gh-issue-116809.JL786L.rst @@ -0,0 +1,2 @@ +Restore removed private ``_PyErr_ChainExceptions1()`` function. Patch by +Victor Stinner. From c514a975abe35fa4604cd3541e2286168ef67d10 Mon Sep 17 00:00:00 2001 From: Carol Willing Date: Sat, 16 Mar 2024 17:01:56 -0700 Subject: [PATCH 129/158] Update titles and subtitles on landing page template (#116914) * Update titles and subtitles on landing page template * address review from gvanrossum * Edits from hugovk review * Change word order back. Down the road we should split license and history --- Doc/tools/templates/indexcontent.html | 62 +++++++++++++-------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/Doc/tools/templates/indexcontent.html b/Doc/tools/templates/indexcontent.html index 1e3ab7cfe02fee..5b3c174f9d1729 100644 --- a/Doc/tools/templates/indexcontent.html +++ b/Doc/tools/templates/indexcontent.html @@ -7,60 +7,60 @@

{{ docstitle|e }}

{% trans %}Welcome! This is the official documentation for Python {{ release }}.{% endtrans %}

-

{% trans %}Parts of the documentation:{% endtrans %}

+

{% trans %}Documentation sections:{% endtrans %}

{% trans %}What's new in Python {{ version }}?{% endtrans %}
- {% trans whatsnew_index=pathto("whatsnew/index") %}or all "What's new" documents since 2.0{% endtrans %}

+ {% trans whatsnew_index=pathto("whatsnew/index") %}Or all "What's new" documents since Python 2.0{% endtrans %}

{% trans %}Tutorial{% endtrans %}
- {% trans %}start here{% endtrans %}

-

{% trans %}Library Reference{% endtrans %}
- {% trans %}keep this under your pillow{% endtrans %}

-

{% trans %}Language Reference{% endtrans %}
- {% trans %}describes syntax and language elements{% endtrans %}

-

{% trans %}Python Setup and Usage{% endtrans %}
- {% trans %}how to use Python on different platforms{% endtrans %}

+ {% trans %}Start here: a tour of Python's syntax and features{% endtrans %}

+

{% trans %}Library reference{% endtrans %}
+ {% trans %}Standard library and builtins{% endtrans %}

+

{% trans %}Language reference{% endtrans %}
+ {% trans %}Syntax and language elements{% endtrans %}

+

{% trans %}Python setup and usage{% endtrans %}
+ {% trans %}How to install, configure, and use Python{% endtrans %}

{% trans %}Python HOWTOs{% endtrans %}
- {% trans %}in-depth documents on specific topics{% endtrans %}

+ {% trans %}In-depth topic manuals{% endtrans %}

 -

{% trans %}Installing Python Modules{% endtrans %}
- {% trans %}installing from the Python Package Index & other sources{% endtrans %}

-

{% trans %}Distributing Python Modules{% endtrans %}
- {% trans %}publishing modules for installation by others{% endtrans %}

-

{% trans %}Extending and Embedding{% endtrans %}
- {% trans %}tutorial for C/C++ programmers{% endtrans %}

-

{% trans %}Python/C API{% endtrans %}
- {% trans %}reference for C/C++ programmers{% endtrans %}

+

{% trans %}Installing Python modules{% endtrans %}
+ {% trans %}Third-party modules and PyPI.org{% endtrans %}

+

{% trans %}Distributing Python modules{% endtrans %}
+ {% trans %}Publishing modules for use by other people{% endtrans %}

+

{% trans %}Extending and embedding{% endtrans %}
+ {% trans %}For C/C++ programmers{% endtrans %}

+

{% trans %}Python's C API{% endtrans %}
+ {% trans %}C API reference{% endtrans %}

{% trans %}FAQs{% endtrans %}
- {% trans %}frequently asked questions (with answers!){% endtrans %}

+ {% trans %}Frequently asked questions (with answers!){% endtrans %}

-

{% trans %}Indices and tables:{% endtrans %}

+

{% trans %}Indices, glossary, and search:{% endtrans %}

-

{% trans %}Global Module Index{% endtrans %}
- {% trans %}quick access to all modules{% endtrans %}

-

{% trans %}General Index{% endtrans %}
- {% trans %}all functions, classes, terms{% endtrans %}

+

{% trans %}Global module index{% endtrans %}
+ {% trans %}All modules and libraries{% endtrans %}

+

{% trans %}General index{% endtrans %}
+ {% trans %}All functions, classes, and terms{% endtrans %}

{% trans %}Glossary{% endtrans %}
- {% trans %}the most important terms explained{% endtrans %}

+ {% trans %}Terms explained{% endtrans %}

{% trans %}Search page{% endtrans %}
- {% trans %}search this documentation{% endtrans %}

-

{% trans %}Complete Table of Contents{% endtrans %}
- {% trans %}lists all sections and subsections{% endtrans %}

+ {% trans %}Search this documentation{% endtrans %}

+

{% trans %}Complete table of contents{% endtrans %}
+ {% trans %}Lists all sections and subsections{% endtrans %}

-

{% trans %}Meta information:{% endtrans %}

+

{% trans %}Project information:{% endtrans %}

From 649857a1574a02235ccfac9e2ac1c12914cf8fe0 Mon Sep 17 00:00:00 2001 From: John Sloboda Date: Sun, 17 Mar 2024 00:58:42 -0400 Subject: [PATCH 130/158] gh-85287: Change codecs to raise precise UnicodeEncodeError and UnicodeDecodeError (#113674) Co-authored-by: Inada Naoki --- Lib/encodings/idna.py | 164 +++++++++++++----- Lib/encodings/punycode.py | 42 +++-- Lib/encodings/undefined.py | 2 +- Lib/encodings/utf_16.py | 4 +- Lib/encodings/utf_32.py | 6 +- Lib/test/test_codecs.py | 123 ++++++++++++- Lib/test/test_multibytecodec.py | 2 +- ...4-01-02-22-47-12.gh-issue-85287.ZC5DLj.rst | 2 + Modules/cjkcodecs/multibytecodec.c | 42 ++++- 9 files changed, 306 insertions(+), 81 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-01-02-22-47-12.gh-issue-85287.ZC5DLj.rst diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py index d0f70c00f0ab66..60a8d5eb227f82 100644 --- a/Lib/encodings/idna.py +++ b/Lib/encodings/idna.py @@ -11,7 +11,7 @@ sace_prefix = "xn--" # This assumes query strings, so AllowUnassigned is true -def nameprep(label): +def nameprep(label): # type: (str) -> str # Map newlabel = [] for c in label: @@ -25,7 +25,7 @@ def nameprep(label): label = unicodedata.normalize("NFKC", label) # Prohibit - for c in label: + for i, c in enumerate(label): if stringprep.in_table_c12(c) or \ stringprep.in_table_c22(c) or \ stringprep.in_table_c3(c) or \ @@ -35,7 +35,7 @@ def nameprep(label): stringprep.in_table_c7(c) or \ stringprep.in_table_c8(c) or \ stringprep.in_table_c9(c): - raise UnicodeError("Invalid character %r" % c) + raise UnicodeEncodeError("idna", label, i, i+1, f"Invalid character {c!r}") # Check bidi RandAL = [stringprep.in_table_d1(x) for x in label] @@ -46,29 +46,38 @@ def nameprep(label): # This is table C.8, which was already checked # 2) If a string contains any RandALCat character, the string # MUST NOT contain any LCat character. - if any(stringprep.in_table_d2(x) for x in label): - raise UnicodeError("Violation of BIDI requirement 2") + for i, x in enumerate(label): + if stringprep.in_table_d2(x): + raise UnicodeEncodeError("idna", label, i, i+1, + "Violation of BIDI requirement 2") # 3) If a string contains any RandALCat character, a # RandALCat character MUST be the first character of the # string, and a RandALCat character MUST be the last # character of the string. - if not RandAL[0] or not RandAL[-1]: - raise UnicodeError("Violation of BIDI requirement 3") + if not RandAL[0]: + raise UnicodeEncodeError("idna", label, 0, 1, + "Violation of BIDI requirement 3") + if not RandAL[-1]: + raise UnicodeEncodeError("idna", label, len(label)-1, len(label), + "Violation of BIDI requirement 3") return label -def ToASCII(label): +def ToASCII(label): # type: (str) -> bytes try: # Step 1: try ASCII - label = label.encode("ascii") - except UnicodeError: + label_ascii = label.encode("ascii") + except UnicodeEncodeError: pass else: # Skip to step 3: UseSTD3ASCIIRules is false, so # Skip to step 8. - if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + if 0 < len(label_ascii) < 64: + return label_ascii + if len(label) == 0: + raise UnicodeEncodeError("idna", label, 0, 1, "label empty") + else: + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") # Step 2: nameprep label = nameprep(label) @@ -76,29 +85,34 @@ def ToASCII(label): # Step 3: UseSTD3ASCIIRules is false # Step 4: try ASCII try: - label = label.encode("ascii") - except UnicodeError: + label_ascii = label.encode("ascii") + except UnicodeEncodeError: pass else: # Skip to step 8. if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + return label_ascii + if len(label) == 0: + raise UnicodeEncodeError("idna", label, 0, 1, "label empty") + else: + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") # Step 5: Check ACE prefix - if label[:4].lower() == sace_prefix: - raise UnicodeError("Label starts with ACE prefix") + if label.lower().startswith(sace_prefix): + raise UnicodeEncodeError( + "idna", label, 0, len(sace_prefix), "Label starts with ACE prefix") # Step 6: Encode with PUNYCODE - label = label.encode("punycode") + label_ascii = label.encode("punycode") # Step 7: Prepend ACE prefix - label = ace_prefix + label + label_ascii = ace_prefix + label_ascii # Step 8: Check size - if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + # do not check for empty as we prepend ace_prefix. + if len(label_ascii) < 64: + return label_ascii + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") def ToUnicode(label): if len(label) > 1024: @@ -110,7 +124,9 @@ def ToUnicode(label): # per https://www.rfc-editor.org/rfc/rfc3454#section-3.1 while still # preventing us from wasting time decoding a big thing that'll just # hit the actual <= 63 length limit in Step 6. - raise UnicodeError("label way too long") + if isinstance(label, str): + label = label.encode("utf-8", errors="backslashreplace") + raise UnicodeDecodeError("idna", label, 0, len(label), "label way too long") # Step 1: Check for ASCII if isinstance(label, bytes): pure_ascii = True @@ -118,25 +134,32 @@ def ToUnicode(label): try: label = label.encode("ascii") pure_ascii = True - except UnicodeError: + except UnicodeEncodeError: pure_ascii = False if not pure_ascii: + assert isinstance(label, str) # Step 2: Perform nameprep label = nameprep(label) # It doesn't say this, but apparently, it should be ASCII now try: label = label.encode("ascii") - except UnicodeError: - raise UnicodeError("Invalid character in IDN label") + except UnicodeEncodeError as exc: + raise UnicodeEncodeError("idna", label, exc.start, exc.end, + "Invalid character in IDN label") # Step 3: Check for ACE prefix - if not label[:4].lower() == ace_prefix: + assert isinstance(label, bytes) + if not label.lower().startswith(ace_prefix): return str(label, "ascii") # Step 4: Remove ACE prefix label1 = label[len(ace_prefix):] # Step 5: Decode using PUNYCODE - result = label1.decode("punycode") + try: + result = label1.decode("punycode") + except UnicodeDecodeError as exc: + offset = len(ace_prefix) + raise UnicodeDecodeError("idna", label, offset+exc.start, offset+exc.end, exc.reason) # Step 6: Apply ToASCII label2 = ToASCII(result) @@ -144,7 +167,8 @@ def ToUnicode(label): # Step 7: Compare the result of step 6 with the one of step 3 # label2 will already be in lower case. if str(label, "ascii").lower() != str(label2, "ascii"): - raise UnicodeError("IDNA does not round-trip", label, label2) + raise UnicodeDecodeError("idna", label, 0, len(label), + f"IDNA does not round-trip, '{label!r}' != '{label2!r}'") # Step 8: return the result of step 5 return result @@ -156,7 +180,7 @@ def encode(self, input, errors='strict'): if errors != 'strict': # IDNA is quite clear that implementations must be strict - raise UnicodeError("unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return b'', 0 @@ -168,11 +192,16 @@ def encode(self, input, errors='strict'): else: # ASCII name: fast path labels = result.split(b'.') - for label in labels[:-1]: - if not (0 < len(label) < 64): - raise UnicodeError("label empty or too long") - if len(labels[-1]) >= 64: - raise UnicodeError("label too long") + for i, label in enumerate(labels[:-1]): + if len(label) == 0: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError("idna", input, offset, offset+1, + "label empty") + for i, label in enumerate(labels): + if len(label) >= 64: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError("idna", input, offset, offset+len(label), + "label too long") return result, len(input) result = bytearray() @@ -182,17 +211,27 @@ def encode(self, input, errors='strict'): del labels[-1] else: trailing_dot = b'' - for label in labels: + for i, label in enumerate(labels): if result: # Join with U+002E result.extend(b'.') - result.extend(ToASCII(label)) + try: + result.extend(ToASCII(label)) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError( + "idna", + input, + offset + exc.start, + offset + exc.end, + exc.reason, + ) return bytes(result+trailing_dot), len(input) def decode(self, input, errors='strict'): if errors != 'strict': - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return "", 0 @@ -218,8 +257,15 @@ def decode(self, input, errors='strict'): trailing_dot = '' result = [] - for label in labels: - result.append(ToUnicode(label)) + for i, label in enumerate(labels): + try: + u_label = ToUnicode(label) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + offset = sum(len(x) for x in labels[:i]) + len(labels[:i]) + raise UnicodeDecodeError( + "idna", input, offset+exc.start, offset+exc.end, exc.reason) + else: + result.append(u_label) return ".".join(result)+trailing_dot, len(input) @@ -227,7 +273,7 @@ class IncrementalEncoder(codecs.BufferedIncrementalEncoder): def _buffer_encode(self, input, errors, final): if errors != 'strict': # IDNA is quite clear that implementations must be strict - raise UnicodeError("unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return (b'', 0) @@ -251,7 +297,16 @@ def _buffer_encode(self, input, errors, final): # Join with U+002E result.extend(b'.') size += 1 - result.extend(ToASCII(label)) + try: + result.extend(ToASCII(label)) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeEncodeError( + "idna", + input, + size + exc.start, + size + exc.end, + exc.reason, + ) size += len(label) result += trailing_dot @@ -261,7 +316,7 @@ def _buffer_encode(self, input, errors, final): class IncrementalDecoder(codecs.BufferedIncrementalDecoder): def _buffer_decode(self, input, errors, final): if errors != 'strict': - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError("Unsupported error handling: {errors}") if not input: return ("", 0) @@ -271,7 +326,11 @@ def _buffer_decode(self, input, errors, final): labels = dots.split(input) else: # Must be ASCII string - input = str(input, "ascii") + try: + input = str(input, "ascii") + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeDecodeError("idna", input, + exc.start, exc.end, exc.reason) labels = input.split(".") trailing_dot = '' @@ -288,7 +347,18 @@ def _buffer_decode(self, input, errors, final): result = [] size = 0 for label in labels: - result.append(ToUnicode(label)) + try: + u_label = ToUnicode(label) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeDecodeError( + "idna", + input.encode("ascii", errors="backslashreplace"), + size + exc.start, + size + exc.end, + exc.reason, + ) + else: + result.append(u_label) if size: size += 1 size += len(label) diff --git a/Lib/encodings/punycode.py b/Lib/encodings/punycode.py index 1c5726447077b1..4622fc8c9206f3 100644 --- a/Lib/encodings/punycode.py +++ b/Lib/encodings/punycode.py @@ -1,4 +1,4 @@ -""" Codec for the Punicode encoding, as specified in RFC 3492 +""" Codec for the Punycode encoding, as specified in RFC 3492 Written by Martin v. Löwis. """ @@ -131,10 +131,11 @@ def decode_generalized_number(extended, extpos, bias, errors): j = 0 while 1: try: - char = ord(extended[extpos]) + char = extended[extpos] except IndexError: if errors == "strict": - raise UnicodeError("incomplete punicode string") + raise UnicodeDecodeError("punycode", extended, extpos, extpos+1, + "incomplete punycode string") return extpos + 1, None extpos += 1 if 0x41 <= char <= 0x5A: # A-Z @@ -142,8 +143,8 @@ def decode_generalized_number(extended, extpos, bias, errors): elif 0x30 <= char <= 0x39: digit = char - 22 # 0x30-26 elif errors == "strict": - raise UnicodeError("Invalid extended code point '%s'" - % extended[extpos-1]) + raise UnicodeDecodeError("punycode", extended, extpos-1, extpos, + f"Invalid extended code point '{extended[extpos-1]}'") else: return extpos, None t = T(j, bias) @@ -155,11 +156,14 @@ def decode_generalized_number(extended, extpos, bias, errors): def insertion_sort(base, extended, errors): - """3.2 Insertion unsort coding""" + """3.2 Insertion sort coding""" + # This function raises UnicodeDecodeError with position in the extended. + # Caller should add the offset. char = 0x80 pos = -1 bias = 72 extpos = 0 + while extpos < len(extended): newpos, delta = decode_generalized_number(extended, extpos, bias, errors) @@ -171,7 +175,9 @@ def insertion_sort(base, extended, errors): char += pos // (len(base) + 1) if char > 0x10FFFF: if errors == "strict": - raise UnicodeError("Invalid character U+%x" % char) + raise UnicodeDecodeError( + "punycode", extended, pos-1, pos, + f"Invalid character U+{char:x}") char = ord('?') pos = pos % (len(base) + 1) base = base[:pos] + chr(char) + base[pos:] @@ -187,11 +193,21 @@ def punycode_decode(text, errors): pos = text.rfind(b"-") if pos == -1: base = "" - extended = str(text, "ascii").upper() + extended = text.upper() else: - base = str(text[:pos], "ascii", errors) - extended = str(text[pos+1:], "ascii").upper() - return insertion_sort(base, extended, errors) + try: + base = str(text[:pos], "ascii", errors) + except UnicodeDecodeError as exc: + raise UnicodeDecodeError("ascii", text, exc.start, exc.end, + exc.reason) from None + extended = text[pos+1:].upper() + try: + return insertion_sort(base, extended, errors) + except UnicodeDecodeError as exc: + offset = pos + 1 + raise UnicodeDecodeError("punycode", text, + offset+exc.start, offset+exc.end, + exc.reason) from None ### Codec APIs @@ -203,7 +219,7 @@ def encode(self, input, errors='strict'): def decode(self, input, errors='strict'): if errors not in ('strict', 'replace', 'ignore'): - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") res = punycode_decode(input, errors) return res, len(input) @@ -214,7 +230,7 @@ def encode(self, input, final=False): class IncrementalDecoder(codecs.IncrementalDecoder): def decode(self, input, final=False): if self.errors not in ('strict', 'replace', 'ignore'): - raise UnicodeError("Unsupported error handling "+self.errors) + raise UnicodeError(f"Unsupported error handling: {self.errors}") return punycode_decode(input, self.errors) class StreamWriter(Codec,codecs.StreamWriter): diff --git a/Lib/encodings/undefined.py b/Lib/encodings/undefined.py index 4690288355c710..082771e1c86677 100644 --- a/Lib/encodings/undefined.py +++ b/Lib/encodings/undefined.py @@ -1,6 +1,6 @@ """ Python 'undefined' Codec - This codec will always raise a ValueError exception when being + This codec will always raise a UnicodeError exception when being used. It is intended for use by the site.py file to switch off automatic string to Unicode coercion. diff --git a/Lib/encodings/utf_16.py b/Lib/encodings/utf_16.py index c61248242be8c7..d3b9980026666f 100644 --- a/Lib/encodings/utf_16.py +++ b/Lib/encodings/utf_16.py @@ -64,7 +64,7 @@ def _buffer_decode(self, input, errors, final): elif byteorder == 1: self.decoder = codecs.utf_16_be_decode elif consumed >= 2: - raise UnicodeError("UTF-16 stream does not start with BOM") + raise UnicodeDecodeError("utf-16", input, 0, 2, "Stream does not start with BOM") return (output, consumed) return self.decoder(input, self.errors, final) @@ -138,7 +138,7 @@ def decode(self, input, errors='strict'): elif byteorder == 1: self.decode = codecs.utf_16_be_decode elif consumed>=2: - raise UnicodeError("UTF-16 stream does not start with BOM") + raise UnicodeDecodeError("utf-16", input, 0, 2, "Stream does not start with BOM") return (object, consumed) ### encodings module API diff --git a/Lib/encodings/utf_32.py b/Lib/encodings/utf_32.py index cdf84d14129a62..1924bedbb74c68 100644 --- a/Lib/encodings/utf_32.py +++ b/Lib/encodings/utf_32.py @@ -59,7 +59,7 @@ def _buffer_decode(self, input, errors, final): elif byteorder == 1: self.decoder = codecs.utf_32_be_decode elif consumed >= 4: - raise UnicodeError("UTF-32 stream does not start with BOM") + raise UnicodeDecodeError("utf-32", input, 0, 4, "Stream does not start with BOM") return (output, consumed) return self.decoder(input, self.errors, final) @@ -132,8 +132,8 @@ def decode(self, input, errors='strict'): self.decode = codecs.utf_32_le_decode elif byteorder == 1: self.decode = codecs.utf_32_be_decode - elif consumed>=4: - raise UnicodeError("UTF-32 stream does not start with BOM") + elif consumed >= 4: + raise UnicodeDecodeError("utf-32", input, 0, 4, "Stream does not start with BOM") return (object, consumed) ### encodings module API diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 9585f947877142..fe3776d6dd9337 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -482,11 +482,11 @@ def test_only_one_bom(self): def test_badbom(self): s = io.BytesIO(4*b"\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) s = io.BytesIO(8*b"\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) def test_partial(self): self.check_partial( @@ -666,11 +666,11 @@ def test_only_one_bom(self): def test_badbom(self): s = io.BytesIO(b"\xff\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) s = io.BytesIO(b"\xff\xff\xff\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) def test_partial(self): self.check_partial( @@ -1356,13 +1356,29 @@ def test_decode(self): def test_decode_invalid(self): testcases = [ - (b"xn--w&", "strict", UnicodeError()), + (b"xn--w&", "strict", UnicodeDecodeError("punycode", b"", 5, 6, "")), + (b"&egbpdaj6bu4bxfgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")), + (b"egbpdaj6bu&4bx&fgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 10, 11, "")), + (b"egbpdaj6bu4bxfgehfvwxn&", "strict", UnicodeDecodeError("punycode", b"", 22, 23, "")), + (b"\xFFProprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 0, 1, "")), + (b"Pro\xFFprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 3, 4, "")), + (b"Proprost&nemluvesky-uyb24&dma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), + (b"Proprostnemluvesky&-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 20, 21, "")), + (b"Proprostnemluvesky-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 19, 20, "")), + (b"Proprostnemluvesky-uyb24d&ma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), + (b"Proprostnemluvesky-uyb24dma41a&", "strict", UnicodeDecodeError("punycode", b"", 30, 31, "")), (b"xn--w&", "ignore", "xn-"), ] for puny, errors, expected in testcases: with self.subTest(puny=puny, errors=errors): if isinstance(expected, Exception): - self.assertRaises(UnicodeError, puny.decode, "punycode", errors) + with self.assertRaises(UnicodeDecodeError) as cm: + puny.decode("punycode", errors) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, puny) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) else: self.assertEqual(puny.decode("punycode", errors), expected) @@ -1532,7 +1548,7 @@ def test_nameprep(self): orig = str(orig, "utf-8", "surrogatepass") if prepped is None: # Input contains prohibited characters - self.assertRaises(UnicodeError, nameprep, orig) + self.assertRaises(UnicodeEncodeError, nameprep, orig) else: prepped = str(prepped, "utf-8", "surrogatepass") try: @@ -1542,6 +1558,23 @@ def test_nameprep(self): class IDNACodecTest(unittest.TestCase): + + invalid_decode_testcases = [ + (b"\xFFpython.org", UnicodeDecodeError("idna", b"\xFFpython.org", 0, 1, "")), + (b"pyt\xFFhon.org", UnicodeDecodeError("idna", b"pyt\xFFhon.org", 3, 4, "")), + (b"python\xFF.org", UnicodeDecodeError("idna", b"python\xFF.org", 6, 7, "")), + (b"python.\xFForg", UnicodeDecodeError("idna", b"python.\xFForg", 7, 8, "")), + (b"python.o\xFFrg", UnicodeDecodeError("idna", b"python.o\xFFrg", 8, 9, "")), + (b"python.org\xFF", UnicodeDecodeError("idna", b"python.org\xFF", 10, 11, "")), + (b"xn--pythn-&mua.org", UnicodeDecodeError("idna", b"xn--pythn-&mua.org", 10, 11, "")), + (b"xn--pythn-m&ua.org", UnicodeDecodeError("idna", b"xn--pythn-m&ua.org", 11, 12, "")), + (b"xn--pythn-mua&.org", UnicodeDecodeError("idna", b"xn--pythn-mua&.org", 13, 14, "")), + ] + invalid_encode_testcases = [ + (f"foo.{'\xff'*60}", UnicodeEncodeError("idna", f"foo.{'\xff'*60}", 4, 64, "")), + ("あさ.\u034f", UnicodeEncodeError("idna", "あさ.\u034f", 3, 4, "")), + ] + def test_builtin_decode(self): self.assertEqual(str(b"python.org", "idna"), "python.org") self.assertEqual(str(b"python.org.", "idna"), "python.org.") @@ -1555,16 +1588,38 @@ def test_builtin_decode(self): self.assertEqual(str(b"bugs.XN--pythn-mua.org.", "idna"), "bugs.pyth\xf6n.org.") + def test_builtin_decode_invalid(self): + for case, expected in self.invalid_decode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + case.decode("idna") + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start, msg=f'reason: {exc.reason}') + self.assertEqual(exc.end, expected.end) + def test_builtin_encode(self): self.assertEqual("python.org".encode("idna"), b"python.org") self.assertEqual("python.org.".encode("idna"), b"python.org.") self.assertEqual("pyth\xf6n.org".encode("idna"), b"xn--pythn-mua.org") self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.") + def test_builtin_encode_invalid(self): + for case, expected in self.invalid_encode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeEncodeError) as cm: + case.encode("idna") + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + def test_builtin_decode_length_limit(self): - with self.assertRaisesRegex(UnicodeError, "way too long"): + with self.assertRaisesRegex(UnicodeDecodeError, "way too long"): (b"xn--016c"+b"a"*1100).decode("idna") - with self.assertRaisesRegex(UnicodeError, "too long"): + with self.assertRaisesRegex(UnicodeDecodeError, "too long"): (b"xn--016c"+b"a"*70).decode("idna") def test_stream(self): @@ -1602,6 +1657,39 @@ def test_incremental_decode(self): self.assertEqual(decoder.decode(b"rg."), "org.") self.assertEqual(decoder.decode(b"", True), "") + def test_incremental_decode_invalid(self): + iterdecode_testcases = [ + (b"\xFFpython.org", UnicodeDecodeError("idna", b"\xFF", 0, 1, "")), + (b"pyt\xFFhon.org", UnicodeDecodeError("idna", b"pyt\xFF", 3, 4, "")), + (b"python\xFF.org", UnicodeDecodeError("idna", b"python\xFF", 6, 7, "")), + (b"python.\xFForg", UnicodeDecodeError("idna", b"\xFF", 0, 1, "")), + (b"python.o\xFFrg", UnicodeDecodeError("idna", b"o\xFF", 1, 2, "")), + (b"python.org\xFF", UnicodeDecodeError("idna", b"org\xFF", 3, 4, "")), + (b"xn--pythn-&mua.org", UnicodeDecodeError("idna", b"xn--pythn-&mua.", 10, 11, "")), + (b"xn--pythn-m&ua.org", UnicodeDecodeError("idna", b"xn--pythn-m&ua.", 11, 12, "")), + (b"xn--pythn-mua&.org", UnicodeDecodeError("idna", b"xn--pythn-mua&.", 13, 14, "")), + ] + for case, expected in iterdecode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + list(codecs.iterdecode((bytes([c]) for c in case), "idna")) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + + decoder = codecs.getincrementaldecoder("idna")() + for case, expected in self.invalid_decode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + decoder.decode(case) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + def test_incremental_encode(self): self.assertEqual( b"".join(codecs.iterencode("python.org", "idna")), @@ -1630,6 +1718,23 @@ def test_incremental_encode(self): self.assertEqual(encoder.encode("ample.org."), b"xn--xample-9ta.org.") self.assertEqual(encoder.encode("", True), b"") + def test_incremental_encode_invalid(self): + iterencode_testcases = [ + (f"foo.{'\xff'*60}", UnicodeEncodeError("idna", f"{'\xff'*60}", 0, 60, "")), + ("あさ.\u034f", UnicodeEncodeError("idna", "\u034f", 0, 1, "")), + ] + for case, expected in iterencode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeEncodeError) as cm: + list(codecs.iterencode(case, "idna")) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + + # codecs.getincrementalencoder.encode() does not throw an error + def test_errors(self): """Only supports "strict" error handler""" "python.org".encode("idna", "strict") diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index 6451df14696933..ccdf3a6cdc0dc7 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -303,7 +303,7 @@ def test_setstate_validates_input(self): self.assertRaises(TypeError, decoder.setstate, 123) self.assertRaises(TypeError, decoder.setstate, ("invalid", 0)) self.assertRaises(TypeError, decoder.setstate, (b"1234", "invalid")) - self.assertRaises(UnicodeError, decoder.setstate, (b"123456789", 0)) + self.assertRaises(UnicodeDecodeError, decoder.setstate, (b"123456789", 0)) class Test_StreamReader(unittest.TestCase): def test_bug1728403(self): diff --git a/Misc/NEWS.d/next/Library/2024-01-02-22-47-12.gh-issue-85287.ZC5DLj.rst b/Misc/NEWS.d/next/Library/2024-01-02-22-47-12.gh-issue-85287.ZC5DLj.rst new file mode 100644 index 00000000000000..e6d031fbc93e83 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-02-22-47-12.gh-issue-85287.ZC5DLj.rst @@ -0,0 +1,2 @@ +Changes Unicode codecs to return UnicodeEncodeError or UnicodeDecodeError, +rather than just UnicodeError. diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 2125da437963d2..e5433d7dd85306 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -825,8 +825,15 @@ encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx, if (inpos < datalen) { if (datalen - inpos > MAXENCPENDING) { /* normal codecs can't reach here */ - PyErr_SetString(PyExc_UnicodeError, - "pending buffer overflow"); + PyObject *excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError, + "sOnns", + ctx->codec->encoding, + inbuf, + inpos, datalen, + "pending buffer overflow"); + if (excobj == NULL) goto errorexit; + PyErr_SetObject(PyExc_UnicodeEncodeError, excobj); + Py_DECREF(excobj); goto errorexit; } ctx->pending = PyUnicode_Substring(inbuf, inpos, datalen); @@ -857,7 +864,16 @@ decoder_append_pending(MultibyteStatefulDecoderContext *ctx, npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); if (npendings + ctx->pendingsize > MAXDECPENDING || npendings > PY_SSIZE_T_MAX - ctx->pendingsize) { - PyErr_SetString(PyExc_UnicodeError, "pending buffer overflow"); + Py_ssize_t bufsize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top); + PyObject *excobj = PyUnicodeDecodeError_Create(ctx->codec->encoding, + (const char *)buf->inbuf_top, + bufsize, + 0, + bufsize, + "pending buffer overflow"); + if (excobj == NULL) return -1; + PyErr_SetObject(PyExc_UnicodeDecodeError, excobj); + Py_DECREF(excobj); return -1; } memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings); @@ -938,7 +954,17 @@ _multibytecodec_MultibyteIncrementalEncoder_getstate_impl(MultibyteIncrementalEn return NULL; } if (pendingsize > MAXENCPENDING*4) { - PyErr_SetString(PyExc_UnicodeError, "pending buffer too large"); + PyObject *excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError, + "sOnns", + self->codec->encoding, + self->pending, + 0, PyUnicode_GET_LENGTH(self->pending), + "pending buffer too large"); + if (excobj == NULL) { + return NULL; + } + PyErr_SetObject(PyExc_UnicodeEncodeError, excobj); + Py_DECREF(excobj); return NULL; } statebytes[0] = (unsigned char)pendingsize; @@ -1267,7 +1293,13 @@ _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDe } if (buffersize > MAXDECPENDING) { - PyErr_SetString(PyExc_UnicodeError, "pending buffer too large"); + PyObject *excobj = PyUnicodeDecodeError_Create(self->codec->encoding, + PyBytes_AS_STRING(buffer), buffersize, + 0, buffersize, + "pending buffer too large"); + if (excobj == NULL) return NULL; + PyErr_SetObject(PyExc_UnicodeDecodeError, excobj); + Py_DECREF(excobj); return NULL; } From b8d808ddd77f84de9f93adcc2aede2879eb5241e Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sun, 17 Mar 2024 09:47:14 +0100 Subject: [PATCH 131/158] GH-112536: Add more TSan tests (#116911) These may all exercise some non-trivial aspects of thread synchronization. --- Lib/test/libregrtest/tsan.py | 6 ++++++ Modules/_testinternalcapi.c | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index c5aed436b829d1..dd18ae2584f5d8 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -2,6 +2,9 @@ # chosen because they use threads and run in a reasonable amount of time. TSAN_TESTS = [ + # TODO: enable more of test_capi once bugs are fixed (GH-116908, GH-116909). + 'test_capi.test_mem', + 'test_capi.test_pyatomic', 'test_code', 'test_enum', 'test_functools', @@ -11,6 +14,9 @@ 'test_io', 'test_logging', 'test_queue', + 'test_signal', + 'test_socket', + 'test_sqlite3', 'test_ssl', 'test_syslog', 'test_thread', diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index b3076a8f548b62..1c10dd02138f3a 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1287,8 +1287,8 @@ check_pyobject_forbidden_bytes_is_freed(PyObject *self, static PyObject * check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) { - /* This test would fail if run with the address sanitizer */ -#ifdef _Py_ADDRESS_SANITIZER + /* ASan or TSan would report an use-after-free error */ +#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) Py_RETURN_NONE; #else PyObject *op = PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type); From 8e3c953b3acd7c656f66696806c9ae917e816492 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 17 Mar 2024 14:58:26 +0100 Subject: [PATCH 132/158] gh-73468: Add math.fma() function (#116667) Added new math.fma() function, wrapping C99's ``fma()`` operation: fused multiply-add function. Co-authored-by: Mark Dickinson --- Doc/library/math.rst | 16 ++ Doc/whatsnew/3.13.rst | 10 + Lib/test/test_math.py | 238 ++++++++++++++++++ ...4-03-12-17-53-14.gh-issue-73468.z4ZzvJ.rst | 2 + Modules/clinic/mathmodule.c.h | 63 ++++- Modules/mathmodule.c | 43 ++++ 6 files changed, 371 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-12-17-53-14.gh-issue-73468.z4ZzvJ.rst diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 93755be717e2ef..1475d26486de5f 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -82,6 +82,22 @@ Number-theoretic and representation functions should return an :class:`~numbers.Integral` value. +.. function:: fma(x, y, z) + + Fused multiply-add operation. Return ``(x * y) + z``, computed as though with + infinite precision and range followed by a single round to the ``float`` + format. This operation often provides better accuracy than the direct + expression ``(x * y) + z``. + + This function follows the specification of the fusedMultiplyAdd operation + described in the IEEE 754 standard. The standard leaves one case + implementation-defined, namely the result of ``fma(0, inf, nan)`` + and ``fma(inf, 0, nan)``. In these cases, ``math.fma`` returns a NaN, + and does not raise any exception. + + .. versionadded:: 3.13 + + .. function:: fmod(x, y) Return ``fmod(x, y)``, as defined by the platform C library. Note that the diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 03ae6018905380..78b5868b6549c6 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -383,6 +383,16 @@ marshal code objects which are incompatible between Python versions. (Contributed by Serhiy Storchaka in :gh:`113626`.) +math +---- + +A new function :func:`~math.fma` for fused multiply-add operations has been +added. This function computes ``x * y + z`` with only a single round, and so +avoids any intermediate loss of precision. It wraps the ``fma()`` function +provided by C99, and follows the specification of the IEEE 754 +"fusedMultiplyAdd" operation for special cases. +(Contributed by Mark Dickinson and Victor Stinner in :gh:`73468`.) + mmap ---- diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index ad382fc2b59891..aaa3b16d33fb7d 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -2613,6 +2613,244 @@ def test_fractions(self): self.assertAllNotClose(fraction_examples, rel_tol=1e-9) +class FMATests(unittest.TestCase): + """ Tests for math.fma. """ + + def test_fma_nan_results(self): + # Selected representative values. + values = [ + -math.inf, -1e300, -2.3, -1e-300, -0.0, + 0.0, 1e-300, 2.3, 1e300, math.inf, math.nan + ] + + # If any input is a NaN, the result should be a NaN, too. + for a, b in itertools.product(values, repeat=2): + self.assertIsNaN(math.fma(math.nan, a, b)) + self.assertIsNaN(math.fma(a, math.nan, b)) + self.assertIsNaN(math.fma(a, b, math.nan)) + + def test_fma_infinities(self): + # Cases involving infinite inputs or results. + positives = [1e-300, 2.3, 1e300, math.inf] + finites = [-1e300, -2.3, -1e-300, -0.0, 0.0, 1e-300, 2.3, 1e300] + non_nans = [-math.inf, -2.3, -0.0, 0.0, 2.3, math.inf] + + # ValueError due to inf * 0 computation. + for c in non_nans: + for infinity in [math.inf, -math.inf]: + for zero in [0.0, -0.0]: + with self.assertRaises(ValueError): + math.fma(infinity, zero, c) + with self.assertRaises(ValueError): + math.fma(zero, infinity, c) + + # ValueError when a*b and c both infinite of opposite signs. + for b in positives: + with self.assertRaises(ValueError): + math.fma(math.inf, b, -math.inf) + with self.assertRaises(ValueError): + math.fma(math.inf, -b, math.inf) + with self.assertRaises(ValueError): + math.fma(-math.inf, -b, -math.inf) + with self.assertRaises(ValueError): + math.fma(-math.inf, b, math.inf) + with self.assertRaises(ValueError): + math.fma(b, math.inf, -math.inf) + with self.assertRaises(ValueError): + math.fma(-b, math.inf, math.inf) + with self.assertRaises(ValueError): + math.fma(-b, -math.inf, -math.inf) + with self.assertRaises(ValueError): + math.fma(b, -math.inf, math.inf) + + # Infinite result when a*b and c both infinite of the same sign. + for b in positives: + self.assertEqual(math.fma(math.inf, b, math.inf), math.inf) + self.assertEqual(math.fma(math.inf, -b, -math.inf), -math.inf) + self.assertEqual(math.fma(-math.inf, -b, math.inf), math.inf) + self.assertEqual(math.fma(-math.inf, b, -math.inf), -math.inf) + self.assertEqual(math.fma(b, math.inf, math.inf), math.inf) + self.assertEqual(math.fma(-b, math.inf, -math.inf), -math.inf) + self.assertEqual(math.fma(-b, -math.inf, math.inf), math.inf) + self.assertEqual(math.fma(b, -math.inf, -math.inf), -math.inf) + + # Infinite result when a*b finite, c infinite. + for a, b in itertools.product(finites, finites): + self.assertEqual(math.fma(a, b, math.inf), math.inf) + self.assertEqual(math.fma(a, b, -math.inf), -math.inf) + + # Infinite result when a*b infinite, c finite. + for b, c in itertools.product(positives, finites): + self.assertEqual(math.fma(math.inf, b, c), math.inf) + self.assertEqual(math.fma(-math.inf, b, c), -math.inf) + self.assertEqual(math.fma(-math.inf, -b, c), math.inf) + self.assertEqual(math.fma(math.inf, -b, c), -math.inf) + + self.assertEqual(math.fma(b, math.inf, c), math.inf) + self.assertEqual(math.fma(b, -math.inf, c), -math.inf) + self.assertEqual(math.fma(-b, -math.inf, c), math.inf) + self.assertEqual(math.fma(-b, math.inf, c), -math.inf) + + # gh-73468: On WASI and FreeBSD, libc fma() doesn't implement IEE 754-2008 + # properly: it doesn't use the right sign when the result is zero. + @unittest.skipIf(support.is_wasi, + "WASI fma() doesn't implement IEE 754-2008 properly") + @unittest.skipIf(sys.platform.startswith('freebsd'), + "FreeBSD fma() doesn't implement IEE 754-2008 properly") + def test_fma_zero_result(self): + nonnegative_finites = [0.0, 1e-300, 2.3, 1e300] + + # Zero results from exact zero inputs. + for b in nonnegative_finites: + self.assertIsPositiveZero(math.fma(0.0, b, 0.0)) + self.assertIsPositiveZero(math.fma(0.0, b, -0.0)) + self.assertIsNegativeZero(math.fma(0.0, -b, -0.0)) + self.assertIsPositiveZero(math.fma(0.0, -b, 0.0)) + self.assertIsPositiveZero(math.fma(-0.0, -b, 0.0)) + self.assertIsPositiveZero(math.fma(-0.0, -b, -0.0)) + self.assertIsNegativeZero(math.fma(-0.0, b, -0.0)) + self.assertIsPositiveZero(math.fma(-0.0, b, 0.0)) + + self.assertIsPositiveZero(math.fma(b, 0.0, 0.0)) + self.assertIsPositiveZero(math.fma(b, 0.0, -0.0)) + self.assertIsNegativeZero(math.fma(-b, 0.0, -0.0)) + self.assertIsPositiveZero(math.fma(-b, 0.0, 0.0)) + self.assertIsPositiveZero(math.fma(-b, -0.0, 0.0)) + self.assertIsPositiveZero(math.fma(-b, -0.0, -0.0)) + self.assertIsNegativeZero(math.fma(b, -0.0, -0.0)) + self.assertIsPositiveZero(math.fma(b, -0.0, 0.0)) + + # Exact zero result from nonzero inputs. + self.assertIsPositiveZero(math.fma(2.0, 2.0, -4.0)) + self.assertIsPositiveZero(math.fma(2.0, -2.0, 4.0)) + self.assertIsPositiveZero(math.fma(-2.0, -2.0, -4.0)) + self.assertIsPositiveZero(math.fma(-2.0, 2.0, 4.0)) + + # Underflow to zero. + tiny = 1e-300 + self.assertIsPositiveZero(math.fma(tiny, tiny, 0.0)) + self.assertIsNegativeZero(math.fma(tiny, -tiny, 0.0)) + self.assertIsPositiveZero(math.fma(-tiny, -tiny, 0.0)) + self.assertIsNegativeZero(math.fma(-tiny, tiny, 0.0)) + self.assertIsPositiveZero(math.fma(tiny, tiny, -0.0)) + self.assertIsNegativeZero(math.fma(tiny, -tiny, -0.0)) + self.assertIsPositiveZero(math.fma(-tiny, -tiny, -0.0)) + self.assertIsNegativeZero(math.fma(-tiny, tiny, -0.0)) + + # Corner case where rounding the multiplication would + # give the wrong result. + x = float.fromhex('0x1p-500') + y = float.fromhex('0x1p-550') + z = float.fromhex('0x1p-1000') + self.assertIsNegativeZero(math.fma(x-y, x+y, -z)) + self.assertIsPositiveZero(math.fma(y-x, x+y, z)) + self.assertIsNegativeZero(math.fma(y-x, -(x+y), -z)) + self.assertIsPositiveZero(math.fma(x-y, -(x+y), z)) + + def test_fma_overflow(self): + a = b = float.fromhex('0x1p512') + c = float.fromhex('0x1p1023') + # Overflow from multiplication. + with self.assertRaises(OverflowError): + math.fma(a, b, 0.0) + self.assertEqual(math.fma(a, b/2.0, 0.0), c) + # Overflow from the addition. + with self.assertRaises(OverflowError): + math.fma(a, b/2.0, c) + # No overflow, even though a*b overflows a float. + self.assertEqual(math.fma(a, b, -c), c) + + # Extreme case: a * b is exactly at the overflow boundary, so the + # tiniest offset makes a difference between overflow and a finite + # result. + a = float.fromhex('0x1.ffffffc000000p+511') + b = float.fromhex('0x1.0000002000000p+512') + c = float.fromhex('0x0.0000000000001p-1022') + with self.assertRaises(OverflowError): + math.fma(a, b, 0.0) + with self.assertRaises(OverflowError): + math.fma(a, b, c) + self.assertEqual(math.fma(a, b, -c), + float.fromhex('0x1.fffffffffffffp+1023')) + + # Another extreme case: here a*b is about as large as possible subject + # to math.fma(a, b, c) being finite. + a = float.fromhex('0x1.ae565943785f9p+512') + b = float.fromhex('0x1.3094665de9db8p+512') + c = float.fromhex('0x1.fffffffffffffp+1023') + self.assertEqual(math.fma(a, b, -c), c) + + def test_fma_single_round(self): + a = float.fromhex('0x1p-50') + self.assertEqual(math.fma(a - 1.0, a + 1.0, 1.0), a*a) + + def test_random(self): + # A collection of randomly generated inputs for which the naive FMA + # (with two rounds) gives a different result from a singly-rounded FMA. + + # tuples (a, b, c, expected) + test_values = [ + ('0x1.694adde428b44p-1', '0x1.371b0d64caed7p-1', + '0x1.f347e7b8deab8p-4', '0x1.19f10da56c8adp-1'), + ('0x1.605401ccc6ad6p-2', '0x1.ce3a40bf56640p-2', + '0x1.96e3bf7bf2e20p-2', '0x1.1af6d8aa83101p-1'), + ('0x1.e5abd653a67d4p-2', '0x1.a2e400209b3e6p-1', + '0x1.a90051422ce13p-1', '0x1.37d68cc8c0fbbp+0'), + ('0x1.f94e8efd54700p-2', '0x1.123065c812cebp-1', + '0x1.458f86fb6ccd0p-1', '0x1.ccdcee26a3ff3p-1'), + ('0x1.bd926f1eedc96p-1', '0x1.eee9ca68c5740p-1', + '0x1.960c703eb3298p-2', '0x1.3cdcfb4fdb007p+0'), + ('0x1.27348350fbccdp-1', '0x1.3b073914a53f1p-1', + '0x1.e300da5c2b4cbp-1', '0x1.4c51e9a3c4e29p+0'), + ('0x1.2774f00b3497bp-1', '0x1.7038ec336bff0p-2', + '0x1.2f6f2ccc3576bp-1', '0x1.99ad9f9c2688bp-1'), + ('0x1.51d5a99300e5cp-1', '0x1.5cd74abd445a1p-1', + '0x1.8880ab0bbe530p-1', '0x1.3756f96b91129p+0'), + ('0x1.73cb965b821b8p-2', '0x1.218fd3d8d5371p-1', + '0x1.d1ea966a1f758p-2', '0x1.5217b8fd90119p-1'), + ('0x1.4aa98e890b046p-1', '0x1.954d85dff1041p-1', + '0x1.122b59317ebdfp-1', '0x1.0bf644b340cc5p+0'), + ('0x1.e28f29e44750fp-1', '0x1.4bcc4fdcd18fep-1', + '0x1.fd47f81298259p-1', '0x1.9b000afbc9995p+0'), + ('0x1.d2e850717fe78p-3', '0x1.1dd7531c303afp-1', + '0x1.e0869746a2fc2p-2', '0x1.316df6eb26439p-1'), + ('0x1.cf89c75ee6fbap-2', '0x1.b23decdc66825p-1', + '0x1.3d1fe76ac6168p-1', '0x1.00d8ea4c12abbp+0'), + ('0x1.3265ae6f05572p-2', '0x1.16d7ec285f7a2p-1', + '0x1.0b8405b3827fbp-1', '0x1.5ef33c118a001p-1'), + ('0x1.c4d1bf55ec1a5p-1', '0x1.bc59618459e12p-2', + '0x1.ce5b73dc1773dp-1', '0x1.496cf6164f99bp+0'), + ('0x1.d350026ac3946p-1', '0x1.9a234e149a68cp-2', + '0x1.f5467b1911fd6p-2', '0x1.b5cee3225caa5p-1'), + ] + for a_hex, b_hex, c_hex, expected_hex in test_values: + a = float.fromhex(a_hex) + b = float.fromhex(b_hex) + c = float.fromhex(c_hex) + expected = float.fromhex(expected_hex) + self.assertEqual(math.fma(a, b, c), expected) + self.assertEqual(math.fma(b, a, c), expected) + + # Custom assertions. + def assertIsNaN(self, value): + self.assertTrue( + math.isnan(value), + msg="Expected a NaN, got {!r}".format(value) + ) + + def assertIsPositiveZero(self, value): + self.assertTrue( + value == 0 and math.copysign(1, value) > 0, + msg="Expected a positive zero, got {!r}".format(value) + ) + + def assertIsNegativeZero(self, value): + self.assertTrue( + value == 0 and math.copysign(1, value) < 0, + msg="Expected a negative zero, got {!r}".format(value) + ) + + def load_tests(loader, tests, pattern): from doctest import DocFileSuite tests.addTest(DocFileSuite(os.path.join("mathdata", "ieee754.txt"))) diff --git a/Misc/NEWS.d/next/Library/2024-03-12-17-53-14.gh-issue-73468.z4ZzvJ.rst b/Misc/NEWS.d/next/Library/2024-03-12-17-53-14.gh-issue-73468.z4ZzvJ.rst new file mode 100644 index 00000000000000..c91f4eb97e06bc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-12-17-53-14.gh-issue-73468.z4ZzvJ.rst @@ -0,0 +1,2 @@ +Added new :func:`math.fma` function, wrapping C99's ``fma()`` operation: +fused multiply-add function. Patch by Mark Dickinson and Victor Stinner. diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index ca14c03f16f706..666b6b3790dae5 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -204,6 +204,67 @@ PyDoc_STRVAR(math_log10__doc__, #define MATH_LOG10_METHODDEF \ {"log10", (PyCFunction)math_log10, METH_O, math_log10__doc__}, +PyDoc_STRVAR(math_fma__doc__, +"fma($module, x, y, z, /)\n" +"--\n" +"\n" +"Fused multiply-add operation.\n" +"\n" +"Compute (x * y) + z with a single round."); + +#define MATH_FMA_METHODDEF \ + {"fma", _PyCFunction_CAST(math_fma), METH_FASTCALL, math_fma__doc__}, + +static PyObject * +math_fma_impl(PyObject *module, double x, double y, double z); + +static PyObject * +math_fma(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double x; + double y; + double z; + + if (!_PyArg_CheckPositional("fma", nargs, 3, 3)) { + goto exit; + } + if (PyFloat_CheckExact(args[0])) { + x = PyFloat_AS_DOUBLE(args[0]); + } + else + { + x = PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (PyFloat_CheckExact(args[1])) { + y = PyFloat_AS_DOUBLE(args[1]); + } + else + { + y = PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (PyFloat_CheckExact(args[2])) { + z = PyFloat_AS_DOUBLE(args[2]); + } + else + { + z = PyFloat_AsDouble(args[2]); + if (z == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + return_value = math_fma_impl(module, x, y, z); + +exit: + return return_value; +} + PyDoc_STRVAR(math_fmod__doc__, "fmod($module, x, y, /)\n" "--\n" @@ -950,4 +1011,4 @@ math_ulp(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=6b2eeaed8d8a76d5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9fe3f007f474e015 input=a9049054013a1b77]*/ diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index a877bfcd6afb68..8ba0431f4a47b7 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -2321,6 +2321,48 @@ math_log10(PyObject *module, PyObject *x) } +/*[clinic input] +math.fma + + x: double + y: double + z: double + / + +Fused multiply-add operation. + +Compute (x * y) + z with a single round. +[clinic start generated code]*/ + +static PyObject * +math_fma_impl(PyObject *module, double x, double y, double z) +/*[clinic end generated code: output=4fc8626dbc278d17 input=e3ad1f4a4c89626e]*/ +{ + double r = fma(x, y, z); + + /* Fast path: if we got a finite result, we're done. */ + if (Py_IS_FINITE(r)) { + return PyFloat_FromDouble(r); + } + + /* Non-finite result. Raise an exception if appropriate, else return r. */ + if (Py_IS_NAN(r)) { + if (!Py_IS_NAN(x) && !Py_IS_NAN(y) && !Py_IS_NAN(z)) { + /* NaN result from non-NaN inputs. */ + PyErr_SetString(PyExc_ValueError, "invalid operation in fma"); + return NULL; + } + } + else if (Py_IS_FINITE(x) && Py_IS_FINITE(y) && Py_IS_FINITE(z)) { + /* Infinite result from finite inputs. */ + PyErr_SetString(PyExc_OverflowError, "overflow in fma"); + return NULL; + } + + return PyFloat_FromDouble(r); +} + + /*[clinic input] math.fmod @@ -4094,6 +4136,7 @@ static PyMethodDef math_methods[] = { {"fabs", math_fabs, METH_O, math_fabs_doc}, MATH_FACTORIAL_METHODDEF MATH_FLOOR_METHODDEF + MATH_FMA_METHODDEF MATH_FMOD_METHODDEF MATH_FREXP_METHODDEF MATH_FSUM_METHODDEF From 1cf03010865c66c2c3286ffdafd55e7ce2d97444 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 17 Mar 2024 16:12:29 +0100 Subject: [PATCH 133/158] gh-85283: Build termios extension with the limited C API (#116928) --- Doc/whatsnew/3.13.rst | 3 +- ...4-03-14-10-33-58.gh-issue-85283.LOgmdU.rst | 4 +- Modules/clinic/termios.c.h | 29 +++++++------- Modules/termios.c | 38 +++++++++++-------- 4 files changed, 42 insertions(+), 32 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 78b5868b6549c6..3e08d804d9494d 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1479,7 +1479,8 @@ Build Changes * Building CPython now requires a compiler with support for the C11 atomic library, GCC built-in atomic functions, or MSVC interlocked intrinsics. -* The ``errno``, ``fcntl``, ``grp``, ``md5``, ``pwd``, ``resource``, ``winsound``, +* The ``errno``, ``fcntl``, ``grp``, ``md5``, ``pwd``, ``resource``, + ``termios``, ``winsound``, ``_ctypes_test``, ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, ``_testimportmultiple`` and ``_uuid`` C extensions are now built with the :ref:`limited C API `. diff --git a/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst b/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst index 8a3185a8649d7d..bfb72db9d931d9 100644 --- a/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst +++ b/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst @@ -1,2 +1,2 @@ -The ``fcntl``, ``grp`` and ``pwd`` C extensions are now built with the :ref:`limited -C API `. (Contributed by Victor Stinner in :gh:`85283`.) +The ``fcntl``, ``grp``, ``pwd`` and ``termios`` C extensions are now +built with the :ref:`limited C API `. Patch by Victor Stinner. diff --git a/Modules/clinic/termios.c.h b/Modules/clinic/termios.c.h index 2813e5e4700352..83f5a4f6e9f882 100644 --- a/Modules/clinic/termios.c.h +++ b/Modules/clinic/termios.c.h @@ -2,8 +2,6 @@ preserve [clinic start generated code]*/ -#include "pycore_modsupport.h" // _PyArg_CheckPositional() - PyDoc_STRVAR(termios_tcgetattr__doc__, "tcgetattr($module, fd, /)\n" "--\n" @@ -53,7 +51,7 @@ PyDoc_STRVAR(termios_tcsetattr__doc__, "queued output and discarding all queued input."); #define TERMIOS_TCSETATTR_METHODDEF \ - {"tcsetattr", _PyCFunction_CAST(termios_tcsetattr), METH_FASTCALL, termios_tcsetattr__doc__}, + {"tcsetattr", (PyCFunction)(void(*)(void))termios_tcsetattr, METH_FASTCALL, termios_tcsetattr__doc__}, static PyObject * termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term); @@ -66,7 +64,8 @@ termios_tcsetattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int when; PyObject *term; - if (!_PyArg_CheckPositional("tcsetattr", nargs, 3, 3)) { + if (nargs != 3) { + PyErr_Format(PyExc_TypeError, "tcsetattr expected 3 arguments, got %zd", nargs); goto exit; } fd = PyObject_AsFileDescriptor(args[0]); @@ -94,7 +93,7 @@ PyDoc_STRVAR(termios_tcsendbreak__doc__, "has a system dependent meaning."); #define TERMIOS_TCSENDBREAK_METHODDEF \ - {"tcsendbreak", _PyCFunction_CAST(termios_tcsendbreak), METH_FASTCALL, termios_tcsendbreak__doc__}, + {"tcsendbreak", (PyCFunction)(void(*)(void))termios_tcsendbreak, METH_FASTCALL, termios_tcsendbreak__doc__}, static PyObject * termios_tcsendbreak_impl(PyObject *module, int fd, int duration); @@ -106,7 +105,8 @@ termios_tcsendbreak(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int duration; - if (!_PyArg_CheckPositional("tcsendbreak", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "tcsendbreak expected 2 arguments, got %zd", nargs); goto exit; } fd = PyObject_AsFileDescriptor(args[0]); @@ -162,7 +162,7 @@ PyDoc_STRVAR(termios_tcflush__doc__, "both queues."); #define TERMIOS_TCFLUSH_METHODDEF \ - {"tcflush", _PyCFunction_CAST(termios_tcflush), METH_FASTCALL, termios_tcflush__doc__}, + {"tcflush", (PyCFunction)(void(*)(void))termios_tcflush, METH_FASTCALL, termios_tcflush__doc__}, static PyObject * termios_tcflush_impl(PyObject *module, int fd, int queue); @@ -174,7 +174,8 @@ termios_tcflush(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int queue; - if (!_PyArg_CheckPositional("tcflush", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "tcflush expected 2 arguments, got %zd", nargs); goto exit; } fd = PyObject_AsFileDescriptor(args[0]); @@ -202,7 +203,7 @@ PyDoc_STRVAR(termios_tcflow__doc__, "or termios.TCION to restart input."); #define TERMIOS_TCFLOW_METHODDEF \ - {"tcflow", _PyCFunction_CAST(termios_tcflow), METH_FASTCALL, termios_tcflow__doc__}, + {"tcflow", (PyCFunction)(void(*)(void))termios_tcflow, METH_FASTCALL, termios_tcflow__doc__}, static PyObject * termios_tcflow_impl(PyObject *module, int fd, int action); @@ -214,7 +215,8 @@ termios_tcflow(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int action; - if (!_PyArg_CheckPositional("tcflow", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "tcflow expected 2 arguments, got %zd", nargs); goto exit; } fd = PyObject_AsFileDescriptor(args[0]); @@ -271,7 +273,7 @@ PyDoc_STRVAR(termios_tcsetwinsize__doc__, "is a two-item tuple (ws_row, ws_col) like the one returned by tcgetwinsize()."); #define TERMIOS_TCSETWINSIZE_METHODDEF \ - {"tcsetwinsize", _PyCFunction_CAST(termios_tcsetwinsize), METH_FASTCALL, termios_tcsetwinsize__doc__}, + {"tcsetwinsize", (PyCFunction)(void(*)(void))termios_tcsetwinsize, METH_FASTCALL, termios_tcsetwinsize__doc__}, static PyObject * termios_tcsetwinsize_impl(PyObject *module, int fd, PyObject *winsz); @@ -283,7 +285,8 @@ termios_tcsetwinsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; PyObject *winsz; - if (!_PyArg_CheckPositional("tcsetwinsize", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "tcsetwinsize expected 2 arguments, got %zd", nargs); goto exit; } fd = PyObject_AsFileDescriptor(args[0]); @@ -296,4 +299,4 @@ termios_tcsetwinsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=7327a2085972bf59 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c6c6192583b0da36 input=a9049054013a1b77]*/ diff --git a/Modules/termios.c b/Modules/termios.c index 4635fefb8f3f5a..a29474d650127f 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -1,11 +1,19 @@ /* termios.c -- POSIX terminal I/O module implementation. */ -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 +// Need limited C API version 3.13 for PyLong_AsInt() +// in code generated by Argument Clinic. +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" +#include // memset() +#include +#include +#include // _POSIX_VDISABLE + // On QNX 6, struct termio must be declared by including sys/termio.h // if TCGETA, TCSETA, TCSETAW, or TCSETAF are used. sys/termio.h must // be included before termios.h or it will generate an error. @@ -19,28 +27,26 @@ # define CTRL(c) ((c)&037) #endif +// We could do better. Check bpo-32660 #if defined(__sun) -/* We could do better. Check issue-32660 */ -#include -#include +# include +# include #endif -#include -#include -#include // _POSIX_VDISABLE - /* HP-UX requires that this be included to pick up MDCD, MCTS, MDSR, * MDTR, MRI, and MRTS (apparently used internally by some things * defined as macros; these are not used here directly). */ #ifdef HAVE_SYS_MODEM_H -#include +# include #endif + /* HP-UX requires that this be included to pick up TIOCGPGRP and friends */ #ifdef HAVE_SYS_BSDTTY_H -#include +# include #endif + /*[clinic input] module termios [clinic start generated code]*/ @@ -120,7 +126,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) v = PyBytes_FromStringAndSize(&ch, 1); if (v == NULL) goto err; - PyList_SET_ITEM(cc, i, v); + PyList_SetItem(cc, i, v); } /* Convert the MIN and TIME slots to integer. On some systems, the @@ -154,7 +160,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) Py_DECREF(v); \ goto err; \ } \ - PyList_SET_ITEM(v, index, l); \ + PyList_SetItem(v, index, l); \ } while (0) ADD_LONG_ITEM(0, mode.c_iflag); @@ -165,7 +171,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) ADD_LONG_ITEM(5, ospeed); #undef ADD_LONG_ITEM - PyList_SET_ITEM(v, 6, cc); + PyList_SetItem(v, 6, cc); return v; err: Py_DECREF(cc); @@ -214,7 +220,7 @@ termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term) speed_t ispeed, ospeed; #define SET_FROM_LIST(TYPE, VAR, LIST, N) do { \ - PyObject *item = PyList_GET_ITEM(LIST, N); \ + PyObject *item = PyList_GetItem(LIST, N); \ long num = PyLong_AsLong(item); \ if (num == -1 && PyErr_Occurred()) { \ return NULL; \ @@ -230,7 +236,7 @@ termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term) SET_FROM_LIST(speed_t, ospeed, term, 5); #undef SET_FROM_LIST - PyObject *cc = PyList_GET_ITEM(term, 6); + PyObject *cc = PyList_GetItem(term, 6); if (!PyList_Check(cc) || PyList_Size(cc) != NCCS) { PyErr_Format(PyExc_TypeError, "tcsetattr: attributes[6] must be %d element list", From 0634201f5391242524dbb5225de37f81a2cc1826 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 17 Mar 2024 17:09:35 +0000 Subject: [PATCH 134/158] GH-116377: Stop raising `ValueError` from `glob.translate()`. (#116378) Stop raising `ValueError` from `glob.translate()` when a `**` sub-string appears in a non-recursive pattern segment. This matches `glob.glob()` behaviour. --- Doc/library/glob.rst | 3 +-- Lib/glob.py | 33 ++++++++++------------- Lib/test/test_glob.py | 6 ++--- Lib/test/test_pathlib/test_pathlib_abc.py | 2 -- 4 files changed, 18 insertions(+), 26 deletions(-) diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 19a0bbba8966ba..15fef747296ed4 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -136,8 +136,7 @@ The :mod:`glob` module defines the following functions: separators, and ``*`` pattern segments match precisely one path segment. If *recursive* is true, the pattern segment "``**``" will match any number - of path segments. If "``**``" occurs in any position other than a full - pattern segment, :exc:`ValueError` is raised. + of path segments. If *include_hidden* is true, wildcards can match path segments that start with a dot (``.``). diff --git a/Lib/glob.py b/Lib/glob.py index 343be78a73b20a..473502c67336f9 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -256,8 +256,7 @@ def translate(pat, *, recursive=False, include_hidden=False, seps=None): """Translate a pathname with shell wildcards to a regular expression. If `recursive` is true, the pattern segment '**' will match any number of - path segments; if '**' appears outside its own segment, ValueError will be - raised. + path segments. If `include_hidden` is true, wildcards can match path segments beginning with a dot ('.'). @@ -291,22 +290,18 @@ def translate(pat, *, recursive=False, include_hidden=False, seps=None): for idx, part in enumerate(parts): if part == '*': results.append(one_segment if idx < last_part_idx else one_last_segment) - continue - if recursive: - if part == '**': - if idx < last_part_idx: - if parts[idx + 1] != '**': - results.append(any_segments) - else: - results.append(any_last_segments) - continue - elif '**' in part: - raise ValueError("Invalid pattern: '**' can only be an entire path component") - if part: - if not include_hidden and part[0] in '*?': - results.append(r'(?!\.)') - results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) - if idx < last_part_idx: - results.append(any_sep) + elif recursive and part == '**': + if idx < last_part_idx: + if parts[idx + 1] != '**': + results.append(any_segments) + else: + results.append(any_last_segments) + else: + if part: + if not include_hidden and part[0] in '*?': + results.append(r'(?!\.)') + results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) + if idx < last_part_idx: + results.append(any_sep) res = ''.join(results) return fr'(?s:{res})\Z' diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index 1fdf2817e236f6..2de997501039ad 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -452,9 +452,9 @@ def fn(pat): self.assertEqual(fn('?'), r'(?s:[^/])\Z') self.assertEqual(fn('**'), r'(?s:.*)\Z') self.assertEqual(fn('**/**'), r'(?s:.*)\Z') - self.assertRaises(ValueError, fn, '***') - self.assertRaises(ValueError, fn, 'a**') - self.assertRaises(ValueError, fn, '**b') + self.assertEqual(fn('***'), r'(?s:[^/]*)\Z') + self.assertEqual(fn('a**'), r'(?s:a[^/]*)\Z') + self.assertEqual(fn('**b'), r'(?s:[^/]*b)\Z') self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/(?:.+/)?[^/]+/[^/]*\.[^/]*/.*)\Z') def test_translate_seps(self): diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index fb467a015a80d2..840fb903fd5338 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -512,8 +512,6 @@ def test_full_match_common(self): self.assertFalse(P('a/b/c.py').full_match('**/a/b/c./**')) self.assertFalse(P('a/b/c.py').full_match('/a/b/c.py/**')) self.assertFalse(P('a/b/c.py').full_match('/**/a/b/c.py')) - self.assertRaises(ValueError, P('a').full_match, '**a/b/c') - self.assertRaises(ValueError, P('a').full_match, 'a/b/c**') # Case-sensitive flag self.assertFalse(P('A.py').full_match('a.PY', case_sensitive=True)) self.assertTrue(P('A.py').full_match('a.PY', case_sensitive=False)) From 2982bdb936f76518b29cf7de356eb5fafd22d112 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 17 Mar 2024 18:59:02 +0100 Subject: [PATCH 135/158] gh-85283: Build _statistics extension with the limited C API (#116927) Argument Clinic now inlines _PyArg_CheckPositional() for the limited C API. The generated code should be as fast or even a little bit faster. --- Doc/whatsnew/3.13.rst | 3 +- ...4-03-14-10-33-58.gh-issue-85283.LOgmdU.rst | 2 +- Modules/_statisticsmodule.c | 7 +-- Modules/clinic/_statisticsmodule.c.h | 45 ++++++------------- 4 files changed, 20 insertions(+), 37 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 3e08d804d9494d..a48db949d8f401 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1482,7 +1482,8 @@ Build Changes * The ``errno``, ``fcntl``, ``grp``, ``md5``, ``pwd``, ``resource``, ``termios``, ``winsound``, ``_ctypes_test``, ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, - ``_testimportmultiple`` and ``_uuid`` C extensions are now built with the + ``_statistics``, ``_testimportmultiple`` and ``_uuid`` + C extensions are now built with the :ref:`limited C API `. (Contributed by Victor Stinner in :gh:`85283`.) diff --git a/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst b/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst index bfb72db9d931d9..ef8a934b435a88 100644 --- a/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst +++ b/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst @@ -1,2 +1,2 @@ -The ``fcntl``, ``grp``, ``pwd`` and ``termios`` C extensions are now +The ``fcntl``, ``grp``, ``pwd``, ``termios`` and ``_statistics`` C extensions are now built with the :ref:`limited C API `. Patch by Victor Stinner. diff --git a/Modules/_statisticsmodule.c b/Modules/_statisticsmodule.c index a04a2a779a5d3d..78a6552c4c9ec0 100644 --- a/Modules/_statisticsmodule.c +++ b/Modules/_statisticsmodule.c @@ -1,8 +1,9 @@ /* statistics accelerator C extension: _statistics module. */ -// clinic/_statisticsmodule.c.h uses internal pycore_modsupport.h API -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 +// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030c0000 #endif #include "Python.h" diff --git a/Modules/clinic/_statisticsmodule.c.h b/Modules/clinic/_statisticsmodule.c.h index 653a2138aaad70..cd5b9bea8db2f2 100644 --- a/Modules/clinic/_statisticsmodule.c.h +++ b/Modules/clinic/_statisticsmodule.c.h @@ -2,15 +2,13 @@ preserve [clinic start generated code]*/ -#include "pycore_modsupport.h" // _PyArg_CheckPositional() - PyDoc_STRVAR(_statistics__normal_dist_inv_cdf__doc__, "_normal_dist_inv_cdf($module, p, mu, sigma, /)\n" "--\n" "\n"); #define _STATISTICS__NORMAL_DIST_INV_CDF_METHODDEF \ - {"_normal_dist_inv_cdf", _PyCFunction_CAST(_statistics__normal_dist_inv_cdf), METH_FASTCALL, _statistics__normal_dist_inv_cdf__doc__}, + {"_normal_dist_inv_cdf", (PyCFunction)(void(*)(void))_statistics__normal_dist_inv_cdf, METH_FASTCALL, _statistics__normal_dist_inv_cdf__doc__}, static double _statistics__normal_dist_inv_cdf_impl(PyObject *module, double p, double mu, @@ -25,38 +23,21 @@ _statistics__normal_dist_inv_cdf(PyObject *module, PyObject *const *args, Py_ssi double sigma; double _return_value; - if (!_PyArg_CheckPositional("_normal_dist_inv_cdf", nargs, 3, 3)) { + if (nargs != 3) { + PyErr_Format(PyExc_TypeError, "_normal_dist_inv_cdf expected 3 arguments, got %zd", nargs); goto exit; } - if (PyFloat_CheckExact(args[0])) { - p = PyFloat_AS_DOUBLE(args[0]); - } - else - { - p = PyFloat_AsDouble(args[0]); - if (p == -1.0 && PyErr_Occurred()) { - goto exit; - } - } - if (PyFloat_CheckExact(args[1])) { - mu = PyFloat_AS_DOUBLE(args[1]); - } - else - { - mu = PyFloat_AsDouble(args[1]); - if (mu == -1.0 && PyErr_Occurred()) { - goto exit; - } + p = PyFloat_AsDouble(args[0]); + if (p == -1.0 && PyErr_Occurred()) { + goto exit; } - if (PyFloat_CheckExact(args[2])) { - sigma = PyFloat_AS_DOUBLE(args[2]); + mu = PyFloat_AsDouble(args[1]); + if (mu == -1.0 && PyErr_Occurred()) { + goto exit; } - else - { - sigma = PyFloat_AsDouble(args[2]); - if (sigma == -1.0 && PyErr_Occurred()) { - goto exit; - } + sigma = PyFloat_AsDouble(args[2]); + if (sigma == -1.0 && PyErr_Occurred()) { + goto exit; } _return_value = _statistics__normal_dist_inv_cdf_impl(module, p, mu, sigma); if ((_return_value == -1.0) && PyErr_Occurred()) { @@ -67,4 +48,4 @@ _statistics__normal_dist_inv_cdf(PyObject *module, PyObject *const *args, Py_ssi exit: return return_value; } -/*[clinic end generated code: output=e7cead17f9f3e19f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0f0c849d51f16f1b input=a9049054013a1b77]*/ From b3f0c1591a85d335c89dc38a177d116d2017502d Mon Sep 17 00:00:00 2001 From: mpage Date: Mon, 18 Mar 2024 01:40:16 -0700 Subject: [PATCH 136/158] gh-116915: Make `_thread._ThreadHandle` support GC (#116934) Even though it has no internal references to Python objects it still has a reference to its type by virtue of being a heap type. We need to provide a traverse function that visits the type, but we do not need to provide a clear function. --- Modules/_threadmodule.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 6c84b7e08e226b..6889e8f6e12126 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -589,12 +589,21 @@ PyThreadHandleObject_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *)PyThreadHandleObject_new(type); } +static int +PyThreadHandleObject_traverse(PyThreadHandleObject *self, visitproc visit, + void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + static void PyThreadHandleObject_dealloc(PyThreadHandleObject *self) { - PyObject *tp = (PyObject *) Py_TYPE(self); + PyObject_GC_UnTrack(self); + PyTypeObject *tp = Py_TYPE(self); ThreadHandle_decref(self->handle); - PyObject_Free(self); + tp->tp_free(self); Py_DECREF(tp); } @@ -673,6 +682,7 @@ static PyType_Slot ThreadHandle_Type_slots[] = { {Py_tp_dealloc, (destructor)PyThreadHandleObject_dealloc}, {Py_tp_repr, (reprfunc)PyThreadHandleObject_repr}, {Py_tp_getset, ThreadHandle_getsetlist}, + {Py_tp_traverse, PyThreadHandleObject_traverse}, {Py_tp_methods, ThreadHandle_methods}, {Py_tp_new, PyThreadHandleObject_tp_new}, {0, 0} @@ -682,7 +692,7 @@ static PyType_Spec ThreadHandle_Type_spec = { "_thread._ThreadHandle", sizeof(PyThreadHandleObject), 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC, ThreadHandle_Type_slots, }; From 3a99f5c5f34dc7b67597ca7230da355d92927c71 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 18 Mar 2024 10:44:15 +0200 Subject: [PATCH 137/158] Bump GitHub Actions (#116944) --- .github/workflows/build.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/project-updater.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e36859e728b67f..9e236534ae3770 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -97,7 +97,7 @@ jobs: - name: Get a list of the changed documentation-related files if: github.event_name == 'pull_request' id: changed-docs-files - uses: Ana06/get-changed-files@v2.2.0 + uses: Ana06/get-changed-files@v2.3.0 with: filter: | Doc/** diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4a70ec6205a05b..ccde03f91983df 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -23,4 +23,4 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.x" - - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/project-updater.yml b/.github/workflows/project-updater.yml index 7574bfc208ff76..56e508d453c346 100644 --- a/.github/workflows/project-updater.yml +++ b/.github/workflows/project-updater.yml @@ -23,7 +23,7 @@ jobs: - { project: 32, label: sprint } steps: - - uses: actions/add-to-project@v0.1.0 + - uses: actions/add-to-project@v0.6.0 with: project-url: https://github.com/orgs/python/projects/${{ matrix.project }} github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} From 4e45c6c54a9457b1ca5b4cf3aa2843b7218d4414 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 18 Mar 2024 05:31:13 -0400 Subject: [PATCH 138/158] gh-116881: Remove erroneous or redundant grammar NULL (GH-116885) In Lexical Analysis f-strings section, NULL in the description of 'literal character' means '\0'. In the format_spec grammar production, it is wrong with that meaning and redundant if instead interpreted as . Remove it there. --- Doc/reference/lexical_analysis.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 40bd477bcaf762..f0b3d0a7458cbe 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -733,7 +733,7 @@ for the contents of the string is: : ("," `conditional_expression` | "," "*" `or_expr`)* [","] : | `yield_expression` conversion: "s" | "r" | "a" - format_spec: (`literal_char` | NULL | `replacement_field`)* + format_spec: (`literal_char` | `replacement_field`)* literal_char: The parts of the string outside curly braces are treated literally, From 762f489b31afe0f0589aa784bf99c752044e7f30 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 18 Mar 2024 10:37:48 +0100 Subject: [PATCH 139/158] gh-116664: Ensure thread-safe dict access in _warnings (#116768) Replace _PyDict_GetItemWithError() with PyDict_GetItemRef(). --- Python/_warnings.c | 61 ++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/Python/_warnings.c b/Python/_warnings.c index d4765032824e56..dfa82c569e1383 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1,5 +1,4 @@ #include "Python.h" -#include "pycore_dict.h" // _PyDict_GetItemWithError() #include "pycore_interp.h" // PyInterpreterState.warnings #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_pyerrors.h" // _PyErr_Occurred() @@ -8,6 +7,8 @@ #include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_traceback.h" // _Py_DisplaySourceLine() +#include + #include "clinic/_warnings.c.h" #define MODULE_NAME "_warnings" @@ -397,7 +398,7 @@ static int already_warned(PyInterpreterState *interp, PyObject *registry, PyObject *key, int should_set) { - PyObject *version_obj, *already_warned; + PyObject *already_warned; if (key == NULL) return -1; @@ -406,14 +407,17 @@ already_warned(PyInterpreterState *interp, PyObject *registry, PyObject *key, if (st == NULL) { return -1; } - version_obj = _PyDict_GetItemWithError(registry, &_Py_ID(version)); - if (version_obj == NULL + PyObject *version_obj; + if (PyDict_GetItemRef(registry, &_Py_ID(version), &version_obj) < 0) { + return -1; + } + bool should_update_version = ( + version_obj == NULL || !PyLong_CheckExact(version_obj) - || PyLong_AsLong(version_obj) != st->filters_version) - { - if (PyErr_Occurred()) { - return -1; - } + || PyLong_AsLong(version_obj) != st->filters_version + ); + Py_XDECREF(version_obj); + if (should_update_version) { PyDict_Clear(registry); version_obj = PyLong_FromLong(st->filters_version); if (version_obj == NULL) @@ -911,13 +915,12 @@ setup_context(Py_ssize_t stack_level, /* Setup registry. */ assert(globals != NULL); assert(PyDict_Check(globals)); - *registry = _PyDict_GetItemWithError(globals, &_Py_ID(__warningregistry__)); + int rc = PyDict_GetItemRef(globals, &_Py_ID(__warningregistry__), + registry); + if (rc < 0) { + goto handle_error; + } if (*registry == NULL) { - int rc; - - if (_PyErr_Occurred(tstate)) { - goto handle_error; - } *registry = PyDict_New(); if (*registry == NULL) goto handle_error; @@ -926,21 +929,21 @@ setup_context(Py_ssize_t stack_level, if (rc < 0) goto handle_error; } - else - Py_INCREF(*registry); /* Setup module. */ - *module = _PyDict_GetItemWithError(globals, &_Py_ID(__name__)); - if (*module == Py_None || (*module != NULL && PyUnicode_Check(*module))) { - Py_INCREF(*module); - } - else if (_PyErr_Occurred(tstate)) { + rc = PyDict_GetItemRef(globals, &_Py_ID(__name__), module); + if (rc < 0) { goto handle_error; } - else { - *module = PyUnicode_FromString(""); - if (*module == NULL) - goto handle_error; + if (rc > 0) { + if (Py_IsNone(*module) || PyUnicode_Check(*module)) { + return 1; + } + Py_DECREF(*module); + } + *module = PyUnicode_FromString(""); + if (*module == NULL) { + goto handle_error; } return 1; @@ -1063,12 +1066,12 @@ get_source_line(PyInterpreterState *interp, PyObject *module_globals, int lineno return NULL; } - module_name = _PyDict_GetItemWithError(module_globals, &_Py_ID(__name__)); - if (!module_name) { + int rc = PyDict_GetItemRef(module_globals, &_Py_ID(__name__), + &module_name); + if (rc < 0 || rc == 0) { Py_DECREF(loader); return NULL; } - Py_INCREF(module_name); /* Make sure the loader implements the optional get_source() method. */ (void)PyObject_GetOptionalAttr(loader, &_Py_ID(get_source), &get_source); From 43c9d6196a8593ebd1fda221a277dccb984e84b6 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 18 Mar 2024 10:49:45 +0100 Subject: [PATCH 140/158] CI: Pass environment variables directly to configure in JIT CI (#116156) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/jit.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 809ac45919fe74..48c6f555fdc5a0 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -108,8 +108,8 @@ jobs: if: runner.os == 'macOS' run: | brew install llvm@${{ matrix.llvm }} - export SDKROOT="$(xcrun --show-sdk-path)" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + SDKROOT="$(xcrun --show-sdk-path)" \ + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} make all --jobs 4 ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 @@ -134,10 +134,10 @@ jobs: sudo apt install --yes "gcc-$HOST" qemu-user ${{ !matrix.debug && matrix.compiler == 'clang' && './configure --enable-optimizations' || '' }} ${{ !matrix.debug && matrix.compiler == 'clang' && 'make profile-run-stamp --jobs 4' || '' }} - export CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" - export CPP="$CC --preprocess" - export HOSTRUNNER=qemu-${{ matrix.architecture }} export QEMU_LD_PREFIX="/usr/$HOST" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes + CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \ + CPP="$CC --preprocess" \ + HOSTRUNNER=qemu-${{ matrix.architecture }} \ + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes make all --jobs 4 ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 From cd2ed917801b93fb46d1dcf19dd480e5146932d8 Mon Sep 17 00:00:00 2001 From: AN Long Date: Mon, 18 Mar 2024 19:48:50 +0800 Subject: [PATCH 141/158] gh-115538: Emit warning when use bool as fd in _io.WindowsConsoleIO (GH-116925) --- Lib/test/test_winconsoleio.py | 6 ++++++ .../Library/2024-03-17-18-12-39.gh-issue-115538.PBiRQB.rst | 2 ++ Modules/_io/winconsoleio.c | 7 +++++++ 3 files changed, 15 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-03-17-18-12-39.gh-issue-115538.PBiRQB.rst diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py index 209e4464e1a5c0..a10d63dfdc9753 100644 --- a/Lib/test/test_winconsoleio.py +++ b/Lib/test/test_winconsoleio.py @@ -43,6 +43,9 @@ def test_open_fd(self): self.assertEqual(0, f.fileno()) f.close() # multiple close should not crash f.close() + with self.assertWarns(RuntimeWarning): + with ConIO(False): + pass try: f = ConIO(1, 'w') @@ -55,6 +58,9 @@ def test_open_fd(self): self.assertEqual(1, f.fileno()) f.close() f.close() + with self.assertWarns(RuntimeWarning): + with ConIO(False): + pass try: f = ConIO(2, 'w') diff --git a/Misc/NEWS.d/next/Library/2024-03-17-18-12-39.gh-issue-115538.PBiRQB.rst b/Misc/NEWS.d/next/Library/2024-03-17-18-12-39.gh-issue-115538.PBiRQB.rst new file mode 100644 index 00000000000000..fda2ebf7593ed5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-17-18-12-39.gh-issue-115538.PBiRQB.rst @@ -0,0 +1,2 @@ +:class:`_io.WindowsConsoleIO` now emit a warning if a boolean value is +passed as a filedescriptor argument. diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 54e15555417287..ec5c298066a587 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -298,6 +298,13 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, self->fd = -1; } + if (PyBool_Check(nameobj)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return -1; + } + } fd = PyLong_AsInt(nameobj); if (fd < 0) { if (!PyErr_Occurred()) { From e2fcaf19d302b05d3466807bad0a61f39db2a51b Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 18 Mar 2024 13:24:24 +0100 Subject: [PATCH 142/158] gh-115874: Don't use module state in teedataobject tp_dealloc (#116204) Co-authored-by: Brandt Bucher --- Lib/test/test_itertools.py | 10 +++++++++- Modules/itertoolsmodule.c | 8 +++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 9af0730ea98004..95e67911db6a7f 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1,7 +1,7 @@ import doctest import unittest from test import support -from test.support import threading_helper +from test.support import threading_helper, script_helper from itertools import * import weakref from decimal import Decimal @@ -1699,6 +1699,14 @@ def test_tee(self): self.pickletest(proto, a, compare=ans) self.pickletest(proto, b, compare=ans) + def test_tee_dealloc_segfault(self): + # gh-115874: segfaults when accessing module state in tp_dealloc. + script = ( + "import typing, copyreg, itertools; " + "copyreg.buggy_tee = itertools.tee(())" + ) + script_helper.assert_python_ok("-c", script) + # Issue 13454: Crash when deleting backward iterator from tee() def test_tee_del_backward(self): forward, backward = tee(repeat(None, 20000000)) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index d377c2c57006aa..44b92f8dcffe4d 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -815,10 +815,9 @@ teedataobject_traverse(teedataobject *tdo, visitproc visit, void * arg) } static void -teedataobject_safe_decref(PyObject *obj, PyTypeObject *tdo_type) +teedataobject_safe_decref(PyObject *obj) { - while (obj && Py_IS_TYPE(obj, tdo_type) && - Py_REFCNT(obj) == 1) { + while (obj && Py_REFCNT(obj) == 1) { PyObject *nextlink = ((teedataobject *)obj)->nextlink; ((teedataobject *)obj)->nextlink = NULL; Py_SETREF(obj, nextlink); @@ -837,8 +836,7 @@ teedataobject_clear(teedataobject *tdo) Py_CLEAR(tdo->values[i]); tmp = tdo->nextlink; tdo->nextlink = NULL; - itertools_state *state = get_module_state_by_cls(Py_TYPE(tdo)); - teedataobject_safe_decref(tmp, state->teedataobject_type); + teedataobject_safe_decref(tmp); return 0; } From 2a4cbf17af19a01d942f9579342f77c39fbd23c4 Mon Sep 17 00:00:00 2001 From: Jakub Stasiak Date: Mon, 18 Mar 2024 13:57:00 +0100 Subject: [PATCH 143/158] GH-65056: Improve the IP address' is_global/is_private documentation (GH-113186) * GH-65056: Improve the IP address' is_global/is_private documentation It wasn't clear what the semantics of is_global/is_private are and, when one gets to the bottom of it, it's not quite so simple (hence the exceptions listed). Co-authored-by: Petr Viktorin --- Doc/library/ipaddress.rst | 27 +++++++++++++++--- Lib/ipaddress.py | 58 ++++++++++++++++++++++++++++++--------- 2 files changed, 68 insertions(+), 17 deletions(-) diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 7b07a34bd57575..73f4960082617b 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -178,15 +178,34 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: is_private - ``True`` if the address is allocated for private networks. See + ``True`` if the address is defined as not globally reachable by iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ - (for IPv6). + (for IPv6) with the following exceptions: + + * ``is_private`` is ``False`` for the shared address space (``100.64.0.0/10``) + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + + ``is_private`` has value opposite to :attr:`is_global`, except for the shared address space + (``100.64.0.0/10`` range) where they are both ``False``. .. attribute:: is_global - ``True`` if the address is allocated for public networks. See + ``True`` if the address is defined as globally reachable by iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ - (for IPv6). + (for IPv6) with the following exception: + + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + + ``is_global`` has value opposite to :attr:`is_private`, except for the shared address space + (``100.64.0.0/10`` range) where they are both ``False``. .. versionadded:: 3.4 diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index e398cc138308d9..7d6edcf2478a82 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1333,18 +1333,38 @@ def is_reserved(self): @property @functools.lru_cache() def is_private(self): - """Test if this address is allocated for private networks. + """``True`` if the address is defined as not globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exceptions: - Returns: - A boolean, True if the address is reserved per - iana-ipv4-special-registry. + * ``is_private`` is ``False`` for ``100.64.0.0/10`` + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ return any(self in net for net in self._constants._private_networks) @property @functools.lru_cache() def is_global(self): + """``True`` if the address is defined as globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exception: + + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + + ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. + """ return self not in self._constants._public_network and not self.is_private @property @@ -2049,13 +2069,19 @@ def is_site_local(self): @property @functools.lru_cache() def is_private(self): - """Test if this address is allocated for private networks. + """``True`` if the address is defined as not globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exceptions: - Returns: - A boolean, True if the address is reserved per - iana-ipv6-special-registry, or is ipv4_mapped and is - reserved in the iana-ipv4-special-registry. + * ``is_private`` is ``False`` for ``100.64.0.0/10`` + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ ipv4_mapped = self.ipv4_mapped if ipv4_mapped is not None: @@ -2064,12 +2090,18 @@ def is_private(self): @property def is_global(self): - """Test if this address is allocated for public networks. + """``True`` if the address is defined as globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exception: - Returns: - A boolean, true if the address is not reserved per - iana-ipv6-special-registry. + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ return not self.is_private From f6cdc6b4a191b75027de342aa8b5d344fb31313e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Mar 2024 14:54:45 +0100 Subject: [PATCH 144/158] Revert "gh-96844: Improve error message of list.remove (gh-106455)" (#116956) This reverts commit 217f47d6e5e56bca78b8556e910cd00890f6f84a. --- Doc/library/doctest.rst | 6 +++--- Lib/test/test_xml_etree.py | 2 +- Objects/listobject.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 1bfcd69f72df2e..835a3a76806148 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -430,10 +430,10 @@ Simple example:: >>> [1, 2, 3].remove(42) Traceback (most recent call last): File "", line 1, in - ValueError: 42 is not in list + ValueError: list.remove(x): x not in list -That doctest succeeds if :exc:`ValueError` is raised, with the ``42 is not in list`` -detail as shown. +That doctest succeeds if :exc:`ValueError` is raised, with the ``list.remove(x): +x not in list`` detail as shown. The expected output for an exception must start with a traceback header, which may be either of the following two lines, indented the same as the first line of diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 14df482ba6c207..3f01a79cc05efd 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -329,7 +329,7 @@ def test_simpleops(self): self.serialize_check(element, '') # 5 with self.assertRaises(ValueError) as cm: element.remove(subelement) - self.assertIn('not in list', str(cm.exception)) + self.assertEqual(str(cm.exception), 'list.remove(x): x not in list') self.serialize_check(element, '') # 6 element[0:0] = [subelement, subelement, subelement] self.serialize_check(element[1], '') diff --git a/Objects/listobject.c b/Objects/listobject.c index 6f919ce02b3ce2..096043bb3d3c51 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3194,7 +3194,7 @@ list_remove_impl(PyListObject *self, PyObject *value) else if (cmp < 0) return NULL; } - PyErr_Format(PyExc_ValueError, "%R is not in list", value); + PyErr_SetString(PyExc_ValueError, "list.remove(x): x not in list"); return NULL; } From 52ef4430a9b3e212fe9200675cddede77b90785b Mon Sep 17 00:00:00 2001 From: kernc Date: Mon, 18 Mar 2024 16:13:02 +0100 Subject: [PATCH 145/158] gh-71765: Fix inspect.getsource() on empty file (GH-20809) * bpo-27578: Fix inspect.getsource() on empty file For modules from empty files, `inspect.getsource()` now returns an empty string, and `inspect.getsourcelines()` returns a list of one empty string, fixing the expected invariant. As indicated by `exec('')`, empty strings are valid Python source code. Co-authored-by: Oleg Iarygin --- Lib/linecache.py | 4 +++- Lib/test/test_inspect/test_inspect.py | 14 +++++++++++++- Lib/test/test_linecache.py | 4 ++++ .../2020-06-11-16-20-33.bpo-27578.CIA-fu.rst | 3 +++ 4 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-06-11-16-20-33.bpo-27578.CIA-fu.rst diff --git a/Lib/linecache.py b/Lib/linecache.py index 04c8f45a6c60ca..b97999fc1dc909 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -137,7 +137,9 @@ def updatecache(filename, module_globals=None): lines = fp.readlines() except (OSError, UnicodeDecodeError, SyntaxError): return [] - if lines and not lines[-1].endswith('\n'): + if not lines: + lines = ['\n'] + elif not lines[-1].endswith('\n'): lines[-1] += '\n' size, mtime = stat.st_size, stat.st_mtime cache[filename] = size, mtime, lines, fullname diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index c3a9dc998e38d0..21d9f96c8c460e 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -35,7 +35,7 @@ from test.support import cpython_only from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ from test.support.import_helper import DirsOnSysPath, ready_to_import -from test.support.os_helper import TESTFN +from test.support.os_helper import TESTFN, temp_cwd from test.support.script_helper import assert_python_ok, assert_python_failure, kill_python from test.support import has_subprocess_support, SuppressCrashReport from test import support @@ -730,6 +730,18 @@ def test_getsourcefile(self): finally: del linecache.cache[co.co_filename] + def test_getsource_empty_file(self): + with temp_cwd() as cwd: + with open('empty_file.py', 'w'): + pass + sys.path.insert(0, cwd) + try: + import empty_file + self.assertEqual(inspect.getsource(empty_file), '\n') + self.assertEqual(inspect.getsourcelines(empty_file), (['\n'], 0)) + finally: + sys.path.remove(cwd) + def test_getfile(self): self.assertEqual(inspect.getfile(mod.StupidGit), mod.__file__) diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index e42df3d9496bc8..8ac521d72ef13e 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -83,6 +83,10 @@ def test_getlines(self): class EmptyFile(GetLineTestsGoodData, unittest.TestCase): file_list = [] + def test_getlines(self): + lines = linecache.getlines(self.file_name) + self.assertEqual(lines, ['\n']) + class SingleEmptyLine(GetLineTestsGoodData, unittest.TestCase): file_list = ['\n'] diff --git a/Misc/NEWS.d/next/Library/2020-06-11-16-20-33.bpo-27578.CIA-fu.rst b/Misc/NEWS.d/next/Library/2020-06-11-16-20-33.bpo-27578.CIA-fu.rst new file mode 100644 index 00000000000000..df58a7ede45521 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-06-11-16-20-33.bpo-27578.CIA-fu.rst @@ -0,0 +1,3 @@ +:func:`inspect.getsource` (and related functions) work with +empty module files, returning ``'\n'`` (or reasonable equivalent) +instead of raising ``OSError``. Patch by Kernc. From c80d2d3263b3caf579777fd2a98399aeb3497f23 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 18 Mar 2024 11:50:32 -0400 Subject: [PATCH 146/158] gh-116877: Update `wheel` to version `0.43.0` (#116878) --- .../wheeldata/wheel-0.40.0-py3-none-any.whl | Bin 64545 -> 0 bytes .../wheeldata/wheel-0.43.0-py3-none-any.whl | Bin 0 -> 65775 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 Lib/test/wheeldata/wheel-0.40.0-py3-none-any.whl create mode 100644 Lib/test/wheeldata/wheel-0.43.0-py3-none-any.whl diff --git a/Lib/test/wheeldata/wheel-0.40.0-py3-none-any.whl b/Lib/test/wheeldata/wheel-0.40.0-py3-none-any.whl deleted file mode 100644 index 410132385bba4d3919508eb280a9095a796f3c7e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 64545 zcmZ6SV{oQX*QKA>d15>1*fu)u*tTt_W81dvq+{E*la6g?dcOGv@BG+R_x)>E?Oo^8 zs%xDkF9imU1^@t{0bCxossi_dINbmKc>Wdgzj8A-F|lUQ)3dO(aMshKxA%~u8J|%J zij}(6${u_gMk15Bme6EZDeC$Vf$Z=bmaSN2Z-Ri?$K2}%^fKxT;ks_6J5)wMNrLz zz<59l*nI`^EZCi$dXJF=j&P-e2^y2&so5Zjr``rkAk3&Ju$<>;gYfxN9 z7+r><7L|if$wu#RTZV?k(HNT-&Ut2@7J!6{vHXAm7PH%w-+l@}6bu7HWNeT_q_m%h z>*8=iiyM!!c&>W4`%F&Fosn13Nk8Q7g&IJ}Z;gWYA?CETja*Wq+4=js!LFF+iS$x20f38M`dOf5W4pbu=hO7a$rZcuQBUaFoz|41*l@TS{ljV8YEW`|jEX zsj%^c#*F&hZv31?aOOqUJR$Xxpu8uWq+f~&_3yU_e1&u`BkBcyzu+e-s(`-^!zsOgxzPC1G z0q0x@0|pu_os4Yo*z{$Fnq8@o=YqBOuo6NMZvK(VriZUfYQ=Z*C(><+W=FNi_O7Ia zpi(#WD^>~DN+1zcREN-U$R@3*Ukwbu_7(e&n~h9?EuUI4TxRU{Xu0=oS_1Zso5vy? zEh1a--u6Kgl=!IcxCQ3t%jswAdzCBY*Z;S*CI!?mO-ukFO9lY=pU3Qf9VkO%3nyp2 z|N8#7ttnm|w+&9$>o0W9$#haj5{E=inFiaX>XBPxZ}#xtC}s;9u@l7-68jk>8Uam( z4qH3hUJL*f0M*b&_ad-G1c3Sc4wuI_!^2tbv1yOd*ui$pbwAMILQuzwDw9pOPEHef zv%9MvZ3;?GzfNnXu1#@%kTK8-$j^ z!z(i@p3Swa5eR3qPIKGQmwfi1iHeofplwX{vBz8yRjmrzogSd)dA_CXv^*pPfZ;B=76?!rwQl~)UJ@$KCxvy-g4o?cC%4f{j7M`tWPXO@a{o`4>`4-B<>H zMJq%V?49GsD$I#p{IH89HmWcY_@&jGBjQbNESt-WEpZn58G%L{jMD4dhoT*=g93v?d`?0S))M6id+}f zz3ELPG}D+8?d{S|j>Ox$`!BYEzj+Wv<$qA9zj#3+`fLu@uCv&zYF1nhB47arKnTpQ zp+5FJmk+J}N>h>9Jq-sqtzt`!*x+H;8iC-5iSCg&m3+)rtjps{F=+LT4Z^gMoeAzU zj){oj>rmkVfUv?}je~<>wV4`_$)Vc0_eP+Z{=?kukcP8^w};2uDQRz6R}uFI-D46` zrA;FwKp(9;A-mu0RAs~KCM|y(Yzn9^aFR^3?HqDWlS{BvcQ~Je$p!zU$aB<$W)4r37oD1*Z=L(YrfbWB7hKkUP0^9%36%-)oL@brhit-!GnFB>}lzp7*fe2B(s)7)6oZfa{%99vlvHv1#?JF~tP>yb1)kx|hh=d?3w6zSGh@wf`mFcqd_4(Rg zhPk0y!kC>N0fAfVXvRNjh{{`H`S(I4q-Nj!9|Kfgs95XR7r7RwKWR6yf+SuUy1X{u z_k4)7vMIrV&!+>QM|<7BZ#<#SMm6$%4h1P3bH>NG_os;V`FghD%9)Az4BYTqmGf+c z$lJuOlg&8rNDP|!yxA%%Rzj~@SQtLM($T;44+=3?{8_NgoMkzcW!(qqHlRC_#b-y+))Cl~Cd3(^(fqE@p!9g2M{}s7X9V6ulXCzoP1RWhY2>-X>ZkN~$6< z*~H4hT@?IlEq?{|qAfYe1Cqy+Nu-xEnIHdh(}EAo%&o1+#lehWL#K#2?2yF<>%&Pm zlKwjWt#V$Ltw^81U5S(K;}NcNo3Yl`06wRx4ts;yQPZPAyI`T zE6guW3V@jFA3vug5*H|HTl{^mfPgc~y*ELqnAXrwO}R??ok1!^Yk3pe(qK04Gl;PD zY*7ztf&;3-Il7g^cE}L`#lvY5$Z#zf3i)fb9XRcfvDk}&iqE)Pp!R{K7@v`ZG^?+M zPap}mh9?5s)OaqQ-w67*4DzIE6YAQ2{Tl~=-58Ufph)1v>Yx+zle}A~b}3roJGd8$ zcSj~;Z&n*re6XfNewg}bfK`<~cJ)(uceTxbRHUO@a<+I*S!n`^_e?)f+yg}8e42>O z1000;=A*J34si ze0QS11bR_n%S>x=UL(2Zxnk4@o&>kXoQU|k|6urZ@i%fs3C3Un_N>^jRQV}p-a45L zjIk?(#4(m7$&zQHQ3f&~Ye?nE-9x{6pJHh*@;g8;^yd6v#LqnB;YfbS1Z^VC7I@R1 zV&JTh!W9e0l8}KMxOTO8^Lvt!9RiL8n=(8zN>dr9h{3&^mD&6QXS@llvvYd6O5wu( z3U3fvLUR2|!FX{suZeQfv{!`B3qCw1&K&Ct+v6y&38>edfUvG}u*wzAjI3hA7sJNP zl`v=M+!cdaSf+r5r6WG==v?F$T1X@_6D*o3JBp|4)`E~IE$n6-?JPPjB5u+Wox~@i z4?MtfQH-61J2G$b)GhT&BE6X7rRB8#i`yiBBaUdK1o)>yNma{XIl1(t6IF6xf+c%; z16bspC1#m+rOdV&K$Q}hs5nCsZs!i_^b>hyr zal&dgPgn36mrTA`c5;?oG&VXG9U!Z|;GUMv7xg^#B5dJUB7QzNKf5!wh1hTmB#sTD za7oCZ%5*+i&hJ8MV-i_QIC4REhL!~LnFs`&-%Lf9EVuZ>P1pZEY!@(YaB*yn5gqJd zLtb>WnGTlYE4#IT$zc%EN!)AN9wLNy-mYi5KqjJQ1NM=UTj3VpLcTe}EA||_1)YiT zVe4DijbkusVC^gu9l^ZQE(QGcMyJqn6(x5-2%B5`0U;bmXp)c}d7d%>%|&SN1E&>@ z$Q(|&JX;9Insp!l2#$52w{yppb}ozC=m_UaNwTjA~1 zeZe2<;W1(*FJmdiUy9RVJJ!;o?H0aIQqQNnvw_Ky~ByEyY5^yxAZHTm;^7gSG!%|Pbf%pFYia{fr!gfe8ORS~^h8Nz*)8E&g>H>oHpnswW z<*63zaUmgnKQrq;^@m1HAU+@|q>W0zY1Emhac|$>r(L`@Nn( zmQQCCeXKwiPnYo@jmbj@RLjm>?=U-f3iEgkc4$dd2oK1HE(*Ofi>cEBVTFvMs9vQ( znK95Z0SY*gOp85T3yn{4o(=>=?PTm>#sOX?Q}xgxC70Gp*?}(oI?rp12K?=(^Q`qz zPEl90mxHHL;qAJ}p6j}R^icGmZN{TZb|1dN$V}n&MSqe*H3lLMo(BR0QOj_pL5&4t z1@TY|KVJ}8?2{99?e2DCJCmpA7igoM$DtQb700WDs!42V%R2cs9Kf|?kH35$=X)TzFR`nO$3WfsZWEs(8QJBP19@R5ihX-#C!NxohW$88G3*#@R$3Gh=&Wau&i%@4mhP6vax+pC1~7H`H=|pd z7e|t}tJr}F$&Yo3nJKU41>zuEX_ZNa?WvfiqT9a*r;&36UZfCYwwSvAWN!@Qtyp;I zUJA?XM1~{&fOaj(V@BM<{ge6i)=jxzbJtt!Ca~P+leTq^zgxHt%74m7&^f1H5G!mQ z-F($?^>98wJFa17vFwgIQH6Nx1+IeG-IjklkFi;f;n=S_D$btG2I$tD3uXn@jtGv! zNQ;XbX7Qxs#GafUx1Y2@weZW=nu+;6oZfv$<|KsI(SW8N&^t^Upw=Bi$lh)Ra(NX` ze0Sg^+?9D%_U$k|!wh(CMbXv-SZ=;@_z`*9A`xlb{-QOto9*24_zaoXN7|><^K5~S zLIj3hEt>)}9oMe-5PG1Zq2#xn?blIfGwGy`x$BS68zfiVcB-Api<=DYmKXnBRT#2$ zw~JLqYpu8I)C>oU=O~D@D{mj(#k|*o%_jwMkT7W4I#34MDtMjg!4DDwTPKM;^r<0z zs#k>Eq^}M)^Z;F-ERyWSAuf15pdi+T3vJIg>)BR&pn2wPNMh+mz{$S&M@QRWP{#5G zbmwQCUsA0c!X!Mb&qeA?c4oMk)-oIRL|#c1uDn^+mY0!J;`8pbpi=cpsl3w?I`)8m zl-n44Ue0ZW*ZoQ0$2yA>DI;h3h?sW3u^PeYw{+LyzD}W)Hv#{B*R)$4n}jKUcX;0HZ3z!-og;{!?bR|=Us?_etbFwwD%4{c z=);b_F^*F2&;{GmYCy1R_g5tMfd>-8}ouk^K)D$b;B=WDFB?>@S zIuc@HJk-?!yJ~x~a=JOJCmF|T7~B6C@T>p0ca*fDe`GG~lX}Phr8k|`AoC%EfzHKn zG!uYiXTaKd%#}m z1co|M>j4EittMhk(2;<^*}F)ER^W_V_W558P~~6rF?u9xw<+sA(`y1fl{YaVd(^M^A6h_|tB5M)w;b1AqB zq-U5igf8SYW?p?p>9==71g49;RBED6JiEcRD?)m%)ByC903fMaT8guG_w2O<4oa#t zUZG(YpVzoQ#YMq%otNLMWVq{2*WgqnIat?D0X)s<42yMFer16$%PTX4P5*@qS}89a zM7u~{JuX-MN%(wX4IbbwC?~4`|9OOG2vTH6mXEg8>*j5`(ipX~aJN^{5wESB@i3&b zk`dd~$Zko|kM(UmO@^-z?tt}i1Y6u2_efXq`eDp;g!YpNRXGwJ=9LU5GoV!z6@F80 z#N_&OZR~WUGI3_5RKT#!)zRnnd~L+-;q~D42mJiV0Yhp}mham~p@zE>ASsU$lDcdl z@7EO~DPu;6n9T*+&-Q5~M=q8sHyoXcFCkC@)H^~oo zV))L(@OtZ3m1C5Xdk%zOyTq=lO{B+#5ja%o{c+id#OZlii!f63$!ML=NHD*eGDULf z8T?6C91D;?z=ggEi5u0_A1z}DX6<+rSWXKPRrX}l=^ac)ug_FN9p9(Csv^q~^g9s} z(H|5&cSue~cpDwUCefv2Co%6-*&{GZXAn-m($>Uiad81pV)R%5t05T#tuaD*yAAiZRH+xmfInme zv7Is&L&Gj2F^|UmVM&i`uD%xfUr-OSBvZsRPRVf`1nM2`k(!3M!teJzZY2(?6+PwW z5mQRe1wx)J*ND9eG-mx7i$eK#DRI)XjP>NwN|jvv)8~j=zuoQ48-EnE2!a>gMZxUH z9MX}NHpz6`^%qZ&PeL#^g#9P9c%v`AD&M%w1`WA&a#Y3PlB6#*cXl->w|#*9ZVr5! zu5<1DQ+LqURP-AVw{_a|TLuV5$syE!e^~kFzTVese`9<4r}eB>hfwZrn%T%ieCJGl z5lilGXZ3aq(qo;E*R6|-gPG$Rk}p!Uj14X~>m0_2zK4uC$g0##YN={A??}P`a7<-A zc^oSa+hV&QMH{JYhcX9p`ap%q<^hB7E#6p;NLJIG5^Jx)D{(OWnQ~$@oRM!hka;A7 z`|35eiU}uvVLwuEOOE*M^}y-h+l_+>qZwYZvQzMkig1Zft%|l+ z!g6aCKuK5pH{qad|Io=1iM5@G6ZeGM$gO|?I`6ngz2)+~nb8PA*9C2bGLMh4y~)OE z)_JBFBluT@DIuVH;cD1FtT%U*R!@TG_|lc`G?~#ajon&#O|D4 zXr8ckiP&IgT)oBi79ot)WZ?%Tkvc5A_yctmXwumTK_%A|9Y3!*SURMx?tYB?&g;zv z;G%F{uT0aX)|_*vau`hSJGVrD!_61%8YDEnQo}?r%BaUP=iHbl!m)3V@gjm@1TRFH z;U0_Kq>nm+BcGET2=(2`=!T^QkMT|!#(?7;8z-BPP!%9Zp}s)5t#Sdkbv{lw?Z|$x z1N}J|BR%4dtYm&LxT_vs@YC8N9UQ&#K(>~40^E_Oc&R*kQQ0B(`a7W$ho~p*R?(Mhj6@RZZ)>-511L=7RcF!CfDB1%c_vz|5D(XcxX=3o!gwA_%xMMl-`Jz7n_ zUSF2h!bj*AXA&FL4xBLHat>w5uX>3DecFX;EZBw5MyT`?M-|sup;ZMjti5s@3>(p2 ziGM{f6I@6s&p61Aq@qY_B%*WVdH7S)Kw)j8Ji~Jp4hVW*=7Asq6$~sMH8^wOoP&)k zm_i76XntwN?D4qzg59h?t#8fK5$%E3G-{7>=2zymozjIXgr}1iBwUi1>}h*$JF`-i zJiMOdHg0tI=$zAmQRaJu{(L*PA77w}q$w4MA(rn!DFQN#ZmjY5G`dCqmV_X7HJY^ z<0P?}wZj$$(0SNTV1kAlZX#!!WE}6I^#jjVvc&ZRrSqsS$h9|>14cWgg3@3pxn|Zl zK}pfps3}$TMeM&x_U>q4_;ftdz+u_R zt(`^aj-@Wd^#F|*(makStbtH|eW-2vfb`^%h#H4#Nc!3PHP~=}Mw)2<`(t=q|ErPhty5w7wSu0|Gd@^k%yZ z6x><7UU4T=pY^&j`Ym^?<#&Xv*R<19f~7KNgcbOkTp8vVQ*IQEA(!`cL=9ZUO2{-0 zjW~_LNbfW7M$m=O1Ehp`UP>hRnpQ6&52W&ftZykcHA4gX+!CxZNqQr$_ZE)6u*r6X zcq~Z~>IWw~DOK+25Q|-&@^bQFJ>*;fY%_u-@au{~FeR1ulm_(UZC;4W%uv>93-o6# zBki)wpQbq}4YiDX8kUkDMY;8r{%}@)qgR7fKM>ms05D$yNMCh=b$}+XuRZPv$8{L9F|eS-CC&E9o!0cgOAex%8)BNh_?EsB?0@faY2s z!0>iO(VBY0!&=UkTQ>t=h|~FEc>RhdW;WSchh+Ix9g5v5B1Rqshnc8$ocJfzhLQOU zgjPDhoUKp@2Nz%@AX#I~Ez@TqmqTf%dr6>;+7Lt0xw=czRl!XK!txlh2N8d4l3?*c zZke16yYy?Co);aG;oN1H*u7ZPJ*x(#0-s55+#j`run}c*h+hoU1&GE{p*a?v6nhL` zOC`)M?~F%vA1Vqh>_H>DWGipJgxl_TO)cHdwy~b0mauddr-N#I{)tHmH~ec^7X#ie zra?Y%_2*%&28Q|6Qu3buw?lG%b@OjQ2f?NtWkU14-`cp^}Y4=I5$8$&W($Pbz+EmrrSS@p=5+=?Ys7ZYieV+xO*r*1&%;l+XKPWEqSi{-+qj?wDt;XG zfsqaCPPyES!1)VR&8NK=Zbx(9Hu^(hBQxw

#cQ?q)_wuV;9k0I6*mdzNny^gk+* z-7k%Q%m}aMRKv!(xW@oW7$(;e@Dr>D@1>sCGA zUtT9X11P%!IEPj+^^K$kXGl9fT0#H<%v^lQM8ZGLf0jK!ljI!pmjG>)UlkvP z&}4sWJBYvwvpJC;<6OB;CxhZXoAQWtCjXo-=G{JzPOr~pV6 zCb6M3`2sq7EVf~Le@$n}!kkV7_sS)zkMo!V#mzwQ-fok+AU|&%)KdcNZb7=dn`$4M z!2`|PsR?(Z(q3uJ1|JGj&v3Q7Ym~(gY#f0aNKmfMgPWAJ5fZJTsw4z~PO`Hc$vxdhuWE!sN0=!c|y$B24csGg-7=7R}w-wEl`q>KjYu?W6IX&A9ie zjK$w_Q-`m&nf9|D6f;OmPwvT^g>ygcPl$Idl8;i9wQnM%bhi3~o`Bi>faS}TP3>>z zFtlZN32hB8EH-D_E4vB|H7i#DIRlBDnQ@t&54j&*_bQQBS&0^pSaIc)1mnq;*2Kcv zvx_Iuxk;O`T4x|kIs@9Iyvz*{uNKsM*(tHH@5V-zQLSMS8|^Kj`v=iJmciJ<2!v>q zsXbuaMpt`eHb|EX_-V8WZL(r)#rAhnj@aHCX&|lPJ{88#il>^nBN@o2Z`0&zt%`NG ztW_S4Nj$#p>k(D8&N{Qb$kT)*LBXUbG#4n5%=VF>pxayeDUj)NqWG}4qsb=_2 zoiLA#uj?B{6Dpig?pjucAqzzCWq zM1#Cew6hgG>XH<(?BLL{`3Li}XCPQ&6!w_N;Gs=2d1le%#HapDn#kQ+fWaHO8=vRC z0OYgp&M*hUenOY`UC}3bRy}pIl+PNikgWkL1$4wF_uTfe+qLqH(n(vF3n` zfnz z7jNkR@?uj)dqb0XXg{(Nk>Lv=oGjwI3iU6mw1`h6O;c(%;5dmkuDU+PsW?5HY)hgP zm`%?{_N+Q>Xa?MAV~C+S0iSfrd$jWz4t>DrX9FXO;|qu9Hyb$i%s-2f6k!YTv1X8R2zjLnf{LE(f8 zO&CrC0yZ{z_}aY#XT>rV1O!BoP|a5mEiG+?CM{W^W8s^wr|kAqqgx$?)l*5a;Z_z~ z`ddq&m5Rbk`FPgS4Z21uND6;Us&p#&3-J!F<^nJ0{v1D>U=Vl5azNUsq?eS{GDW)^ z&fv84*DsiA*Q_+G|AEb;ZGxmv3g+Rd|f)&6J06Df^<4fkA4zT|p;Jkpa)|(yPX@XmIA}izHvy zdf+J;`ax<*E?gFS7sSKIQ6M~BHy$AY*=sy?%i1LK*G%$_3qRSHK9=O$8m2Kwlm?uQ z3MH9K?9C;PR0@GXPt1VHKDT4|O-TlK2zK=VIiXKeR?@16w*AErsQ~W9*&d+NWT|J@o)sOP)$k zop_WZan(;NL;cin`=kK9?tQKB zD~V8Jth0k3^$z|oWt6?4=gj?(ZGC@PctyZYbK7y74~M>(viKrli~Le8a}X=_bf+&x z4F&EJm?4xNE_; zDM#yJ;to?>^@dGimaw!x_lq_f84k(9=|adzh&_^o$ra{`QyPkV2H=yIxK$Z4iLfoF z5wpt(84-nVuFXcvp{@;=Dh1gWdHojBk#9z?-@j;FbzoEwL6>`@W}E%nFlsm%HzHHn zev*Q`c9Ig#ZnG!2ueTpotW@7ECFo}MsZs<^Tlv!%kduH&8&3xGvJq?R{Ww+8OVI^R z7|i@it=WgU>|;gYjD_sJ0F7B)qMPf+uje7NJBJx!gJiQf0FN?tXuJAxZiqmNfa6wO z5j$=1Qy;&~=5P)n*V-${B~8~mDriF~^o)`O(=syeXMYzsp(4*Q9G=sKrxh(J>xYt? zh#k+~t-|9`=4Kf4wy#tkM!jab8-r4c5$pl2T#_H3i zw_a|*EJ<8YVv=~ z!c#6;S3$C6ffq!QbT6c^grF@JuVY1_OOv#R!42{ABUmVfR7Lz*iSb#c-t3X=#C)(G z_44g*u&4;?h|I^z=M9Y%Jpo4CzScMWw+q1>XU9FpL!Kp4{2J{J%Qk!fTW_9E5mX*6 z#QV%5jH{uxq}xTP30*`%NORFXnC}UduXHiW8ac@g3Ow%s46UCM4ybM4b+NKCF+D`9p&F6>C$s{5BX-Ypu-?NA7B=vUh8fMb4kYL zFrp6J2?4Ev#pJ%9Z)|ar$#0#BWD^fpuFuSK>=vQKW0H4*q76<^0YiH60OM;;l+RkI z9f#i!kSB3{GlwMvvL|0OS)Efm%!8ChvZuhCWu6${6&pAzIxkEx+$-#Llix-749M!q zPQIQ6g|%Y*P~;1OH!T2nDESPSXiBgKv_@hPwT(v!mIDR~%KlwxXQLX_vLvG&DkJxj zAH+Oe>XNGG#xlhw@J}E3ZZ-R%Wu&O^y-N<12<3cPMqBc`w^30|sEYR*$2H~Y>ChG@RNix%$#{f^-&q{%B zRwJt_`h{y6VQ662Gp3bfhc;Fhq-fu->7y{<8G%6fv%(faZ+gLkB{?S+&Er-tS-){` zDCJX5O7oN7YJM&6pKf4g@4vDAM%(scc59ZDwcw=Knd)T=Wk)wdo8XAL+f@_6hfHz3 z!1%ScC*YzpzVASstFbb z{SB2Ic`W?6wRo*3ph}2kUtHDb&14_UFr$FdD9_L6DI6~6P_q6oAR!&HVs(BR1d5SE z4lYS?HMCptl*Ow62(|h;$@zT0V&?kX{!J|^zJnJ3Wd=fa2O`E<98?Qp&L2+T zJQ8tG+|$fKuGjZXqbCdV5dRo%rmCJVFQ+^zz;S*hb<5c*BHZqXHDwLi@Mw1q$n?{} zCFm+}i?AH=(Ejo|ai=!0Htst${l6f zv`FJ5IF)BAOdn|{?*wPV;&(pQOx(9<@qEllf{&!_u0}tOOWs|BgbOq)dr^FLF1qr8 zsIhX@of{lXvHzH#^C_Na8gDnK_DBv9ZMg|r&1kN}nJHLze~)~vGDc6y#GCWIJiJKQaAVH121bBo+;j*c>4GRD?nnu1r( zkuJ*{Uczm7Oy2rUwA&N^3!G-rxU%V#IB_u55CBHVU25mIfE|b$309|7oVVRXSmrBuJyw@ zE8pD4P;f1;lW6V=(G=L4=UCJ(mHo_^4Mug71_ETIHFRR1A0J}j1cn7 z(}=bzeVaw|h{AJm-9BuqUVKUZcu8sgdK{4+I({UXFA>l^I52pAGr8nycQv?rQBl_= zaiMno*p^99i+1p+So^WS4_I%@H|H3O7jf&L9sTcHsRhKIQZSZ zm!EsJG70^5QfX%TYa>9>>Z$s4wNU4i@~2DW;R;p~YgUIdJddN7`(1FXsfZ{fudX&| z2~Ynlrp`>CX2Gv8wCV={WC#v9&0@E}$DzYs%ja5@`i9F!u~w`K7dyw{H+$z0kF16) z!P;w0X=!Lk!ZCu!woePo56<@&2VD-?a+uWyW}td6aS6#*STrmPNQ(MYH~_(|BAQ4JwZj5L9{e1w5|b-N^Z0kzuM zo~n$GS|Fzudad|8)7~Rr3r8b^#L&%=G2RSO9XwPF_PJ=TjZajm;xWq(PTV}k~rKsJDdY!^4XRV zU5oJQlUB!OIHs=`jJZU~S1J^*4)H}XzDv#fze^~KPxYXv-N99Qag+mnP(Xno=k*-L zm+fb)uGE}mB%@_$E0a)0>x`fthU3a)bE@xTjy;#CVQFEl)rK@tuAZvQJCX!NxYw@F znpWhcKhh9065Y&lqeh`NcO2aRRco;6-ChPxqdHQ`-&V9!aj*a+p_Zb4Xt=JD))1Ro z--(&&FC5C$)77Dm@xH^Q+36L3wAU{3E_t?IPBInBVro1Glcf#X?bnr8qh+CC?!J?x zW5zCK)h6JVG@nFW&n$qDoigP)#AkR?4VXX7I8;Ed_4n#mlBe4H0gG>9tkFnIESk+w z?kBopEGap}70%)b!D2ft6v`8XuBI@IGKC~06iHc{V6C2Fv92pfZ;l;aR*?6E{wbzN zFA>M6amj@A`uAdBV`xA1=4C5$;l^9&sE#_!Ll>0JO-vtp#KFf+FXuC;XzvE|=yhmh zM_{aL2B_!h#TlHtjyQfKT#|&hyVA-$=Vk`G0)*H z;mp(X6L9w*j1y23IQ;y3zGj^=n^a|v4yq_mYyevEld}B)7 zB*=tz`Go=^4dsOjLhdY$4#Z}(f;se|F9~qT) zY%wD$zY9)-;yRRoV?PH-zWIx3j}$%}e66^gbb-g~fk)|x@6UBnt$o9^G= zj$C*?uEpF#<`in>E$UB1@KUrv(ZR^df*tN0>1uB*=S$)_2vn%#} z1}G4CbwHb-*n!Diu)%3uXjS(}b5IDrnXBX9UrRaHQQ-n^K$(aa6M-U0zump=_3$7c zOkfY{@prfuM&}>8oJ1CzzJkmc5ffedqFkHboezgS37UKJGTFI>2JR#rM@>-s%eSU2dl0ih(fR*BN*815~G_JZ(KX4)xr**V{ zgmwWie`==^8jKKi?0V$XRi2#iTMf?6)~2$cdlU~GU3N!|=g?h6@Qo;iBM(=8luhzy zbo0Mw8puc9T&fXMdn5U+x&G)PA+JHc8V{bCCx6mX&|fjTRBc&=94^!; zke=eZGKo*M{xY=~?Fp9JV&Nu^WDpral;BkDgQ8`_x-}t+*N!@}i4r>QLc&ZE^P#Go z0<4@$Yv#MmVmJy8qBVN`H{ZuW^Q)Y4i5i3=CEOiMB>2K|wPXBAjfk z$(WCf2>z@Ht>Vg!0~;X&on(9kSv+VNkzTN1ys=7Ja)yJQOcNY<|GE4lr2rv_CqE}K z^Xb(!43-CK$kDZ~F}@rKPrS&p&%|?kBguz3G{zI$H#& z&QhU-=iFc0kK%M&@VYx0VGr9WG{e(sVvM0B&$GcS1)Ev^QP|yUQ(v43x)N?YnRei@ zAktY=XC4UCig%b72bZjPv0t!B<~n5Q8sZXOY=afK3c=a$#A+`L`*@TkFAu`|*zK&f zC)i^7{G8~kgSPwl%f)}2sSBxBOrt(4#uWZIkXX;Z)s`1yHX#k!R# z{x^@5)mzhL4H(E7L=(;03fAcwbP$v1b>8fz{cbXDMdaOsBNSL-+ku)1o60AuUa&)F zn`=g+`7CHSZ!_bI>zOP5a(-zYFx{FnR4**+evAJ0M-c=r5@I4*LN;GRB9CjOE9{IW zY27}5x#(2nO_?GB#sp=8vc7(%LFU zI3@Sl$Md1;i-IeCANS^P>`<$-!-!7(^(-wWDM zZuBfQet(^_jDbJ4ynx49a+`vSN}j&l<(J@g6VS$9@j8CY6%o4>ZMQ^URjJ>cuI+*$u;1M6$ZQCz-H@Q$eul%7oU92zCV; zHl5&8xcrQn+dei_Jx&_!YMQ0X=vz>FeGHax@wN)e?CIm`@XVlbEA&M?U!l1Ur3MjiBak?x%EhoDuKk%Km5R;!uv zdHk}ngK#WKyyISwgQlc;Kw^LUlS?7(w4qPbYp&u){mRtgnalPz+gND^eASg7_^QAP z7x})fjjOQYgLsJq=U%yDgM|I5nvGTgHz=IJ48U@6#hg7|w70nMX;Oe>b1qFZb4FGX zwt^76UCL2ns(S-$-nr_toKjaiHv^!ycsEM}OC`BC0gw))?>|UfC=4bBSsN$)CO=xp zOq6<$6!&=x$HPN}eqMwPdcQuh9BLmud43*uY(Tkk%NAF#0nZP>$MaGJ^{qs$4(mXh3tiqM7`Zm`<$GP-Yrg*~+r|t~DCi{l9QpDWKuZfP5 z4x$81GSjR*bdO%(ibb@Nr3Qz-~H!JBj> zCBt@Vt4Y%7^08lde-i(t+WN2E!n<>cnoS(}38GzO=U5C4z{lxSR{Qssf3SaD&BmsA z6g`mmd%rm9Kt{hDQ{X~}+i|zYujVEO#_5joMIDg&3H6@}F>G$W{tFxc&;|kk=>JC{ z8d+QXzxnqrH7UCTcBJleH6(CQbsMs#6#zfk)&mHuXxv3XDkv!396NG4ngn(D_iwY1 zBC-v~<+dtd$YA#hMI&MNy_UMM%jz(oPdH;Eb_CH}QmK^IgWA7isnk=@Zu>)m0d`A^Z6F zI&Md=C9|D)P=@%4pCRh$Rm|rvp#)kKX2S;W0 ze7Cz!M017|_*xARU3);21Kcml=oKg_k_7Y3!gN?$uZ)Nm?byN@@!ghfpMV zppQ1!bWI62^ylMK?D5&mdk-P;mVL$71i@gWae&1u#+I*=&jkCKie<&mG26$S9DTJ8 z*=5UWV=zsd2_jrj!&|K}yjES)8C~_-hme%fkXi5pfqQ+DcGHk64 zHwB5-Y)!Up+qP|+r)}HTY1_7K+qP|=w)?a({r&&w-if%gANJd>sE3N&D|6+_3iPO( zaPJ2vlHlI+!_#_{YbaoF)=dQ!0V&n=y30wu%(v@4-;mISo>paYLdx*J-TP}f>;BW| zLv4#z!7dh^v{%C)TC`gG14ltr?MR4EDm*y#*mJ>~ZqScL2fso+j&-bWbmc#8{>R%$ z0luQ=nl{KPpRWYW9D?zwX$>?_WnI*gS0R0=cS5W4%uVTiv`h%g{a1$)i|p~63WIB8 zwRI>J%pUXNx#@c9RQ-0jl6pKd4w%0%9L>hP6K zp{8{d4&e{!BHT|F`+b9b6_v&R3@LeRPQJ#;OLM@G*GI7MO{Ln)fKkDY_t;?35L_m! zE;)Aj=DmlhOJDNVv`}qf4~lYxy9sfx2L6fg&b4J&=<5VnuqGwAyU~}k!llcQv@uii z&|WLG=X^b^gaKz+WH3ak8(^LW z^-3Gwr|`1i9s@{%qah&vD1bo+g|R|++}im}uH#1*!+E76GLU>{7R+Md>-+mIM@LAB zJmA~IyS2?fZJz)eIeibaf3^?V=*q)svBtglUEsd=d&r*=Q_vC{XI!b2Ybe9u$0{um zlXjA($QIJYYFEc-P*-sLvE~_4iQ1_`Y~Il0e-=u%4JC{AYF{}Rns2I;Rbii$IrGb6 zq5w1~pE5$VK7dYYDWm_HWD|I$q16c0FN(0 zWX?Aejno@Xws#)Wei{!5Oe3GS%0a%~fSU~@aB0&M19q?JH}Jm#W{n5HGBOMRz#_^2 zoo(e!kx|%pT|7aB3nl`bUBZyx$diBLtIH<}jj*`7No~#K=aB#3V#B2w7F@#lR z%NAZJlJ=6+=l$Q^vr|3xlII6tlo161@x*RsrYANh(d*5B1*=v&doHw^b)kLC&s&fi zD$eCxJ8Gs3_g5?v*&8>PT2!u$-DAJITl{0fdzjSq_sn+SaP2~Vfdo1c>-lsodyg?%QLg@2! zKAX%_)_WwML#y`qB-s36*y>TvIdACn=JD?I_3#wF?da7&`vlYLJu*3I#u@WPd}{VY zlKlPh5n9{p`^lq4gB~fWC}lLnrF7nr*lO|E)0Du6)If7lc={xTQml$

}x)#O>} z?osL5K+o+^Nwqv7XA{hd1Hpx4M^qMWA?*Sn>q<9|Z6e}y{^XvrtU4A=Ml_;W?&ht% zsg_)0%d5Y*63j$4N{A)K0@On#Q_J1%Z*-~e7>wV^GsRi;=aQ#Vzsem>c{ zndx;vvyEPZ)M}9(_Go9oo8H)B$s9eB{z3}T=S!)ez8gSNc z=m~@ZM8>7c4Vd2*sx)^8Qgi%7CQegKOV^O=H-WxvjFxtEKCoW^iCI7K!x8cI9?KA8 z*Li}QvZDJlumM+xkN;G@}^%Fqkwe*Ztd8xsE z9%e#HE7+G-d31!WM~Ad+1$@Jj7SWhBM1ce8HE(75m$# zvh39Po3hUC*o@a*mj^=;v?d9^deZv1O1%zMG~8=js4rSCqfCCJqhB~0*y5}@MA>PwqkwyT>p*eTGZx-rKu-NoGt z=8jZY`!SMQIDxpK_$;!-7mqpNO_MaBG7o5J%<*z%%H5~JLK`H9gI!fc*92kol#jxg z2QpjKnEjkAZ^@zp*J@7>WR7CtGW+6Y87eU~n2ZWJ+cns6=j~nKKVr4V zsv!8r_esJ+&&+Z?G_qjr8IA!0WM<>MzOGs;GgIk%}1YA3? zr(Lu;F~xI?jBJXcSZ)A=V|C6*HpN%_Lph7qr(jCGf^y|KG2Q3h{t3XBYCmBsiwrzJ zQgvNO$>%G4ofsCxq)X|6>C?aUQxYtv&7y1`6)*u?C27pD#b5VIV^St2;v;Sn7AoZm zYIzw(HPz4XAuoan${G~2ITO$UwWLcAPdbox(d25`Rfm!xz-dfXg|V8ftQ1w?vZ?KJ z?l9a*J&VAceIY=b>4s&z*TKIyjyz zg>z!$Hv)8F+-7z)B z#8@$j5L#+Pv&Q2THszcsLx5bC1^`Vq`l3m?()CK;(;kJ1dP?xLv=x4Sn}zVEkzZY% znm(^P{e-9ORyTHPSl3dN)p6!e>RH$l2Pi%}ZDI6(tyw>C8kTPl&W-HNM`bBo&plXElP2Df+8 znVk?Dur;jX9d_Pk5HqAQ5>~gfjH12gl@M-SkPdmjA%4XjT9~>{ouz{jo*AnG+K}f{ z$UGK=SPe%WY}-h^Z<7iTcb4qKm&M5H&N1Lu2i^ILLCTAOcJMHiKvfET&O zwWk`QT=D{Kq52M-i`;s1P0Y_mBg15sL&B>rRf%k5VgBqe*nN#Z8$oXwTZ1DExPF1tt< zu!}wI+5Yx>=yk|@$Q`!MkzDp18^lcxqK#tZ$`V%Kh3`5zZ`)*c0KrhgNO2=!%x|KrOkoS3yoOAr!Ohm#3xKQU%Dl zN9BGwX0Q-VFQ1%5|4E#zwl-Fgd*=Aa--(qDGh#5RG4}(?axm~zCQ@t5h!^{Qil}!i z6KlC|?)8xP^rzysj9-p%9+5$BTrkH6^Yz(~ivjd;e*zu(cfN27Z6^D*(3cBb7n8kO zc8i?a=3B0|!@;2$+VG=tt?y!Mx$yNJ$?Ob@dv1k309}v5@j}=AodVgL~X@l9sg;cNPEpocv zoS&YGIKCfk)lR8=mLof~<1Q<@Hh8Avc_gKD<729Ojf)k$Z8Tk42{y`vm40ep4(D3x zBU6I1N2)SsANpt!F6* z3m!>6u=%gAA7Mm8Qwl$ncG+EbkCSR8?p)d{Dp%@F972H>ZycPb2P{{9P^;lXS;xZ@H)=abcGM~492xwBbb8o;RW{%R>j~h_bJYN5aiF=p&S&IdpuY|Bt@MI zV)jKSQpif3$!lQ-6X)W5EX`MPA+IJTuQGQvrdDvhvhOYeMK*uUIX1X6kE&lpYY}Q+ zbT(fMzVatna%TtBpWfBc4fqN?klJxk2m2F>#3?z$P7{3s#^5JC77bxzfp7G|J&}%MUcL^V~mv{c`#u_3Ei?C=mPWI1Q6qUs`}ox@W3pw8q!; zXS(7~Pxu~0c)wZrevMckJtQ4Q_kJ+9z#N|`cW7o2gZaWc0tCGHv)mUj zjf%^fU`aC7gCk*lCSi0YL2PzFGjPV2J__4VKP5rua~FKl0t!|%O)BMws!Oe1Dtg5U zUlBICG`1)%+|tm6|8_jNZP>?=q9Y5{Z`HX#9nE2?>Bf}k?1E1c&^k0KIJst3O zRB9cmIwxilApC77C z7*(Up`9&b+g28K&)m=7u1S2$AAy4?SYfFP$bu*78?%OC!^JXz(pavDsep@t>>B9t3 zVv38rnMrGXL4N_*q*VHv@_`p;4LRu^Tj`u)nc83pr(Cm6#Y;;kug8`iAKTxt*M#1} zm-w0PaB91AayT=1>wAd)DW(A%H^P#_1*IT2*lY^i3Bcwo%5ID(Co(@Oez*D1?4_~e zM}7C$**u!up$8_hcRan-hSINq)Yornd^h_OQ|2a{UUXxlr`%I&7gFVW65G|JNs4@K z;)$8r;8tE2fPx+=L^|WR4gh@YJdQHZPWI;re;&TNWY4u9? zz1GE)&Vc3X8g5_4=j7L1bJY6sEJdco3N;x1%xfl(*@oX+2}-*0SW-i*O`w=Yij|v* z32~G?Q>m~%p{mNGR9Zt(-{eqi{N4^$vg}oqn$yk_7hnWl@}}qqX>6?VrJf84$)%`% zw@-7Aw2d+q`d!j!=tDy-nlb+7XpQ+pP{hm!;&Yy*jL58Lt;k~#uxEJC%5>u9+1shl zn$_Eh#gJ+?x!;LnFx#?61!L5T%(89x*q9Xm80#j`+WuZP1#Oml8^?@3-9`ozVdmqf!P*r*n8KUy`mH~dq=9X{W;tIsEQEcgr`y*pwVl!$pIk#q-Y zN|SgXp~GxMCHO6(CCILNx*d?Br=9brNEh)waN_edCK?e2<0Dv%fKyDb2T^dLj0+7F zzGKMkYZ#y3*;BJX_w524$IRKr-(maUl6uVIxb~ z5r%}j8mG!8c-iJL;!Wdi9V2%VdlVPhsb~<1J@;})Q4c$yjX)+$w9|J-1=(er9tM(c zuw$8LdYa4g@n#2!BHd@D!;kq#lGkML99=8=CKZ+QkrY z5~may5E+B!x5x1UGvf0{3D{+Wf)cgUJs#p|UabkkBB9@AD$?G4m*}S7+JiHrJGfQI zi8sy$ah!N#5)kVzR;tZv*E>*%suLLY5Mb!7O&}$i zB38v2`aI!^y4BXKE*;RYCg4}?zCUiM`BVIIWme+?v|g_ce0n`K-}WHSJrvEo{V{x9 z^H`Q`NT*^iQ`<+%J@>Ae`jNPK*9rkE85Rp2BoREO0GJcBdrlM5&XUC;Kv!LfCwNLe zo{&sSfxmISw)1x1WtXw*fTi%G7UcC4-S#e9F|jJo+VHFRrPyVb*Z2mW3kPI-iCy>_ z7GE8U&`(&tamkycP8bwQjW!GbaHf4M_%r5Hm6j74D1Yal=Fz8-2a@f+BN?D9+R-UY zI&gkXn@sYDzq7{re5U;!9i)y+T7rMVg+Byvw5}&@;%*PhA2gM^tJ!!`^^4Z}8ul*l z%n(Yb%X02aT)k*l{r89lukg%L#2>HYColj2^1t+8U2Oj+AJ9?mkI7^}_z4Clrcxm= zk@Ye!BqA-XNR_fFwrw6A^LMa^W^0f&@0jRy8xcsUg2x!K+v)N!N@~Q#fwP1N92^96 zUe!Vuz%mMi*}%#riD2x8k*$n{-UIiUAs32*HLP@woIFDY?@=I-Ou#~gDM?+@#=>ZvmUNg@*#+@J@%7pPjZFQsw0El5R1Wg$Iw* zOwif}WT$;fO4;T`%9;nM)ZP4}KH`M2IXbbj4#jLs+toK>-^pov9^c zX5?gMr=(k(S(#Rt_kjLwp`UO6_ZEIyWBlFMQY3Fj_ZfQbA@ zjLAbsJTl1RfcPTfXUQuF$kH(qlULfY+_0n6nQeY!;G2OPR)@J zu!H`tJ>)1ZjXx6k$b*dS?zK?rt4~UQ9EIyoF6@)DzhD3G8gb`9e8c#rZlC?cd(C#9 z{51D_*Ok4JuQ=nYW^P`y%--~oy~q-yrqj>eDe~nI+VA1je%{5$n9kw#NutC91An@m z-5+zrRJq}s7|ZDl{d`-^grM|~j-38h8zZHFf$(A`F}kXQZiRA11}1*sMAx1@$r-SM zfxs@@ERAdiV|^IkBzBAspNP>AqsU?|8qB&9rd{QRY-Au05=&Rw2&y@870=c7r3Q6F zJj(2Z@7LBv7E-Fo`#ot(Xnmj`vQ|VDZ9hW%-p+cNMCTE-awYDrYR!c=1`N2|n8}Gf7 zpoXwc;U$$b>V_dnG_t1rX9feEk#7Vf6yTeA1mzTfc!`I#Smo0?Ks`bbRmkJPZ|^kW zJ)}!Rmz_YzSBB?#@a$};YH?ASo!~lDviWuoH%~zTtOp;X;&Vsqk&miFM6J1r8-Q^t^rEhr!4EmbxR?$5CA@O=FYPS0gG2cJEHcl~x!@3Tb$nJYZzybY%o z8uQgRbttuMAxu{?6s@(!@KefTPH899G9b`C7RUoAS0_w&oE#~0#SQ9u#%KnxV@Fzx zjk4h+Psg=J8kXwzD7x{%T_e$iYQ;X-7NsG$vlZGXXFyh)a%b^Az3T+lJ^Y&l5Go*F zcBy4GvuGKz(+2_TLZ+Kw^?~3^SOvw(F2c%x;{vaOTmjM#V9??iV9dD5kQfMtLaYcG z{Cd5b1BM5X8ocr6Ntk}KBK7Ejp5eP+a7uLVEYvmPu{nP<8`A_&v#^L(Vfjyk=&wvq z$dr4E%EXLgOf8v{k8N(!vpFCZ_p8dL3d{`E`#dSkufoI22J%rr^4H8}FgajF0yG-vIqe2dS!LX26fe?2m zM|7JK9p27uz~S@dhc;5*IyYXGaX_-!$=7(O+t+w3`i)$Tuj#1uhx>27mE4uI?~CbTyEhJR4o>#)5a7c zDC%e!G+|x}Jm7d#YkL=;l#`;rnJ#3PhqXLZ(+_omOcW&0a+R?p$@t?%-j5!5wP?V1 z6afu;4-zb;nS)v|L(nY;9&MdrO1cbX37|X%PW?iPeK#UwQL$AZkQ1r_p+sh9LcQaY zK_6oW#vWJ@#Z*fLV@vbqMNqn9eb9>n9&zAoVd9RY205SejzgR*z3hJ*?CX^)YfKWw zo_c_s6q7s#z;L7K(tgoMLKnt7LCD)9(eGix8`qnTjwV}qmU=DHGXJE85C#Y5NcLPz zFL_T6t+mzXnCWbAj~~)9qa*53J;AR$KO3z)+}CPuI=miU0kh8853(NZR}Px-9DWa? zp?lTBvHz_*zz>F36Y-P689Q-WUk#ab$E#|g6r+1YusQ52=I@Ehg4OZngb2mPFpexm z@N8iMa7_V=hkQTc>EU{Wf5x4IElvHyI2o`aOmeDt)s4r`m75=deredBqCesn9t&}y zKb%;sAcGb${1MAy=3XiAo6ZY?22nqd$mCh_tq)H%5BdOLtr|dqCh;c@7KTEvxJhbJ zl2TBSCG!`azlJ)AiXh8CVVPzz&t4WV=nI`z%3gwLX0mn-NQ(1w_+1NtR==uLGUEbV zLS_^Zfd2s8h_(MX=t{pAk83grkiNY%9tx?eZs0r9ElUt>V;s7W9ja8eoktH?6daJJ zRZ3l~NdHa@KX-f2-R=zCZ=S!qI@-EE91GzOs$>#5FAkwBOH$i-Suv@Kf^&gkqYrWT zf;^6+emJ>+QZw?ttfN>g;VbuDY$J$(nYBeA56f?jYz2192ZJUK`Y&z`eTM>_cG<(X zBV@Fr4v#m_?NJVz6YKF|qCa)|amfn1XI<5=M`o3r;b$&=dQIXX0N)fO&tq%2MVdWo z`$iz!5uhYoArCuKW_Z#O_Bu(B(D1R-_%8JMje`Dt{Rxkp9q6#6!%Ha(tDQK)ddR63 z?M1ob0AC`MG0ib9$*?Z4Di6677JZE&=r)n50~Q+M{lAXONF#J7T%slyhRI}weP21^ zXWmT^Xo>bbJDK!P=!fI(I8pjdios_l7wC=SlY!goN6R~=injqewF^iF4CFTIDhrb7 zMnv}KvbtQlyuFpqvck9J#eNR@3n+9K>a^U0{!X4(z1<(ft6yO#jy?`9XE0VY?r8#y zD~*S2o=@9cEiqmX-P$JcG+^6I$0EW;DYS}m1ld`sqACqbCQ~aL#aA@;%4u$sf=x*u z#Hon&_}KbrTj8N(XrVbLf6E*B5d> zY(Vu+0jpc1VGJz!3dK7IDGw|JZbFCPh~!x&L0`(%G0rj`?^D;UPtT*mX=)(l;nfLH zMlmalN?tO&VITv}UxSd_sl+06G>r$QAOh-yMMC3q@-f{=Em=TFpaxB5j7CQ;f%VkF z0n7VH5Yx8;F$RA5dPzKZMZ5=Vr?l%^Xs9=ZCX2Kz&L^aJsMt5KQ z=VIws7i1MTCW^~@^rOCPM{4)jCfT48Y6FE@RS1N6xEb<~C#e6V& zmx#R5BvDFmKROc9?;Ro`C3Of9GSjlE$d@yfNrQ+`@IuC78Kkyaza>IY-z4cUhrK{! z!3}NkE}MnGCzWjaboAni45Up+qD&)Zg|^(oUO>r@h-h-}p&7q}-50_FVF;PXJB1P* z#~Tjn;Wy9BuM)s5=ar`-20usv!S>V(so0)!Ov`x1S^5(lMWNV_`${kOzrBZ%^9n{p%fZe4kwXT}W7FHiulS;4P zTvSW8PW5kj$=6L0XPdeV){5Nwk{y48ej6)ZN8_>%MUTBgmg!IH(p1cgrZ&)FGAfeK zze&-zPO&PWy&`y8tIa&wx&y=IrVvAUpf~3Sz4I1UWtn)2yYZ!PI5j|&@gA+l_!-Z? zz&xq`de>>PBnutWnPy8{Bw8zF58KDo>m*xOa1S%P5XI%p#4H0&Ro5}Q%(X`H#nu>R zHnj20djXy1j52Ei>jbLrF|?~kj7G&9^mz~I7Fp;7<%L0UOa}n-+bbZt6C_a)OaMrx z5eJ&}@%Sc<^QvS8chewHGWIaIl$~yqySl-mW|seP)ZYSO8Hm&;9Y7=s1uE`nT1wA_ zMX18dl-$BmT53c!Wvy`_Xgpt+UnXS4y^$MD^s!@d2*K+_o13+CyV+Z=wB1J-d=>~} z9A_TO?uWnWU3?C_u^a8Iz4_w&h%MEd@2^mc*?bx>*nBut-JNIw<@G$(`IFr^I`tUX z!B=)8u8gZ;-(S?h;pyqMdgfneqKQ*4PpiSlZEH3Y%0y{Q^P9PV#b}TZ`btjw!;YK% z!wSJ`l83g@zp&UoM7Z@iV`#w25@tn5nSMf=f3J=XXKRSOt>|J)&-D)-r8i*{b#yDa zGAC%ZO^f2xJlE>S36O1>-NQ+eIxa34y3Y*a)t?k?|_=~FdY2jQ%yHPF0~e$W+yAE zy@W^kuV$A=S=|V!OH{SDfaSb=h?-n2F6R&%oZv}b%A%Yz8!csxOIL3-t9MsNv#2?D zj-k4+YHH0^7i&`TMwqFsXaI}c3Oa@uuk4WWNjOb#4^_ZKdhDFKBmFCD3LTm*D?^fbGVpQR#6$w4rxAbeL|`6@)s~5?qbo4`Ez{V+ot&tuUQPw*W=Nz^hGdu zc=olwV^HdITWG8M&|w4{ZlXQ1%_T8^tfZD+(x})hG)Y7a?FB5q5R?I)z0wLeuXZ$Z zRv9Dw78lptv66RA94565Lx8Y-7A{Ir9=@ppaJ(orUxddYC&&9GXvRJZbIpW;SMp@8 z^K05thitIpv*+0c`m?5YY-457#hK%@L(D3Fb7#0iT zqnc4@QERQ{cV%M@Bo_yPdX+7zU@qTb?jhp)z2U8mS?;w`yP^lsR9wYoaxFQx&lG-( zEZP+43tp;|xG(4lpxIjExV)?tJ{8?^95f%Wi7pjnr~hI%3}v*>-rYE z2apS>#engn9pSj`x56f_Ivi@HzBU;7g z$FP9#V|jL1e!6p#++W{s$g(4GL)RH85h~jP6P5gXXD_~lJ z;I7QeH>ZWU*S2hwsrfRsD`Het&AWc7ZSrB?LxJDp9 z6a}}{oo;rYtERM%-)15|7VHA4)H%2!AisTic4&!{C7`z2t@F_xMC3gWcm%0M z1{q|>uzKFL^I5c_ErL2@SG(=5rZN=!PJ5!Xr4`m~Y0ALqFuSh$b+k?&_)ge)&0rAP zZT;WV>E$KqoeElX@dzY))#M=J*#BI8_FP`D|!)npjNJ3Mwrc zeB%YOe>6#b=Em={-iliLVR|U@8|@PZXYC=&Fyu1vTQZk3X}udmk_u!B_1-l0xhwYr z=ZVNd9n|}=t|yI57x@&%t}AaIywLE@3R*qur@==z^au4?H*#kua4I@Rs7ztG)fb!B zMRVv4U1o<;J#vhSeR1~(lCB~WQpMws>jA;{XD8`%MAtof)DFIz0SsQ*4t|4tI0ASB zFyVDs4*{4TA(n&E?dt`H4}5`OD7^}BiQukAD6Jr2GKoIm{`Okv60AV)O<127z>-1)^pr*qu)Fg7E$2Y(p~x(ydhW zu{O8;{J`srODXxZLRYQWA$VRkFp|)?;m216eF=%yUt{Lh%HFyP zecDvC;-F%Fhr-d^)0jOr%keN^Pe@IJHT{mcM0E`*9DD+v>Y#;3z6a0j7#IL~(+e$Y zh?o&I(`iPaLm|6JgjA0h)V2Bw!KYPVAE9$0awW%X~0j_sqTJ=(yC}(IW5|esR+V#}f zm8v`;Y+3(e3);JXbTugsIUV7o%jO#5pskv31QF&tOV(wKyfKQ%b^ds(1?bI9qcxz% z&)znm^D*$pzb2A&9etBydg?>^wmEQ>?ZKe}p>vG-uINSOKE>W=jE zAkp(175yg&1RZpQJOXH?%_O3re=dRuR--8eAFF48rOq#wSEICB7fWYSxQgd*m^q6C zX0n~RHdXv?WfCF?7|uZm0D#sX37Pmm$|Uv%j!q_yKN`uj=C<8t3!3k$UVk}!3e$3bT_~W~ zwr;D7no)W7y4*VgNPy*HID*E41eeA2%6se0tuPU#;upn=SFQh0!kCjgpYKUr!_E6m zqW7nS(ujS`M391vLcO4s_JMcw!!mLKscvy53i8Ht*D6X^>vjs}&Ol&L z;xq5=GcPgj1ZCtL%lg4zM{pV<7HZB=lE^qu`{coH za?VF1)vZIGPxjODwJzLVY#kk%!`J0ucR-QlwiAy^ntT-5NLD;|2tKPWK%GS%69*iH zEE}T7IAd-e0sVLcUSCKjC)QNom!zIF{}9I#vs*x{VX?S@>JpN)D+yZ$2FUh1I7QnJ zbYG}*4m8anM(tz$5{@sd$Dz4Z-Is+`@BY6Q3I2a{q7st~72;&6)b{_T#R*}b8v)_) znQ`o)L4%JvsZKBvkz>FJKG+?#oV_YZ_o_W)CV%^x<0Co7LY_kLq2{~joZ(9P+btzQ z4r-+l7u$E+f*3Ci2z zOO}rtPAllr54)C%2Lq{f4Jv_8GsFBdvhOg%e1h3wR8+C8rHl3C7a8ZN<<*BPai4_ZT?pq*-G%twx|9%7oLsL;vX=a_~^+ zfQ9i=HWF6@6Jq5)>@h$psBJnAOlFu$$T-!2tI3}AD4lq&ayn1!$RBLir{H#hK(qp! zpoNcCH-CKd&Y9DL9dmZL8V26MVk&wSgGyP9R<{`jVT*Iy#$nG_^Ce$RJg*A7E+!|u z*Kbx`!6>*kHFPSm(4F^j(%2<;%dj7e6z0xRTmy)6RnekY!l0Y2szUUXS>d8?O6u9b z#NMOU1{cp1upG@IpD0-*qj<1n`y;vjtyZDf9hPV zNSYOL0E?Treou>!K+lw|6R9!bA>)}UP`6)*%bs#l(R|axTJy3+Z8IeE%4hJK3x<6I zaI^OiTKz(Q09Kv55_gL0gnn&s_1Aj{urpp@^n zs)`U_2ZTad)!&9)Zk|OugimB|j9hPWB+jdeuGMGW;cYD_ExGKI?Fo-KL%wwC9ZI{& zu}-!5s24&8gqXKCx1xtY=QTU+j=?VGD5h$lC6uWK)*s1}Yc$wHMeT155Pgrm@ zX?!qu^>BHxeJ{4v4~V?$H2Lu4^L*HJ5683cC>c5UQPsxDAI58ME+oZhw@#=M?Zii@ zn``WfQex}A87l>PCIAZVwbs+;(WG~gCUFs47%!@vTI_@l6aD2V2{qch7xldo0zqiU z(raD~V*36FxEG=SEwXx)aQ~yPf4So)YjQ@Hi%ILcDVrRU;lWSop@+$GsM`cH!J|;l z6xDVZ>lRyStG?~4_PQ1c!u4zr+ATY0HigHi>KRT8z$#2~!L*RM9tuy~TYqmh_p!4) znYA$iY0=RttkX$Q9c1Yq6jL2n%VSz)$Ymu@!_DPyO0p`>tpbq&8$7l>IgeB1%dabg zRi zM{?|!R`*|EI%lW;X}ld3Yp<>vzo;S&OtUrqp6)@3hi={jZw{REdrBA*v3AFnHFYwH z6p?}s4!(;(I5kAT#2K8UgK-(-u-;amwz2>1e#O+*)W~D*yaLs&Vy+h60nua?e2{bB z3R-P5!iVF)N*J@SZz=lqisTG)YM}*p>4xX^j5})II*YRPj%l!O#Lg-v(f-6D7jtcY zaATjfpER*5EC#_tGSo#YHJUeUk^7iIGjm&#?ap8y!Pq1tD<=91>)obqE5M;xbN(wo z-?j)eM$?MRVNG}KYBCnmq2pe#3+96(Ltd1TSG)tl(jN zbyACObl&PQzdOTthgs{DW-l&(Q^sVSES^+E2`)_T*^jkXqj|FWcXFJ<$}B*HM8Yrz z9+tCez_{97R(+oHGZPvbi?X>Ymyg9+qBtJDU|8j^6 zW6-d?_oBlkw;vwM8?P@uxWdTMob-b(cy&JfE+5X^`1JzI)%;nNM9i#Z_B2&$Boa|1 zDv6M2%xfgyd(xUOW7J$E#3z8B@Qc^@H_;=$1N8R-kC2oi@$>h;1wSzq-Y4k)L};b{ zn9RxlV-4iw?C4_T?BZzhk4bSlk=N=cTr0%wJ7Q1+$6n?P!#p3K5=1B^s79!Mwk3k7 zeW5+hS5HGOf`6gkI2>*LzB|_W;11@gxGoMleD6=yAEY7)MUAOERZOfrB>y=a+S9Gx zu3tPZizT#?`^eF#nxM??GI)l2R1YS#ozK#;K-?m|Z5qAREuG6A!pEyoD@zUAkT9wn zPz!&K+}qjW&W%Xp5tvBmiev)_BuHD)R;bXW<}7?G>@oeYG5_?X31f^9MKa|M9$?a> zKRGeODA}L{rP^aSCy=(9uv4N$C&!l@x;x?raZ-A$w zc|mkx`tK?u4nUcNSE3N-hUJHoN4-XxR;?N{wvTt0dx6uCJ#&yy(9DGCviSOn$1{ra;~D+WOxxMc%EZ>f^S^4uTx)n_lMveI_I^qly8e-5&BFb>@%#e=~+{0}&fA~_y ziv$| zlc&-5y2{*ditF2lW|YBx#z-d!DS`E`5!+@%O-knOy+;3rE)4p89U_H8Z)KIeK_GXdkn={0HC4=S5qyrsIw9VKn#xJkZz`Z*6$M=Vl zzPFw1%2n;{OM9=RcIVl!;D?KM5(~bx&gA{UD)cixcRxV>Em~^9ZKapYBUyr)UCZ1zNXk?sYfUEWdEy9s3yG7A$|rF z$3XULBNygb!@UFAzh3` zkNkfSc8<-0Kx=b7wr$(CZQHhO+qP}nwr$(?oXOluzT{RV@4wi)pVhsH>>3mtfV3MT z7Gn;RazaHBGR2F5PjLTABh6-+@{ z51l(d>;ba!yn_$R3)wx*_^Z#3z$B}@R0%d#s+rhX zJ&TbgzaSyyRgaT(n11F(S5CmIAMs3Mjmmz(c;QS<^@8f|s;d0zz6`-+fZx4S0uc@1 zSimY0aK20@mt^Bi^A<5Zr?lF8iQn1@VM_5xINaqbw5Ve&;F&YDyB@yNHa8Kiag=O` z(|Y<-e@kueTKR)DHI~BOTGeffekvqO!Nyf0+aYm8CS4^2gHoy#b`aw1&90T7OuPak zVNHy>S3_9SzhmAQ$&~y4d7?k{%yG~9ZIGEa-EnypSx-ROu4w=>Xg1gmsd))0H!3iO zSPTE%-VDf&JXwP~Uy-1QmT)6cG~u^&iPfn_W9qJwLzDKLmjZ9eDUIEAnlj0}vvn0aG1-wd}{%N9xk3)_U()&iVfgh%5Rd`Coj5#`|u5yiSgRe;3s z-dh4k0zGvi4c5h>XmuXA^LC!Gs=87UrR^cSB|v99pysrOaY%h>$*N%`EF9I7Y>eqx z;>5e-R9gEaLLrxpzEUEp=?8x#Wco68^QtD=xY(9pWIE=%lrV&KE|>uY2WN{P6)4cO zFPn9`*-#xGN?;02AiDuIDob_nU^if~#5@MO;h(h=rcYWU8G6_eA&oS*jz`%CwuVMmpc>! zdqzO@tbIMuZE-M_PkLOHb`q^u>_@}MW=P!jcd#7C3;uQjR))5}V;XI3FC0;Y91~L+ zRm^i7A_CD7b=NBd7_!!YQ|avpn>{{Il=*cv)nn>zjH2~(@7Z@O7u|Ou6tAmdgO-t+88`D}Mq}0|f7T_2D9riA17v@i*MBP1h z_%@g}aX90^@tlo$72>nwDBIR!2=cp*Yb!BKO~z6!Np4jZvB_wZv1Luv{&wmrP5tp6 zO`M(V&sf@t(v19M`)MRsQ9--OKkpC4Ldt1A1NZy%HoMvV%uKJa!AM%_*dqu$s#V#U zcm?p&rj#gkw;hHlW@ScU@4}h==u3@CW0_ebu-Sk8u)?-7u;uHq4$iwx)k%J~;N&Rz z^I+|mk<8v=c_9k?{Sm2d21jN6@!iMTK)N+EIbJt9DG$_s(UFp!Mg?2;=;K#<6;-(| zc6a{E$pCn~P$rD6IGT&n?b6(C5}2E)F7v<-9*0l!_c@s1r6TD{FX9~i?ayC~%4nF|dQLl>}`jTKPe(uJwFW zHnAY1%?>Jq{0DLYq3B zmhp@$u{Isc=8Hs6(?@_1e-+$pO>p3@(V3=D1w#zMV9eYC&Pid8VacPlaQQg zUS=QoM2F#tI6YP)x6aaHN1nMfxsQyBrfP<*^yxiGj8F){p?Z_#ks5CoLnkv$v zaZ+ZwJdosEGS5H(>h`m zqK4NxCZ{;RPX1gLGNe~9H64hsOJaQH=eg#eOg1bdi0LR^HGDr}Mgb0b>EkuajKrEB zn$C&Z3=RXp8ywG|sg7fe# z$OFu2ymWJ6u(66RXNnooHUijP{go9_PLYoxArJ@=3n!Bdt~+wqX2ehF0vN26SjggO zlZ?+pQ>K}|k3;gw28fmVq{C@H4ouKUeP9a4009Q@qQf8!<_R}LY4%fbnl^$h4sR@1 z447z8OpWUL&LY!zgDoJxUH>SG<{FZfumdi-*jA{8GD847 zuSk!JVla0cLNlMfVDNxMnLI1)kPKlWkXm=85ibQe3RYD(+i;_VoyoVzuc!lOaIPa_ zp^@CCdDAo17jGd!&xfp|il&LbhgQJ;Lh3gRIWe>gxxmC6^gMHhnD4{!`^kTQUhSDI zB0eseh{^qyb)2&x0-h_m#;>j&uOh)H}PiVqLiwnef-~m zOnBd2>L>2G$!yMnlnVDFNQ69s$t|C*%b(+gv)VrHSh@@vip)Jr|mdH6y>*?xO=tsfN{k8*t>!d1Zpvu7pARNL^nBgusY0xyl9 zxo?Bas$HX=RB)UuZk=mFar-)|G(KDbKYTyiZnMmsZ|%|xe72@QCtn294Q+yrHx<5$ zN%%p_=I-=TyuUPsng+?~zRIO0>oqS!H2UWF)N!7clMVzfn=OayG?$4f=lB7~680^S zZTTjAx>{H^kaJmXv8Mv%CXd(KMsk}HvcQf{Go2XF<4EzJ zK6^5m8{%=u)-=5fC3d<0k!yy;Q_z%+nY-6Lb{#T(z*v;T{gBj+^9aG_9T>L)=Y)?K zQn(B+n8+%42gh9gF0}Mw0Pir7xf3g^4DWq8Ib=cd(xxUUN2PyUPNohfkx$-o3X+s! zC><>jQ2R1^#gk>LRMB_OEXHft&Y=__O9GVKx_M3FW zxi`jJ&Kj!4f40{w!3FtKs5qqnM6yZLs}+5pAk3a;7K!|XC?#s8!#h2JA9#)Lff$nR z&$^5%({s5RF|?LPt0FeR>>QGbKeMu2C2V?h34 ztHx7-<&o0Ljx{yW2v@zx{ri+&=rtv;EGieR?)>!Ro1Ne16n~d>i{cXePB8{%R$lOQ zxiXS+GA^Z0hflp%MX*`|W^B3F@E#>-T$}@`MtmQ!=iBV_A_@Y9&eZ4L=xh093b4$8 zA~50yZP*MEVK8>Ape3jwmz=I7>4C;bKf2X#+BU1_l`D6j@IakG6rFifu{SC+up5^_ zAbW?psS&Yja{xKpx4BW^d>t*q`&KHhv)u#h{v>=H!cXCJqdU6*t?`5~K6=86Zt$|5 z;G)5cZM{<+@)$;+W?G_W;oB1=S-q~WTT+VP=A1!A)|OzpnQ0DTa5h(klkzp|-W$Ns z5eJCIHSb_86z>c9fsQ3CME_Vc`r=-(MqPYQC`tWKhyHPnA#u zP*SvwXFAlrnnj4RUB>r$vV=@r6MrIVy(2D100^Vb@n;YSz%sQ4KzMpX=1+5Z?6rKu zy@~4ALRMAKoQ#v?orqUqk>hc*AHBnvIif3rauhkKx5 ziaL?5AqH`F;1yd8+Vn9TTi6~pvUCM<9fQ-*?vTbGp!OY{DUiH3cY?||ySco30pbQO zF5(FXi#}1NjNkvI#X&Z1-2M-6G4K!VruhG5bxx*^u9i-wwx)J2|9R<#s!9KQ=_2&q z*Jn(HO9T*sfI^{cbJbyg8w|w4a?8Sp5iU?qO^XqsCOvr>fAqVDtpqPkRlrq~zRt4W z6UHA*t6@-xjsotBPE%Ekgh>xWTbK^AZqR~HyV?q*+Bt4D!1mGc(uCIQU>BZTLHNz_ zl?<318u;(!y=K7Jak^)xqvy}@@8`Kgm2NLfY0k%@S?o~|*DQITP^X5)R%J(N!-2NT zsE}#bitJL@7Otw2;x&a?4y%`{v2OShpAft3%b)UzLdVD+RyA4>Rx7cL)bVgk=a$?Z z+NE0m_M&F3>ulkft+LaN#v^*(p4h!FHJv-xrrss6F7Gp-@!rt4%MhkZt=9;J-v#UA+IS+)JhJvWYZQnCar472@5EVzrD|v)>EQ$ClA6RhtY$Aq=?kQ*HpaSH#%AI zIqoNV0t10x&LjGCa&>DBlam>Wx7snNX~&gT4pw0O0I# zJ?ZPp5&i0?vv9ms9dJC%7Y(nOESSP?@6o*AgaW{~T2RT5h6T!LoJVqi6;}^N_(Vw@ zo{QXT8M=)nP;}GoxJT2Oa?Cdswj5D#C(jc_hP)VHl(JdfJK zq$0=z`Q!3a*7z5@v|_tB?ztCR*7gili_lSRJiJU95VSjKTgnz`fM*n34PlgN_$Jys zbAmivuJ=*_)2k&cosp@Yd0vN5VI(a02oCxPZ^$AJRWnob@bxI7oGfE49bem;%fm}) z*j69o*IL137mPBn``cuSHD>JfBtc-xVbdA@0y8=vHn9G;pesL{AM2FV0z@gSES9fW z6iGh)wDW{J2gk1y;h}5zb^Pp6o7|L#kr=WZS*!(H&JOI*sbwaY&e|RLg4mzE<>wF8 z3-ip%hys^*Myp1Gu2F;NGR3z#BqlabjVB^_lO$6<{F+IPWd}_I*1g`DP5rx~lf`(( zY5S%{iR|Dc(DKp<6zvzVp4UzfsaI5;23(ij5XxW@zH|{Yu-`Sz!@_!ksPnc}WO)mG z8*)wwo_&=4d0#SJ@&0h+!~7 zpAEPB!Li~oi_AQimjxU+DKF6p^ZashH;#9xg z)NV9hPgNrnbP_QH1%d&HTGQIw_xqE<4(2nlHzKKqZv%<*BJ=L@{_+wXzPMf85=FNx zBl7UqCe=hSO-@uVQnO{syeQ_18gaEqM@7+5(w|40O!?#98*eRsFS1E0`sd()!3!t< zt(YW#vBQ4fryOi_5u5Ph@SQ%t-{bWxG>5U=+*EM|1}9A+%2ct`RdElfXSyh+Y}6<- zQcHwck=#;k6g*e%oFL9nQB^n6s}%!>d$vqRxlnDpsj7c6-jD)dx1oY*QCk72NmMycO?)-6LfHJaArqLShCQHa;$2) zpMMFS%p>IHP~3ovG`uiR2huH5&s1|ylneJ(arTb`Kb(svALeo^#D%2{mYLLx4J?;w zDyzV@UH3^hrFtl)i&7dAyC44YlTu2Zd`X=sDy2BSR~k* zP+|_7n=ZmkS$|@>7g=v$=NUljTbAnBW;q}^8Q$+)mO-+l-E{EC=L>=$d^_CJc69c zSJU>EB-8YwDKdUa%3No(RD-GNqE_3W$%X)*h*6AI83w!!gpSSyxeZ)WRo2;*EcNWg z{4*+2FTytAQf|X`+;n3+s&}UNsQ|b&l*2@*g%NexT5^bybP}R9(MkXSMH|#oLzaa;Jl;`$GX?8F?3CS^@l=mtEN7HEd)(MzY&cYnvrW!u#~uX@G~MK zB4r?U+Q9?!YjO ztYNT{8m6-XKBp{T*UlU)c&H7Brn-a5wJL8SATlhOb=E8CL*cbafo;n3S?3=kbQ4Gh z!y20$gc1hQJ+gHQs7^h&D+Ll^(KB-T+>|$>Dk#32D4dP#@tXRj%T%y1qn1#Ywu-{d ztS=oxd2(ka&hr$`SxpYh8X|8{DUN*8id_ImQ%MbQGJ905p2{}Y$8%EL1Ls6wNTvZ; zRZ&PQI$FcTfo_|Q70NB}5IBiFDDknyzo^u%IENI8Gd7IO3%{V?}o!`_USw&3`Ka{J6q7MS4 z!M_NoGTpNmY(@atdIka!JVH10D>v3yP7>fdEP{3hWV`(f@e=rWGtn?>K=pMZ48bS|uSI zE@0FY<{+YL0=UHi()6ewc(O>9re_{NG>E%*782piu^;H``iPDh0VZn%Dd`P}Taq_8 zgaB=ixfK|B(KE(_Ln!c-DHMLr2jyfuo@EV|20t$+{q=Z!7QV;wRA`{DLzMY>Q1(6< z&-A^=S+w$2W>s;PBneINXFEo=qes}}4p+jX3qPp=wf02-A-PX@SDKX3o!LAL%g;<3 zwVozv@9W)JmVx4Y{B$I&iCREq@raGa_>)fh)~)O94s|{Qw3zLdF#Yd>i| zV5Imh2|I+-dQE6={ZS#};!Wlv9ah85TxtPcRTS@d%d+chcF9{yRT&4gK$2H5c?QMt zF>HN}u5K<$D`sw}>d!v%x-dX@-IoT<7lBVTj9SVS0fd8Ues*izCDC|CUYAAC9$^{y z;*LD0d-~jL!qnSw%iJMI`G#I#W-u?M7tcR-`7{nD0?f7$l7l{g?{WyL~ZOk zm%ZErt=}&)j58;ZZM^NV)}rA2(uN8fsTT1ZuGaV>1c)6;+A+Kms16GH6Hw37<}pl2 zK^kfnE-E`&Ol;~CeDsEHh$(5JUpx39K5|h|r$+j(Uceokjryrbiwbv>8_|!HO##5$ z@w=Zgu3pQt6V12(av2ui?u~f7m}WNQ(LDvX^AX4GkQJf>3omRJ6#jtJ^V}WR`2cw0 zPXGIId(BQi5EaU=_xred&EME8?eIAW`1j{M>Q~;s+_99Ay)9)J{ahRI&a7v^)Zb+;1fZUOh_4w$R&`fOrMzqJSe~@F7I_Y zAf0ycrOxMnke%_MdgnEu@cjzeXrhXK0g`1Dlikbx`#33QxgA)UNX;0x3FFg6(bwX; zjTXeEf$6%eF|tUa_}-Cs9jn^^Y{*11?>ZBNrA|ayuC5DWtr8UTy5!+b+j3x5@YaMJ zrRg`q&xh;k6;|nCWo8yNo3Bg(0KD4JP{)iei@*G*eOnQ$a5*~hJ#AO7-?w3BXF$4V7xY^jR7DvUCWAm_}kU0*lYp`*RAOrh`BgI=^i4BUAi<99LB!9t0d2osk zZhr8G187fVKO59Ksq_=(hWSOigARM+{$Kr^IB9s5U<4Sa*S^P4`R%)?V=NjIY*^4N zO4ZW4bJ4$$x17n%S(*{l$x&_K5{yu1sZ~+EicsMJ=+4)^W>4sQ%8ejWJ#Kg3)X4$_(~)1 z)8SgeUjcg-%OF&;uMzX$#gAfzxx!dCn|F6?YaO{il;cF=F{$_<^Ds z`UaTzhC}kb&2h6z+Dm15t=Ln5V@ZsqNXox*3l5rGku)8;9eD`X8MsUHZl3fb_}K!W zCAH6w%j5F7A4szwUEVe!_3LFUVqE=6W0+T-V1R5to1+e)Mt26fHpkdD{%m z*ic{ffAnp6{EZ6d#%A&i{1oT42R5GK%yK1vN%06cUuVXZ86$%|?wy#;5WSG?^_mYqvEe=sgA>3&KkGQB_FjXAk@hARlyA zh{unBIrWmP!$~&gZ`h=l{n{&Wm@Htxkbpf>TFR(yh85X~_BAQRo+g&ntDlWLA1jyZ zuCSc7At*D(=*bs>X5%}IS&^~2(i$3N0RL0r}``R?Ek_l+}?iR}rgW2815P^cRaqXEUb zp+Y!!>KYdW+Yu0|JS!gcU`o0IxxuN|a)!uJCJ_|rIP7iJCQQT%Hm$6V(Xu$&B0u%Q z2y1k(WJreI8ANr>q|WYe;lwjOLVLZIRux;jdSv)3AgWwNk3V3zD30ZRloP$q9n;iY z)zO1ST%Bmw)Gti%265qdTVOiueI`+Lj;!BgU%ebaL7n#;Dq0)Nn=?@fgc8liDkwux z25r^32`6x#)qfDueReDn9mNeEm-er$A~0PJ%HiJ_O- ztmmHUV((e7x6)vyH8ElB-3p&ma{qlL^kR&+DseM;OoO_1YJ^Ig2Bq%^^Z~e z1ku*(t|6XJ$S`~Ec%kv+%McsNYtW1!Pd zA?B7j&B`+wV&Vg3xX*Vs!@k#HZAheIA1vx)yAXV z`J8$!AvyaLw>rZj+Gob-bU94WWFCvXL*11wSpLpxh z%KPG%ZpvzgN&_D5WuO=P_SKX)&b^4`BK?!++0{H;Nftw6~Ff6h&|(wrLX9KZVYH=(X$zKu$}~Da&e?408qu8<~nk1^NEq>tclkkof?Xdi?PT~Us9PqhfEROFrdKbZ&PHZ%xVsj zA{fX3F&7ZVt7-~>k^4=%CYo)^14m-dHlWMDsfnAKd|?wwmzD9fd^0qP>~U69(yVK; zg0fbdJ;>|RrNS0G0lRAS8dht9!cZNm-Gcs`|{S-LjW4DBy~j;5KKj{%j19kCLn5=o*2+Ly{ z+)`8ht*;6qf&Eo%S=dxB2(3ZNp|zR_LiDo6=QRm5ight#Nh%JPGtQx^iNJd{$7VtF z>5)XujQ{J`bsX|$13RzoaO9}1Zl{$^%qAmB86MBmxUX~8m z=z&>_DgU>ji@SpEO>w_%GHU&2USXeW{nhzi6@#_u4?SaAVY==XVAOi?pDMhqL}gW; z+T-$_m;mlqg$+%*VKog0eE0JKu(nrdFldqb{SC=4lOb#|dTsY!o%4N)~>G#g4h8@xdfm z(bn7L#OS&i~rm6gm#~ngd7>YO#HI;?vh+D z&X_q^)CGMV{6uxL`~s$xY|}~#YqnDvt)<;8)1Po-H=2H&s^GwRoU!^-;w*XE=0#4O zuHTYZrZ?5g)%`m4%2HJqO-WZFQbu9k1YaB;8a{$Y&a#h!9Yjk~T_8)ueuOvPG6aa+ z*-Vogd>Xw3Je`->MbH|dwn~M*J&KJ_VWp>c@h~nhn`5@CU z@cJVS^iD18S%)TBS?_ytf*D_GfRSo5*XZYhH~uJU&$9+VZj@z^i-6`-kwe<2l|sGY zUFD2TGwQ_P_jKYiLt|deeS`^nJNvii_xXK)(NZi9jnSBtA~{`0MR?RMZ&Nr^WUyG% z$TN-%8tsIKh#27a8=j+-o(B5t2ZE2QtAl-G9E#!B98*f3i`O5kM*TvZMW5fRYHVm< zKF-VhU0)D_*uxD!#|F!G#)&swV1eC%qB}`HBq{(~BrPkJlrT^F*sQVI744D?g_>eS1ME51EoA;TW6F%BUQTOo+X;f+k}~&rq(6 z!1ayL^MW}^&ztfEL}M=Fp>p6nDV29&>31DEyBZ4G_PopCYw_{%mFr-M=p5B+McbyJ z&nmA?te!4>{JO!*)L{*dBQzFu)wWFMEh90UdUn$sMj;o64LGJPmO{!+w*$FE6bSti z4Y_vPBl+SE4-R&C8y~-b^uBR?#IZ>y%IAzt*{#thamtuvRp#0bunm5Bjd}AC8PUSm zjY756UbPy^+`~HW%(M6j!9OdL{%Sm;+L_u4hOxGA)=N(j9pQ`N})fz={dd5Mr-lbk9h%>{iJ!{kd~bt$zA#QM1RaV zI(Ktk7FC{-ilGbajPcqyG=550ZD>q@yD`_cs%kxVwff7ok4{SjcC~I9%jglX8Ma$( z&||fAE|=_i>b>M9GxJ0@lwZxnQ7N;V>7Mdrp)DySVDJVSx?C!dRc|fFZ5rY|LWzkU z&SLd;W&}1b7xI^R(0_{ZH`);2bA7Gb^c>LaXlXCfPo?ac`a4oJ#x!=Re~MO=q30V3 z=&vC5_Ih~jdyY(#0^r7eH6$mY?fjg*XInXYN|n9dw)k zK@S+y>Kw*Evhxfb3w~}sP%JolZFPzGOWbRrpCsL>sa_=n8yQQL@Vu6Q6meI)0XtWD zZuA94$&v7T3w|g;zQ`;q*hL3oS@@x(1(B%qi`_26E!3x!L07K;Ze3#@a0tMD{bWpIyN$-=m;_&GW=u|)u=K-nw5AMH4T*7B zw!-smY%Ka6uQAd=^&Q_af)0P*n2#F+MGl47(jzLSK1p)JbVkCxDJ1Xls$O960?}?h zilrb9b{+GL$dZ*v))OTdmh$J9vQzr9#-)HJFZWzQ-rlmX2fFFfgzF{}5K5x8*_Hxm z1#ds^yS|2BFxs9yRD#-1s5s83R`tT<%(}KUUDDWO@QZUD3YCoMZCLyb%xp%h?ftta zO|{g@igC&wI6$m-Of#0j5g(2_e?NkD8Xej=*(I|RU{!?Or!eGjuIuW--*NK)Xg{4U z0g8HPog`!44>a);dqc5xQeR2NsO$SSN8`$lovumv`XMVC9t4i@u{4`@IMXt@p9}BO z;_CPWzTlj+k?VkT{bk`f)Ng`T*j6{3xIu8XO;*CZgF&H<$~+-*c5gE{>@1D48-U5H z<_oqONxz4BQ3}&;XfEv2!Yo_(1@s1S8bZ0&b9nwjW2j$DtlSFyqipGowB!osE{^)r zP}j%Ccyo^eIkF|g#Ls9TJ?L;3Vyt(p>JtmwT(TXs{JpV`?BuK#D8jN0deqZ1KmiZj zhGcJ|43L$oV-)*>(_^dB=v=yQLk5)W5QQl&lpQbKK>#~b!^DUFI z1VBQrD?qGH#xS=3(j;+_P1a*g)!(O6X-{eqy=1{I$(B22yyp z!9&Kei05nKmeE~XIs>Qt7uYS`ap5}^dkUn)Kw3GoKOKYv)-r(!cbNI(FERJ-i3H!Z z{k<(RNNee=EcJ^*cXl6*ZG8ihgBa~U;kZU9*JQXiTR>wf18_j|qW#ZMW;8DLC*1p7 zA>|Id5U2j$sBHNXnV&!CDm&O)_f|bS%pl?jb9Rb>^LMd-bX=a4>44PXqGI>b0dC6i zm;Adz%3!AM!!HvXMSbqNWBeLbRdf_H`MZ4MPI@h`{LT8?wTID46uW0VCy7bncG`EY zRf3Uo2Z@lvbCO_qSfP(Sx%G-xoV5c>U`NSz#Sc4;aXcxD>+duFB5p z&!t&2hwA5?{x1>9)>#-PS0s6hB4qRwjrBx1WSlX(DHpJ|(r?3O0#;3(G^wAa+6I8| z`W|ex=q@a{_J5Zp3;TrD=gG}IYHM%=?)F)$@z%$e<^s_0n>2H;{#y^9zjoTXR7woL zH@b7-3h}l2rwJA;qs^%3e{7@e$ev{NXRZ8aZThriKw5cSf5i1*^82!w3>p;m!_ktV zgrVN3T%__H(``T1)^yDCJ2&xObPqRavz7wsiPk#Z%({awJjj@}bF??=@pYrO2~#nu z(&umMa-9e=i2kI}m7oizka8mKs(H1cZLsuff3+WF=|)+8bC9~r>$L&h|F z^;DgR>xbG+SWo@Y(-2>hi83yzI$x(M^1`>U9W%*9Z=w-WXU;HKK4F`RI?4jqSjbu= znyxrWf{DcTZ)B%z!I88*5=QF?;kwZj<^OGGb^t0?&dM_86EqFT&jQ`IH{RHg{WXAWfi zZdTe{97RVJ$2e0FYuZ>HAklRVd1rCexC0?0@V2W4j4Pt9>F*r13)64b>wWYLxYJ&x z9Qy9&yzBafGwoTFnzQ$v>-rf+XWQJ_oJVB?4-DAFo42xI5HCiX`c^NM&&1X?=H?6} z)h2IG!5UQUm~vPBvzT+uS2B@rY(c3jccx`m``iv?$^y>&4`DG~H|-eh{!vqS(-zz` zM^7nKMHzqOm(|Q{d+PHr-IPmUUr)K!mTGgI0z-QbezR6d;%xRUG-CP#dC>`%tAR_8 z5*6fQX{{@c!TG_bid+9@&y}oSIefDHS&N(W{hSVc%~uK(a|XR`cN_cp`!T^g_v@C) z*w3whMk6!KFHP^Z4K{Agv`GbBm2axIUY}oOB#|8INP@HeV#-LUiIpSO_NAg!C`oi) z&ljCE8psBQn>I$EZC(fP zg4KfNA6O(=+XpRlCgRjZMn4;5ZrUXsWZdkk*uII)Yxz+rPEh?x9aSHQN!toK`GX(0m=Agx|xzIC8d;V z2%wXcN|d?Fw3IO&CDP`dR-gGhs5MPyeW#Jq)rg@0M6_GhM$bXfp*c&%@m=2gpe%dZ zcMsF5wa?i3B$U_lbRA&wnLRq5mX^!jm3|F1llqe+^{x9Zb(ch}6H?h%Wzg(5im!cz zZ7_s8D{VVt-iU9!pBP9OcyU72?6S2$lM|pc9ssFn^1KBk+ zw;6SSGiiL(GoIvx)>X0qt?B!XLMpMAQARDT2Mh8+>F0iZ_qh;L^2}KOt?o{zTO-ZO z%j>J#TWr&yJYA@%Q-U-2z9dRH?VnYTL|!A6w8!9-)SQ$aE$P(xtFb@E{NYtG5O!0( zCxK|*3)_Z#2djxg8sg@5(i<4L33=}FY<$=qJYOE}B?NPq`l&~yBQPk4-kEZ$M+30W zQ?Eg@Pou2ogyztcMhWxH%cyZC`d>Ic8ahEYGjEEO&R}5ZH*M8L^Mskcw5Qpb-prF7 z9lh5xfiANvJIbOnuq84Uo!U1J_v;9Sf{*7LC)elm;qVb3jLcf1Lu>f2e09(G(SzRX zB%zKU_2ZtG>+t6p0Rux``R*r(v`Qa8bVWgY8tDV0{AwNYZ|%{hOgv|8n%PG>Ou@=2 z%~)e_9uXzWibynvb}3P{U6_OJWJ;Yk?+J#4i;)Jp-oq2|?zOlg8odV!VBN8*XQsJU z6@U!{C$gsJ-Z#qpmADA%9zK)T2KSGT*TeB+Jop1U*XQBI`0~Z`Z+UPsuhDEKO*COJ z|2e};8X5wPWs?DXeGs8B_RT%c2EAFK7hg4?*|dQL7$5^&qq|6=IcrJ7d;g~QK7j9i z-TJlYaqh+Kq2);e)JcF)z>R&$M{e^va@+IGrU!U{xg`xLcP#xIN*_VNz3<9$kaS)u z(r$*XzEfV%RMG~s=GcJvX7l+)N8#TQcPT=K$_1iB*0a!-mP$TT-J5ds2zfW-%N z3|x^1)y4J(?4JS7U}sjoL+1fPCPgtt{U;YWAvTUT3`yjM3+mu-Ke&wx*zX*3okv*y z2tt=ifTkRM!z}$^JUFa}0Q`f4`>b;_H2|h`+6h<(C!DmPXkfd5nh#X14iGt@WzQ*e z0tpv!Mhb6Fup>V6bt2hKlsvsrOH^QYOz`yqsm24iqbDXh!;}-!4Wle6GO5YFu*igc zCk-9{hg4dQ3Iiq(RRjh<#Cim3AmAx!hsFZH3oYFehH#P9$&8#)%`iVQv+0i@;#-61 zekmpThvEJqSpd{i(yKg>7t*hKXrdWG%QV!cdt9NZEh8j~In+RtH1+EYF z0*lFLCsWoXOnD5+<$yS?#RE-M6I);HaKtiy7%~9u@Ii$?7%XfuoNQzM&1h-CA_Hwt zm+XAhkgF{7%UJ0d%`iiy0K2MtCGx~!|B3*t8O*omJv<`_QCxG$ddXFN>}B^;wnY_+ z{NN>G=?4~K%!)HqicNo#{}`7>^ke|{X$0Yg;PJUw{3$Ed>QURe9!q%DcKB#TTY9E; z3vq78<8;uh_YtDyz-Sb6fVhkULF@+ZCgFp((_Yz)wY%NAX*C?tJ(vq%JUgV+LyqHq zV|SxQ0`++Zdw7vxJ{Ed{&=JP}t>Q@Ci)(5%+Z&--wOfjOO z*hsWzRYVQWmGT^L7d%PLq8dQ;I!VUPIq<~=#MzUMQg4b=g@6PW>I}jB9;8qZRw&mE z4VpMGVmvTB(7+k>R#1;2Vh|^f31nUthp@HOe<*;-Ik6_gx(`o?Ykw_reQmEo1yJPQW)) z8Z78zUUHv7A!u?$p%>iIc~jlO(yc72hU?zaDq3*L>EZ9g%{xhN$&q1!^&p73AeE^F7E?ngE-;fT7_+SRjm0waS!$F`GD- z%B2+xhD0Hnlm>s@+?)Fp=(bc)p5al4&M^l|UzO<#pr7TEnIQsRW$8ZfK<(xC_&&W4 z)M^}pGz7!>>yxIQ;H7wll1^H}MeU#sXh8AIBnKPtrjC7Mo+!u%2I@0s+NPNl&3Z9t z=B&+$K;7b1P)~M5K^PLnv$*jg3#kXZ*%08v5Nbi7B9P&vkexHNqA;wNcB*v8>+c6B zU0X(I;01T8`Qt;GHXoXTnm#{%tbPpJ6;@thUwoT>oV%Oe?Z4S59Y&dGtB6FWE*WsO zq3B^ve&m3Pw(RcL6<2!ihy?J;0zBT0wFn|0EG|TsT}%z9%mXlwXA@!L*tFUT%y+ooUIN&hL7FW}bK8U#K;$`W|ZI(M1SeGv(Dz7{&!K>{8^c?CC zBEx0td@6hKdV|S_b=tETFA3O%IH1zpE+|A*%r(HPF918$mtqvHF28ny%9JsfT%rDJ z)fub}&HNH-8<3e1T|+1+<{CSBRoX^vg}}IYn?-<%Owns~3(iM-r1i$gcSd0*7&MI3 zl_q=Qg)9pfY-}zAGi@m)1Yjn;uGY(v0W>1c{R zgjK@_Wqe%|_LN<1*&x;`FUw`n#2IH9M6cN^pu-8CR%B>&OcJy*SXAefoR%UWUXA3q z4}6t#w%O5XoRyak+}U!Z2)V?UIuzT&xOH-%#UPBjFyGg`x?dMHk#|JJT`@d=|6Q{lN_9@Ns8KDT~>Kx8|sskpKF%w_ZX_w zzIn!~qO&+nj~LLBTlha(@IO{mAQE+TjqxymXX~XX3DalV7NBw?6v(7#b-Z zmTl$=bn3_I#iviF9<<~rG9e6TqoRo8lt!B{ucr_TCXSSK9fBp9$u&xhyDP-`Z-JAn zZx9d)8>#mJIDAQ~K6G|nR+(C_E||=mv{hZND;C-79br|?!i^I(h3&gT`R4{-s(6*gmcjl|C@jvq)f z|5st>0AAP9{r?l&Y8uj0h>mixuSR<8}Gt`c@7a zUqbdRXGocWaykaovWur{WrC|-S=jGu3t^tIkTrzRhA1Wt(r@)NoIz&U}&% zjSU6&I<1?w7DTDKp+ zxbdCFq?B3Mv>JEjv$xH^QS4(BoeppVbmkRb;Z=v=4Ep_;m?HRYaZZnExyL=}q`>vU zdSS*V)HcD?Sb8u;k9pioNpGIw_nhz18WugsqoVZHH*ZHb zr)$l|2AZ!r&4lQx)d5Q2+0=#01xz)?Arq5Ie}971Y4h(Gv8;97Mg7Q48g9cEVS5Rzc)5?KdTPkUC*YWY{D)0Hokt)FYA(;NyOe5I~H<*wWhk4iY(LC=$Wl`}?b9tlxg=9-~Mxrowfp z=}sofAvh1ee=$6QMDPyH)$w+MokJK;z?t?aPD4^a@$ofM)#NT?WYb+m(Wl(>$hBgY z`o!4tcEhhZZKiLE(Lr29i(jLDgX%+NE~ z`Rn;kNYrTUp|-w4+m)M+BDeM0U&J4MS{ZX5!z4U91UHc$!IBSrEQ~zx6W+9R6P18V%$g|8>o#c%$DzlO>nBqnK)~ZEQGm(!tUBc~ z8p72MlffJR*6t@zVutv&2r-Lx+blaqf{TSm4^LVaDujLqY>f0|#Xfxl@(H`C*--1JeE zlhZMDBPFvl`g^StM1RJIEe_uz`6eVDqu_X6m<7J{3ZEIzgB=0FpuU|2yDi}?zC%Ih z^-eJg(-N~K|G~Pl5j)4Lv5}*m;=|$)f+Nk*wtR)5r{a729(Bh>e3CJ6(7yhy)o)wh zgm5Of^C@>0KMyyb5*i~@yXp}?%6lJSZL!(=?UA$NHFPjna?dhjE^2Mk7%~_T<1D@1 zY{N3QE~i;bZT%q%r!`|kXXDMR4AKRjgn1-GMY8Ul0Vdbn3jJ9@_X?3I_osU zkiuNH_kJAILo1jI~e()G|`Bzxvv*xWK%GV@}HJnc5V z6ef@GST)ABRPzN!yn?URd7p&KH`zMu@FX!0U0VR!Ac4u*WP9%sRDCB@5x*ckuyuYb z(Dt^EsYokAGvEeJbq$>S=JXM6>qP6;M4@8hG;gv{i5H%G@JJH%Ex+$`)`lc|3(L>G zGynXVBdDm3n60&WaL@h%ak;tA{PJ4VAu9XAlsEj40teZ@GaqG30y{y0-g zjP8TUytcvOTu#&Lk*aw5;7t^SeAgPLTj|U@7$xhAql^e2UXdHe=gJRtE$bilNop%U z6qD|d3ViuRCszdT3m8r%VylcAd z>nm|vjZ*WgZj6_S92BG7_3NRJd(5Yc$}~T}Q#8f8a{jauMKH3_FN<(HS$aYmv>)t* zUM@%_S0;6;PBjguKQ^Fz787H*8!^i=nq(a!SJdAu%ZJrAQFnIs$Qy2>cb3W&m(7-W zVX){=yJr8)avtP@=}scs_gO_g^PO`C2ld<5U7B|+!#r^+i}d4MDd!o-MENTw2YYEg zFcnarRkUL{X|~bqP!yZmT5bn#8$WaGY>Aty<%&oMDm$aKJG^+iC71h{6}M! zuS>)iH!VATvmm;asi6>KD2F^{3AoN5jft2-N{1B#wTURvZD4$j@Yg>|NgEtAXaWqB zBnROR05QE%0I7KF8V@Xjw`s`q$m81&%TKe*1E1YAQ`vf2O6^p@L1pOEwvAIiDJNgwvjYTXNvq>r!xy*^cU+PqanuVQU2@ejQ)E&+zd# zU7OSrB9bD@8%}eoA!lhy9Zz6Oi2N|;@{Xrc=@M^9>RNPk@Tm#I_%$AxR&wwueHSAO zwkJ%$Es($RB>kcZ-K_$EQ(+{yNL?*evYRoABS=OC7DuDMhfUNv1#xis@SO#h;03?& z)}6R{I90U^J!tN27jlq2{Bd3Oi1MQPSa+>4_N_?dY{4dk41!@zH`zM!M^KS9mfaDw zb6?zd)43-qEXx^1bKjsfT6GmwPiUCiD=Aq&qn2KTf5sq~ej+r+(QgTTnOk?2TS zMl*@0Qfw*VpL&e|JIP@UszwGod$B&{r1)tSr{T=A>Q+}k)=s6#uj+|2P-7p8#)Rz9 zQaN{92i+G|BjW)5ys%`7$trQKnk7AYD~lRTLdu|yG8eU7$WFh><* zT6_?5j#RV2QZ_B&$bFeq$56X<*g1)(GExkc|+#vV$;+lGR(9l$BW0jI|kRG#oJe0 zV_Tld@&We`&uzIlubbtU(E1mH9CQBp4gi}4>w7uFr^ix+Gl2O;ksA^=7j+{>s1hf( z)S47sbj3nn&LQK;PCk+|=QkvS87k+aA2B%>yn#DGp#$4>-8E|~Le1#=!SifcN(=&^ zbx*94ru`SbPM`5z#C0Yw8y!FF>M5LbmYqPg4`!r5mcH0CxBF*dp|+p6dh<_k#mmoD zhaeue?~3np`6FIDQdi*kIKQ?1pxozsF8R?*Ya3VIJ#&KDn`A=?g%J zBlOD=Q)H2xG7LeW*3X$kjoQi%>Id^*Ge%LotO)LC6q6i$>V%tEG>L&YKPh0wcXn- zOQgyWeeBLPUST;N238;JCV8U*q$kNI@<%t1FMMKpP?rcqDBjU8ZN|cAvdW25-wJ5< z%?A#_mmZk*AUepfZfPMGqIauvLFCY~i)hA`zq1!dt8$w1rw7#7>J)M{)0O5<(a2hX z9-?*K%0*6serJOBSUM7zQ4t8CKn%Hf2K(#s)@2$pEFAE%Q7rKP$Ghq$S%o)niC;M6ysl3hW$XOl($I;l{Kl$p%9al)8Ls|L9o+4-|3 ze=&iBI&W|+0ozjC3u%G$Rry^8UT8kij+U$;P&B5Fk%$K^82{Dt>h}I+xh9P<--3A( zP7a4N&N#WOV0aJmaNK1i&$mTdj7XCfHmiX86hdeUPGnOGLmK^L-gM@K>HgE0??_-1 zj(?`<;#P%u6xo;>J=~i(`Z~!1i|{;NIWg646EgxbLZ|oxs|UeI{AmVu8bbSs&ONJ9 z3#zG?YGL0j#N}o0Fq30an($NKj^612bI#%-WO+5BMWvdL!dl!E%e*5A#w@g5nGWiK zclx^5#4Q-}o-Z)HH{P>UjLey}BJL=_hw8tb?)E($9eujK-if^kA%c^1e>N@{s-jq? zQ%GFmZ#lhzmkJ`qh$AjyXk-ahVDKqc&e$*Yq_F?yfv1MX+S&}*dT02seO2qxnDvq> z!Mhfol!FO!P`e0DR@UUiT7bew;5Q%uHT2p758r|E;p77vmYre+u4g?S>w(Ri@JA@# zw$l#1%2#@J!cTqf%D^j^A4K#R(RIVi{Y3&o>Okrf9W6CITC&9tEUffI zqSdcQx$UCOyU@2P6hrIYhNB8qLCXwB%M3d8w{HJI8&=sT z5Z=GY`{KKQ^LiPYJ5i$O3XpB$J2vF7DY^4-3$f_bCere!m@y?M?Jsi8C-KIlbB2kB zHHL*K2p$WbaRK=u%Us0~MAS?BfUvs@5f_}ABDf!WbSHzo1F2z4_;CQVhcJ=~WF#=# zTv(a#EXM6Ao&T|l24&0V$ITJVC}a%1_)|cOFOw&`=XmRnKsa!B%8X8Cu23QXt3X)z z0!IitEF}J%`v=$?OQBn*db3AJy1)&N@4S6_rwRgLDODX7;jgTG$Jg(cMZOc}LuFNB zUV=rYLcQq?Xo_gYfp;hWsMVp?H=z+6Z0G5bCP><0$6*FKey$TdYimn!>*ZIH@7dK> z;N%7IvN;4+vMq$ziOT6yH=nmXQ+*!Lp{ zP!(@q*Pgq8hKZ_?j|zqeNpQzpB#xiNkF1+oj8CFcH`kZ?8An-4npsAjYhBv{)Tg9eqsbLfCUj5OyVhZDt~Btn&IK@#m>cVO1c5gQaU zeS#U?0N^WI7$(;{x{--NTR*+_Eq|NGsF916_xJpG^s%vr@v;R;bp%ihJ)@QC|%BYBsqh#joHfDYxNBy{s;j;)yvDj&H6S??` z@@tMfkXSTPP?VMT9!0J>(;;6=c-ZaL&7XN_$EZ)qLpk`vpJxG?V-1(7-1Q^@JD;z@ zV_GFzEPfgjRwShn9jB`zl@i%qV*+2yX1-> z^fM5)_5<9!eMdWA+(Vs0EpYTKj}my5@cEnwV7NzUyeP=Gr{;|dKOdY@m?AjSSG7tA zNtsI)JKF8qCJVah$`yj`=Z+iYOHnptplFpX{^o}X(pD825m9BvLoxG@?R+}~Tt1gbVtkF-Fykcp#^6aj`R zjWS|8);;|ZdUD`)^$aRr$q&>CROXFtsr z9YA!VEf@%o10E!cEe;f!Jq|e?n`*%h*P!ZM39N7hR`*7|sGe>S`$sIe;WEO2mXK(W z@6u1skP(EoOJR<}?LL(f2U+ z&5_PagNIl}sO$IZ)~R6R_6Y`Q@#niJDPS29Et5=~!Q)(PP6e;n|dcB-&R? zsV8B}rIBeyotG3ic_Md$e?HM)BtTm3iR-|%U9n3+A zW2H4n8T`7=e^JeUsiU>N-fS>;k&MH?8QGFJu7NuHC47(RFxjzrril}lz$SFm(8t$d zcu;R&JVOi3;`U8Ubh;$-RSho=$YFtw2ZBsXQbqEmR!Ug6j<#%Q{U>uLzA@Z+-Vs_s z3yU`7?@NY!sBP^c9vlyCZwx+eZQx8nz}HOW5^9>Z9S`Znb_wyVJMEf$?KyGyaE-nO z&@Quf6!BMOsV8d$ipS>-j4}C%_TZi`K4C%4_u$2=keBXJ2IsNMr=I8jeXq>DIzRCrPEv;-n@Ro5eq1gg( zAKgph?q)Bh*3+3)cwsfINBen$z#Mgh9*yJ4%>9#wd+<{W1Uq+RFZ1D~SZ&iV)#ei3 z;Yy`^5Ep$psAJEsrZdQ2B-`KH-i48+a0uq|ZQTb@E_PC1D4^SSejr`y@`prtX-9T# z5T;8uBPqyx3TDRg{sPxAB0Y6{--Z{Dn(aP!<=Kwsa%(D^3C;&I-~5Ewtk-5x@l{m8 zzO23}4ITFf$MT$1fjg5m+uDRW-E0x!b0({r0OV30s^YQ3wB#x;*^&^`d_mhu;Z-)~ zY4N>6it@+j6kU6Uhl2>!xz_f6OP(!=QyGo$Vw&$`Wo6Uz7~=hIgATdk{r;0f--WOJ z#h5MkL*&pYq{G%J#)jWvS(&&@Y*1Mz@R66ID#dUN{xH*W+D6^Y0c!@~{HjH@V_%5N zTbd-fzmg1kBKV5r3VQvKkfG_QA7WLH0&0!hOY1n4JGg1J=Fr0Rnu~Q{rr*_fHh;OO z+upgD<45h?5)~*4`NI3gM40F31L-vBFDEDok9p>v?_v;M;%-PGW>Y?_FN-j9OidK- zxXk*(@xsB7A!ty?qFOT+W`z`#C=;~9%()TvN9{}HWAoJWt6piZwIO`mD29%v1h4HetE=Ns>G7jiYike>fKhRWnL6 zvRP-02O;c=)$kAw5e0%ZNP`~pQ;zf~c780`ZHZHE?x&3_=BJqy`To4=0Mc6$>VrA-(Xx@;Vy@bHM?Q&$9@*YkC!E>Tuci=P z7JexD?`&~nZZi`rilJ9s25-Vwt}j@Nr47ME<7LzwR-5n9&FwgyXwhn;4c6gz7n+_7*^zdgIG)B z36-xGWYOOzWN>bg#*o^kDs-mmL{7~a9GG+QQ$^^ZywJVjLYSNH7VB*xzYy(mAry}3GVIhG#0!FUrpB2 z*o0ZQ{%{^{Pg<`SL7#i{Q6ps`Jj3HY_F|UqS_kFbDJ>MU51>Qc8?m5r$10stQ5PN? zD!xxb%irP7ZwvA+W z5<+iI5D*J1sbY*4A}hemC=(++xlo#fRuwY6e2ezc1vl!GN6$qhw?c680D)ISg z&&QuAGN7s3EbD5IcK_5LG>ixQdAhon>cqn(Tep)UxUGC@JPxFM=a77gCwVKXc(DcC zMTunwHm|^I-qI8&`%>N;{bT=>yi=vft}WPB4s3#^dOF&B=1uu)+BI1j0;bgP^hCE* z-M);b!`^9BxSAALOFGU_+)|SRa}IHrkJ3%cqEzX+#myQfSWs@{u;XlnXoE$cnWTj| zIN80X^^}W9X$f4{*N&s220w-0q+Gz~le4JJb0wxk+_+_9xrg`4E~T@Fs4-V(W>ezq zT<7^}NYDgh;xZN%=ISlS4_RPnUTa<e)#;ez^#mbO&&`=+rONYZ?nn43^-q>t)1WnO&+g1Rbm4^|F7f+>9930vCk&%% zWR=|XWPVd|$`*ZS*ew$ab||ys{c=~q8LkZWXdBcqRC{WF%PD((`W3IOu1d>TDDZ9$ zk!r=2>YPiwAbl(S3HfDFU#hW7pOmCDZs$HvZV5bo7j2($&*7N`7LJW@1w61A{#MC@m#b%pUcmq}+H7M56ZySDJcDM4Feb?le`} z09hef8rOjL$U{p@gcp@yp%h_{rzwIYvAU!!J^e2#pl`cyc49#R02N?jO5)#^1pe0- zSt$`Qc@?prZs;lNIDBG6_rI&+995yowLRHkm2>H)T7V|pVI9i{Bljfc?48JaeZu28 zYLyeN^`+k!w`~sXt@n0LkDJRfuyDcU<*kps{Qhkg&D5INJ9IV1KagP!dBUN?`L*Cn z^QnS?kQ`pU`&Av6ZhIykQA3u(AeRXRBO)Z+V$XPDlobMcrXB2Xk z<3e2@gwN%utT}fuG2JLkqQBr93!ebBx)|S4u9df8o>R-W&NGP`{FgW8bBOrQIY!f~ z9GI~%LJ9I^C||&H?-qi~zJ+vnrZ$Lx*>y`gmf$hyOmG40LMo1FoJi@jT#Xa7Sf#_EZB>jmq5${vt!Clmivj7$IX}G7(?jhb9^(Q(zCaOM zKQUA=`uOY=jB=CWB`l5S!5tMZHkjRdlZ%?;!xMA+wtOqmjhxdt8zz298R|(!f}zR4 zuok~?oir4Rv#aa4V-GUyE7Bu=>N)ga*=sEel=s26Ed=v`Mwi{EjeXPy7f88=k37_G zE+MUzIxeHls&f$5v0zS{zm!#gXCT)n`IIaOj`8^(n0*91DZwqUD#G+~xbX4>7FE;|B5W6NS^%#`K(mU`Xx)ZLc?vCWp#|G-MHS2~625ZPQ3SbB zpHu6L;}=;~G2~U6Wdt%&gdgqDF`-Y5Mi<)5;&A=ey;mE3GFN}ng#0i)X_yNm@MZC5YeBLdG*K zI|`|G0JRYz4<$CZ?)gT`pZxr(G1*_E7h1i8o$18%Mye9U`alQ!0i;gPHUpz;$Nr=p z)<>l{q^Qw`=~$}AVoBOgI#sf4y4zu*A9B4kJo~~Ln>gyawF8|)c}3E|w?7)5zq{&n z%tGdvu07dxa&f)zgeM9~X+%%|M8@e|z|B{a;qVga=628IE)+EQQ&YOxt-&$f@Ds>JN>**|Oc`^fec!x$pyGe{c5f0}GOg$t)x; zflJ#w5!&1pG5GjlNP&G;j!QV?iuTYbIM1F0tl-nv2(>T#>v}1S_QT9d#(uDJ(R!3o z#0YZdN~KFV&T7LAG*g&FPUY3j2wrG9F1D82rCD6mX|WrO?$H_HQzT#boZw$TS{7=K zG`#k{=B>CnSF2j8Bib6~W03g`jC?ztntN~LqI)Uv-YXyMO~+L|TVTvC0q?@$EbU<< z%l#XMX6})@LOJ)%YIaJOvI31_LNLm5D)7yx$Qyh0O`wCFQ4?v^n?+NO)i(ON?bGOYTweQ{R{3sExLnL2fU!Mkr6?>ZixyvF z#1cumFwW*>14@sm?yKEk_U+=`pwU)^>IHONSYFmj4p?>Q1+h`O2ZE#$X9|k%x4WUt z>kdhWnhi*<1<`wp)GA(-kltvIrCTL@%&XjQN?c&|x|r)N#p|drXeW(o7!18O$J{@8 ze1>beJ`vt3I;@6!PRxuqk2icbha^cHA)jhbsNtkY9m<(kJe=j9N)3Jdxfhh$ifiaA zFrV4+3?9=PO^04w{9u9Lq>Y^RH#}iqShOG88)8j1DJ$6zJz~5E-HNQfKKwEENFYpKaaHt^@S%q0e@;w|Lo*Mj$z0)`{;Ozbb z{}siQ*!_66%WN+|-kQIiQhl%j*4ZKI+u5$Q)000_=L_QlXVyUl>bApQFYG_yPkYyZ z>+9`6DHFVZ9`VXzA_~f)6$?!t%QP_oQ|(<4rQgU81qsUq0|78pMq3?FLHCcj-lgl? z^X;XUsF>6Ht3J3cZ^iaDDnHFHAGSK2!M#yN>WK)D|6&3o?<`7ec=7G0mc*W{$v z^dx!O4N@}tp7DhdNBnN%F4K(z5rLvE_6glNQ-D1bb*Lke6Ka0|>oQjXbefIFzmD<4 z@@ptcQq#@m6SZAIR&hE*6~vl^F+&e?6T_Y+G8WTepOgk_px`Fb?#4k%K9{h#6k|R! z9A?3`XK-j)G&gRxrH90B?>F9PiIF#6O#6TY!WtxM$~pnHHz$vx+Y#p6QHBZ0y4?%| zZiN=LqdP*JM3N1eA5()p!vRU+@tK&cH!R0Pc5MgP#B&DxVDkOkA*1{OB2H=c7t+FZY z5_R)1yHaClUA}BLizh}eiEWVLw`MbWx5*1 zwv&&DJ_@QNQ{zBRS+-=(lTq5TXh?QI0+Ab!XaYGbyW2=5p9A;Cz7`^HK2?i%mQ zBaTdHBC}BGT2T_uiXUZUZ1KnHZ&pjkuVr`sx)&YeKQ7|WcPnzbR9%lXD5QB%k(fKt zjxy$nEubWp0^)nLp_d={@WRoO> zoRaT)D`2!WFrl4bk0ntjQ-Uj^N5A$fh&&`$WqAu-9q^_k&5FefyS2p4sDu}jDxtb- z;wa`m@!GRULLfhD1ieH!bojr2^uyWpwRb;AZ70gvL-AR5+)5vomb-#i;__e zsW67LOezYfepizn>~x&(=i+MV%BS%vhY7wI8ly8P_kBt+bGm}Qa)a)jfGrJWuK^#V zt;kgXtoK2z9LoIiqt1N~wP`1)@rU((xYgSI!y>=BEWet$@11B_q7cGFsai&LH~8Xy zlN$O4qA&ce-%5CJcT2fVCi+fiRB$ThB-M;ImW8uT(>L82b|BW*>smkDU!3tD7nR4B zmf+tJbX%V#iR14WaP`%I*Il#rc-v`Y#?|sTx`oKAc2?(Hj9;idi0Hp#hppv&;(M(N z8TC?UW*UUAJ~&7a42n&>42AdaP63Eyza?W0Yvx>K_ausbUjM=T&G<}Jpwh`aUrw;D zwPnASuLQ!sVOun6#l_pkS0^-Fq~(ksmiY2w?~bgoI3EOd$&wG@?5hRjdwnWuF@vkt za%5q#Sq$t`EwasEuupchI|9Qi`)_KC)|V?QM4d3gN0s7_PL6sBGF&BFkhR^U+fJb!j&q*ojor^BWQIH5a?rf|OBz8Zj-yRANV?9~Lc*d=xxxNzY3oFKCJ=SEo(m0+0X z@Xkq>eJSTCP)rdI{KOx%Q;dwCW9MzM@h!67*&=aetdyE47+FtK&*44#G5%ecb=rIP zLgEtE(Xnmy;<-pU8L&X3_s=n%5xB+Au#2+X8unsYF#~YA^qgPEBj7^bUs)4mtf=+u z-fOhwGg;GIUPva_Ar6JO-K&oEw|Z%C_THQ|uI%!v7TQcHKTdtmpOrdRao$R+!@}tV zgANTi(dSfe!v)!_`k6Kx1QZ<%_&6v48o=dcuP$&c_?8>^i3fN9SJ8hj1wlXGdRUs7 z*)kXySlL^-8W;eV&VMy86tJ%V0|5YrfX83Wt^Q1pU6Y#(P_w6obu5OO54i2_12DT0s7FPBaf8$8K654xhEC3)w4gmOByWdJdkP`4t z_rE0LpHo^ICHF7oj8*}_*Z43fn&Fz5gf$+Nr06;-d9LsO!KexdB zjp?I&BHk4Y0PqOBzN7xd^egLcrvIt>KY0FrdZy+TIM>!i0st`nV*kwlo4tvx)xULy zN-w}X8c;)Gmh`XZ;4Fwv^v?(r2YcXV4A;M@1h;KUem?L%o7KYagB z8=(Dv3b4=oUJ8PG^?w7no7tN>IGdUNTSML;Ch`3|hrm_)Z$@_Q)=5;OQ8cYapM=;z@2H^+?OAu88^oun6dmj6nO_*bHtt@%H6I|N@1 zQeq&@j`+Wc`+sw?AgILj-+Z()viGvJvUl_Rn{E(<7Z3GQ&tw%i9Dd5${#FWt5S{*& z>E>elPfTKHc&(;%s&JXe+B+3H}eNDAmLwtzlhKLO8r#< z;t#5D%D+(mq6hIS^Vie~e=zl$|AqMvKQF&-a{hw`GX5{L-?lve>fEmz-~Pby&i)JT zk8N;&2L4_2_je%3``!No{%?@z*I4kMp+E00ehuUP4h=%z`$y=Xg1Ud^{aqaGcOJ;j z!9Vi;6IJ__^lKFC50b;-KazfniTw)ub%Ol|%;@wVVgH?OE6P9sFIm940sj(!zgS*Z HKR^9HaRT!n diff --git a/Lib/test/wheeldata/wheel-0.43.0-py3-none-any.whl b/Lib/test/wheeldata/wheel-0.43.0-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..67e2308717d675f953a9bc89d05046091cc9d4a8 GIT binary patch literal 65775 zcmZ6yV~i+JlP%h|ZQHhOowjYBwr$(CZQHhO+uirPc{g+Cn;(_z{i~8ysibPHRSMF; zASeI;01yBPnO5qz;d?q<|9;&60`XtCnwywd)9dM3*jhO0>CxG{OH)tGs)WVLQ`1t? zN=3)2jZM*x(Nif(OifPGOVW%_%~GEnM8wid(^7#o_&>BjNr8Egjd*}bX#*(yN1|eW z+iGAy0D#ed@jnyU7+Bc;S0WvSe%nC;nC?eZRS)yO6qC;JALxlLWmF=_W`e-nKm}~R z{CO5^)8t=MkDjwCD&ez`^Fp&T7Fc~)LSa*AmL%Sd5RX{oWE`ArsksKlW%$u$NNQ0z zxD>2(4)x zQ5LUNpLXBLskyTXDmv*$Tz%k!u=#CKFg}DFwziSWf2enly&LR`xqq10#c__R4oO^i zT4Sx9MK44|Suujj&>`&xMZ#J(s9VBE{z7ebPjO}EUYLo2^4tb0oKhnr@ zDUD|rx6DZ$rz>f_Wq%re{V&8+as-{c{-JdN1OR~Z|4QR%;^b`aWM^mXsAp|wW@ce) z_OD98O0srY0{_6Hrul17jLJ_if%lQFTJcvx^EnJ`#BVvBF@q61zwEbr2e87%0|Gtj zd#CYx3f74SQS*%0Q-b1=WQuM%Ce(kx9^{3j0MQMu*FHVUsn7>4eU9au#&M_w@BvvI z5E&>E@gOe3aMDKe(@e8TS^X+s(fZNaaLy97{?Hu074!_*IRQg|dKqr^3}|qhPUY9@ z_3iF?-)Hfh8&eoX7B!AOUJ`y+Y^l%x-d*yKUK{HFlz&Vm=Obs-V>=FUgXg*bqnXY;AI@78Znr zKdKtTMphb=+Hz{4EOXfo@WSg~(^eQx7nskp*$fwkg*$5~GqBG_l8LE5@8o!>E`9tN zelGO@#Wvk$S6}e0*(myLDUXhojHe$`;%|$r)IOTjJQ!H59~&T@bw_0vEk^usSiJj) z_Te(v7I#$9FeX|*BQ1Tq3cwqeoUBT_6!h0SGLj6$D_fBrZpV4oNrH}d;C*Usj68cj zm$!?hm)Da&+Z>&IgfFe-hFk8km@J`S5|#ed3FM$$x#W;X-4*`kh_H260d5rpa-bD0 z#V;?v)+YYoj|XOLFbt` zWPQEQ-7}Bv$EUcHj0YxJQd78XH?L^h$h+A8kp--Pr%Eq%NY;GmM}Y9>m|?H7e9SyB9X5jQu}=sS@?}DRc6+P(-g?CB+ftB zMb5+E5hU|Dth2phXV+sD>f6})>ErRCt|;4;If~we4-EkfaZV2(8I7*Z(FIP6)GKH;yPfGf zD?E&BJUe)Mcz7E0^iy<~bbs3|FAc&7=i=f4m=^)aQX|!$q}B_sHzRW!Mwo5^>-pRX zMz33s;#FgqkZlAa>8r~MhOXV(3cQ42%`qSF+YQ|D9?zV4^FF>Ptf0uyjpdV25Y=Mj zO>Bz0-VOh{)At>CVUBOrE1<8{6`{|$eZA0bMc_B>zJ#mZF7g`^l7$gOt}GG5zUMtp z@sm)HGc5>?hctxo)I0hbi>uqPEH!F8%PbR>0%X2Nl~ivdy5%MEpbCHakX#UxQfz3L zW>UZFrLDz&Il6m?r>~a8U4Tk(yT1rtohFC^>l~ICSaZq#Y+~o-@Vc4m;_>x{!_nF8 znIO|j?S9**%>0I&!fwlsNaDyb0BA4>$2tG%32oS)mP2ybO~pLRF-al$p7EN;=pKWH z0o5@k=-R|McqN)?;Iw83OjZ1AUw?kpVEG=@0Qj+Xd0mRth}#|Dk8eR4X+SyQOv~V@ zk3Sph%VX3g7=0ibn&Oxf(5Si&#+lhB)D0XuOPE`}C+r0+Da14DR@RK$2d}K-6Q$2- zxO>hm2WsB}HRxE>k5)C~AoR7TCH&UfCjiDx5CBwf@cDXoZ_qpFGC()R%20$P<5ztM z3k=U08kbHkssae#Jq~ae%4Ro|;hVAvKiaRHm>ISEVC|wG3)fY1(o6aGV7Ur8pYYs` zPKJKP6CWk2!~k!V0G~h#Y8HIO`2j+K&r6n1AXd+sjtyLpyu8Ct$7Ot?(3#3lz|=or zIE$2JD`1L>E55*AqV25C92O%KJ_oP4ohF%FYuI^9KtHH@|DNk-;e?1%=>}7dOw{rG zyKm!XV?!_4fM91B>|qL@zuTYoI$B=}*xgioP5;&d(fSZ9+V(XhMB^ zx+;C=v~e=GFj8*^x7XX#=in@{eb_i}n+8jn-Ufl_ zxcBbH;WqTs-`ZIUm1nCI{`E5S^QHfCFo@U;+8L(-NP|eYk#dhbiOB~o?tNz&P7dUp z8&=#D<Qp)Uk80Q!zO|Q=JuXM0gBUul5GtS7cLVel?*Q5N0j4jKWP+H2`%F9mDm3gXYoth+dO>z_@th07p*V zxyxR4I$~7JG??cakf(KUfX8pRXT|4ion3JOz+^LtD){4s>4_ zl&qf0K9u&vWf77IF*GoJ{U){Pb{UWSc8Yq@1B&4^7LRPSOr^oKH@RM|Z5`H2Oca~o z?GsNP`e6p%Z3Yy}q2SHJ3PESUoY0!uw&gCdW=VOODMb$F&!Rm|Ry|qJC_)tUTM#9Z zqA)AbkWM7pASAaSz@KYM%ay77$DGSI-p&|5uB6&igatz53OG2g{tmoQtCRIeZ+&4H z%0tA@5;&fhXq~}4hctgf>JIj-)IUYLqBo{@0H#R96B(@*jHsMvaw&?rkp`RK4;B!W z3S~3|9&Kl~CdD;YT@Kc~(wdati{lXLsx7d06rsveFXu9Iag!Sf4;R>qat=g5LU@22 zUSKc6@`!?CrxnppyL?oFz|8<&?|i|O(o?sAxC^x$OhW}=zvk4QWY^+bbJfFW+(a_M z1cTL#($}Q08phLsv{?=P=&pH!z^K;S;HMn27kZKBBxyD~mEJIrdZJVi=5>{Du|>g2 zm_)dm>(7OA8UdC`z)mSQf$SXD@P)#e)`O#gB*;2WScd`XBN4-hoxTuP)3437oA6x=x9x`HKEe|E^ef+;lSS?u~9;F`Qamhs2W zj+`Fuls8l3D4?v-;loVF65&+5#Ok9gJ^^a2 zFjY@Quqv`j^q#B_j4j_=Q7ppWeR5qliq(Vn zIwKGhnRsrB$155IF6=N=0v0}BF_Lfz6&SpcAEgH_(0RQp;SpM6kM`iO?N78|jbf$a zjK48<=!(ZFE>SQ<)eygOaxHL=DaMu1XJWpm4_?l+7$jJn#Z<-DNTi>YO|iu{ghE^y z;1tSTAa)GFM61qTI>k1G;AVs-M^<|U+M@o98(5F%s|b4y-S`2?XIrZ-dzONb1AvF|Q6MmGX}QyjTRi>Kp$ z_y{C-T!AV{(R7+zsku*ynG+WJZf{y9J#DAMEq9AMYLj9jXBmx&-FEISab>L+q%XLM z8672ae|MZl^CZ7R?`Em0D6V%Z*8}wbqGndGT+nyPySD~o9uszSdh)pK#oa^9l7Y6{ zxT&WD5!084m#pV?S37}}ut_ipUc^N?#u>xQQtt)F!e+w!gdlJwt)>HNoAuAm4*D>nR#}ZG&$Na zJy-I}qm?{#jTz7{L=0t&&BqKhLJ7MUaM3*agiWL(S;SGV>j!4w`3co0$Q*8N=Aur? zzgUemu1Hn5X#c0Fq9P>&%Yd%{z z3B2(bvXy?#_93_3#y1HZqR-SI#lACw?y_yw7_&x6o_PwkunV{nxSrULDZeUuoeJjN z;m={vaai{nE<-T4b19gob#_?EJ#(QlZ|b&_Xj8{#T0qF}57)khu_uwWO^K&^Rs~(0 z2QHk-71qXvKLxU(J10IS1I#=)@}l3U%7c&D+t}p|5DI zJpE42@59a0w~3Sc*Z%Z5X(uXUqFUow*x`|mml20x!xKp4pJi1UV5IcSxysExWgKe>0X8|1eUyajLS5fD{pFNT0YPdl&8p7Uf!}g$Oq8yZ1I2=?)ICyxOHJ`SRF_Wuvmc?l&yGd<*(*bbjpe*40JI1rVCP1n?m?4Rr8C&oD z+@1;}c4YAn@;(|f=EnIvIRMwfGBS6!MTAkp>8#3rOrw04&>lM)H8&Tj3J!_XR6FRfv28+3fEu={Ag-eMvtYXR*hOcv z26hDH(m|0Sfl$(UM(I3~;m@z}h}w28=W{t;a%;F`{t`!p!>R+V3AaCEGJk9VBCt*I z*`aB8)QB`J>gu5Uo{|pO*u97y3sGbLgsCkt{TbThWNN{$F9xgUMRwfQ8I1L}f`;8D z{=?BHG7Z>O0`9s?-T zoW})k3Do5eFH>!T)!qAhn@4cOurJ;AYi8CR%NATqy_ApSQg+5EmG@V3qOzaWEur{& zQJi7b>CndQYwy-D2t7m>0cJbugsg7Ix(GXVKfbelJwNH7o*9gk`qF@$o%V5FBo?TH zQH^rgn4Er!PWTnE(_l^Q6&)+X<8W`s#TGvBr~25v8J^Xfo{TGrU@bjhg5!xZ$h)t= z&WHGh3SQgo)#&>@^!y(U#8*MPm$-A-j}1%qaW=l9etGht+RHapkma0W8qIz(6@lI> ze!(MrSWMgYhxRDQk?*HXtOINgw?w9IR$u~BMqJ!5>pmST_U!z$kQ6rJFrKzPw4A-? zsouLD94iayg_xEJSuORGu2H8Wifyf3vwM7(=@oMDB@k0t zdwZ$vkscub22H1ht{OMNhw;DLsiPy-KVCy376%@~<~RQGZRcCq?RRcF@ZIH32!1Ccn;?5B zi3>&#P5|S|IjZNEK^L(pUez_xzGlPlQkfT*ajq0lPiKztgh^Mb z`~o<$=aeWMO504Hi|gc02lj>U*URH$g~5GL*okzb8}_ju795mAdN(I z+qM;6wK=ADEP(v*{mKI2z$Ocf0O&fz0#1q8BEji)lP5uhC=JucBLx|4o zVm(|860;BI2L&TpKook}w-Bz0P;>j}^Oh&Ai`WjdJTuRJqxAbPeQ;)~9F$4zFF3b+ zJEytSXjp;Z;yv>*y<*U9g0xHYD%fyovKU3inH=2ZL1Y(2NNYBIxj%UhF8mjLk79m1 zb_yY+MUOz-&S&yxTM9kx!kG-7$itMgLjE_3=2v2HP@V-(MbzMaT}1*plQENrjO%`9 zo97XruHd?Q?)iW5Yb`kHl5oW7tYF#+Ypt%}ncG$^DFr@!yOqOzo3t=W%lT(i4=xxo z6E(|YeYjOGzza$S=;X(Y=GQ=y@@)hB9WRXcx_P-dy?vb@?mwQis=noB=zTane?$0s z`Fn!|Kis~~A5I_P@q1Y--69N-P=XmjoiWhIF%1z!2j|OdI75o0$mN&UNBLWVg`#Ah zqhD_-iEO)AS2=o%k)RaW( zs7~`3beN}%3#dVICvEay_$RX^zm8bR5|!?Do=N}uOfh%Hg$ZRv6ud}>HZokd4Jl|r z+5_#0n9$7)rc1yDtOQVZLr!N3NR`R)>Vjo|KL-pqdldOXWx^l~@V_&&LQcYuH)phr-pO-qv?;GLU-N+e0(D#O%YTYFoWpV_~-^_z3lA}Rql}i z4Eg@mO{j&b%XlhrTs$<7UemE^g=3zF@h}m7FNbu4(2$5KZ~O5|igYz!piLFCoD57R zmrx)SIKei9SyMH9)*7g7c5$-&5qUwr4g+fTVN<#(=dOd+40Fo7L9z6+FdtS?uycC; zcl}iEL)44Mrj4E5EAJ_|0K8|9O~I`DbPUtpRH70~uYQVr286LHIUJ^G7E|=^8A4`s1Ey zGF+!K$KhJTk$V8T@B=oN1@e$}M}T`Pl7efhgW+Xwr=_yLb<6=Y^9vEt%t}cQJwpfZ znnVuDnUk6BCQV7Laiu;3`nB^}F<8;eh+4?n@nfLPYk7H-<4Sa!2&UX<8G}fE^{L{Q zr>lqi*T>t#8gncz71vCOCb1H~4Lh^%@(dfW{s#3sHkS)Ntr$DH!6R|T{4jtkj;}J^ zsp80^Nn-{tZl~4Ozg>JT2u6Ndrv>G@E(0w&^ku4cdz*E$r85dpP-r@{c;;; zsjLU#Zht&wo;?%j4PDl9cqjtJr|fa{{IxqWNsqFIKEN8VC)gFGR?YpiR~|eTC2&y3`rt%T6ppW>cms0 zv*H6ut}Fe0zh!6cl)gzoIp(lY+>`Tz4bOVDGE0$ObQ>27aAT(AShv113g~&3^lkAWnBYMqp zb!S88>^!^&I!e=YMGP0AKwGqEGX8m=pKL^q)hWv*H=~c8(I9fuBJfLNVFeO)Sef%)--=A~BIw-SImit*FA$caz>? zK8k_^zHbf$6W|dFE?yoq=R!S$8do-g;dN0}ZNeFJz4`&$YrbyoEY=bjVAM7Xj&l%F z6LehELMlXKmKLSm6dfrjf9g6nS5i3F$tXkNFF@OHh$$ge5QoY?QBUZ@`DxN#xXD35 zvj#?RG0sOOne#^`A>3@}jtwDB51^6k6NUIAf)Co4tTApR#y?m57=}Zi&+0jtL2$28 zVSXp1@hIDpd-mGM!FS&}Qt2xAl-NP=MhoVwQs{%Hn?)_`!P2 zxJH0p2GtpE%HB-CbgM)t1W0NYi94(x#H0su3={=%lUuCqBMK#wqK z;SJb;MReXRPA*l}qZ(M?;{99bVqqeqZuNO?lzi;oM_a(M2+ErUsO~+l3Wl63qEOm4 zZ>fs;-N&Pj`rhvMncmNX?G=7EzZyxaYsB9k#)#L7_iu)wvS_RU z%uBceX_g)z;h!$Iha>!51m!|dNJ2^?2ECb{ckq4x3%&;+8IvOPFsNM(Msz`F)pf~$ zN_={jMu?SVG*!~{7DE3+bZs$Q1fov*0S~>$zmiMZ|^KwdKBGR)Hf|qN}h# zXGK4dUwjB(bpmyO4PFtwUxj_5RYctuY8CnTfdhXI9(Y@}_pXxd!&{Hp|f70g+&J1lRhq*kq3 z>UnEPt(FKo$;MVb=@rnTlB+stFaWG3gn$i!<9kgt@>tc>LcY64RYGc3UI2#y=X)yn z<_137KJpZL+xQT6h%bbLU-?z=%Mi+7)%n90Tmt8ot9dFf0)?(C0rsy?(r)AOID66+ zx9eR~jzM-FE%vgzJ)d8u->!#KjeeY*+#fC&!&319D(PWOUSOYIKrw`|#W0I`2`K@D ze%C*TKhba2BUr}l)ld%zqsfrHSn7geTQ%MB`CGt??^HEk_TRYvn)|j>9tj(n;XNX; zAANE)Q%rlj!mNU%c4Z!z{s2?imZEvzo7v6sZ53umg;?X!SA1-|_7{;StFs$)m6o>> zi)LmXE%0k97~j*whStK8L+e$nRXMmJLL|GBKsE0N1j4B}5)@)vq(&^&WM34Y|5))AN<`JlhV+s^E z1H}2bPwEExzIRa1@wdAN?DlG^eQpK`H1DXu+lxwjr!gCPDop*r*6yiM5kIVkg4aL* zcX1or;-H#TZv$5)!V7ein`3V#jVWB`fF;h+JsV9#4YAo{eX9+ZPZSAWX6TJDeY8wv z=6+)_uXSvlh>hqQOAr{Q^jLy-A6FTVzvrq5+h{i(XgVyWmywy;S1=3bdO4UB?_MGs zBQNXNf<5W#@&`HtwjqV$&7CMjX67ASy%vEdB<11Uj-=MiK3>B^uwR+ttv9#{P z$5Q-L&nPr9U_}K4sc}A!u!ZykVVbWy&ppK6a$+;VRQv;Ax(w#9sqVxd6VZUxIUQ>v zXl%C@D<*`mlXEB-BxvJN>tSnv{k~yZn^0P|rnk8O=#EJ2Ej`n+V$yTL7R<%l%WWZ`Z_ZS=WPr3jsRPcVpxn8DAr?J)vbH&_fJ&1ca%C54ZN{tk#bd&6 zgkKorVr(+&A06HYF6887=m{+Aej}ON$w2NIn?1cClbsmXTAhgPtz0~NRF8g@R&j_M z9f-S0{0?!MQ_H3xas(6W&FX@Gm#0H$xFvjgku(uloHXEuop<&`2zTv8mF5ry6-#I! zK(V>yU*g7P*=CxW^OjF(8k+r*hx5GYSgDL&@WHk0y0o%AAfk|XI(*@v%FuR>wTeGrqyK{ zTLY;Cp(gHq%0qvLp>_C1md%kAEl0?`v$8KBBjhchAm-rBhrkHu$0C2R0%lyCSO$pO zGiWNYMm0%iku$F-Dt*+k{T^XkX3KQ0!Su8?>$+W2+{0xT96zx2&8tR5qP7Fw9_t5v zn`df$lv97?a>dEv;S(n$K^bMD%Oq%4CRdgy9w`55A7SKl+{vcJVvGh4-Xa?$o2ZhwYdt< z^fO!dBw|}y#cn@1cInE?ok@bBjkl|2eT2tBs;Iot$9C`(`+4lSTxKq4mJGw+7rhtDu?(Bgl5}Io8j8YHdcb{-b zX_h%roA88&>%W^I9_8B{ls-s>;-7&MK7m90oiUnU9HKULSccQ4byKGsd_h^D4dBUx z#iTxQZWiu54YB~jfWE2o8(+z!HyVa+E0*D4!ep0P64JduAxF+Krr4K{60|t9lIEoeoruS_IG| z>^KYUeZo&tL45JSRd%6C1~{@>I;#SVBF1?G6oaufc~8me)U5t=RymRC ziMOkXx}r`W9}-f!uX(s(S%F@C2Un!0uI~!Fj#{HX{x@`rz;K6-sny70L_MdHBXG@(+rkP6Mw0~bsy8;7eF zvnHrL43r_B5gLd+$lg1-NX=WrgLjE*b^`&hP(!=MbG$=+P$FQm{ns1PCw5D~0Nl2L2S^NnN8Sd#Kn$3zO6?(LEwo372dol_&ZGa;SnKGyJ zt%9Ih&}x-JJF8CSp}-o!Yb7FM#3pqfaUKB6#T;gHY=ecCWrHLR+E6&u*$|HbYzOQp zg*cjT(CH(UT~w#1!KYS-UZ;5mc)!`SWLqR|s}c1^>kyAJYMsBTL3C%CT|{iN&+P5Q zd>yF@Z!xnwjx-F2CDZn*zg$$@?9*19K!x3S(Kd#jjPSJgKbeS_*yE5_Z~%ZWwEt<_ zwJ~urFg9>9_{TwHt9!;CwjloM^2BbiiE z|DlrHu0h^%y@kO?vT#qd@^0H^Wxai$4vaYD_;A4RklYZA^VqvhSHR_u%eR9n02^5U zL%7^-b~J=yQ&wPt0{mwwVqc`9B72EMT_yg81x^TY;ythI0*tP3ETYd}h63-A94Sb7HX%N-pf!v(4v&{I}@NpJ|E^&Y^V2tS?u_Qo^h-HL=%4?6J z^!rp~WdnP%4#rhb<%#;+BnO_w423MdDw0a)`jQ1GFyEgT`I>{!GGb(yzGa6q*m9GZ zdIyRyH!-md$nxW^{O9z%0nqfJgm~gYSY=x*yX$c}w(veA zL$M;26A?pO#VMK%WdP5b&vL;}XZZahPk=Vd;qK0&s(sWmro{uc<4dg{?ZktYg(N3 zfg!PIwtRd11v>w?b4^(!mQ0%~#5a+-Cz|n_grNk^#;Gs?#}(tZ`I9(l1IEkMI{H4yJyMGBbJEJngZAC3uOa(3o&X3wS67GmE$$!9hVds2U!ekU z)b0j3v@8iNPQZb?CGJOmk)be>?7tY!yL^;+_-Ogrh(sUeW3c$4-|}(mZ`P*;RV~X- zdgl+8HYwSDnBwCPSh={w55NnNkqIk=b+sS)>zar&(-yWnF zl-xm57Tf`Xz5H&alaFGGM1qG|_hR#?cl{pI4}@gae!x`ErrNoCNFH~k5tfbGRX5Bte$aH3m zJ;rZdo2zYJ_>G-)`d0WnPSs?9!zbeSfB%GgQlf4?c?Xn6y ziOq&@!A3=yFB)U&+)r02<;b07ldWJkJV$SP6Cm`(ONmfB8Cf-@<0UTfVr`&uM_*Vw z#pxv&(}JkS+K+;m3$WjQV!9hFF5d|fa#~=#fQ73=$K9VO< z=z-`va(q!1ZBGeSv%mNTQKnJe+^dxJ>*cIiVq({(JmvM+lg>`9-S=E^-k zA=dePfgO65q8F6&mr>9GY8e2Gkgmmg_5HNEf9dS?iY@jdeXLfI@lo3;xFxRu^3Q-Q zA(Ho|Cb;jyv;W7}kuRh9_DxB865opt7T$vi+3tiErMt@}XevR=imPpOmoddIuBEGe z8G|v_@aUI!=WNH}#{$#r$m{G5>Og*ET2)3{*nPJ5EjKo_=7c;q_FR(|xT;AE1lt%3(ip|6HGO>ZGBd-K_JI*TtVhH^Oii_#1PZ zPR6@fD=){+rf?T}`8w_tpLmXLRJIFfw>B^11 zgw6qQj#F-LZYuKur`!%A<3TTD_qH zDU4BZwFR$$xjJwO-s{VF`2#yMFY*iepHl)}vL-72zf%>Pf2S(2|E*UzJ6Tx&n-G!} zCoKmV5V|g@(Lf`%kn}A7LdZH^D8;jg%!sUm*^CY7ktO9CN3PmL@28xfJ2lC#{Xovd`Qu8gq)0tyU=A9N~(h{g#>!{FYD@pX-5B zxq+zkVJQdtAOXSOWtsRw|K@|`gxI9Nf&C|vTZPE@`;Y(taCiU!NdJxGe;tneU(u&p z62Wnqix+b(2SWGZBqKPBj$+9NcuBz zGB2v4`J1xLJCR_pM~$Ym5vA2=%hpjBQne~Z&g-vWuZlQ1zKC*)@|nf1MtVckMtd)A zc1SI{osYYFz1wx-uKGI|Uqxym(J^zkX=4cs^g-0R3ckHQAqe_5r1LNnH}N5L?h;X= zqSWB=eqqoEPkuL{=H8rji}239p&coOY>fn-$~{&SIS*(X1U+Ik1@UBt2_)r(mn5i6 z2a&WC)r@4;yflV`Bi!2+f~vL{rgW1K2rO+umQ;g*Mu~|hAZMuwNkIfq5%ouOk#-z` zLBocT{zK$?=s)V3et*7`oqT^iV9G!h;js%u<0h()7-#f-+uPv-Ms)CG_WJWNw%0eo zGxxUl?!_L0pOrYLawG|JmY>w)3w+iyUj*s7zUzWk(P%yaR&0W%BFC4j_tqPV?efYl zFRFIApuaYRRmx8cl7+0qB4j7hI?0 zx3jvm1FHxi`v*@)%UYwiwp2^80tS2TI}*lt-d6SZ;}9ORZ{P`n`J&Yie0e*g-=(3s zfS(+ZMN9j^{%09}Eb4)^dK(y|kf>Kk0KG*bwG}IfMQJjB*QlgOIDJ3=-u52~C#w>G zFeoG^rCjpo(CTJqb%zK@OLUdot3lSPS+={Ng{vY`lYB%lX5K{tW zwl-uk(ht=}Ib~&bfKDSIkrm1W(pRCc0ZxX|rq0)yQUa3fSceTukSHp-BnZWYKWAtM zRKOT4P*LaPHB4|>D9+^K7ax2Qskyd&9g04j#)2GK?pMA0=yTrjCJ-87C)m52{}Nhi zB37}8#eq<9BTn#(ar9n98iXYH0U&HOn@U>mm%8h4!cBzEMjP|z2f+a#s_&FNx6kDW ziIej!-=JY($w|D-ET2vA#`ghLBa$$NTY)USO~d`$M71|Xkc}X4^M^w}AxINlvW1F> zXXqFsyham0l+6cBB@V9VH2*%zmP@I@r*|2z5YOFxN8_=^n z(O+_?eS#3IJ|{^^`*~kwORYlP>@flDMnSZ;HOwc{LL=UHff|(>8EJIh38o!yxf3s< zN6`T2zQPF)JBHCl8OJV&RB!}}$D&RE8UciAYo5o;gEgg+^ur#+R-q3^&a40mSK;r? zl1j(x(7o8-u6n4qJlviKQHskg@-c@#WKj7T_V!7-G_(l$smKV_X8@#6miMmhO=A}? zSdy$Kd-ECct3I;?=$HCY;J!n)O_@nX>o?X3%Z_097tDLZOBpYvQ*Xjcq)Fw$r*cCjbXaFafewW^P>0sj#5&ZajN_&K=Ns5SpioAQWl(&c({!-g^$gH3c7q_dN=bncU za1T3Y>*c3DI3x^N7S;Wz2sOlbtr=Syr`?BKcz<*Q_c{$gKy12Qso7EKJq4U(dbhWF zX0lm!iNks8J@2iKyz%a9TNjSf9B`v%;W$r*)y_VpL}{wnx$;H+Ldl8&$#&luDDxNS z4~An&ZuUPF%PkR4kYiD@dN@qtDv|xWx<2}Vl!13lW}D)T@CZE!(`&3v9UN1@0<$h z4L>E#A^v(fT=${RK&crOcMLJ=5rAA7lQ~eUEjKTMdy*Uof>WE9uq`F)0(1N`=z(A^ z%q0HX+YQp{xujSMP77VEJmVF`l;nJhFvCPTJFo3Facvpl@uk}VCijUPqD{oQApxFU z{u1e}iLVzHm=RdB5q-oVNrdiT9R+7^?cT@eWI)_J?Y*_9v#;W&T=x%NR1^(f2tMOb zwYN^<)(4T1CXnEs{jta>a@)MjxI4W85UhgK*R9|}X$>56bxPt>x>LB$O-6%pB(o4z zMjFH}e?L0TtpYk*S?*p|j;(!xCa|OgF+&qmA^qneJ`Q}mJ9|5N9*FQ;oFMi^{5+q& z7k!Ns&*3Eo4+ZY$RTs4XZS77t*Y%scKG_z8xWOW0>UL#9| z*V(3b8X*$+B8UiaY~i%Cyvg3VY#ixyK15v8s?XNuJ?P`(=kZ!{ep^4lUzyapTL6e< zWO5qy&4}22St`_O{2htTUSiZ(j;U(CXReC0b?&Kl`jiD$?H+7Lj$fff%)lbBftrYc z+6eG;#f7K{wzTY-&#vLN~fH{&N6q^4hkB^STd6N;Ca{-sN+V)Y5kjMi;xk~ zgu&)^UF4f)F0rRDbN{+^pU?T-c!C}i$8m~v73wvd*>%tEnA#WnW|6$xGO$U&)ouDn)W(EKb=R44>w@?=YNgXS3@VTj-UVlW&i9zsQ*0> z8d+QX&m;YAHEFvoHiVu_H3X0Vb1i9!TCf(s?KT@f5Qql>cpE}{(i=L&a3qO|nV-)Y z82p44&Cwnw_?9tuuPku0kgUh6+QKB-%Fi0)4ob~pQtWKYWbR~4aE}&t@6ifN{*L*v z3d0I)kHfUN4%dP5f99%4&l3g+BNOdcHlOvx=LjNcWk(}8Bl70YzgS1vjNRi6Rkp#+ zQE*~f@7u4Ma1hPyGZDB z@X)I3F5~)|E~}%=<#4XBP;SI=zFr74ofaBsCoO{ci)5+4;<0Pdv_N>W{tsdA6eLQt zZ3~ud+qP|2?Xqp}vTfV8ZQHhOW0$S2`(EqZj<}s4`IjqJMy{AK2jtHc-9?Z4F~VrT9SU9@&LJg^lAjOkN1M z=D66uvNVO@l~fKYgu)WAFQ;xMoSv+w=?!o-;>U>~Q?rY!tc^2P>UY%NFJYFmNSHRp zgL|^F+`hfPLj^4g;K%pV#?gpS|8@%_Qyr0p_8M%O^2G7_P~j(V)AfsjJwqqZ?A-h3 zl@`x^s`So1=PN>%#SKKRiwxc_51z=%1?5FQZxDa)O>fke<0c+lFxIm z0$<7Y%{BRg))qA2FLW)X<`cHhkiGkpTLY_>b;rTlHqahjf$X*wn4s-x8)s!PN+11O zB*@ffd{1;mU;;tVJ9E^JTupfS2}$kglk+`^5RORB1ba`2-AX7ts7yD3XaXk6vR`{B z*F)=_EzN=5X#3pKH6YIkaI!pH4ZDs=KY-S{{msqinY!+wJ;h*I`+!9JR8~F;jVaQe z3bDn+4b!IEx82UOg8;PJoXG_)SV`0uiSf0=zs&ozfyXyj^RT30ybYvOksJT|jST_m zjexvjsD}Yxj>$)qlwY+y46yM}xUAjJ1z1h_$*=PwhiM=|`)w(`p*{@gdDoY+KxlNt zk!v8zvLw^#qNq~}!qR6HT~!Z_DcTy9rvRhEpdl2|4iRQfOd@meIHwkEwrXu?BDW$< zw1n2b`7AxLv9&qT=iE;Wkmnu|=h76-&TzG|P@Lsw6j`s_P~Ls{-{SGg-L*GD&yCfZ zc1~NN>TC(aBm7feDeR|9IU^J}m$t*C^^)L9rw4MM>(?Ewjot=5;Z@Bu_J8p&yu~@N`pSy@2l&itY%TR=2+oj_)}~~SQ-w9 zVrgZl^{uq<#f>Dd0`gVg@IyXjoZ&4NmD|)wF(4p<{)6`|#61u(1zGH1nsiCV4`-0E zC4^$pq6xo$1Y;~U(_?`JLvSQTZOQqBU&co?g!DJII{Mz5m~|k%n|ZAns|oF>bFDuN z*b}e~^PP@$pWJ!I=o)*PriHqO2mXqDH*3qCI`I4{GYL4SeDyjU1IC%^N-C{lw%*N$ zi{BT`d=d4suf@}#1qUas>o)x)09jhjF4=$#vkw@7E31p3IOll3z)$iy9fQ5~rr=*{ z8sf>mnSb5b7ZNcZD&O3tzSCw-bRqNYX6?;Rua1ANX*+aZEPcFhTo5)2`2n;uLmcw0 zcveLr0&G7+#&AUQQgc8S*c(6$7d;`db;HXVzd*o%pXg?$<2h=Ts#B-f+`}PO34sTk zGDU)IiqZ{22{q37Nux?ui#hP}9Lq6r(4>I>InFMixIwUEh%SZv=`DoUGmJVO0nD?i zEX+(+iKqyTLbUHAjrsnO?LtWr`^OdPY>Wx*b>IJl z?IesvEATJ?0J|jr@6?C?s?tWbHZCR(PQN_Jwx)IL<_O|P4l&z7UJPMX*|LQvilm(+^+o?r_v}=Uo#e#<7-d93Ks>RlndzzZY4m#YlVH_q zXV0Zpvo5r^`9%wIL&b%xOGnL=;r@zcB74K;$ExZZdE*VY&U>pYZbmlU;6MIm@9lZX zR+H{KGjk7(wI`lktLF2S(Qd_J@URpflwJf*_=ga*9Psxy^Dm9fm21QK=F-aGzF5Ji zi}#Ys%LTW!-u*OHTMs1Hd>>ln#c=E$Lm)&tMv;b$telwx`lah@O|?#sZ7O?M+G9~~ z^FI9v;|&tYS3>Xi``TolGWv&6xm0Qo&st5N`pvGTEc1HKuZAzuACr$Yn|9uHG|vz{ zzT@NLMI7-T%tvO=gb81d=fhS0E{_!2l*r)%ieg4XTuSHd@l6&FJxxh`NR5;y1t(9^ z2*pY$wrWvWSX=C2qc&>q;rLR=hfkOCij}Lj+i2EI>WflGW_3K1LUY_F8gy!>1Fb7+Mp3?>v%cjU-bm!tQzMpC>LQ$xhBWp5g zSiskEN|VsgVm~thLr)+Spc1Z?PJevv;6=F`km@5xG6~v3T6#uYeuO%*QQDf3IY0ry zgvPyi_j^QFJFEi?UFY#m%67J3KWf0xSpvA*mX71qVs&rKdJ)&8E$j;id*WS*!9V^{ z&ZmJ_5xji`o{0~d7Z&Oq=3zxA)q*jqmW2hIIsXzeFS==-QXrVH1I)F~uhH#D8x zk{{TWWbC#$?^-#}7e7mkh4A9f-4AZ3Y(nTz-z;j!Xp${XqiQ|t8n$I#*dVZe;&OAYrEOUp3#u6)6lEh;= zB)oofAjDHibVZO-OX)C_zVEzQiP^`=>V`Zbc*X8yPX;U!Dx*JUmbM&4lU_fcwN-^3 zQ_2=G{l(LMYTniX{tZrNt_qTSWQQ~~__!$7O)~?=k>&s>SYkHD`|-TFJRNz;Pcmuc z<tl>eu@S4{C@B0zow{LK0alUeCG| zgKVL#Ia#1Ys2HFwC-~BaJ>|T`nJ$52ba+h!#d_@z9IbO|yfL;S0NPQYJ_%F&34}Yx ziTNhy=GYHUy3LHeG%V=sK+SD2sgOJG1$vkVgD#^7qR-$aFrK%BHj}hr^lRYyo1h`r z8hgbnonDTJn2WeUn6Jbur1g0K#lkS%gRBT9G-FWM;*?hd#F{!KB6U~HMT@&xOAAVd z5VQV=I;8n{d6|eJyG2c(ONa4h(rGyAG$-8u4D%nqEE~CH4qHQc<+koJc!jru02WoK zePpumH;wNZgIZbtKL-2K-AZjQ6;Z!nk*DDMl>F8lWX0Qhn;VhB4csP;9C-$Ult4#J zgHwG;oQss(KhwqZo+0b2-k4$jJeF29v(%#F<>k@y05g~eOfMU*&)5_QR8q&d6Fz%N z2N0^{aDl5<%7h>JR)7eKqJkSjEGh3UyBcQ4J^e3o{p`$I&%zPFcxU&yG-kP4Gge}- zAb5t#yQ*_TY{aA%ET;MTlym(q)_U@>NM#&5IUQ6e@g$N-o8!nh63cVUOjf)Y$mc~>_@_Xgqx?#FMgt)KwM(u?^NGmhq-i2U6=ub2%)t~G-E7LW?kBjBJ9s)x$nnhy(fmWGgZ6jE&Wc2 zsHad%OGja<*MzTmw4@kOlJpLF+Ool4^4!o z!{0nx{+4K@XZ?X;dIumCi-Sbe>D81Y8uf|oh!x9fJcdzS82hnKw=tFG<}|%K0+De9 zRouPc%u{VyyyHahtDl_{V@@DA!(??1p${w~ylj0<3OjCsOj#qKE#GNjFkF_dQ}e<_ zLPaU6XbKaND7f|PGrnkDrd{13-^D6m=$y`e2ZZ$dD9K7`9l>NvbpL(JJ@WF=bSB}4 zy=Dxdxd4gHQrmnKPH-VeHPHkFjcXGlB121jOD6Fj?THEm0*vGFvVLa)b!oVC8`#?f z^=n{dVm5JGe_}Sq>%f*yYU4vZ1NORA?46F=Fk;$dTHK0uhEZg<+9JZgCnQ6@F9_eU zI~H#4lSk>mI47n`zm`P#Br=d|#?!*^@$|wuoFBI81Xtnd3&rX55<@m8qhaZdNdLL6 zZ2aad5;m6WW--OZ2Vh0+v8_LhP_Fd7KNn9g?0V#8VBC&mA$OPBkn{fVEsKxD{jlcS zHS= zCVC(x-1|H@@1qIW_X<}cV}T09pSix@8e1(6$N`jA#fUHL72H%y!Abb@FXPu}XD#0? z?YJ&Rv)BbY{vB;8PIu4S!Y_l~v#;4U4yCf@xLJ+?6I*8TS1uO!AaB==$0(=}ueQ1? z;!XaCIt7v$hkpS<7bLik7te@h$zlMvWGYW|F%chCy>N14*zGulDkraj4WoYRY&oUfh!h}Xy zugD8CD3}4={E>WRqTrkJG*R`2oi+*&;2ZMHe`ycuV32@Lr1RQ%G%| zG)~E*o*)1K_`e|H|A|NZF92a@VD$IDR8-YJj+?&}&x2Z?6}+)EV{N9^`r?vHrp2W( z&g@ZM8;*pwl(}Rau%9YV%(tg2ARmC}y|jDW*f1ZzSEoREM#i!Gdb+G&4zogQG*i@{ zbqpI3VrP^mrn2_3hL-Z73;Q@$uD|W;Fu70#2{>HKtP=8DES9(L&+~rY)^YC@v}#yF z-SDa?oYmK0c@wJ17|ML{ds5~7(Mol_#bpHYonnG0Iz1n5slA=;KDoQP^_(Rc)fpYb z#UjJLaK~tVz+gusaBBcrGP5XkfQj{Fzkt|Q-gs>oT`HQOhPW< zwnCaf9>Zi1Y&TSe=@wQlm9ms^rizl_2-iO^SDIr3ma-gI!DQ~T4p3F`7|_%urAdp_ zK*~~L=I2#H$jY+M1q;Lz`J_zwl4h^(rHBqp9HM3x5jv_2xhWdc)0T;Q8P0C_Pju7- zTKuy~*^O#uM(V@CV7254OyAJ7E4uDwH7Ytlzk1VFmkviIC3}nwgcxRIEqTrFJ)mdi7ubHxgkr$PGyI$QBY_;pWhKD4X z+>0=a*T4F$#<}E=j77!nfgOIcki^^;xaH?nze3&i+Th_-T3;tI9J|L+c-DdDLMXph z)7m*1>z7(`zuH73e`sf#>s zf7&@4P|dQAJ#T_x=G?L+B$Za3=&`N6=EVU%q*|z|Zeck6X?`yh{9BfaDh(VE6B>u2 zin&U4u$86I>I`V{nLdB?aDm@rPC zf7rL~an=M?i^4IdqsVfltx)#Nvi`4uW)KsgA#LP8uSQF<+9T@|t_eQ)mD z08_^P;KetpwM-bp-DpfV>DHu+{OjWJ?wNVdikz{Ptcvn_CCC-&^gdN!u$sCLx~`~s zN5d|@Z_`-R6*$^eiUKKsbJ3c#&QBxj;QSC{k1H$d9}b)srMYd4Ru7kHxbV=P)K}E? zrBO=_nk9>PnrPOOZA%OayAQ%6^mf16DekcZSN|0Sf;e(eeWS*e^1%p89! zRkONv%pI?uhyIHggR})u+?jgSr2Xb$q5w()ys$w-Majz45{q-^vMblgD z%1*gtN0X3?8Eu;5eut{5LL(Eo(xO>JnW$3OWYSfWe5_irQKh<_Q`Sb4D3;9R`RKU- zWt~hFy(@33k~blpTs$TrIMABD>Iv@FLmLe2jMdI2n2eD{;;PmmfK;Qp|f0~tEcXbAD*W;Vu|I*_ArJ6n|;pPfu19+IwjQP z1{d)-4lwXS>?72DOuCqHLmZLZ8v1fIo`!s&i9d5SYe4n(NQgj}g$OK*vaxvKaJO=` zK@2*MZ~A&=%M_+)j=>S8tC|}oNz^tfp4gNfK%N*3CtPHMvDUmhi5Z>ePs+%I$0<^v zviA}d^2lDv5KMVzWq?Lsard>CeYA$s+g_m=md(qEmfVMFRMs5O9lpgNCrXi@t_JlK8{NiVK2uP@&`z0P!LQfHxA+#gUUH@!6>pdp$$!J#&T z;kf)Q0qdqJi8vkBT9K)UJqDUw^<}Kc{eu^Iz4}McO5Kddk_I{y`3{MuJ$H2y&Y%P~ zr6>^3@qpZ(Zy(POafGLU?6ESw&>HluC~VRekEt@m8eY^U8H0^ES**@$!M)wQBaoNfi$Ur!z4nOE+Ohlk45D#GSM{e?6o(2EH z0N1JsrH3rMKJB>xl5Q(FzY-6L_(Bgzz)~A`86k@(d(eE`SMq=x&p*hY&$Ag#?MzM!BB2z`+y}-R!vA@ph6255bXEUbFSa zObadSVCN+O>OeIbASqFK&_!S}8ut%@h@5lsrQ@&d?q4_DnEqLnw$QSV@H({PeD#V{ zsNKfgTet8@h{u9gWpZ{7w2MdeuEh~TAhEB9D&9Kg^ItDn^=;~&BR$`#q1!X=nmPeX zj##u*n%+w0Vpi#vK=)aAlDEe!Q0Cj%D^QgcYH18Lc|dP3uCR^+<<@01pcj0bC*Tih z?`$!stKKEBoP21Roye!H3D#z%F^!iPwEI&YREIhN)+;WZc8f%(Y_lL z<~dw$2{v?wN{7bVgTYD+YzXD#mfqOgmmd7dbR9l~?$Oa_xIKq>f(j9>*c|D$tXTXf zxbO_n2iW30Fi;th)T*;PFkS7mt3<3Z8LRaKIRoV|_FUQ7TtArYqpd39R45`92FuB^ z5c4CMg^(>2?dFkya;2B$Q?=|ByXfE8Ib3L`rgCj|zuwMU_kI*_#O_V5kEX$4BlIhv z6>=xXf-;w2_Z6H-AaXShHkb2L&piQC=flM8{Qqg0$$GVRi&UNS!<^vgeEqi$)H6J@ z6!F);_XP|9fczgqqqEKb*@NmR^~Yo~ApAyj6H}=Wn8RKa76*zR<>8znX3;=ox#1P%@YI<0D<3t$-q!fasWl0-0e!^l*| zLhpfl&yWj6!5UUNMNXb0gZIc2NG4#R!jz;g=SFc#-S%7OiQykmUlE{Mz`=XIn!=_) z&Pgpks!0Pv#%q%>;<@KHkmW%#T(kre&s?bs*WFGHtsbn8!PUagnmMh)Dsco+qPaJP z&JoyU@DSbtHfswH1;XK-4J*AkZa1XL%Hk#6VlWF29;cb0wGGHjdzX~5&5M*Z4^pYS z`bT}n31f3~Vr3nQ*_5`cZ^V9-l}}7hn(M(W43?2)y4AZN{b7rdl6Q}N@)z&7w|8(k zYjheZdZVq>wb&dsFvXe3SBJ2j+p$1qVucsUM5d%C&ov9FL?GG*$nZ#AsM+Xe?-Xa> z`@?I3`?)BlF&7ftFrb?Fhh5}K4e}E^t4av1KeKs;r+u);KNllUeAQw$@qY4mJo$lU z6Oh+aG#@&O$5+m+Vg}>9LM-&L)p4Hoya>~HQ#TKfuvao&b{WVO+$#MxS6qMod0*^7v7FvG7=uyL9?S#D&-xm9-c#MHOUp>(fr@#Zc5e`1*5Lj&-8fd1Eo^?U!{(bp~}HpaFN zCdU77y&Cwxg=R~$3FrNn+woW92EhFPKKZ|wpmTC_qLz@Jk(HjE`qSFX$~4cs2lU?- z`hE8Q-omdG^WV4dU+381zarEBr_C6`fYxmPLa~8gDE2?EVPa+af0{CrRX1%mIS{;8 z)#@%Hn^cZpH+Hx-XabokbaSR7Bx zIoN+AI1U?;2i$;2N)`QKa8FH#QO8Z514r{9NEL*ViCSAJ+yeqyTSaQi=#KBY3BZ8< z5m2${5FntJ6}e3yQ{mFFNQe;=a0y5Gy6YMS9VwFh6#*jhA2B8m9dXYfj|1Y1h@T~| zARtS}NK9U7$8zQRQ{&Wj*Q{(KCX=xJ;(KSPQu}G~WT}qJ$b34qKiOtj;2?0B(PwgQ`acTUK$Y&m8 zY}htPfxulCC>KE`wouXhq99vJxZ?d<-T1E$Ii-^5r>XXw}a zY9<7we{|&ZkJ=b11q_5IGl|i49ds*{3o}k%xUl<7N!p+jiW-!)=@l9ff z`0$As4Ka!==Ayx@Yhl_|ZpcOk@*uHv#f_kvQy1}EZC`3oSHz>tPWXOpU1TA}nmnIL z8$zoC{gAaHs%X0r;*WOLt0X%2pp|QJH&rVxyeSu;EET3&`LQpTmzaFq7toKX>hm5|gVvkRj!rOSSodh+6bqY_ZoKaT{NurT8 zrIQ&9bVj}rkWhdh<`I-L0OBPc)?$?}s{r)~K~y342fw|ugpZIe4PABu9bajlm%;P% zp{m72C3b>;p_0wFd$@V>0$@G(7!_YTT9163o!=<`hA%a`3dwfnBi&9t?>wA+)YQ~i zebii3+$h+qZp8x4UY{;LE#1WF51L(6H7r9Nhh1Tt56%y}EmEHvh;Z|YEL+d`PGr72o# zjp3)1#vId5sii@ny)BRjP_9pz?l?J8=87BC^^DOBV8@QM78_;4NuG~ujWjIP?ND^% zgS$qe3Dt_du`NnNaAzyDQO<$>Zpxm=d-tvrSoQF45(}*J;eoH2e|;GAHbl+F~FE{lOZt>42Aq9Wbo_tYz`P6Kx**9pC@7R zVMXfE13kxgyX2JU-dU(?#A9>%Y&NC|o@QYYt-|u32GL)co{%p05S5M@$Cz3&Cm-9~ zq-TE^AD+HjrRT`zQ|Y~wFPU*v8h{NdGKADU%=kJnvEow&^m@TUl-+3>jQCVF{D6&8 zEzopL^I3&R(Nr_P=;csLLp9;x2m{JshHi8zX~tvbO`D#In#138n_*e^sLKZM6vU>p z=yx@1B>SL>V?t#@J$xF*4pl>35;`Lcz&}{M-*W85T?)yAnvk}g`EOD;;CZnt&9BMd zi4*{U?0+n=)&@52Ru(qSZvRXCw)my5u)hE4`74-emQyb#)RN?M=&aT&%~G|H%Wht~ z@v?{rAzA8@_N{G5e}C}N1LPRS$o5u~B~v1}z3!k>AJR;-LN8Dt`KMSEv>3N1nco!= zbJDA!i6Wh`L|4#SI44KmOI246i|3FtX;f5;=8Q=YD zto(x)d8-pIYB*KHR1hw134$*^p9j`h5zsGcsEhNr2FUJBj7L69#4IyQpe6JRACgq`zR>^?u}5bm|l=(z-!Y8RSJULhDku@<%i} zX~|s@V?nsGDW(4M6(Y-AO5)tyQjTRa(2D}@dEi`Ou18V>c+ajW+*!i+o}IdAbS+f# ztWJuHJE;0}PVwYcYYArok|xp|ut5>gWkDxbBwykF5S(LEjJ5#a_ zXtpIMyx2yU?54R7gf}Fyo-TAmJ<3%Z5x;Qoy2E)71_$3hF$@*ibj_b)IX?Hy8<%Lu&JBD{P#Ki6T!0JyQr}KW}L)m8& z$BS^B6#^3YdZ6glqjXhrH!(4_9~kzhKx_db-yk{%pvt8P zF_xo7zTtBOs06R^n8pT>0`*IkUJ**-uOC9qM+u=!NdZ%&NOt3vtECqOD3NWTA|WQZ zB#Meac+n(CDJ20XA5$I!p5jpS;WwK2%d7F5lBQDo6JMVWsV_s6Fpw5UU|3*S=!qfv zAsE1ug;+;BGT6lG^p605)$aS#-5#+DIfk$R8Aj3^X{Lz>FDMi7=mC$sQdDmcU-6ni z*iPnVZ*@KB&C>aBJ?(01>vr=pkq%c&%d75P!m%w-Sn|Ar&=bkb4Tgqt zj02FKmiB74gg8SUEW{~{AOddYl7M_Hwr467+zcHI#`ZC6(QE8Em}^%^H5?hEIWu6S z^{%!@_0ujh#D|FrL4pp*WLrDwYvi9{i-ruoXzI^mxl#yxX-JGEH*zXB`c?N$u}npQ zDl1XkEL)xAzrLe!h3hjQQt}GzBUR&K!N65yot_5*NOHQVO_)F6QX;|oZ>0S7P|VlQ zWw2d9OE5$tx(522lSoFZj%E*=7^aaGB1+OFOQj>UKD=c-H>E7j?f=0YyYMzcq%J); zZfx7LrYBvJC|L(we4Ui0+v3BP$;ZEzBcpd^y+Sj)Hcdsl2j-X%f=p^Z0k#6{+I{}u zuXb$<{Z&@YudYUWyubH2;_LkuX_T8Y#|=$-9Ct~mgBs&esfF} zS%lZ}w+#7Lh_CVCnqT9gchZnkVO6y?SY|=RpiJ5v45QWGb;ML)F%^z5vYHXT3@4s_ z=*aIf9RG5O^4d{}V)dCK-EhqKkpa}hz#g^5#*{)UQH`^W zEW0lwtckB+h2U;zsO@JvJTQhbmD)RHT39c6TGP_Xj)Oq;0#nSbfhsk8B@B z6SZ*ANa=a=VhI7LG>|_ue%;{UT1wg6PF%TqGHVq0v_95Lrv#J)C{xJz-mea!!NSHJT5+Ls$ z)U0;+7Ts!i?<2xLzX8@E0^kQ6Vf(eUt2E-81p)x)SQ3&-A09p?<5*lINJ$=hCc>ef zp%N0({y?_&m2yjCMNH(CKq3~fAh7XUMXi}`2#}2y3p!2V&Wx5o3v^i6*gA!Rp)-2d zElwOBB0+En=b~o;(9akW(Xs*<4IVu-<5#w=04xp$8&rQAWYKcG;vfr~s2O?XyfM}k z@?=t=00glS<72Ut4w7&WGJ&A6&o%R*po0-RCNaEt#C0&8d7v#ZX3M z#FXPP9P#i!4FomKVa(UoTt$tvxiEV95ok#>oqQ`!%A=2FSxv#c(onvXO}kN;ZbKtR z-96FMr!BxX)SFd@?ojhZLZrKvSET{a^VLXQ<>zfme$3|NK9TxFn!ZlX_q?S0FYz2M zGnTVuHUpbaa8@W++?Vl$i(UCh?t6s{(_dDViRd!u{?^_UbP9yWwXW-MJum9((Go}X z&dWLoFGysvsKNvxu&?zN>z3(kHqDE~72FL=`+k)IQpP(uSHd5Bss4Pa9y?r=8(jG; z=$%R1>+6L_c4DnxO7xwwD8tR^=$xUMdSZ;dS1Iw)#5I>`Lk_;Zr6 z23UchVaU+`+$Aw8!Vuw$(kC=g3dswBX1VnH;d2Th9teb$gb9HK1esvkgT*;u_~Uq+U+m{@3W-sb4t7L_8i$pZ-v(xRt8J+;a;C_@?m21QKxGq5#Va? zl69dO9zULwz#7xC_?2*AN^Epb3ZJb-9Y*9X4dqj}0y~to+~{kq#a3Od!4>U8(^v?w z@UT7$mTO8l4b5Q8kA_TQHwFEtpK#|4H=^W5$(MzaO=yrDhYRJ+i78bHLH#sjtclf< z!zO)O=JnAm<-BgLPGjKt&YCukEZMSjMb1##n_p%xM_i%H$9j5SS#!fI8;%WBsd4Sj zi#O<&3H(zdRpP_BR9;+(CWY`&=^H#}WGs}EH5g+xUUjW@O{Z2bo z^}@WJ>Kbey?5GS5C>CPLA$h)`ySje@2#_SyO^+3DNYn>rSuiLzfpp*;wxO zkN-exFsI_mwqM`m-xXYo&_U`0Bdo#sqG(}fZ3#Ey@!;&hF{PTX%0biQI* z4L*kT->@_WZ(!1K{~+<%j)?jlq%(ph+xNFp2lyIG2I4h$tj*4)Nn~ zsRd~w3xVw=$fLuqk>X;Or$ve;ciYq_(U?IlK`85K*P$K*{E8Q_f6GIorVlR33|l+- z*e-VQJ~8^@%l&1lH>r@DtxlXKE0{gaH=R8}w(uLnY5Z@dNxoAyTJo+>ufZL5>^Zj0 z`gQL-DkfTs9=mxRi5kH^ya;8|!Ko*c+3xk9wNXSpg{tR;v3k07N&FMHZmvIeH+8H( zZ{uB_d&|(^@-&A;Hs}ozCBP*zb%o^)2|8z0b2dQ+|=SGIw zUB$3e1c7A?i#R}xgY5n_=bKt+7Rq^xHpB&hW79urHIarh628T#q&<7HcoH;^s=sS^IB1 zJ#)VMAqv2fKZRWwTGWohyxOv6DMHMq6a{p`h<;Kj(&w zuK-xL@?5X>bA>s%m*F-+ZMapJAV6y7%OuyiAJLx+gdMa(Wd*HU7p0p>`qIi1%1 zyH5q5xZ}-5YKcnFcmEA<@pFu1HUH944p9HUc+1(*>i@!9(W=(AzjlFNv$6dmpY(v0 z0c3|h!-)ZK1{`Pyn?RtCJxqFvEE{^nk_1(_!ej4EOp-CTbsZ)oNzc~D!xp8?V&)}w z+xr`o_OA2DJCUT#%YxwF3thso7NxavP9tDU5g4D|P0J@(+UR;;R!zuCiVqpzK z_<(45g@rLp_HpfQx4ZBpi6j+iR^UCo!-vu%7k9-JI^PEs4iT-S;>pe4fOy8kE#6U8wV|@z}k5yJH^JACHd~ zsrqV}yY6I%($M)n`dI@J>%?Z>t$tc)U|se|{}*?~%mPkBM-i>DWldwzbzL-P*3f0P zh5mu3RQiVxuvETP!dFHd9HX#3zsP2DTx#M2*#ITw$?bw~>|f#=c&n zgeqh%c}q2p_sc;=w|kwF1wQ_fkf(!+ui@Sr%!}Cc-guk4R`(Mwg zS&sW?(*Xxjp(&f%qJRggeufv+`+!5NTeeuW3N5GZ7%4|080 z`SccI3`JB78}dMXCUyc}L0Lub+;q^ztcpFQ@u~~j4nwJJ+H∈`x3J7tef@K?ZfD z5kg#;5zufb=MwTcqc$17xENDKp7*hOKXabMkQ!??!U2cId@2LtTvI<#1>?r>ydA3< z?~e;U;(!p5-TS3NwVf7}2B#yHy}ec@Q9)_TR`T>5d}bChX4M>|H}K}SeEnGmcipms zC`A@xU~LFry;dPEnwX=E_a_qOJa53;FOEaZf5Mps8#sH|z`br_L3J?OVV1+%;bUnw zcfV=!UZ%l225cVkL0_0t^soHM3>!Av^I|8?&urfJC2n3(;1^ea3J{GmBXM>z~AmS3GjS@5Eg zJkvS74glkECdRs{H0&WuZ^HQu+eOphDt!{|$yf~HFPCUK0>^gc`j=4d3;3UMLeMnJ zisx?>!e0si0OJ3VXtFbKa5QoFWtpZmwQM&z(0pEM_sZZ?6qoz;@zEGAT(Xz4u?{yd zH!aX1fRgLaMZ#z-C@3vlHomSM-Qp#rlIZ)L%wU`1MREomzh($YNA3Cc+`c3RdB_>1 z7^fjb$HA&4r4Crb)#NDtj8km|1(|lsnz@V5acP)v!X9chNm3BGIfn-LC4!NgD-Bih zNEZ>A2DGa}sP>C^o31r0#-7(f+byQLV8Khh5)+O5lL2 zL=byaV9cQXuC0@yJ+9sCznSYb@b(}Xt&AfEVgmhM+8q&9;z?u^^0^FlZQ)^)G$&8+ zr!>jaS|PRW7cf{cDet>}6XRLdEx0 zM2CTv#Fwv~PlLNKPsUpe%Q?ajdL5QXPBjth83_)QK#B{&aP*9>nMpa0pDDcqXvi=l z5^+pKCl>t@R0JDu{p)Cmr#NV4G)}q%T-|d_zZMra8G5O0JSvSnGx&~OCA}uxvO_df z-%77p@EFvJ~YN=zj8Mf4C$8SLUfFmi@iW{7ep*3^#KWp;hI|%h0BN}`5 zCYKK7sD~7hTeTy6f??I9Xvij^A1^{b{qDM$E;Ed&SUt@o z1=X+uSb2!L2#^WQnaTi^8txHki4tI8w4c;Z>>e$d$`IOX`&?5gs#qZ6D*(f)XCqt9 zKW$iaVs>N29et<=0`)8%16lniP)xvTrNAKmaZtAmIrGuu)GCSQm0{P#WJvY|%sR9V zK#9}Br<03~d+ZgtImg;Ung!a$sJBl#z;LcAS_DhzHnUY#h@2hgt@xxD{|j8$JJ4O< z*qjAYoK_QhV+u$46DhO#_Q!~`@SJ;)BhRZ6vj~UjF+;vT>HwNC*aOXe^ROmiJxLi_ zsCRHs9)RNrJXC%cifFYuUxRM&)-Igx6#2+U7Xvvc&$hc&g=E=xH%XT#W1FnK=)^h^ zFrcmm1q!CvhRUoVM)Y^Gix=$e&TAWqDYofYll^%(|g?ZEgg{S~K|!(~ck0Y8)5<4220pETgfqZbTWcXz zsydJO>VnPqPl6|OM`0VFKGDc6@sO+L@OrbWTQ!4joyQw?nyo3QCG+8UN!}85On9Q; zV+z{-&fLmsd|FX_F|%fg>rGC?c{Sm+`phf5>;Kmv6ci=w{+m0?EQts!g4z+-LrnGS*sbKsiYH>5DNhQ>m+2Y-k zl^L#!^eVuIvSK%>{p((C-@my3SV=g@?OBl~zJK zo_O|YLs;OITfzv?C_+0&Rn4Ndg|yn&!*QUf^~HOJ&q9u3gCx3NQwSG@Ni3Nc-mVTu z^OdyTz*u@)pZG>we6%y3ua!1yBq<58NotIUkN(73RG!UjAWo*ty(^x8+s<-V~`CTY02@ANtK-p-Aa^ zoRlwgcOu)+y*SK`&3|DR$`!sD@??UcKg@sJ^#&e$apdrzM`u)<-GG=3nE3q>Cds?c!c|~LAnGwTBMC% zd2v9J4K~gn)wax#XzZ6+77}|27xeDRTC;KG_2Fu1PIQ_D-8;q~+;E*H!?K~UOEpMm zmH!Y(E)mvV9bBGvK6%Hv|4x<`loffc^w-Sfq59D>^_AS;!j9z(4u!Ovk0ito43U zIhDQc1>yQoHs>SVpY4365sa4IQV2yY?rGeh*0?eTGpwaYn}Nr$Y@4{mWNx=X+D!ya z)L?PK0E1%$L#mXmkiZuRk31PeD?Xuuhqkixl-%cIBr|&=(96XEr(Csb8NJTiUxco0h>_MADSSQ^cYnxFLxAtPfSz%59e|<_2nd4(P<; zf0(Rcpfw$5J(_SZ%=__Lbe&X~?jGveqNNVBo8Zrt(Py&5H&#ocPX6H;U&nZd8~+_3 z;8tgY_ol|mc#Mwzweacy21RI!Q>O`Mz1p(0{Miz=zQ8xfqWL5`$(O@cx$G{1c`0QH zr;yrFq2Rut>?0Y}*o6)_BaZ_Lf@kR@_8c~AhG8VDVY^Hdt~(p6VJQOxhki(DJml1Z zg5DU}{l;L!jm(aXLitJKoGp*JGJ=4{e}8?He{GPu6>5Xws%6gVy@qBH zD{IPLY7Rf6BI)2ZQK6h1uKAC_do2QKMK06UC2JKJzx#P!?$!+8XQ$4W@14Fk_-9~E zTv36x{s-hgdpCx{`vm>pK(N$bqbT|R*toRiY)b@D`$Bu1ubzf%1ph+6aX8xgeRr(W!5z$Vaa|mA_})p? z2~v@Sg2q&yDkfGQlK&hI?b%jumoks@VhL^JK5{gwCMdI;G@jue)q_cG=gS`%Aa0S~ zHjUoumd<5&;p5e)m8FJlNEp=(sD+aww|2I;3nS8a1SS%?BAEaJ3DQ=y6)JS8ISbzk zJ4`=p%#*$}VT=)?NT%Gu15BFqXGdljMQfCxR67i(1kzR$c1o1!1R0qntNE1aPIt*u zCzuUq)08}Ty|jc-+@=)b4e(SnPlzr||6L`-0VtF3N)+PUu>5fHsJBSds#Rmg_VMm= zPjDKt7Y-5%nwcmu779Ta*=4h2yYVJ}BZ0UmN1+?e26< zkOg0J{==lcvAx;qY2zAempw#89GPZzMHo5YPOFU4Z%K_zQ7764l}<4o z1ua#r`jKZnVapVLgJHW2;l-iTh$hKQxXS?Zl`>=zrm979gn6bANnb9Jj$!LAhR;K2 zXy6GCO}1T0ot(` zwr$(CZ9D1MPCB;r#7;W4ZQHh!j%{;#ubD5i*P3~M!S(H4b=P@R6%&U+{YB9mfy#{b zWZmgL%0eowyu>x|?R{ytE6ugrwUqpCPH0rOZFT<#NA}gq6eqP(XQ4aG0G2xj&gHTjjp;u9Zhw=8j8oN$Y;%(KKn*a-P)%~b@C@*tOqPOejI>s47|2?JZEwF8*|NmI|Xwo zchewUhVdI-1;MEPJ@K=CX1*{DXY7s({yTQz_S}7T82_P^cV)elmVCyZVfHNzy~av~ zR=f!`Pi^OS!S>^|`Z>NDXFvCdi+<=;=;Xi!#UQ!kc{6Lsa{icnpTTQ$3Hubjtsr>n zuF&nLeh=1Mn1cGTy3&L_rPo`uBunmJ_Rh}k^9*5;_zsGcGl4<-@%uFyGaSuO$r_!w^LruiiW1tyot`D`01AY+RC%K4MvBSjrYAtoSyok=CRhZwhM-oV z^%Q(tEF&6O{2vZAZQQ@c;z}k@ZBxZ*wbEyWRv6G*H|oVm)x`ZMRO2q~Al_pLWgwL< za(asN#)6FdTwH5V#{C*Excg z$;PDSJ~nZ#66;sChTIXCUdFjUPRur7*%~j?H+GQ25zoaGM~i&o#Dj-)`YN|491Bx;qPr1hh^`O?R2MY!c`}VzWlIH_e zg!oJo2R!$!CH85GZ+c}ZoN#j#bIQc~OC9qC796iz#4a7%wR&x}J9|g6+bQP=*tu{a*mpNV}pi&2}(EV15?n zU>F?>)eIa=GCs{dyuo>{_u&jiFTXsquwSJP5KXuN?iwv{qH@NsXnqejM=HxULdFZ$ zxET)9`38-r`DFimbgar}8C`N+$wLBZoYtNAE{2CVJIarahWeEK#{%-U>~ zYfE4ag*yuy!NQMK6h9$Ow`M%7HM;Moq?F#ub2`Kv{!DxjjagR$PL)^waHOZ}+7kZ9 z&&gEwo>w(fww{Y*H*2FhNHy{I$mJfR!){hLuiyhnD88x+&L}RMQdLqW zZuxVc11Wm>QH@Q8QIS#Dy+#@UeAiW_KQWdXCV8+ELGhO?eARZSd-p0JCXv_p86Puk z2(cIAgWOYeGgZH-E-u%@E;X+qtoMc&RJY(N(dUvk>yyk85G$+gs%FI$bCSde_32l9 z9#y_7ah*EfX=bsz>wf)n({!cnp|&-lkA~>fVzNAQPpXzgLlCio<2elWW25)o&%(P4 zE9x+t{101|*{;uY;L6g?^P#P=n*8&?R8KZoXt0RONc^PnYzO+?*`J3_nDz2ILQk_QhiZWQyWjbaGX;?VFPP(~ojvwq~Pma%Q1tAQ#8iz;vd5q!1b& zctSP~qCW!h_CuuT8M^WHgt#}@gupkap4D#_pwEeYV5pxmua-Il2$vZ+Bx*QN%cg>G zy~I`yS*$c%Lt$mQ%S2LV2KT93e6Rs{QeCfUBA&p!S|eIG=L>YwN_Fkj3-=gdR!eL1KDQWEdr`d?vt6QieGO&Bzs1nn_M z6rZ+%GcWQio_+lzR<`#8PJn_H?};JKu%mC6g?OnYTx};}Eb};@iK&m7v)QI)BoVEw z%et^T+{n9rZf%?vg~4dkBa3ly`{ALxiuKPRK2pWfNyj3`F%J00x=QLA`J-Alau3c{ zqYT1J^Ivob&G&S8;9@4|G)zJkbUAvN@X%Pn=QDv!nCsA1XV(S!3}eK7igY3YLSht> zVHpR0DxA0pyQEgP!#e@`b~?yELzt?YcP+f8>2PS_o80Q+tG zM^QCaktc`k@h|{fVdhHAfC%2v{?19lUvq&>z4}NZ0+D9$Ew?~1hl@e!+!RGT<>4t= z*WheKHxsodZKFP+_n#oR3`vGW@S5dLPF0@1gaqC1u@4IyCHxGgKzxVPtoiX_>mKt$ zN;nGKafPdIAqx5_em~UuCn<{#DW)MYgIeSAEo_Ff0Z~R7JYs0#t}w3w%D5MEj#knY za{2ss>J`$qcWjHP{uMpGSTG8${8oI4`m8q_corl=??{y*3E}ier0DhI`Qb9R0^BGR z!a~qONW?HU<3`k4<(xRh-psRwV znD;Cg#sd7yuUV&dn}E)=!``q@o%4nrE{d=Edaxu#-M!ULIaiHA~=*rr_6kv z@OZnt?_-%~YoIH-oC4AQp8?Wpoxv1AWEVRivUwKr`wr{mj+(CS?&L}z5FUkob7{g= z#R9cqE6q~g@U$e)hYbQFN0_;5g~_H{t(shFoGWUPYeIi{JFGrDPz&34GubM?+p^r; zSr~M0MXD%O^3nlkf*xyTb#znOx3s#aAy2q zwdG&C2~uX*K?<&O-3xqi?)}@pa4!>Lmbu2dO}^B2r%~xT9Aa6OxkTDa%^lC*2AeK0 z+|we2&msm&zJYF{J(KRxEJ!hq3nwdA4KL!dG(_;Fvjp}!6j#_Y)`P8ut3k}i5N#R|B z&Epo=CuForC7jL{{5x^6t*zR^C-iPGYsU~qe$5q~j!!$EKsTRYEpf|5M+4lJ`^43F za>O-Ycs0Yds{^;1y8HeSgGBt%$+nZGe5H%gEl3P$%0NLj8ElyVGTkChn{jq$%h5$P zCout$YN;4nOYAq+x-V45pxgUeqmD3XO#y^lXeuz@?ZG=yc#G$4GHpC@b?j4%3tIQz z2Nk#O4MsI^1|2fKWCo?}hn52`7F=|0vhFQ)11#>On;JW9=hz##F$mo16t0?7X^l4% zge)Lc_v2_3biCpG5s)~%Yp{NQ&=D&j7|gC>!dRRu2CeU7dq2&na_K7Ch32ApKh?c7$RIPUEUZR9e3bU{^P;Vl3 zd|Q2P#ld<@b`)>#rR$>>ASa zOAmO-K4D76j45<0TA5KDTR|zLlDx%f0QKtNik8n64ZiZqS){(HY;|s$r#}wfz($VvpmZ)de^){%K2UBMUBW{2_vHcD49eA-61$>_`dEyi z!yctV%VrLv#H3PCqM^I$M7qFI;;Z}5fgBw`+g&gz_DOTQRROMs8tu zoEsTYz1FsaOSszjyt{xA`%d=~iTVqV@FtD#4E}GAsm&?xFXcaa#TE9S^!@)CfzD=5 zu2#-wc4qc2|Ixmo8r%QjSNvZLoRkP?;475ju$VmL8>_}e1L&X=TfCt_0_zo+H8WhK zX<{`G{de)p88^<4AtxojuHAPf3HwV^nv}+I0&~nrt;>#3Qif4*)C+)HMI0?xFkFs8 z7h|**4p{n}UxP+us?8;DA3lE}h@=oak8W#+XAI&O2zGV7dic4z@8GC?Pg!1paC#a# z7FN4#Tt?I`K%{zgm(LR?pN3Rj`EXQ5>j!Vz6?uWQ?^qt}sa-j@{Yy5LA3JtC9VV3h zX1UY}^{hu8Q4y++N`ndB#j$dXnFy>?!pm}-Q1BNg|JQftz_+=rzm9 zZ9b`^1&bL~X6DH|6z^xGyEAUf4Ejzcy`NheRk8P%X=W%@McJytA0fu9^iqbC-Gt^E z3ZC!pF3^^^g_Z*fy|YjIonb@g7_KFwk`*O*4V{j81cM9nZE=&nf#~ z=XeL+QO?!Fm6@ert|4!@E-}3VIam`(Wn4j(yF>Z-3RdXNco1kI(Ay}oPlrIKexGE% z;iQQXwI*#51Bc;Lrk_VCy(@ilsM=6xiVv2g|3YHuthjhb-GVL^d*ec@F0?y{g}#~; zT{5_X93L_*zo)&=GHhm6#B^Mxd+nW$h9oP|y!TmMbTHPE(FWRB5;{xGu-pCTb4|p5()ktWO zm&l*%TZ`J|GH6x3)OfzQTE}BD^9516YAvphWCN>65QDdXpnDX>%ka=`P3KbLu(GN{ zDhQ&rqu}3_!zC_N3S_&B3eCP;G${Q`%1GaZ_wiNIhBanuOyU>S84A^BizRM7W2Qs* zb?mJ+f!>aE(G$kXDb<>j9f^ad71T)`-Npul9Z?jr?@l&)J&%*S+QgA&@`%jPJ z!+Sj52B8dJ|G*bcVS^}y{kSxuU-%9F9(yEGw)H0Vd}knWAhds}8c%RgybH2Qsx;i-B9YeB$z8>Z!N+wZhSnVmSyKHQtVi$9m5hg&jh%jwc? zFPjmO%vNY_Uh%Dao_`r93-%)S$n_5)`*IPC`P%MAl}Euud&STC9KO6(d@Sf+{gV(+ zlrNM4O5~mu#YdeIW#lCUIdKjzwhsAWXnI!e{XeT%fk(E2qD}(ev_xD2IUYCLPyrE< zjK%X^_09nho;aI|Q$ROyCF~x)FIp%i%G)a!0m208hyV4lasEd3MF7}~!b z%}lJ!t^W6K_7%U)Z`{#@-8Whd!*i&yz0BmJ32@3e%1s#s&Dg^zw9Xn!2rVE{$HHNT zhE5_dz`aAS-;}q-UEbl4^b{gWd-_$z_1X2=8D?zZDyJ3tuhNW&y`OcO@dCQ+s4kR7 z%amC$>}5`p3i0-`g8j^Ij}Ez#+g(n+Dneh%2^#wQ!h=&~pOv^oABn*ppQmOVRLR;9exYd>6D03P0ImEX7gAN~k!hgIFclg)xea>4 zMan~5sx)+BOk^3->USEa01kEH`h zlE1@+QXEO#&DF()4Rv+iQYXT>qW*m12QooW)Ws{m4tFlBWK|g3aB__H{ywQb`6UzC zt#AtAioh1tY5_7rvGC{f782(7hD7DWWcEf5R*z(&N1@#LzI_wbUm^Xz-JnN3bS(Zz z9hRv&4bfHKYQnFIfCwtsI3(V^iYJivFW?gNSJuMg|lQ(jXjuS^*owZf;lmHcF-Vo-|U7ACZ_pOO9P2$eT`aO|#Oqa!`kPFIdfFWav?*9ApA|(U?=2Y zC*@!#?CZ-l$Tz{W7yPqI$-Y~k^IWLk7$IC%@7+oIqcqRpL*E3 zD*qzKLU8*lu9;@QhK)gCz2H5{sarw%i5wv$cMu#N*#qk}uSApkdmHihOz6yrr_QOp zmJN|TzRz}#uXotvi#mJP`|%kn!oY^oJ@e=IZcO7&t}ZcB#neUfrJE)?@&I+{O@3IQ zKUD#p4^}Oi{m$_odGZ`CukB0L#>-#(<4CO-`CVce79jPWsB2LW0fZIdF&1h5Z~5=F zJIkWY`O@6d!d{axC2f3gKOGgf)|!3yfdpkT$6pX^xXO9-j7mUce!$P}G^yDY7jfAPc3XizLC#djLR@52&+Vb*jhj0|3XED#VLfI`Fgh;buSZ6T6^Qu*e}) zU<#47X4)tJNo^;8Xd!ItEJ22a9SR5@PcV8i>i`Mu-VAa;@hS{ooO$FD3o9S+${0hd zOVQvu#3!J#cFY4#h{Z-}0(K4Rii{&xNoeyGK}~KJ{P>QDcw!U{2JOeQ2@`e44>{e% z(bMg{vr>`Zv8Rp_eR6>3?-C+nsJmap+v<)e4!z>}F-`1QQG7$3x!(SjxG`?2t_s^?Esg8@H=Y^`SwF{_QaJWtBJ6>K}!$E$H4~$|QVRABR6Dw%BC1baZwa{#wyq=v6_cyF=!V$_0G`v|~rJ z^7@0%*YRg3WpSsQcyT4X{yKDb1Jlk8)^-%vFk9xhQ}p_1*zIncSLmVUa$K@&iYfEz zBAF5NpxrRq1rJrzUY7!9j52NfEKWubE7@;B)Nz#FYfN|JOO2$1If;jSPy;_>p$T+Z zS+eap)4r$CC1)i?Z4}rNMN!H00UXcAsQD?WvaukwkhQMDKYjmwsGs57j}hGm#kT-X zHTjwh#!)Ff+Z}Ml(ch8VW*@vpQUbiPE6ewiFpHQveY@{Ad5DqErx%bD#7*nV{f<{U zNr;OEy(Wh0tOMjfS4~RCZD&4S6|v0kAoIfD;Ts&ros+;eT=P(4m4A5ZK#he`i*f{C zYkal|#D*g499|Au4FmHG+H=3Q43*n|wzfwP_GyMwZ~)3^7QETo~M; zlj5S|@C?C8d!M93gSWpnOnB`KU}$VGYKa-o?Ea1TjTjpw<%Y zx!>ow^K9UzDO>l{tnK&wE!BxDS;(engU>2-CFZA^Cso?m`q1Qqk+GPd4@RT~ok+?@#)z`n6Y|(#I2Yy{S6pDOjfQ zQf4>r_uaUH)n;IM0xfgwI-E}jl|ZZSCPomC7Pjl6*3dke@=IIJMT~mygAohatm{+| zjwT6pDd#UR8?~V5rv(pp`ldbeyyp(oDD9p>!JS+euds3tYjbm$F#{DUAkgKyx*AqY zdBVl7)=g!c{6#>*OX{|MuW#Mf)|{3N{pfrp&RaFIXrqZ=HJ+L$*ZSYQeAep-%Fgl& zGxpa<)wbNNDvB~A!C&_0e@K2@>b30!e$_NwE{5(IHe4;yWf@mm)@1DJ_JNUnB z{`h>py`6x`Np2w7cWXxpC(j7S519d%rJc#?S@EMpH@7pVn2Z~fo7Kk)rW_b3M8ndH zp`%_M+~QSbp&|jh+-cWQXlKs$!HK;@TwsH=STDesi&b<+MQOz(f#Eo;;l9c%clCb| zj%*3D0o1K{7w6>*@S9k~n)2tsJymFts9;kCey?$!!o3(|=ifI%nFTV_vPaO)`O-2E zNCn9g9w5yW%JMC~yTCo2HJvX)ubnI&OcZ16CWVX23g-b-I--T?2bq&pl*s+o(k7fV zfWB4-f(tKKROkZ!B&$eV#YDo+7<%_JvDc(oQr1U{3slZoEnf{G@^&9@4=IU$yK!^* zy0{+FpX%~)**f&>h*58o04m=?uLsazUge01CeBY>4O>N(cj-l zQ7U*`9f2~%8vr(9O3Oou88D;m<7SL-PnoaC=+=Fsl_qMudqpJ5h{pvhGeev-_X^ni z-kV-*C9nCF#WrURBgEw}!?CS#9~F(9eBF@RlGTCjYhPo(jU;sg*lbAiTz7oK$8l?n6$V`b9@! zRfD()Al~K+6LoyA3zh4(IoIh=Q%i~7p&%axH6ECU%4L8hR+n(t=SP}+ zV0?GOxoKhZrq_hr7?E^MjZSUT;>9v4qj*0RRTNsg`eyg2JLz16jVKct3ST*Y+a!2d zIHzj3YHB77x!y}uIn3qq2YLTfAlP2JePw$j293B?tHCy>Z|ZAik1iaI#>!OBQ~2!S zl_YI<){yK9EGrGRnZbp<$836< z5_hEapCEJS*HCElTtQazzhs#-bV_{**A;2`Ic8%{NbnO6veOK>9W?zNq!hzKZXlT- zZ!l~BhB3&uw7l^ODfdZ|XOq5fcd18je$42PT?&F(&1 zUza#KJ7Za*7Rs3vqWSkKvPm*M=jIp7+PY6(HZ*U-Qy+I3VRV&ZrLVHPc3U>cN*e_6 zj6&LYRSMDqyw)}vxgk9wMtoDd>8_P>>HrT1RtcWH$Hp%{Ij-ND;5gXG0f=;|PfVA( zAKP0BIrZp>Gi*fWVUzA)kdJst=W>r@qeAABzb709voA+XP>PdY+x{ic-vB0Z*QqoG zo|_v*D@Q=cr#gqXS*7QW(4u%cAx%}AcSB>XMT zaz}FbU!-%s=^f)ybbF2Oe za}&bs_ITsbDPUV^R_KfMk*qm-y(whsjBIfn1@hA*s8`R~hfE$CnvcU4cj4FN>Qkv# z9HRHe=ZB6879BC4>90+OQ@S_iBxe`BCL`j}ZvBPDk8WurkS#EuW(Ip_SGL>p|Wp!$K5{3U{od(0EB1XqsdW7enb?Ey~AY zhL}L!D4Vn9o32H2leM5Gb6J&1Ql!qBSXNsx0i?to!-FmaZiFDqb_u-L%`^I8( z9_2wGsVD%LHERh3S3EHiyLIA2ZME=mO|^nPFJyf$TP0%LuE<&|Zu%9&WEW|x6R7kN z0}ZD}iQN8fYwjv&_f*Wzfu@C21kq-wzhv5n-juCnBt5LU(7!NnoHUS7L$P|%j|lt| z9=5qnpJtE22LH#~+CfiLK4Z+GlH`YTStt?ghfd~nA2Tncsv;j>XDkTW>jH`0G|)KI z#Rx62Fl@#+i>4}^;Lbu9O*v3n{wj0K=QB=KeAYRm<_bm6e6rImPSc`d{KB?I)>55F zE)uZ#N3Udh(|O@!$f-1kwc9OXc+zq@=e<;YTiDep;<*K0qif$k;Cbb&ozSav04n*) zJF?!n=W+o->5{No^Jy-2pu)=mA8_;yy`>i+vgVAf`L~U(6%o1_+KNLOicM-R&(tN= z!oOO7RakqD#cwnQEg0AGSr(j)pp?61Ly4FS?K`q>(%o!}Ir0tzq3&zIunLN(^W|_& za}eP=_AdylLBfx1hMGF*(B|JUQW6VrGDaNWaS0bSXbH(sHpH;G{Hm~RRZh{0TZ*#v z!_N~BcioMcU=iJkB>U4qIz#44KYKXnaFc8ejBxg$*Uq};!s{S9dv~{VC=(uPcP!^- zQ1Pe8c&WH*Hg5;;>bEOD<4`_~tSP-jI8M8aDr$SNg8*30Vq*Dj5vBY`=n7=SW^OIc zjr;T(zJ3ip{f+>0i$T@wY?bCOUjyqKcqy2hoJd3?Ai8kKAgHkujlA%KVI+hF(3On$ z<}2Wr)~xP}3g+mWt!I58Kf4Wjdx6T_VhhH@P#7CaVfm>Hy$@!$qvK-SVHff8ON_di z!uS$D)A{wNf;pc-2}XWdLI|m542?n7_0S+5#Wl1$^%U^v$n~_mB6*s>TtZ`xFx$D8 zAI!sMsu|Qa*d~$i6Ri9ApP&lhijbG?p7 zL$L=b%1&JS<;&GFC&d$XPYN0c-7PsK`tLnFolg@_Y!zrR)C^@6rBoITu0`RY;X}Zb zY`bWJ z1U3Pv%<^@|Pyi7%5Hth&3G_?;lw%!3?+t&{CB?D%lzn~y?MI&F0mwVfcMNeRKSlwxa#{vgCT?BUSU$Dcd%~}(1Cwy z{$tzXLuvm;xS!a=Qz4}_-RIY9k-O6mdI(dH#Y_Kq$^1(6x@_dIuxe;@UF!Ht#sqUd zNBP(sD)-}Mzs=O>%ef~*wa51#b_`MXD&XwEN>iNU6lZuRUyy>k9d7UT-wfn(2_hie zJx3taZq`OBuBw2EeOgxguvy+R+&tuaSdf3++eHNEP4aBx5<63IO&Jd-ev(cE5Ei#>l2^~yHHYi~& zf)+!n^)s#F&I}&|u#QQPnKmQ~Q)Goln6S0XV5V%q3D#IjV zS1WyZuy~~xO_UgL(SiR|mkeY?8MZHG)D*t@Q`0cXLWVi6M2b(Fz0a4$6!5PQ;2r>Y!t&N2)0ptE=PT|RV80Sk-ROH&Iih;h3B1rW7GoUnX!a+DrpDOL~Eg^k8 zR9hMlQ;O%>#$S!Gu$^-}iLT#{F~Rb@7F>c`II8M+Lb4OyR7EEdhhbSZ=_9k#Qgc}y zL)VpW10knfvY2z(R_>+z7EKf04o8u3nr}+Xxn?QXBetMD z>2Wtj+c!TxCItkNW~nFfHwn=e=VT`9QpUK~P$tU-s$2PHMx<^N0_vwl&`+}JCno5h z#LZ@bhDJSBCp%e`2zj2~a^IT01#z3?Z*>}aut9H1;#)$pbYJ=`ii@pTVFo;#vU@`s z3Q&xdmes=D>o#r({^aN0oQR5Qy1dnR z=t!F_}dq-2!vp8{#VY$N=2h23}5||D{YQI9@e7-b(fIqzzZW0%V?Zt$p z!SyK}N?(#rP}%knZNSa=0=%DtK?-gxp~=(c9lNfUv0y;WbkEA>IUa3oD4r5o*4&I5r^WXfWx z9Wj#r!!)1Nos!2GZ4G@jg_;P|wxF6f8$7+*17=KmX(*O+(vPLS)Vz)_HB6}tg}-jr ze=6ML^}tVzi!;UMl@+cL05c17mnI;6u0^Bz>z3chynHPtW-r=>3;bRA#ah5%x?iOB zm33Pn^`sAaFL4)#^-O4Ri^X@^Pq*;*Z#~y7HBte;gNhTD zsBnIPrFf3v_W&@^BxUt{}vE zs)Qm{>ne*myvjXkmiM*wu33*^PAgQMNTNFrH1WHZ-YZ#Ym{@$KeHsF3m%74@iKWd6 zPoWfk=)5%~)xdB;CH-T&m9C57&B*_r*GutkLGe;v62W8~XqgqGj-l$I=XlaF+pqr2 zmRv~N+a8@ZTWrtRk4vf!aLYLd1LasOTv3o9FuG+!KRr;@L2{ZmE&< zmitm%0~!~;vAFg`ix0y0jr&$c*?W`|nQ(!d-TOIl!&r>;S=rt8qlU7Uq%6@m&5xD) zLg3YPl)urUyG1iPE!AYa@1J{2rTqEx}IQ{fV{(!}WscGmkD2aaOlYJ}Y}7{NGd ze&M;E|B_^XcRD`A5KPT&X{L|++Axy$jMllMxM2?uv31`p=~dFj}>)B@nR#CBI z4W1esM$J4Oa+O(eEm{OC#d*=F|1f$WVjmAomHBMj-T(?W@EB~taAn84#xcg2-x;*K zNowrQ-XtJ;zP?&*3OINj!hLX-UiQ zzKO9dcevS8wDvu4yZ^IWwmTXQQnttT zXDBux1w5*^N`0y&_R}NCRP}K;UkPsh^Jq0HwD?fp%%oFjyEMo)?&u2TLevmnNwcL^ zzs(|YeFxB;M&nF-#~fCC*s}gofLu@q$Adst!Ma+RbfUx1e9>v$reMg96_&pb@gZ}P zY>_<{P=RL^I9S>Ej*=3=^0%atj@nFICZt(ZVS@?c_?GvLNqdRb-u+YkG|WeJ+(F8X zY}#}NB5K&LW~1u{bo)M??m7cfaGBlO6 zT;694Buyz?rZRQv2LyQr;JP7D#M6s$+mLq5Mnx>(DjGg|s7X-fjJXZ>H%s?9OVVcB zz=UM$p^RoO%lR$b|I~iQ7?LS*oUH@+%2O6y7llY2>Wmo`Yq6Ir$u7u8+38=Qz_K*J z&Mt7E*~ddsHT+565TxM5E$^1AVxS=h-cRv8@#{Qv7S?BVKlNFJ zf~NwwLBF3>TjoUbmfn%gs)-$}$2V*$g-+TL#>Mq2&#=J-nqZp(a&-Y$?2lOmdUMLl zJS@gbP*=Cw-5kttwkk(m8(~v!q0oHB73nh7xuVdl>*>(BN4^r;p^feZ&%jv!OZoZO zwYwM_cOs8m&!5!|YELfr_PT4;zzIVEXX(@a^xe4dt&64FH0+1QAG47Ow#UXt>sD0P zCWfTqj20fa3u zJ|BG1X~H;U9Dil4c5tcF-p*nab4-RAE18b#uMa3!T7J9Sc91!nIW7J6mP8$2gUG4K zLl=4dOz_zWj5M%Oh3^JBm0hl@=jhWA5`}Qg`=J>GyL(Mm!zwysl$Mf=M@ZDh3KESpRV>67PZ ztj?i~+#sUlUGghijVqouG38)l9rAVtj!KS=nMZ!lir)P7nG?Rd*w(IUrp`y9e4a;~ z8snL)fH-=3UI(_J)zEI*H~xy}&g+yda>=$9P2Z1Ux34I{mL<-0fRgubdK*Fp2nu3SvVtm-(d+8?ekT@UiZJD$hu*^#Y7{)ogZ9$Vr`%Chef^!xqjE5qii;eb4%Y4G~KA4a~su|u9+Y; zVaSA3$wy0*YC2ggmG*9tBZdyAb+!w>s%fUa^f2*RVd0rD?Wvu~YxCipQw&>=M;B!~ ze7G$(6@*@;UDTyj>3w-hSX!5Oph!4)cH({$`{TrxB3j{!AZA77-?#UmF;$8DtDClA z*OTYy{Dhf_Ba?jV9b7i8cL$&%>^nib=a4*Rp!%&lz1WWPTj#{<^#m>N@Q81qelU=X zdTuGClIj;FNp)}RUIzne2LW7s644px^n&4d<%`QK;vk#O9TS)y<;6LRY)TWb7OFGF z!nw~aaVBS61WlKq@uQ>T=iTG!XcA%E-mc^OU@4*EU|cw1Y{|z^CM(h$RIv+!sDmZ2 zzCx1chsRM2)~509Z^Pbn$osc0;PjInDMS!MJmc#K5?Nd+qicWWmmZ+^{wqT!C`#|Y z7YB=TB$LGA*n#icU>TTZCF2q42GI-mww#w%FhjAAVGp;Y$4k>GzoTDzGt zq2E^|#04D{ZuDZ~-as59)YObl)A+RGi`Zj*B}l;O&Sehrb45A)h6T*2(|iUIjiGF5 z0_ZL@<4srdq6abf_8_@l-3Nd3(*m7MA{=6P2qDP!4+C=wX?(&=cY!H@EP4)T5=lL( zBU}2oLhK7ztddBt!R6?SOXCB(V?+M*(bVr_?4Pq#7^;oaGL4Z@XOzjfArBRNCid@t z{sq@{nj7>0s~|G`q|qVO0E0+Q+w&I&ox9nPGKxh}r!+g_lTmfyK%Oilm)z*j^QV{Y z?vwt3;RIGwODppx&#Rq_q>p5RE!NlQQ-BeW2-np_iDuO!vcxST0jFnbn6eE1YQ%!g( zSI0=`96*y=;mplb;qX-gnkpUsFbzB+_JbMzVX~|s3>q0?njCp`7#U!Rhg#F%_!u}r z&(Ac|Uf4r{O%X+gzPNwfeG?+wC2SJiv)tbcv1w#bq%Fviq0q-S@)H@qTioWmLwGO;vgm?kfvK*236Sab8Lbe{uppj0Dd6OI8J)r&_{ z3`P6`YleWQFCH3qmJ74Ls+@40GzmP@75o^K)T6XRox~3^I99V1Rxs)(2sR;Xu_-)T zBqIci9waov6+M*+#jF>_Fv`$LFs^j+hp1vd= zYLGC~SOA0wD~*U^{00H=7E3wShS>@wyL68YL;S~u*N&7#yIR)@&w?`TlYwx;8xMm` z;&Ql;Wb?{x&nHE(R5A3|2f>HY;6E#OK9{@n6}@+%lQv;@i63eG+kLq#PDJ1m0x&$# zg+>QYX^>ic4|~8#!#X>5p+RFkS7At3MPn{fg32J8hS4lwsenzznA`LntLHGh28@XZ z(Oe*b7m?}J1h8CY$MTY<6#mhT_Ma9Of!m&mn0^?)4k0kC(7X*`!S4-N*z4WzeeHJ_ zRm6E&pJ`X7jZUuQJ)Tno8BwpJYv6ft$A?#^^O5`B)kn!teo3|=pS9xF=NI!G~bMHb(c5UjG4Cl5{ef=E%!|s(mJPRHvXBSO@O(q-K{H{FL1b6kl=h~)xV@UcnBDbuenJr>Baf$`JJyDe z|Qpuo!jgHi7mB=N@E*@ zwSlW1Q7N0L5+uz1M#}lT|9kuOq3D3OT)%F@)Hkq=8zpHkVl01J&4;TSz9vnZR<6oC zYYbI@B+Y60b`m%0AkfVZQ#$(NxslY8I1 znVWZKzGC(2b69Kt&MxU*)w`?qKTVuD<60I{mM7{b80Zi-Y1}GyzQMN<3`C0EW4VcE zZsI1W=MWV)&-{b-c78T95-I`$y1l_h?7@mz)vBeaxYlj=eGZqgkMbhKf*D|?+@-r) z+G6p7pI~3PpXT9+@_~bq)(g){qkWKYjLSyi6dIp4O5`jb>MQ1;fJ-;Oyx9>VMf;1> zYgc{8>I@?7?eyTRW)n0}^Fbt-xA6AMv2TW-s`IqV=kt-U%kGkvkgx#OCkn$1Iku8}rCNlElzpHmxNQ!x{vJd5-Da#ir#aOb zJ8VN_Rc+q88~drLY!7_0(2(RBM#d{zlQ?rs(eLE zS#I2V(MDWmh?^tfycx1PYGCYGFuxJVgss@lq+Nms**-0$9PXT4#(N2V-03nqRIJPp zNag7>&rrw{@;-y)DdpO8WN_9+sjj=911MdJeAak+|*B1q(Sd9q7+(K$cGcI?+%$EYHn6bRtI}>0*MY^1m z#-@xt*XlMs1knNHb1iaFq6HA%u}&~Q>il-uB+fzAx?JM3`qZ&m zU}+IAvlLJiUHDhl`2u4gx~cS1A(NWUI;&ry1ru(f94RM+x;g4Pw-H)+mLNNo_ChRJ zsZ=kYk0EU#N_-2`9xpj{LFOSx*_H>k&($OGl2zH#>jK0%5(vv^?B?_7cfOJ&q^_Ni zVs&*}v>y5Z@dT}^r9bcMbxn~W8vq@1$LV1R%#?ea(qqLO-JVdE1e~Pz^>4}eaAIFlR(hk3h(i>%kKrW11$sG_37m;1`JQRE_81lWNSsog z&33l_bd+kIPzId_E-1ywj0Z0&QiKqDqfZL82m|C&(AAjYozKOjVcEH4P621<5-&O zT+h1G^UEl%_5jC|c+ANCBAGA2HwcLY3KDEh%#QUGdE=3HD{+EZR!GMtF{gmG+~xB! zPxey9rr*Cpcw*c^Eg`#ZH2CFr#9W1K->r45Yi`5@m;2;DLDqb z(FRQKcZd0JNu2;;C+3&LtU#D24*_!i=4YCAQ! zM$51x$9jS9fG4Kx6zvf!y@%m!QMj`Hm0$73H??Wpgfh;Ah+r%;v^-dH zPiFa4ue<|}GS9fKIYbLR?m>ZF=|Kda=3x`QZzzK8g;C!iTx|Bkh9gX~q@Sb;3dzpC-h>obO!SzHTrKt@U5Z(u`cZ1BbwQrW{BJudV=c&pmAI za=>MBqgi6swjdfs%2xjnkHj}JL5OlXK_648wRNn4LRiO6o^8I( zu#S^5j302pI#&l*gp8me;;e9FgBMypbSrIV*l84pD_9a`{E8J})6xijg$$vtB5$pD$7NwmMfD5v>~_uMV;9r#v&-uK zw|<|2amKZ#kW+I^{R3zR>!j)Z?1c5(v(52bu949z>#RHl+3vGbKFe((8lLbwYGkk& zWOjU8S}cqJ^GJX;3&nB6D%hu!wQdBmPp{P3qYsoK?F<{u86N3)FbaKVxJv*kLzlC>JU25M?AmrBX9(vO!8S2jf5CEX$~4abU| zxQ3h-rp8imK`QiG=Li;ad>!0_^^#TXzfT|l}J4LMm^H7yu%OTc#B;G8Y5{R+j zq3L9$Qdg=nQ-l^WJG=JPs-uuDPPYm<9w&WSZ!awpS(C#bp&BYfibaBFf>@J&XBpKL z&y7$FA#kOJqI3Z#?59gQ4@nuT;Z0$n?hIk?OVfN5G1y+k^qIuNjN)aGJuZGI1xt_o6J ziU>Dh9!uWU^^Yy?tDzMbGSQ+1o1q)gtGJJ&hHEU1UWakL?)sitl2W%_Z52y8O`W*3 zSjev}bJYHD!PfD~nH-{0%7Pf>tX5j?6eq*Ng>;XvkdJR32hwb2zrzQ3s`Mne4I8HN zY-+qX=9W5}%8WYQRdloV(}vHJVzy4#24o{>`b{!2s~pj|S3Se|(n~}C%*v~huWtd1 z_UQOXv@oglnB`Y2rAiE8N2`D>>SbgN{UnFFht(m7>bkh+QZ*)}J1PXq4r^H zkR`>NI-Zs&n>~PRFJA)R!T)oEa?Ekw;3hZ_Py!$c9^pSlcb%Qgt^SpjN6p-JRRr

mq=(7Uo`kWm`Sg6|TK@fE0`I%guyOz>KMC!@ z3OloH$TE3ygi@?}8gTbyK>_CQ4CU?zWQBbwJ0gtFB$tH*RzI^U~|1RQ{K5GE&eq6RRCPZ$$e2cQpy*)vb`k!Rtn zKg6>P#s=-m??z(k=B09r4LCuS7!cpBoujByKSu9JF;fZcY4cw5%~S*-9Ynqee%Zob zbiO;F1h?E3i_7!yP?m5G#A);)E$j*nQ>$T;MC?wLV3;-Us(6z=fEFhe+&519tfr|0 zvpxHPz%x*M%r(lmki}^u5n5fof-3|K-3sbx{ScRv@7diF6yw-o6V+c1YZI}XF6T5} zH4;SKz?w`+fe1R~xbA0O+--kYqa_vLKEoGmDb_D?i7X=9=G<#PN7FOz-*Ikz!`>=8 zmL?g_MIq)>-(g3A8z zyttAfz7%pDiEdnY{_MEGSCVDG!LPQRWf8&yb|_5q+ugf{uGL5iZHh&0Gf5l)%T+}8 z4_)#1kPI^o`I7dJuSQfy8uN7E?S#p>6xBT6V!{@`24QLfKSb)o*X_2LFjsqycBNEI zNeXk{Xg{)uKsV^G?&nLG_vTjRA!66#up6_wh&7@CRK?o0Ovayf_RgsTzNVY;Y$ zL1fIUf8QnL;1WnUZgLK$T2rF z9KUsDRg##eE7?7c8h$4~%UUNpp)XVX4Xs0<@H6Qiv{-ywbhvb-@>C2^dO@uwgi094 zy%ID}hEO!^fqZvS1T;`SY5*6pC%Ms7ExE8*#D{Cpm)#E9=bB=V)g{l({-uRnzKAo- zXIDY?*O8euLAQ2nUB}a?6@m)&@@-Xr>#}mL25%-GP8QaJJ%(kK*xMd{*%FB|eXG`Y zc{wr7GJZ7UV32n?q=s_wR^mhMm@PLF{aUxdT-)qQLM=Np_6_uu^3HPITV z>Qv)$P22awUt9b=@TkEO7Cqwmji{B|ftIz!zZGWD6{azVGKNylUod#=?VeIg6|$kL zbT!+bQi)+9n1LJ(ju~sMlVz)B$1_C3XDu7^OCAMZ^*)}?uPzevHn-iqgXX}6oVjs{ zvAPbN*yv0m3wR!HAi0k@i6N-;|U$WbqNC-m3?=fQ}0st-o6Pv9%%-eHk=*bIhm zwkJ&RR^`CC6IRSvg5geZQo~%dTtMtzU46ZH!EX9G(xmNj=lLvt<6<*fs5>9AXQu50 zTQ0B7hr^!#4hO1LX+6j{x){aQ6XC;^S3pym?#_4Z0rlCO@ziqSI} zy=`@_XBu3KyK%<*%f<|gW%W3=Xkd$g>{kWZrvc9Gs#MQ#!-rvVFO(gN&9`Ey76So` zrE9k-630fdT9DH`ZsV2$wc;WS5K9q;x*ll-#)cot!o`^A#Yxa@s8JDIsLrKIQJVQ! z4$g69@0>?c6nofct{$uvC`EWw5b*f@_7q2WV^BMVJV0P}R?|eOQX9N*S zoky_%TCU02zQfZM14#Tbrs8p>CEeqkC~Qgllny;&^TQT=TgqVg_zxk5$4C_(eQ}&{ zrLfS-88NY>NwCpiEsCV^!@=peFPB&R@lU7ZgvuF(?YK?&u-*f+p+i0#zP9EcTftP@&>P`DLS>(B`x zq28v@emTXJMdp+KJVU7)dT7wtcY&YG3AqVFZ zKtcUTlG$&b5$N=CX3*Y;*9SMb!!6&mm`$Kv&7SI~5D&$zm%Jf6m^~rOmnAb1O-C2~ zDbs`RX2{{yz-q9X{0LX}#}Hr3@g1KLp{yKUC}BIj_}S=3IN;1`FLY2~#12iG$pVg1 zr8k{5OmOK|b<3;{)ri=16F2-uOrb7Qa1SEetriz~#NTst!38|sxMoRb{&eN?itZ1R?Ti6M9^ z`0G(HlxnziWTX#fZXlyGxFdB&%rwn2r7%%5@=mM!+{dssd!)tWD1E`WLR8r}zjoPa z5;Nc6l1qp*)Ued7R&VC)K@JCyCReqAdNz2ITTEmHnHK+C*;HxZE`Ln^;;`OcL1KGX zR9KK2#|G}4B=!aS#1O%87I63uyQ z{V19u=eSzXEU_07nw*bk?0F62hcn8bjAm4cQsrs*S&}5MJ`uyogHqV!EW0jYN@KZD z_r-xT1?c>UJ=j_}QR^=qamNa`1h+JDvveDiDEIV0r9dtwC! zLqe0Ev3|n}j@~)ig~k(Zm%?N>taTYWu3{eQ7{XH-5biE+6M3T!jyuYDQzmmgWHN?T zIDHif-r0~No6)heJ|d=qW24!js8^ZYE^Zr5C z(6h}HS@%@ns?AhxG)))t7wlHsp%2fz#FLoKqwT{7TgR0*V?)^_UwG6fo``)j1bA;a zyo0{D);14L4qnQK1Xc4-{N*saEOB0ODLs}Ez+}5m4s>O5# zG+hEIR+TV83zPNb2Av?R#(mhay?B)M>2yk-IR!1whysD41w8Gk*T6IMFNHd zKuS}0a0JD5uKmwKw_O-YF7e1>S|8S(z>rCH!jV6c@l<^yQlLl#2kv>+iBPx5s*DSj z2Fqp7fdGXoi^Hg@E`;o|{^Z5;siT9e@o0KDrWlX{x|6%7xLv)+7IilR$BRLytIza? z1KHCxbZW4jDyb2%%~%_VX{BO1$O}5Zrx!<#7y(gF1E3kzP>L6f8f7Z`3i3Prq_rv6dw8 zcu3|zE7(2c`%y+=J+nv(rxx>%q7cOBFT6nm>xuSurRDVDH99Pb8)1M2b&d0+2-J)@ zP$^RIohg0K9*iYAH!MLa7Eql))s02n4L}XWsZ|7y*VGGvl3-nlu=;XgWKvXm^l{%T z0V>mzAlH8i%%q}zhxz{b4iN=LcCqz*rqH7T$NdK(YJ;Nup^StO|9nJYvCEQd3D#$Z z;$u<4VUy3Q9O_I>k+~LhZ8Mm+U0vMJX`i9!dfn9@G&Mlz!V19&$ z>b8xXvRC(LnaP8GNaOOg5{vCTZ3UY6g(WNxIUCUy8zB-`pri-!vn>&ma^ubX!3{k> zwnkP4V8Gp91Hn}@T7(`N+j(| z>zbO=kW+5J`7C*)(}{pHoQACYFjaB<^GxWmLj1YAhLV!JC;PEkoq>{MK*)v zvCxrQDT02)xTMbQ+Hr5+=ZlqU`SbXrMWqu%YHij0I0iW?M7LLhsuyoo>tSE4mJtI- zm3RaHfI^ZkrkBCa*2`dJ0YPqKK}OH2RdB<(KX*KVR-i%dkKbsdol(S=eBcPA0c3U# zEa0bWo+|^4*#n&wArW4E{xI!xX!i}ajQuW2ko;x&qDUyyo%AP$(knHCn{s|v=s24* z9mRZsL=ooDhwYFWdxXa%$+3xOeS#`o2%6-Sp}i_tRw%nNN5OH(#|-#y~{R)g|{V z8o?1Xtenkzj) ze$XeCFD!nske~lZx|&Yc-gI`cmmD{IG%-eY;%vY??h7{8iOQsl;TYaiYUrB|4V5cy z;aEZp?DoPju0QX@t`~CFmydC_n=dV%M=I*NY$DRwQ03w?UxVt;*33H&Go=6o%x$Lm ztdDvMQQcMJp?4?BxB9SL-fi;0y5wgH`R-hUDXNfQYeJ8gS)gAR4wT7n|4~u-u{<`^ zwHRoP;qBUCmy~~fgbw|M*&xv1{`A4!IRO$LJ20%YET{LWMgR0kj5)!uX4KW2?HG%HH84FuDtt6D~iM69*(&Pt#lJMC#@i zo@x3e<;B)f!@hg8tzYFlYcn0+5vE3}hF2gv6;_A%ipr2_%NG*svz%lvER7=MrS4uA zwZ+Szjs!Vs=DX)MB%78G63)}*xP#p5Svplk;y5ll25A{Wy_ZDtD9E20BhB}zE`tx+ zWZUl40h@-h^=x2tE?GFcRnF*nrGk=s%jqVi36;^%4n^MG#VmgAdFQ{Et$Vd$WOx6} z+a8A}q>oJbAsI>W%Um zJr9~O4^(!Z%iwl+(2_%p(E6gbrl!;oqwAMmoHun;K|uSha1|43xS<|>b>S>y*3{8L z>4ruHpQ3Ms zXr4)JN3h|D%(n+g%ulpylVRC(#WaufscTy9lLn=SD_0-t&r%L*=(Mpjs~G93tG_HR zrXZQ*!M`^s)S)%HE#^2Pmdc@MdhzH6`i z7aum4a`4RrX+uZ;bc?qmE{rmlT)cyIW!rS?4>Jj^Wk7-1_S6*0weFStGkb-Uc+NY@ zUO@cz6Sqh?_1_RDVV(CV>;|+tjHvB9bz&DZf*!DL%A~M>4TTG} z%ix|&)Xht>!j|7=95S^5GBtDPVhL`MLl?dt$N~R^Q;>*0Md~kr!(vPwCX)q4DXg4A z>ab=i&83H@oqer^fR66@9ajHqbEMm5S-kDa1Voay2VuJqVenJes>X(LG6AMdEa(`8 zkW2l}$O$V$u}$&g`H30k62Tw^%c+xF)xlTpImC3tLNPeOKvnMBGdG(O?-gtoX^KfU z%N8{C{xA-32luD^6z=&KL#fKyj88QkkTxIKphG+pGs_P?u9Y{O+J8BM#bOZ5IxY#DLVXrENA5A zTVAzzDnu$HYDWdQo!wTI_E>V`1SSC!nX1Yfjk3#m32wYWhuf^b2887Fu z*PErCFUonf|2S-8^?-G`jlg;6U#Ix!gZ((5@573uQKp75=X6^JV@*7f~`E|-)<{6=SLYD1(a@Q1z}WUqZVQzl`tiEPVZ;9`j@;>F>0DF7#RY~3BpM^Wp3sOMtIY0Cm1s5 z*M=5Sb8VC~1KG60i%KQ_QVcC1oQ~ z!N%sNLZ!16{z%=Kw}?sDZgS8s@1eN+Y`fxIu0`9o%0iy)LTMw?S;S7d&eq6Y z7EuQ;XHZ=W+j^}zKGm>!=yAMQ@V;_VQpta8p*5SI8zjTvbTYuhsCuL~GI-w)k6|=# zLRryoI%rz0ix3FCjK?iNQUMdo(a@7WUGho6t94TeMyIe3XLXTA%yNEXSTsI`^w!L$e*td051&0#<7tRMFFo3%mBy# z3LVih(6KPnG0+*AJ37&t+nCtW1G1Vrxa->4n*Yjf>g47WAu};aJuxXBMK41)LPJ3r zy+<`ECOuvY7Vk04nX1wfmg=shHBFh?Pg+Qt%GvKR{M6DC=1%#!P@JIK%?M76NJY$w zj_x;!0jy5!otO_mK#G8TtAH$s|6Zv6e|?b>7ZjFJ6#nIgwt^;LVJj*?FOy?LkviAv zWQRrCv5Rs6l3<5rG#`Y_jfkUXBJb@5m;0zyTA!yzqpE=$+c5r>DT zA?EV-Y8J)Fg2^LjE!xMAejRbbw%y^aV66F6R+nEIx54GQo>S|4CN5!Pmh1qhAvps) zIPXa>HI6B(`#R~=q(n%l8g5@v3~6cP4Z2c&|D7(`)SLF;M(*GUZ5Ppt%nV0p(YL?QSWJf*{si zVs<6C^x9LG3wYzhYahkbVlZ2BUVlvSPo;JZyA&Z5M>kC*_nNK6el}gB#inVMi_|9v z_4cY^=HQJ6?#?+sywlc3ZYLV$1U6sGhWI$ih)X1??9=Yv54K zqg7jffF-{KRkYB6?6)F+&MgwScG8pszEhb~?v3RWT#8G!^rx}vm1q%CsQ~uuhGF5P z4L^*sk#C25Ca64a-Gr|sKXZqniA4^RY`z%CyLm)sv=8!-@_LgrOUffadsO)WXrusLg^VN~I*azJQdXb7kpP(lU%3BvAR9Sl`cV$)6g_*@IH9!r23>SG+DIK zjw)*HM)~6agLnd*{!2zXj)M=uirP!8U>nRKNG+U?7{n2f5Act~csqBfYPIWQO~1jZ z#_p9B(W!J_emN3f1Ejc^2X@X#s_m&BD(8v`*xl%NuH>7tS@7HAh&eUkpK*S6e=0 z^D`OcqoEzAophr})Khvalj;hLQJ>{5P>Z%f9ex!6|3cczVc%+Q=_Q3^g}L|gCd_T8 zCZj}?U!Yh_z)L}6env3`o-}(x`JM0GzC|oJ1++vArwKiQEkC)4J#N1$V1|p1sFA`A;=#3&j$&mmUt8oPX__aiwBjOiWPT{g2=r^Dn)k+ zaF3m~ydNdFOlxX~%P!E?-^$E0;fhhH)Y5tkbh}8?3?Q{a-TEpx&%F%a6q1iX# zrm{0C;s-M9^{8qL*9(mNHiVs+eD@U!!j9i=0_aWvRW*R1VLL#^2lwx90t&){vI;_# z3oS|IV#q*potb_$gZDS-k%-z43u27?TEi3>4(q4uA>V@*T7^^*cYEJr6= z{fpjnTWCO|BtehsByp&uDj{iS<8NS(6ROK6bp^ATM|mvM+$z%zrbJzr#Z z_1yr|{mM!$a$vq}rS6KN!U;_>7FZ`$NmuW>BH3s;>Ed=r5JOOZDnccEzKdD_RVS%C z7+cOtbAtYWXi|S$Hy*yDaLs75-ky}{*&cYwKI0K~8q zv0Uu7iVsLN_HB4WNu3q)`Lf0Yq2)h5`@cT>h)d(JPFs8boW8`A_s^$G3!lDQNR0|? z&Ofiw(T!z>MUe!ZMt#AQRoZ)B(D(af62;jz@$rIFTHCzUgXd_jM$Eh?@iJHH$a(lQ z*MAbxUtW{eA|2vz@J!xZ{($nuGh?ho{3=Hxrep=Q*GbCx_+E`vwaSZO`8?n=jXxlP z7EHbaF;oUlDWh*{*ZFiV_9Qep^VE{9xD%q}hBm>;W9lRI{Uw6HqM{_D3 zt|d=oKMGVgeeX7A;Wh_f-|>z_d{}RjqbF?cvKWO%icw-l_P@&)I+EmlON`DcjKgma zYpNGXlBfu>i+rX$<+56!vMn<1O}|CV9`ftB#_PeO%Q7{wXdh^8S{>h?$7RZ^lt|-{ z1p}_FhI$Lj4mqZN7lIftuN@P>(?;p5@!#9^mNzkQflc)G|xHaSiNCpfxZ=9Esb2dZpIGS1npQLVJ z(=OD`kLId1-CR`*M_HZ1HOADyvQl4WcL~6IeQP}Jvaj*xaxlMP_nD6dN-n0O?29M# zvq&&u7b)6BdvFK)SK};Ll1VBHsv{2kcznQy#SOhCdFOr0w2TQ|$G3_fVA?Xt=*mEV zl*8Nptm~Sjv34Wr1BeOI^K(efe!AUDrltq3^%#e1+)d8E&|1BE)+;JkKeW!MEdhshWRr|Ke^R%#H0Z7EUSnn6Fyi7bCnKt&xf zoaolIp+9k{2wKLY_^b#GsqBu{=8zB zlLY>N3IbR(1_TKdmu{(c5w@kx1^C7dIFJCxAEm(m*C$sqV`D3NU0rhB#oSuo+~&WUD`)XLc>zq10gr#b=E{GR0{?WtfB)y#9gUrw z?VN0FtsHf&Y)wtgZA|}*BVReJq)jVB8q=EprMudKMe*F#)1U|SRkNH zlHdD4RQRt5Lt7ibx(lcOQVANT#6eF$7yutY`Oo1L3h38Aov_n4wEQpTpXH!5O96FA z0W`pWnlq{V)%<^Uy#GF!J;TzAkpM0K6EI>>{fQzw>k*$NV(LXh$6q{hquRa8H1rX*h!2gs2|B(O7`8V)aSmgKm{IdGP z9YB}-Zz=Gf{J)rARnq@;as88H2C!gN>wy000rby*5>a&jPX+%%G`2GNn_&lhROm_r z80t=>|JJ_v!^r}FCZm5MTI<`mTbbK9yZx78@QGs}qY>bx17O1YOCb7>QsAFw|Bp;( zN2|YM292{UcmggjOMn^>@gG3I8E~g^`6s5GzJuex^rwICoZqF!|6msQtNQ&D@IQpd ze~11qjr}L|)&HNMe-+C9o%y>0=$}mUuzz6wsR#Ny@ON#cKY@A){{Z|=Y3X!J zc>62xzr}?A1b#mHAK?Fu%Kjc9{wws?{NnfE?w`=HzlZ)M)caT7znuIny7nh8{q%30 z{8y~)chc{%u|G-VXMaojBU1J|?DuEvKVdXie+&Ef6Stfs7+@Vgpl*P_IKUl7^x@aH F{|5~6n|uHO literal 0 HcmV?d00001 From 1d95451be1f3080904c00cc4c4a6cc519efdf321 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Mar 2024 17:13:01 +0100 Subject: [PATCH 147/158] gh-63207: Use GetSystemTimePreciseAsFileTime() in time.time() (#116822) --- Doc/whatsnew/3.13.rst | 6 +++ ...4-03-14-17-21-25.gh-issue-63207.LV16SL.rst | 4 ++ Python/pytime.c | 38 +++++++++++-------- 3 files changed, 32 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-14-17-21-25.gh-issue-63207.LV16SL.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index a48db949d8f401..b665e6f1c85915 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -570,6 +570,12 @@ time instead of the ``GetTickCount64()`` clock which has a resolution of 15.6 ms. (Contributed by Victor Stinner in :gh:`88494`.) +* On Windows, :func:`time.time()` now uses the + ``GetSystemTimePreciseAsFileTime()`` clock to have a resolution better + than 1 us, instead of the ``GetSystemTimeAsFileTime()`` clock which has a + resolution of 15.6 ms. + (Contributed by Victor Stinner in :gh:`63207`.) + tkinter ------- diff --git a/Misc/NEWS.d/next/Library/2024-03-14-17-21-25.gh-issue-63207.LV16SL.rst b/Misc/NEWS.d/next/Library/2024-03-14-17-21-25.gh-issue-63207.LV16SL.rst new file mode 100644 index 00000000000000..1f77555d5e7d31 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-14-17-21-25.gh-issue-63207.LV16SL.rst @@ -0,0 +1,4 @@ +On Windows, :func:`time.time()` now uses the +``GetSystemTimePreciseAsFileTime()`` clock to have a resolution better than 1 +us, instead of the ``GetSystemTimeAsFileTime()`` clock which has a resolution +of 15.6 ms. Patch by Victor Stinner. diff --git a/Python/pytime.c b/Python/pytime.c index 45be6a3dbd3341..d5b38047b6db31 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -55,6 +55,14 @@ #endif +#ifdef MS_WINDOWS +static _PyTimeFraction py_qpc_base = {0, 0}; + +// Forward declaration +static int py_win_perf_counter_frequency(_PyTimeFraction *base, int raise_exc); +#endif + + static PyTime_t _PyTime_GCD(PyTime_t x, PyTime_t y) { @@ -895,7 +903,7 @@ py_get_system_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) FILETIME system_time; ULARGE_INTEGER large; - GetSystemTimeAsFileTime(&system_time); + GetSystemTimePreciseAsFileTime(&system_time); large.u.LowPart = system_time.dwLowDateTime; large.u.HighPart = system_time.dwHighDateTime; /* 11,644,473,600,000,000,000: number of nanoseconds between @@ -904,18 +912,17 @@ py_get_system_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) PyTime_t ns = large.QuadPart * 100 - 11644473600000000000; *tp = ns; if (info) { - DWORD timeAdjustment, timeIncrement; - BOOL isTimeAdjustmentDisabled, ok; + // GetSystemTimePreciseAsFileTime() is implemented using + // QueryPerformanceCounter() internally. + if (py_qpc_base.denom == 0) { + if (py_win_perf_counter_frequency(&py_qpc_base, raise_exc) < 0) { + return -1; + } + } - info->implementation = "GetSystemTimeAsFileTime()"; + info->implementation = "GetSystemTimePreciseAsFileTime()"; info->monotonic = 0; - ok = GetSystemTimeAdjustment(&timeAdjustment, &timeIncrement, - &isTimeAdjustmentDisabled); - if (!ok) { - PyErr_SetFromWindowsErr(0); - return -1; - } - info->resolution = timeIncrement * 1e-7; + info->resolution = _PyTimeFraction_Resolution(&py_qpc_base); info->adjustable = 1; } @@ -1063,16 +1070,15 @@ py_get_win_perf_counter(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { assert(info == NULL || raise_exc); - static _PyTimeFraction base = {0, 0}; - if (base.denom == 0) { - if (py_win_perf_counter_frequency(&base, raise_exc) < 0) { + if (py_qpc_base.denom == 0) { + if (py_win_perf_counter_frequency(&py_qpc_base, raise_exc) < 0) { return -1; } } if (info) { info->implementation = "QueryPerformanceCounter()"; - info->resolution = _PyTimeFraction_Resolution(&base); + info->resolution = _PyTimeFraction_Resolution(&py_qpc_base); info->monotonic = 1; info->adjustable = 0; } @@ -1088,7 +1094,7 @@ py_get_win_perf_counter(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) "LONGLONG is larger than PyTime_t"); ticks = (PyTime_t)ticksll; - *tp = _PyTimeFraction_Mul(ticks, &base); + *tp = _PyTimeFraction_Mul(ticks, &py_qpc_base); return 0; } #endif // MS_WINDOWS From f139d840fb543f8357ce9fc8f845c4e945a0ce85 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Mar 2024 17:14:42 +0100 Subject: [PATCH 148/158] gh-116869: Fix redefinition of the _PyOptimizerObject type (#116963) Defining a type twice is a C11 feature and so makes the C API incompatible with C99. Fix the issue by only defining the type once. Example of warning (treated as an error): In file included from Include/Python.h:122: Include/cpython/optimizer.h:77:3: error: redefinition of typedef '_PyOptimizerObject' is a C11 feature [-Werror,-Wtypedef-redefinition] } _PyOptimizerObject; ^ build/Include/cpython/optimizer.h:60:35: note: previous definition is here typedef struct _PyOptimizerObject _PyOptimizerObject; ^ --- Include/cpython/optimizer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 6d7b8bc3c1433a..df83e6d16a429d 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -65,7 +65,7 @@ typedef int (*optimize_func)( _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, int curr_stackentries); -typedef struct _PyOptimizerObject { +struct _PyOptimizerObject { PyObject_HEAD optimize_func optimize; /* These thresholds are treated as signed so do not exceed INT16_MAX @@ -74,7 +74,7 @@ typedef struct _PyOptimizerObject { uint16_t side_threshold; uint16_t backedge_threshold; /* Data needed by the optimizer goes here, but is opaque to the VM */ -} _PyOptimizerObject; +}; /** Test support **/ typedef struct { From 7707b14489644073ab0153f5751c6ddbf3fc6f91 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 18 Mar 2024 17:15:29 +0100 Subject: [PATCH 149/158] gh-115258: Fix hanging tests for threading queue shutdown (#115940) This reinstates `test_shutdown_immediate_all_methods_in_many_threads` and improves `test_shutdown_all_methods_in_many_threads`. --- Lib/test/test_queue.py | 132 ++++++++++++++++++++++------------------- 1 file changed, 71 insertions(+), 61 deletions(-) diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index ad31ba1af03b6f..c4d10110132393 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -317,97 +317,107 @@ def test_shutdown_all_methods_in_one_thread(self): def test_shutdown_immediate_all_methods_in_one_thread(self): return self._shutdown_all_methods_in_one_thread(True) - def _write_msg_thread(self, q, n, results, delay, - i_when_exec_shutdown, - event_start, event_end): - event_start.wait() - for i in range(1, n+1): + def _write_msg_thread(self, q, n, results, + i_when_exec_shutdown, event_shutdown, + barrier_start): + # All `write_msg_threads` + # put several items into the queue. + for i in range(0, i_when_exec_shutdown//2): + q.put((i, 'LOYD')) + # Wait for the barrier to be complete. + barrier_start.wait() + + for i in range(i_when_exec_shutdown//2, n): try: q.put((i, "YDLO")) - results.append(True) except self.queue.ShutDown: results.append(False) - # triggers shutdown of queue - if i == i_when_exec_shutdown: - event_end.set() - time.sleep(delay) - # end of all puts - q.join() + break - def _read_msg_thread(self, q, nb, results, delay, event_start): - event_start.wait() - block = True - while nb: - time.sleep(delay) + # Trigger queue shutdown. + if i == i_when_exec_shutdown: + # Only one thread should call shutdown(). + if not event_shutdown.is_set(): + event_shutdown.set() + results.append(True) + + def _read_msg_thread(self, q, results, barrier_start): + # Get at least one item. + q.get(True) + q.task_done() + # Wait for the barrier to be complete. + barrier_start.wait() + while True: try: - # Get at least one message - q.get(block) - block = False + q.get(False) q.task_done() - results.append(True) - nb -= 1 except self.queue.ShutDown: - results.append(False) - nb -= 1 + results.append(True) + break except self.queue.Empty: pass - q.join() - def _shutdown_thread(self, q, event_end, immediate): + def _shutdown_thread(self, q, results, event_end, immediate): event_end.wait() q.shutdown(immediate) - q.join() + results.append(q.qsize() == 0) - def _join_thread(self, q, delay, event_start): - event_start.wait() - time.sleep(delay) + def _join_thread(self, q, barrier_start): + # Wait for the barrier to be complete. + barrier_start.wait() q.join() def _shutdown_all_methods_in_many_threads(self, immediate): + # Run a 'multi-producers/consumers queue' use case, + # with enough items into the queue. + # When shutdown, all running threads will be joined. q = self.type2test() ps = [] - ev_start = threading.Event() - ev_exec_shutdown = threading.Event() res_puts = [] res_gets = [] - delay = 1e-4 - read_process = 4 - nb_msgs = read_process * 16 - nb_msgs_r = nb_msgs // read_process - when_exec_shutdown = nb_msgs // 2 - lprocs = ( - (self._write_msg_thread, 1, (q, nb_msgs, res_puts, delay, - when_exec_shutdown, - ev_start, ev_exec_shutdown)), - (self._read_msg_thread, read_process, (q, nb_msgs_r, - res_gets, delay*2, - ev_start)), - (self._join_thread, 2, (q, delay*2, ev_start)), - (self._shutdown_thread, 1, (q, ev_exec_shutdown, immediate)), - ) - # start all threds + res_shutdown = [] + write_threads = 4 + read_threads = 6 + join_threads = 2 + nb_msgs = 1024*64 + nb_msgs_w = nb_msgs // write_threads + when_exec_shutdown = nb_msgs_w // 2 + # Use of a Barrier to ensure that + # - all write threads put all their items into the queue, + # - all read thread get at least one item from the queue, + # and keep on running until shutdown. + # The join thread is started only when shutdown is immediate. + nparties = write_threads + read_threads + if immediate: + nparties += join_threads + barrier_start = threading.Barrier(nparties) + ev_exec_shutdown = threading.Event() + lprocs = [ + (self._write_msg_thread, write_threads, (q, nb_msgs_w, res_puts, + when_exec_shutdown, ev_exec_shutdown, + barrier_start)), + (self._read_msg_thread, read_threads, (q, res_gets, barrier_start)), + (self._shutdown_thread, 1, (q, res_shutdown, ev_exec_shutdown, immediate)), + ] + if immediate: + lprocs.append((self._join_thread, join_threads, (q, barrier_start))) + # start all threads. for func, n, args in lprocs: for i in range(n): ps.append(threading.Thread(target=func, args=args)) ps[-1].start() - # set event in order to run q.shutdown() - ev_start.set() - - if not immediate: - assert(len(res_gets) == len(res_puts)) - assert(res_gets.count(True) == res_puts.count(True)) - else: - assert(len(res_gets) <= len(res_puts)) - assert(res_gets.count(True) <= res_puts.count(True)) - - for thread in ps[1:]: + for thread in ps: thread.join() - @unittest.skip("test times out (gh-115258)") + self.assertTrue(True in res_puts) + self.assertEqual(res_gets.count(True), read_threads) + if immediate: + self.assertListEqual(res_shutdown, [True]) + self.assertTrue(q.empty()) + def test_shutdown_all_methods_in_many_threads(self): return self._shutdown_all_methods_in_many_threads(False) - @unittest.skip("test times out (gh-115258)") def test_shutdown_immediate_all_methods_in_many_threads(self): return self._shutdown_all_methods_in_many_threads(True) From 849e0716d378d6f9f724d1b3c386f6613d52a49d Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Mon, 18 Mar 2024 12:07:25 -0500 Subject: [PATCH 150/158] gh-115119: Switch Windows build to mpdecimal external (GH-115182) This includes adding what should be a relatively temporary `Modules/_decimal/windows/mpdecimal.h` shim to choose between `mpdecimal32vc.h` or `mpdecimal64vc.h` based on which of `CONFIG_64` or `CONFIG_32` is defined. --- ...-02-08-14-48-15.gh-issue-115119.qMt32O.rst | 3 + Misc/externals.spdx.json | 22 ++++++ Modules/_decimal/windows/mpdecimal.h | 17 +++++ PCbuild/_decimal.vcxproj | 68 ++++++++++--------- PCbuild/_decimal.vcxproj.filters | 68 ++++++++++--------- PCbuild/get_externals.bat | 1 + PCbuild/python.props | 1 + 7 files changed, 117 insertions(+), 63 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-02-08-14-48-15.gh-issue-115119.qMt32O.rst create mode 100644 Modules/_decimal/windows/mpdecimal.h diff --git a/Misc/NEWS.d/next/Windows/2024-02-08-14-48-15.gh-issue-115119.qMt32O.rst b/Misc/NEWS.d/next/Windows/2024-02-08-14-48-15.gh-issue-115119.qMt32O.rst new file mode 100644 index 00000000000000..f95fed1084cf4f --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-02-08-14-48-15.gh-issue-115119.qMt32O.rst @@ -0,0 +1,3 @@ +Switched from vendored ``libmpdecimal`` code to a separately-hosted external +package in the ``cpython-source-deps`` repository when building the +``_decimal`` module. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 2acfccbb004d6b..6df6401835c6f1 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -43,6 +43,28 @@ "primaryPackagePurpose": "SOURCE", "versionInfo": "3.4.4" }, + { + "SPDXID": "SPDXRef-PACKAGE-mpdecimal", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "93118043651ffa33dcaaab445bae4f8929fca25d2d749079b78e97f220c3d8b1" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/mpdecimal-2.5.1.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:bytereef:mpdecimal:2.5.1:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "mpdecimal", + "originator": "Organization: bytereef.org", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2.5.1" + }, { "SPDXID": "SPDXRef-PACKAGE-openssl", "checksums": [ diff --git a/Modules/_decimal/windows/mpdecimal.h b/Modules/_decimal/windows/mpdecimal.h new file mode 100644 index 00000000000000..77bc6229fbc119 --- /dev/null +++ b/Modules/_decimal/windows/mpdecimal.h @@ -0,0 +1,17 @@ +/* Windows mpdecimal.h shim + * + * Generally, the mpdecimal library build will copy the correct header into + * place named "mpdecimal.h", but since we're building it ourselves directly + * into _decimal.pyd, we need to pick the right one. + * + * */ + +#if defined(_MSC_VER) + #if defined(CONFIG_64) + #include + #elif defined(CONFIG_32) + #include + #else + #error "Unknown configuration!" + #endif +#endif diff --git a/PCbuild/_decimal.vcxproj b/PCbuild/_decimal.vcxproj index 490d7df87eb1c6..ee7421484b5312 100644 --- a/PCbuild/_decimal.vcxproj +++ b/PCbuild/_decimal.vcxproj @@ -93,51 +93,55 @@ - _CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + BUILD_LIBMPDEC;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) CONFIG_32;PPRO;MASM;%(PreprocessorDefinitions) CONFIG_32;ANSI;%(PreprocessorDefinitions) CONFIG_64;ANSI;%(PreprocessorDefinitions) CONFIG_64;MASM;%(PreprocessorDefinitions) - ..\Modules\_decimal;..\Modules\_decimal\libmpdec;%(AdditionalIncludeDirectories) + ..\Modules\_decimal;..\Modules\_decimal\windows;$(mpdecimalDir)\libmpdec;%(AdditionalIncludeDirectories) - - - - - - + + + + + + - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + true true true diff --git a/PCbuild/_decimal.vcxproj.filters b/PCbuild/_decimal.vcxproj.filters index 0cbd3d0736c241..e4bdb64ec1fb9f 100644 --- a/PCbuild/_decimal.vcxproj.filters +++ b/PCbuild/_decimal.vcxproj.filters @@ -21,49 +21,55 @@ Header Files - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + + Header Files\libmpdec + + + Header Files\libmpdec + + Header Files\libmpdec @@ -71,46 +77,46 @@ Source Files - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec @@ -120,8 +126,8 @@ - + Source Files\libmpdec - \ No newline at end of file + diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 60ce12b725e233..f5b7e114c98ded 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,6 +54,7 @@ set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.13 +set libraries=%libraries% mpdecimal-2.5.1 set libraries=%libraries% sqlite-3.45.1.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.1 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.1 diff --git a/PCbuild/python.props b/PCbuild/python.props index e21f1f60464bc8..a8d08073fbd11e 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -74,6 +74,7 @@ $(ExternalsDir)libffi-3.4.4\ $(libffiDir)$(ArchName)\ $(libffiOutDir)include + $(ExternalsDir)\mpdecimal-2.5.1\ $(ExternalsDir)openssl-3.0.13\ $(ExternalsDir)openssl-bin-3.0.13\$(ArchName)\ $(opensslOutDir)include From 76d086890790f1bfbe05d12e02cadb539db5b0b1 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 18 Mar 2024 11:08:43 -0700 Subject: [PATCH 151/158] Cleanup tier2 debug output (#116920) Various tweaks, including a slight refactor of the special cases for `_PUSH_FRAME`/`_POP_FRAME` to show the actual operand emitted. --- Python/ceval.c | 9 ++++++-- Python/optimizer.c | 43 ++++++++++++++++++++++-------------- Python/optimizer_analysis.c | 18 +++++++++++---- Python/optimizer_bytecodes.c | 1 + Python/optimizer_cases.c.h | 1 + 5 files changed, 50 insertions(+), 22 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index f817f288903694..9dbcd3da3fec27 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1017,7 +1017,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int uopcode = next_uop->opcode; #ifdef Py_DEBUG if (lltrace >= 3) { - printf("%4d uop: ", (int)(next_uop - (current_executor == NULL ? next_uop : current_executor->trace))); + if (next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT) { + printf("%4d uop: ", 0); + } + else { + printf("%4d uop: ", (int)(next_uop - current_executor->trace)); + } _PyUOpPrint(next_uop); printf(" stack_level=%d\n", (int)(stack_pointer - _PyFrame_Stackbase(frame))); @@ -1113,7 +1118,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", exit_index, exit->temperature, exit->target, - _PyOpcode_OpName[frame->instr_ptr->op.code]); + _PyOpcode_OpName[_PyCode_CODE(_PyFrame_GetCode(frame))[exit->target].op.code]); } #endif Py_INCREF(exit->executor); diff --git a/Python/optimizer.c b/Python/optimizer.c index 88c45f2e73c682..a6d6ffe5378636 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -313,7 +313,7 @@ _PyUOpPrint(const _PyUOpInstruction *uop) else { printf("%s", name); } - printf(" (%d, target=%d, operand=%" PRIx64 ")", + printf(" (%d, target=%d, operand=%#" PRIx64 ")", uop->oparg, uop->target, (uint64_t)uop->operand); @@ -528,7 +528,7 @@ translate_bytecode_to_trace( } #endif - DPRINTF(4, + DPRINTF(2, "Optimizing %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), @@ -546,7 +546,7 @@ translate_bytecode_to_trace( uint32_t oparg = instr->op.arg; uint32_t extended = 0; - DPRINTF(3, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); + DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); if (opcode == ENTER_EXECUTOR) { assert(oparg < 256); @@ -606,7 +606,7 @@ translate_bytecode_to_trace( confidence = confidence * (18 - bitcount) / 20; } uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; - DPRINTF(2, "%d: %s(%d): counter=%x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", + DPRINTF(2, "%d: %s(%d): counter=%04x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", target, _PyOpcode_OpName[opcode], oparg, counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode)); if (confidence < CONFIDENCE_CUTOFF) { @@ -617,7 +617,7 @@ translate_bytecode_to_trace( _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; _Py_CODEUNIT *target_instr = next_instr + oparg; if (jump_likely) { - DPRINTF(2, "Jump likely (%x = %d bits), continue at byte offset %d\n", + DPRINTF(2, "Jump likely (%04x = %d bits), continue at byte offset %d\n", instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code)); instr = target_instr; ADD_TO_TRACE(uopcode, max_length, 0, INSTR_IP(next_instr, code)); @@ -716,12 +716,12 @@ translate_bytecode_to_trace( expansion->uops[i].offset); Py_FatalError("garbled expansion"); } - ADD_TO_TRACE(uop, oparg, operand, target); + if (uop == _POP_FRAME) { TRACE_STACK_POP(); /* Set the operand to the function object returned to, * to assist optimization passes */ - trace[trace_length-1].operand = (uintptr_t)func; + ADD_TO_TRACE(uop, oparg, (uintptr_t)func, target); DPRINTF(2, "Returning to %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), @@ -730,6 +730,7 @@ translate_bytecode_to_trace( 2 * INSTR_IP(instr, code)); goto top; } + if (uop == _PUSH_FRAME) { assert(i + 1 == nuops); int func_version_offset = @@ -738,7 +739,7 @@ translate_bytecode_to_trace( + 1; uint32_t func_version = read_u32(&instr[func_version_offset].cache); PyFunctionObject *new_func = _PyFunction_LookupByVersion(func_version); - DPRINTF(3, "Function object: %p\n", func); + DPRINTF(2, "Function: version=%#x; object=%p\n", (int)func_version, new_func); if (new_func != NULL) { PyCodeObject *new_code = (PyCodeObject *)PyFunction_GET_CODE(new_func); if (new_code == code) { @@ -748,6 +749,7 @@ translate_bytecode_to_trace( PyUnicode_AsUTF8(new_code->co_filename), new_code->co_firstlineno); OPT_STAT_INC(recursive_call); + ADD_TO_TRACE(uop, oparg, 0, target); ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); goto done; } @@ -756,6 +758,7 @@ translate_bytecode_to_trace( // Perhaps it may happen again, so don't bother tracing. // TODO: Reason about this -- is it better to bail or not? DPRINTF(2, "Bailing because co_version != func_version\n"); + ADD_TO_TRACE(uop, oparg, 0, target); ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); goto done; } @@ -763,9 +766,9 @@ translate_bytecode_to_trace( instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; TRACE_STACK_PUSH(); _Py_BloomFilter_Add(dependencies, new_code); - /* Set the operand to the callee's code object, - * to assist optimization passes */ - trace[trace_length-1].operand = (uintptr_t)new_func; + /* Set the operand to the callee's function object, + * to assist optimization passes */ + ADD_TO_TRACE(uop, oparg, (uintptr_t)new_func, target); code = new_code; func = new_func; instr = _PyCode_CODE(code); @@ -777,9 +780,14 @@ translate_bytecode_to_trace( 2 * INSTR_IP(instr, code)); goto top; } + DPRINTF(2, "Bail, new_func == NULL\n"); + ADD_TO_TRACE(uop, oparg, operand, target); ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); goto done; } + + // All other instructions + ADD_TO_TRACE(uop, oparg, operand, target); } break; } @@ -803,17 +811,18 @@ translate_bytecode_to_trace( // Skip short traces like _SET_IP, LOAD_FAST, _SET_IP, _EXIT_TRACE if (progress_needed || trace_length < 5) { OPT_STAT_INC(trace_too_short); - DPRINTF(4, - "No trace for %s (%s:%d) at byte offset %d\n", + DPRINTF(2, + "No trace for %s (%s:%d) at byte offset %d (%s)\n", PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code)); + 2 * INSTR_IP(initial_instr, code), + progress_needed ? "no progress" : "too short"); return 0; } ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); DPRINTF(1, - "Created a trace for %s (%s:%d) at byte offset %d -- length %d\n", + "Created a proto-trace for %s (%s:%d) at byte offset %d -- length %d\n", PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, @@ -938,6 +947,8 @@ make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *depende assert(next_exit == -1); assert(dest == executor->trace); dest->opcode = _START_EXECUTOR; + dest->oparg = 0; + dest->target = 0; dest->operand = (uintptr_t)executor; _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG @@ -947,7 +958,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *depende lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } if (lltrace >= 2) { - printf("Optimized executor (length %d):\n", length); + printf("Optimized trace (length %d):\n", length); for (int i = 0; i < length; i++) { printf("%4d OPTIMIZED: ", i); _PyUOpPrint(&executor->trace[i]); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 9fd4b1967ecc3b..0c95616848a85b 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -35,6 +35,7 @@ #ifdef Py_DEBUG extern const char *_PyUOpName(int index); + extern void _PyUOpPrint(const _PyUOpInstruction *uop); static const char *const DEBUG_ENV = "PYTHON_OPT_DEBUG"; static inline int get_lltrace(void) { char *uop_debug = Py_GETENV(DEBUG_ENV); @@ -377,14 +378,20 @@ optimize_uops( _Py_UopsSymbol **stack_pointer = ctx->frame->stack_pointer; - DPRINTF(3, "Abstract interpreting %s:%d ", - _PyUOpName(opcode), - oparg); +#ifdef Py_DEBUG + if (get_lltrace() >= 3) { + printf("%4d abs: ", (int)(this_instr - trace)); + _PyUOpPrint(this_instr); + printf(" "); + } +#endif + switch (opcode) { + #include "optimizer_cases.c.h" default: - DPRINTF(1, "Unknown opcode in abstract interpreter\n"); + DPRINTF(1, "\nUnknown opcode in abstract interpreter\n"); Py_UNREACHABLE(); } assert(ctx->frame != NULL); @@ -397,11 +404,13 @@ optimize_uops( return 1; out_of_space: + DPRINTF(3, "\n"); DPRINTF(1, "Out of space in abstract interpreter\n"); _Py_uop_abstractcontext_fini(ctx); return 0; error: + DPRINTF(3, "\n"); DPRINTF(1, "Encountered error in abstract interpreter\n"); _Py_uop_abstractcontext_fini(ctx); return 0; @@ -411,6 +420,7 @@ optimize_uops( // This means that the abstract interpreter has hit unreachable code. // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but it's // simpler to just admit failure and not create the executor. + DPRINTF(3, "\n"); DPRINTF(1, "Hit bottom in abstract interpreter\n"); _Py_uop_abstractcontext_fini(ctx); return 0; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 54abbcd74d7934..ef08c0d8897c9f 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -544,6 +544,7 @@ dummy_func(void) { (void)callable; PyFunctionObject *func = (PyFunctionObject *)(this_instr + 2)->operand; + DPRINTF(3, "func: %p ", func); if (func == NULL) { goto error; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index cf36f1ba792775..610d1b1aede9cc 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1597,6 +1597,7 @@ int argcount = oparg; (void)callable; PyFunctionObject *func = (PyFunctionObject *)(this_instr + 2)->operand; + DPRINTF(3, "func: %p ", func); if (func == NULL) { goto error; } From 7e1f38f2de8f93de362433203faa5605a0c47f0e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 18 Mar 2024 11:11:10 -0700 Subject: [PATCH 152/158] gh-116916: Remove separate next_func_version counter (#116918) Somehow we ended up with two separate counter variables tracking "the next function version". Most likely this was a historical accident where an old branch was updated incorrectly. This PR merges the two counters into a single one: `interp->func_state.next_version`. --- Include/internal/pycore_interp.h | 1 - Objects/codeobject.c | 6 +++--- Objects/funcobject.c | 8 ++++---- Python/pystate.c | 1 - 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index d79fd3b6039ef5..942f47340b3966 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -245,7 +245,6 @@ struct _is { uint16_t optimizer_side_threshold; - uint32_t next_func_version; _rare_events rare_events; PyDict_WatchCallback builtins_dict_watcher; diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 30336fa86111a7..3df733eb4ee578 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -415,9 +415,9 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; PyInterpreterState *interp = _PyInterpreterState_GET(); - co->co_version = interp->next_func_version; - if (interp->next_func_version != 0) { - interp->next_func_version++; + co->co_version = interp->func_state.next_version; + if (interp->func_state.next_version != 0) { + interp->func_state.next_version++; } co->_co_monitoring = NULL; co->_co_instrumentation_version = 0; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 08b2823d8cf024..a506166916de48 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -236,8 +236,9 @@ How does a function's `func_version` field get initialized? - A new version is allocated by `_PyFunction_GetVersionForCurrentState` when the specializer needs a version and the version is 0. -The latter allocates versions using a counter in the interpreter state; -when the counter wraps around to 0, no more versions are allocated. +The latter allocates versions using a counter in the interpreter state, +`interp->func_state.next_version`. +When the counter wraps around to 0, no more versions are allocated. There is one other special case: functions with a non-standard `vectorcall` field are not given a version. @@ -247,8 +248,7 @@ Code object versions -------------------- So where to code objects get their `co_version`? -There is a per-interpreter counter, `next_func_version`. -This is initialized to 1 when the interpreter is created. +They share the same counter, `interp->func_state.next_version`. Code objects get a new `co_version` allocated from this counter upon creation. Since code objects are nominally immutable, `co_version` can diff --git a/Python/pystate.c b/Python/pystate.c index 9f142223aff340..eedcb920cd1cf2 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -630,7 +630,6 @@ init_interpreter(PyInterpreterState *interp, interp->sys_profile_initialized = false; interp->sys_trace_initialized = false; (void)_Py_SetOptimizer(interp, NULL); - interp->next_func_version = 1; interp->executor_list_head = NULL; if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ From 9c7b3688e6ff071456e0ee623b82870755ea7808 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 18 Mar 2024 11:13:11 -0700 Subject: [PATCH 153/158] gh-108716: Cleanup remaining deepfreeze infrastructure (#116919) Keep Tools/build/deepfreeze.py around (we may repurpose it for deepfreezing non-code objects), and keep basic "clean" targets that remove the output of former deep-freeze activities, to keep the build directories of current devs clean. --- .gitignore | 1 - Makefile.pre.in | 43 ++---------------- PCbuild/_freeze_module.vcxproj | 19 -------- PCbuild/pythoncore.vcxproj | 5 --- Python/deepfreeze/README.txt | 6 --- Python/frozen.c | 28 ------------ Tools/build/freeze_modules.py | 79 +--------------------------------- configure | 3 +- configure.ac | 3 +- 9 files changed, 7 insertions(+), 180 deletions(-) delete mode 100644 Python/deepfreeze/README.txt diff --git a/.gitignore b/.gitignore index 3e1213ef925305..2d380a441d2394 100644 --- a/.gitignore +++ b/.gitignore @@ -142,7 +142,6 @@ Tools/unicode/data/ /profile-clean-stamp /profile-run-stamp /profile-bolt-stamp -/Python/deepfreeze/*.c /pybuilddir.txt /pyconfig.h /python-config diff --git a/Makefile.pre.in b/Makefile.pre.in index 3cf4de08a0c842..5958b6bdb05a57 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1400,7 +1400,7 @@ Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) ############################################################################ -# "Bootstrap Python" used to run deepfreeze.py +# "Bootstrap Python" used to run Programs/_freeze_module.py BOOTSTRAP_HEADERS = \ Python/frozen_modules/importlib._bootstrap.h \ @@ -1419,7 +1419,7 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu # # Freezing is a multi step process. It works differently for standard builds # and cross builds. Standard builds use Programs/_freeze_module and -# _bootstrap_python for freezing and deepfreezing, so users can build Python +# _bootstrap_python for freezing, so users can build Python # without an existing Python installation. Cross builds cannot execute # compiled binaries and therefore rely on an external build Python # interpreter. The build interpreter must have same version and same bytecode @@ -1433,12 +1433,10 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu # 5) create remaining frozen module headers with # ``./_bootstrap_python Programs/_freeze_module.py``. The pure Python # script is used to test the cross compile code path. -# 6) deepfreeze modules with _bootstrap_python # # Cross compile process: # 1) create all frozen module headers with external build Python and # Programs/_freeze_module.py script. -# 2) deepfreeze modules with external build Python. # # FROZEN_FILES_* are auto-generated by Tools/build/freeze_modules.py. @@ -1584,41 +1582,6 @@ regen-frozen: Tools/build/freeze_modules.py $(FROZEN_FILES_IN) $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/freeze_modules.py --frozen-modules @echo "The Makefile was updated, you may need to re-run make." -############################################################################ -# Deepfreeze targets - -DEEPFREEZE_C = Python/deepfreeze/deepfreeze.c -DEEPFREEZE_DEPS=$(srcdir)/Tools/build/deepfreeze.py Include/internal/pycore_global_strings.h $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT) - -# BEGIN: deepfreeze modules -$(DEEPFREEZE_C): $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/build/deepfreeze.py \ - Python/frozen_modules/importlib._bootstrap.h:importlib._bootstrap \ - Python/frozen_modules/importlib._bootstrap_external.h:importlib._bootstrap_external \ - Python/frozen_modules/zipimport.h:zipimport \ - Python/frozen_modules/abc.h:abc \ - Python/frozen_modules/codecs.h:codecs \ - Python/frozen_modules/io.h:io \ - Python/frozen_modules/_collections_abc.h:_collections_abc \ - Python/frozen_modules/_sitebuiltins.h:_sitebuiltins \ - Python/frozen_modules/genericpath.h:genericpath \ - Python/frozen_modules/ntpath.h:ntpath \ - Python/frozen_modules/posixpath.h:posixpath \ - Python/frozen_modules/os.h:os \ - Python/frozen_modules/site.h:site \ - Python/frozen_modules/stat.h:stat \ - Python/frozen_modules/importlib.util.h:importlib.util \ - Python/frozen_modules/importlib.machinery.h:importlib.machinery \ - Python/frozen_modules/runpy.h:runpy \ - Python/frozen_modules/__hello__.h:__hello__ \ - Python/frozen_modules/__phello__.h:__phello__ \ - Python/frozen_modules/__phello__.ham.h:__phello__.ham \ - Python/frozen_modules/__phello__.ham.eggs.h:__phello__.ham.eggs \ - Python/frozen_modules/__phello__.spam.h:__phello__.spam \ - Python/frozen_modules/frozen_only.h:frozen_only \ - -o Python/deepfreeze/deepfreeze.c -# END: deepfreeze modules - # We keep this renamed target around for folks with muscle memory. .PHONY: regen-importlib regen-importlib: regen-frozen @@ -2933,7 +2896,7 @@ clean-retain-profile: pycremoval -rm -f python.html python*.js python.data python*.symbols python*.map -rm -f $(WASM_STDLIB) -rm -f Programs/_testembed Programs/_freeze_module - -rm -f Python/deepfreeze/*.[co] + -rm -rf Python/deepfreeze -rm -f Python/frozen_modules/*.h -rm -f Python/frozen_modules/MANIFEST -rm -f jit_stencils.h diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 3a8a417a6bf47a..bce92c91f1ca0d 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -476,25 +476,6 @@ - - - - -$(IntDir)\deepfreeze_mappings.txt - - - - - - - - diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 88a4a7c9564309..9131ce87db6c84 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -632,11 +632,6 @@ - - - - - diff --git a/Python/deepfreeze/README.txt b/Python/deepfreeze/README.txt deleted file mode 100644 index 276ab51143ab33..00000000000000 --- a/Python/deepfreeze/README.txt +++ /dev/null @@ -1,6 +0,0 @@ -This directory contains the generated .c files for all the deep-frozen -modules. Python/frozen.c depends on these files. - -None of these files are committed into the repo. - -See Tools/build/freeze_modules.py for more info. diff --git a/Python/frozen.c b/Python/frozen.c index 77f51a7f750965..627f2ff9413562 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -66,34 +66,6 @@ #include "frozen_modules/frozen_only.h" /* End includes */ -#define GET_CODE(name) _Py_get_##name##_toplevel - -/* Start extern declarations */ -extern PyObject *_Py_get_importlib__bootstrap_toplevel(void); -extern PyObject *_Py_get_importlib__bootstrap_external_toplevel(void); -extern PyObject *_Py_get_zipimport_toplevel(void); -extern PyObject *_Py_get_abc_toplevel(void); -extern PyObject *_Py_get_codecs_toplevel(void); -extern PyObject *_Py_get_io_toplevel(void); -extern PyObject *_Py_get__collections_abc_toplevel(void); -extern PyObject *_Py_get__sitebuiltins_toplevel(void); -extern PyObject *_Py_get_genericpath_toplevel(void); -extern PyObject *_Py_get_ntpath_toplevel(void); -extern PyObject *_Py_get_posixpath_toplevel(void); -extern PyObject *_Py_get_os_toplevel(void); -extern PyObject *_Py_get_site_toplevel(void); -extern PyObject *_Py_get_stat_toplevel(void); -extern PyObject *_Py_get_importlib_util_toplevel(void); -extern PyObject *_Py_get_importlib_machinery_toplevel(void); -extern PyObject *_Py_get_runpy_toplevel(void); -extern PyObject *_Py_get___hello___toplevel(void); -extern PyObject *_Py_get___phello___toplevel(void); -extern PyObject *_Py_get___phello___ham_toplevel(void); -extern PyObject *_Py_get___phello___ham_eggs_toplevel(void); -extern PyObject *_Py_get___phello___spam_toplevel(void); -extern PyObject *_Py_get_frozen_only_toplevel(void); -/* End extern declarations */ - static const struct _frozen bootstrap_modules[] = { {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap), false}, {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external), false}, diff --git a/Tools/build/freeze_modules.py b/Tools/build/freeze_modules.py index a541b4b33c519b..eef2d0af046f51 100644 --- a/Tools/build/freeze_modules.py +++ b/Tools/build/freeze_modules.py @@ -20,8 +20,6 @@ # If FROZEN_MODULES_DIR or DEEPFROZEN_MODULES_DIR is changed then the # .gitattributes and .gitignore files needs to be updated. FROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules') -DEEPFROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'deepfreeze') -DEEPFREEZE_MAPPING_FNAME = 'deepfreeze_mappings.txt' FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c') MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in') @@ -233,7 +231,7 @@ def iter_subs(): ####################################### # frozen source files -class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile deepfreezefile')): +class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile')): @classmethod def from_id(cls, frozenid, pyfile=None): @@ -241,8 +239,7 @@ def from_id(cls, frozenid, pyfile=None): pyfile = os.path.join(STDLIB_DIR, *frozenid.split('.')) + '.py' #assert os.path.exists(pyfile), (frozenid, pyfile) frozenfile = resolve_frozen_file(frozenid, FROZEN_MODULES_DIR) - deepfreezefile = resolve_frozen_file(frozenid, DEEPFROZEN_MODULES_DIR) - return cls(frozenid, pyfile, frozenfile, deepfreezefile) + return cls(frozenid, pyfile, frozenfile) @property def frozenid(self): @@ -508,13 +505,6 @@ def regen_frozen(modules): lines.append(f'/* {mod.section} */') lastsection = mod.section - # Also add a extern declaration for the corresponding - # deepfreeze-generated function. - orig_name = mod.source.id - code_name = orig_name.replace(".", "_") - get_code_name = "_Py_get_%s_toplevel" % code_name - externlines.append("extern PyObject *%s(void);" % get_code_name) - pkg = 'true' if mod.ispkg else 'false' size = f"(int)sizeof({mod.symbol})" line = f'{{"{mod.name}", {mod.symbol}, {size}, {pkg}}},' @@ -549,13 +539,6 @@ def regen_frozen(modules): headerlines, FROZEN_FILE, ) - lines = replace_block( - lines, - "/* Start extern declarations */", - "/* End extern declarations */", - externlines, - FROZEN_FILE, - ) lines = replace_block( lines, "static const struct _frozen bootstrap_modules[] =", @@ -591,8 +574,6 @@ def regen_makefile(modules): pyfiles = [] frozenfiles = [] rules = [''] - deepfreezerules = ["$(DEEPFREEZE_C): $(DEEPFREEZE_DEPS)", - "\t$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/build/deepfreeze.py \\"] for src in _iter_sources(modules): frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR) frozenfiles.append(f'\t\t{frozen_header} \\') @@ -614,8 +595,6 @@ def regen_makefile(modules): f'\t{freeze}', '', ]) - deepfreezerules.append(f"\t{frozen_header}:{src.frozenid} \\") - deepfreezerules.append('\t-o Python/deepfreeze/deepfreeze.c') pyfiles[-1] = pyfiles[-1].rstrip(" \\") frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") @@ -643,13 +622,6 @@ def regen_makefile(modules): rules, MAKEFILE, ) - lines = replace_block( - lines, - "# BEGIN: deepfreeze modules", - "# END: deepfreeze modules", - deepfreezerules, - MAKEFILE, - ) outfile.writelines(lines) @@ -657,9 +629,6 @@ def regen_pcbuild(modules): projlines = [] filterlines = [] corelines = [] - deepfreezemappingsfile = f'$(IntDir)\\{DEEPFREEZE_MAPPING_FNAME}' - deepfreezerules = [f' '] - deepfreezemappings = [] for src in _iter_sources(modules): pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR) header = relpath_for_windows_display(src.frozenfile, ROOT_DIR) @@ -673,9 +642,6 @@ def regen_pcbuild(modules): filterlines.append(f' ') filterlines.append(' Python Files') filterlines.append(' ') - deepfreezemappings.append(f' \n') - - corelines.append(f' ') print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}') with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): @@ -688,36 +654,6 @@ def regen_pcbuild(modules): PCBUILD_PROJECT, ) outfile.writelines(lines) - with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): - lines = infile.readlines() - lines = replace_block( - lines, - '', - '', - deepfreezemappings, - PCBUILD_PROJECT, - ) - outfile.writelines(lines) - with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): - lines = infile.readlines() - lines = replace_block( - lines, - '', - '', - [deepfreezemappingsfile, ], - PCBUILD_PROJECT, - ) - outfile.writelines(lines) - with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): - lines = infile.readlines() - lines = replace_block( - lines, - '', - '', - deepfreezerules, - PCBUILD_PROJECT, - ) - outfile.writelines(lines) print(f'# Updating {os.path.relpath(PCBUILD_FILTERS)}') with updating_file_with_tmpfile(PCBUILD_FILTERS) as (infile, outfile): lines = infile.readlines() @@ -729,17 +665,6 @@ def regen_pcbuild(modules): PCBUILD_FILTERS, ) outfile.writelines(lines) - print(f'# Updating {os.path.relpath(PCBUILD_PYTHONCORE)}') - with updating_file_with_tmpfile(PCBUILD_PYTHONCORE) as (infile, outfile): - lines = infile.readlines() - lines = replace_block( - lines, - '', - '', - corelines, - PCBUILD_FILTERS, - ) - outfile.writelines(lines) ####################################### diff --git a/configure b/configure index 9b40c48979b1be..07dce38c92724f 100755 --- a/configure +++ b/configure @@ -27318,8 +27318,7 @@ SRCDIRS="\ Parser/lexer \ Programs \ Python \ - Python/frozen_modules \ - Python/deepfreeze" + Python/frozen_modules" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for build directories" >&5 printf %s "checking for build directories... " >&6; } for dir in $SRCDIRS; do diff --git a/configure.ac b/configure.ac index 02913cef06a5f9..3e676c56693a3c 100644 --- a/configure.ac +++ b/configure.ac @@ -6826,8 +6826,7 @@ SRCDIRS="\ Parser/lexer \ Programs \ Python \ - Python/frozen_modules \ - Python/deepfreeze" + Python/frozen_modules" AC_MSG_CHECKING([for build directories]) for dir in $SRCDIRS; do if test ! -d $dir; then From 590a26010d5d7f27890f89820645580bb8f28547 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Mar 2024 20:15:20 +0100 Subject: [PATCH 154/158] gh-116869: Add test_cext test: build a C extension (#116954) --- Lib/test/test_cext/__init__.py | 92 +++++++++++++++++++ Lib/test/test_cext/extension.c | 80 ++++++++++++++++ Lib/test/test_cext/setup.py | 47 ++++++++++ Makefile.pre.in | 1 + ...-03-18-10-58-47.gh-issue-116869.lN0GBl.rst | 2 + 5 files changed, 222 insertions(+) create mode 100644 Lib/test/test_cext/__init__.py create mode 100644 Lib/test/test_cext/extension.c create mode 100644 Lib/test/test_cext/setup.py create mode 100644 Misc/NEWS.d/next/C API/2024-03-18-10-58-47.gh-issue-116869.lN0GBl.rst diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py new file mode 100644 index 00000000000000..302ea3d3a82300 --- /dev/null +++ b/Lib/test/test_cext/__init__.py @@ -0,0 +1,92 @@ +# gh-116869: Build a basic C test extension to check that the Python C API +# does not emit C compiler warnings. + +import os.path +import shutil +import subprocess +import sysconfig +import unittest +from test import support + + +SOURCE = os.path.join(os.path.dirname(__file__), 'extension.c') +SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') + + +# gh-110119: pip does not currently support 't' in the ABI flag use by +# --disable-gil builds. Once it does, we can remove this skip. +@unittest.skipIf(support.Py_GIL_DISABLED, + 'test does not work with --disable-gil') +@support.requires_subprocess() +@support.requires_resource('cpu') +class TestExt(unittest.TestCase): + def test_build_c99(self): + self.check_build('c99', '_test_c99_ext') + + def test_build_c11(self): + self.check_build('c11', '_test_c11_ext') + + # With MSVC, the linker fails with: cannot open file 'python311.lib' + # https://github.com/python/cpython/pull/32175#issuecomment-1111175897 + @unittest.skipIf(support.MS_WINDOWS, 'test fails on Windows') + # Building and running an extension in clang sanitizing mode is not + # straightforward + @unittest.skipIf( + '-fsanitize' in (sysconfig.get_config_var('PY_CFLAGS') or ''), + 'test does not work with analyzing builds') + # the test uses venv+pip: skip if it's not available + @support.requires_venv_with_pip() + def check_build(self, clang_std, extension_name): + venv_dir = 'env' + with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe: + self._check_build(clang_std, extension_name, python_exe) + + def _check_build(self, clang_std, extension_name, python_exe): + pkg_dir = 'pkg' + os.mkdir(pkg_dir) + shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) + shutil.copy(SOURCE, os.path.join(pkg_dir, os.path.basename(SOURCE))) + + def run_cmd(operation, cmd): + env = os.environ.copy() + env['CPYTHON_TEST_STD'] = clang_std + env['CPYTHON_TEST_EXT_NAME'] = extension_name + if support.verbose: + print('Run:', ' '.join(cmd)) + subprocess.run(cmd, check=True, env=env) + else: + proc = subprocess.run(cmd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True) + if proc.returncode: + print(proc.stdout, end='') + self.fail( + f"{operation} failed with exit code {proc.returncode}") + + # Build and install the C extension + cmd = [python_exe, '-X', 'dev', + '-m', 'pip', 'install', '--no-build-isolation', + os.path.abspath(pkg_dir)] + run_cmd('Install', cmd) + + # Do a reference run. Until we test that running python + # doesn't leak references (gh-94755), run it so one can manually check + # -X showrefcount results against this baseline. + cmd = [python_exe, + '-X', 'dev', + '-X', 'showrefcount', + '-c', 'pass'] + run_cmd('Reference run', cmd) + + # Import the C extension + cmd = [python_exe, + '-X', 'dev', + '-X', 'showrefcount', + '-c', f"import {extension_name}"] + run_cmd('Import', cmd) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c new file mode 100644 index 00000000000000..cfecad39f8af14 --- /dev/null +++ b/Lib/test/test_cext/extension.c @@ -0,0 +1,80 @@ +// gh-116869: Basic C test extension to check that the Python C API +// does not emit C compiler warnings. + +// Always enable assertions +#undef NDEBUG + +#include "Python.h" + +#if defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L +# define NAME _test_c2x_ext +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +# define NAME _test_c11_ext +#else +# define NAME _test_c99_ext +#endif + +#define _STR(NAME) #NAME +#define STR(NAME) _STR(NAME) + +PyDoc_STRVAR(_testcext_add_doc, +"add(x, y)\n" +"\n" +"Return the sum of two integers: x + y."); + +static PyObject * +_testcext_add(PyObject *Py_UNUSED(module), PyObject *args) +{ + long i, j; + if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) { + return NULL; + } + long res = i + j; + return PyLong_FromLong(res); +} + + +static PyMethodDef _testcext_methods[] = { + {"add", _testcext_add, METH_VARARGS, _testcext_add_doc}, + {NULL, NULL, 0, NULL} // sentinel +}; + + +static int +_testcext_exec(PyObject *module) +{ + if (PyModule_AddIntMacro(module, __STDC_VERSION__) < 0) { + return -1; + } + return 0; +} + +static PyModuleDef_Slot _testcext_slots[] = { + {Py_mod_exec, _testcext_exec}, + {0, NULL} +}; + + +PyDoc_STRVAR(_testcext_doc, "C test extension."); + +static struct PyModuleDef _testcext_module = { + PyModuleDef_HEAD_INIT, // m_base + STR(NAME), // m_name + _testcext_doc, // m_doc + 0, // m_size + _testcext_methods, // m_methods + _testcext_slots, // m_slots + NULL, // m_traverse + NULL, // m_clear + NULL, // m_free +}; + + +#define _FUNC_NAME(NAME) PyInit_ ## NAME +#define FUNC_NAME(NAME) _FUNC_NAME(NAME) + +PyMODINIT_FUNC +FUNC_NAME(NAME)(void) +{ + return PyModuleDef_Init(&_testcext_module); +} diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py new file mode 100644 index 00000000000000..dd57a5fb8e1e1b --- /dev/null +++ b/Lib/test/test_cext/setup.py @@ -0,0 +1,47 @@ +# gh-91321: Build a basic C test extension to check that the Python C API is +# compatible with C and does not emit C compiler warnings. +import os +import shlex +import sys +import sysconfig +from test import support + +from setuptools import setup, Extension + + +SOURCE = 'extension.c' +if not support.MS_WINDOWS: + # C compiler flags for GCC and clang + CFLAGS = [ + # The purpose of test_cext extension is to check that building a C + # extension using the Python C API does not emit C compiler warnings. + '-Werror', + ] +else: + # Don't pass any compiler flag to MSVC + CFLAGS = [] + + +def main(): + std = os.environ["CPYTHON_TEST_STD"] + name = os.environ["CPYTHON_TEST_EXT_NAME"] + cflags = [*CFLAGS, f'-std={std}'] + + # Remove existing -std options to only test ours + cmd = (sysconfig.get_config_var('CC') or '') + if cmd is not None: + cmd = shlex.split(cmd) + cmd = [arg for arg in cmd if not arg.startswith('-std=')] + cmd = shlex.join(cmd) + # CC env var overrides sysconfig CC variable in setuptools + os.environ['CC'] = cmd + + ext = Extension( + name, + sources=[SOURCE], + extra_compile_args=cflags) + setup(name='internal' + name, version='0.0', ext_modules=[ext]) + + +if __name__ == "__main__": + main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 5958b6bdb05a57..404e7ee3e42054 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2330,6 +2330,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/support/interpreters \ test/test_asyncio \ test/test_capi \ + test/test_cext \ test/test_concurrent_futures \ test/test_cppext \ test/test_ctypes \ diff --git a/Misc/NEWS.d/next/C API/2024-03-18-10-58-47.gh-issue-116869.lN0GBl.rst b/Misc/NEWS.d/next/C API/2024-03-18-10-58-47.gh-issue-116869.lN0GBl.rst new file mode 100644 index 00000000000000..71044b4930355a --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-03-18-10-58-47.gh-issue-116869.lN0GBl.rst @@ -0,0 +1,2 @@ +Add ``test_cext`` test: build a C extension to check if the Python C API +emits C compiler warnings. Patch by Victor Stinner. From a9c304cf020e2fa3ae78fd88359dfc808c9dd639 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Mar 2024 20:16:58 +0100 Subject: [PATCH 155/158] gh-116869: Make C API compatible with ISO C90 (#116950) Make the C API compatible with -Werror=declaration-after-statement compiler flag again. --- Include/cpython/longintrepr.h | 3 ++- Include/object.h | 3 +-- .../next/C API/2024-03-18-09-58-46.gh-issue-116869.LFDVKM.rst | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-03-18-09-58-46.gh-issue-116869.LFDVKM.rst diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index f037c7bb90deda..3246908ba982e2 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -129,9 +129,10 @@ _PyLong_IsCompact(const PyLongObject* op) { static inline Py_ssize_t _PyLong_CompactValue(const PyLongObject *op) { + Py_ssize_t sign; assert(PyType_HasFeature((op)->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); assert(PyUnstable_Long_IsCompact(op)); - Py_ssize_t sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); + sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); return sign * (Py_ssize_t)op->long_value.ob_digit[0]; } diff --git a/Include/object.h b/Include/object.h index 34141af7b7f7ef..b0c0dba06ca139 100644 --- a/Include/object.h +++ b/Include/object.h @@ -343,8 +343,7 @@ PyAPI_DATA(PyTypeObject) PyBool_Type; static inline Py_ssize_t Py_SIZE(PyObject *ob) { assert(ob->ob_type != &PyLong_Type); assert(ob->ob_type != &PyBool_Type); - PyVarObject *var_ob = _PyVarObject_CAST(ob); - return var_ob->ob_size; + return _PyVarObject_CAST(ob)->ob_size; } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob)) diff --git a/Misc/NEWS.d/next/C API/2024-03-18-09-58-46.gh-issue-116869.LFDVKM.rst b/Misc/NEWS.d/next/C API/2024-03-18-09-58-46.gh-issue-116869.LFDVKM.rst new file mode 100644 index 00000000000000..9b9d943f2e6d19 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-03-18-09-58-46.gh-issue-116869.LFDVKM.rst @@ -0,0 +1,2 @@ +Make the C API compatible with ``-Werror=declaration-after-statement`` +compiler flag again. Patch by Victor Stinner. From 415964417771946dcb7a163951913adf84644b6d Mon Sep 17 00:00:00 2001 From: "Pierre Ossman (ThinLinc team)" Date: Mon, 18 Mar 2024 21:15:53 +0100 Subject: [PATCH 156/158] gh-113538: Add asycio.Server.{close,abort}_clients (redo) (#116784) These give applications the option of more forcefully terminating client connections for asyncio servers. Useful when terminating a service and there is limited time to wait for clients to finish up their work. This is a do-over with a test fix for gh-114432, which was reverted. --- Doc/library/asyncio-eventloop.rst | 25 +++++ Doc/whatsnew/3.13.rst | 5 + Lib/asyncio/base_events.py | 25 +++-- Lib/asyncio/events.py | 8 ++ Lib/asyncio/proactor_events.py | 4 +- Lib/asyncio/selector_events.py | 6 +- Lib/test/test_asyncio/test_server.py | 96 +++++++++++++++++-- ...-01-22-15-50-58.gh-issue-113538.v2wrwg.rst | 3 + 8 files changed, 152 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 06c5c877ccc173..d6ed817b13676f 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -1641,6 +1641,31 @@ Do not instantiate the :class:`Server` class directly. coroutine to wait until the server is closed (and no more connections are active). + .. method:: close_clients() + + Close all existing incoming client connections. + + Calls :meth:`~asyncio.BaseTransport.close` on all associated + transports. + + :meth:`close` should be called before :meth:`close_clients` when + closing the server to avoid races with new clients connecting. + + .. versionadded:: 3.13 + + .. method:: abort_clients() + + Close all existing incoming client connections immediately, + without waiting for pending operations to complete. + + Calls :meth:`~asyncio.WriteTransport.abort` on all associated + transports. + + :meth:`close` should be called before :meth:`abort_clients` when + closing the server to avoid races with new clients connecting. + + .. versionadded:: 3.13 + .. method:: get_loop() Return the event loop associated with the server object. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index b665e6f1c85915..0553cc97c5c75a 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -270,6 +270,11 @@ asyncio the buffer size. (Contributed by Jamie Phan in :gh:`115199`.) +* Add :meth:`asyncio.Server.close_clients` and + :meth:`asyncio.Server.abort_clients` methods which allow to more + forcefully close an asyncio server. + (Contributed by Pierre Ossman in :gh:`113538`.) + base64 --- diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 6c5cf28e7c59d4..f0e690b61a73dd 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -279,7 +279,9 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, ssl_handshake_timeout, ssl_shutdown_timeout=None): self._loop = loop self._sockets = sockets - self._active_count = 0 + # Weak references so we don't break Transport's ability to + # detect abandoned transports + self._clients = weakref.WeakSet() self._waiters = [] self._protocol_factory = protocol_factory self._backlog = backlog @@ -292,14 +294,13 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, def __repr__(self): return f'<{self.__class__.__name__} sockets={self.sockets!r}>' - def _attach(self): + def _attach(self, transport): assert self._sockets is not None - self._active_count += 1 + self._clients.add(transport) - def _detach(self): - assert self._active_count > 0 - self._active_count -= 1 - if self._active_count == 0 and self._sockets is None: + def _detach(self, transport): + self._clients.discard(transport) + if len(self._clients) == 0 and self._sockets is None: self._wakeup() def _wakeup(self): @@ -348,9 +349,17 @@ def close(self): self._serving_forever_fut.cancel() self._serving_forever_fut = None - if self._active_count == 0: + if len(self._clients) == 0: self._wakeup() + def close_clients(self): + for transport in self._clients.copy(): + transport.close() + + def abort_clients(self): + for transport in self._clients.copy(): + transport.abort() + async def start_serving(self): self._start_serving() # Skip one loop iteration so that all 'loop.add_reader' diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 680749325025db..be495469a0558b 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -175,6 +175,14 @@ def close(self): """Stop serving. This leaves existing connections open.""" raise NotImplementedError + def close_clients(self): + """Close all active connections.""" + raise NotImplementedError + + def abort_clients(self): + """Close all active connections immediately.""" + raise NotImplementedError + def get_loop(self): """Get the event loop the Server object is attached to.""" raise NotImplementedError diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index a512db6367b20a..397a8cda757895 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -63,7 +63,7 @@ def __init__(self, loop, sock, protocol, waiter=None, self._called_connection_lost = False self._eof_written = False if self._server is not None: - self._server._attach() + self._server._attach(self) self._loop.call_soon(self._protocol.connection_made, self) if waiter is not None: # only wake up the waiter when connection_made() has been called @@ -167,7 +167,7 @@ def _call_connection_lost(self, exc): self._sock = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None self._called_connection_lost = True diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 8e888d26ea0737..f94bf10b4225e7 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -791,7 +791,7 @@ def __init__(self, loop, sock, protocol, extra=None, server=None): self._paused = False # Set when pause_reading() called if self._server is not None: - self._server._attach() + self._server._attach(self) loop._transports[self._sock_fd] = self def __repr__(self): @@ -868,6 +868,8 @@ def __del__(self, _warn=warnings.warn): if self._sock is not None: _warn(f"unclosed transport {self!r}", ResourceWarning, source=self) self._sock.close() + if self._server is not None: + self._server._detach(self) def _fatal_error(self, exc, message='Fatal error on transport'): # Should be called from exception handler only. @@ -906,7 +908,7 @@ def _call_connection_lost(self, exc): self._loop = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None def get_write_buffer_size(self): diff --git a/Lib/test/test_asyncio/test_server.py b/Lib/test/test_asyncio/test_server.py index 918faac909b9bf..4ca8a166a0f1a1 100644 --- a/Lib/test/test_asyncio/test_server.py +++ b/Lib/test/test_asyncio/test_server.py @@ -125,8 +125,12 @@ async def main(srv): class TestServer2(unittest.IsolatedAsyncioTestCase): async def test_wait_closed_basic(self): - async def serve(*args): - pass + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) self.addCleanup(srv.close) @@ -137,7 +141,8 @@ async def serve(*args): self.assertFalse(task1.done()) # active count != 0, not closed: should block - srv._attach() + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) task2 = asyncio.create_task(srv.wait_closed()) await asyncio.sleep(0) self.assertFalse(task1.done()) @@ -152,7 +157,8 @@ async def serve(*args): self.assertFalse(task2.done()) self.assertFalse(task3.done()) - srv._detach() + wr.close() + await wr.wait_closed() # active count == 0, closed: should unblock await task1 await task2 @@ -161,8 +167,12 @@ async def serve(*args): async def test_wait_closed_race(self): # Test a regression in 3.12.0, should be fixed in 3.12.1 - async def serve(*args): - pass + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) self.addCleanup(srv.close) @@ -170,13 +180,83 @@ async def serve(*args): task = asyncio.create_task(srv.wait_closed()) await asyncio.sleep(0) self.assertFalse(task.done()) - srv._attach() + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) loop = asyncio.get_running_loop() loop.call_soon(srv.close) - loop.call_soon(srv._detach) + loop.call_soon(wr.close) await srv.wait_closed() + async def test_close_clients(self): + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() + + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) + self.addCleanup(wr.close) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + + srv.close() + srv.close_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) + + async def test_abort_clients(self): + async def serve(rd, wr): + fut.set_result((rd, wr)) + await wr.wait_closed() + + fut = asyncio.Future() + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (c_rd, c_wr) = await asyncio.open_connection(addr[0], addr[1], limit=4096) + self.addCleanup(c_wr.close) + + (s_rd, s_wr) = await fut + + # Limit the socket buffers so we can reliably overfill them + s_sock = s_wr.get_extra_info('socket') + s_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 65536) + c_sock = c_wr.get_extra_info('socket') + c_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 65536) + + # Get the reader in to a paused state by sending more than twice + # the configured limit + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + while c_wr.transport.is_reading(): + await asyncio.sleep(0) + + # Get the writer in a waiting state by sending data until the + # socket buffers are full on both server and client sockets and + # the kernel stops accepting more data + s_wr.write(b'a' * c_sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF)) + s_wr.write(b'a' * s_sock.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF)) + self.assertNotEqual(s_wr.transport.get_write_buffer_size(), 0) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + srv.close() + srv.abort_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) # Test the various corner cases of Unix server socket removal diff --git a/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst b/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst new file mode 100644 index 00000000000000..5c59af98e136bb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst @@ -0,0 +1,3 @@ +Add :meth:`asyncio.Server.close_clients` and +:meth:`asyncio.Server.abort_clients` methods which allow to more forcefully +close an asyncio server. From ecb4a2b711d62f1395ddbff52576d0cca8a1b43e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Mar 2024 22:03:55 +0100 Subject: [PATCH 157/158] gh-116417: Move limited C API list.c tests to _testlimitedcapi (#116602) Split list.c and set.c tests of _testcapi into two parts: limited C API tests in _testlimitedcapi and non-limited C API tests in _testcapi. --- Lib/test/test_capi/test_list.py | 29 ++-- Lib/test/test_capi/test_set.py | 32 ++-- Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/list.c | 150 +----------------- Modules/_testcapi/set.c | 172 --------------------- Modules/_testlimitedcapi.c | 6 + Modules/_testlimitedcapi/list.c | 169 ++++++++++++++++++++ Modules/_testlimitedcapi/parts.h | 2 + Modules/_testlimitedcapi/set.c | 189 +++++++++++++++++++++++ PCbuild/_testlimitedcapi.vcxproj | 2 + PCbuild/_testlimitedcapi.vcxproj.filters | 2 + 11 files changed, 404 insertions(+), 351 deletions(-) create mode 100644 Modules/_testlimitedcapi/list.c create mode 100644 Modules/_testlimitedcapi/set.c diff --git a/Lib/test/test_capi/test_list.py b/Lib/test/test_capi/test_list.py index dceb4fce3c077b..0896a971f5c727 100644 --- a/Lib/test/test_capi/test_list.py +++ b/Lib/test/test_capi/test_list.py @@ -4,6 +4,7 @@ from test.support import import_helper from collections import UserList _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None PY_SSIZE_T_MIN = _testcapi.PY_SSIZE_T_MIN @@ -25,7 +26,7 @@ def __del__(self): class CAPITest(unittest.TestCase): def test_check(self): # Test PyList_Check() - check = _testcapi.list_check + check = _testlimitedcapi.list_check self.assertTrue(check([1, 2])) self.assertTrue(check([])) self.assertTrue(check(ListSubclass([1, 2]))) @@ -39,7 +40,7 @@ def test_check(self): def test_list_check_exact(self): # Test PyList_CheckExact() - check = _testcapi.list_check_exact + check = _testlimitedcapi.list_check_exact self.assertTrue(check([1])) self.assertTrue(check([])) self.assertFalse(check(ListSubclass([1]))) @@ -51,7 +52,7 @@ def test_list_check_exact(self): def test_list_new(self): # Test PyList_New() - list_new = _testcapi.list_new + list_new = _testlimitedcapi.list_new lst = list_new(0) self.assertEqual(lst, []) self.assertIs(type(lst), list) @@ -62,7 +63,7 @@ def test_list_new(self): def test_list_size(self): # Test PyList_Size() - size = _testcapi.list_size + size = _testlimitedcapi.list_size self.assertEqual(size([1, 2]), 2) self.assertEqual(size(ListSubclass([1, 2])), 2) self.assertRaises(SystemError, size, UserList()) @@ -98,11 +99,11 @@ def check_list_get_item(self, getitem, exctype): def test_list_getitem(self): # Test PyList_GetItem() - self.check_list_get_item(_testcapi.list_getitem, SystemError) + self.check_list_get_item(_testlimitedcapi.list_getitem, SystemError) def test_list_get_item_ref(self): # Test PyList_GetItemRef() - self.check_list_get_item(_testcapi.list_get_item_ref, TypeError) + self.check_list_get_item(_testlimitedcapi.list_get_item_ref, TypeError) def test_list_get_item(self): # Test PyList_GET_ITEM() @@ -119,7 +120,7 @@ def test_list_get_item(self): def test_list_setitem(self): # Test PyList_SetItem() - setitem = _testcapi.list_setitem + setitem = _testlimitedcapi.list_setitem lst = [1, 2, 3] setitem(lst, 0, 10) self.assertEqual(lst, [10, 2, 3]) @@ -151,7 +152,7 @@ def test_list_set_item(self): def test_list_insert(self): # Test PyList_Insert() - insert = _testcapi.list_insert + insert = _testlimitedcapi.list_insert lst = [1, 2, 3] insert(lst, 0, 23) self.assertEqual(lst, [23, 1, 2, 3]) @@ -173,7 +174,7 @@ def test_list_insert(self): def test_list_append(self): # Test PyList_Append() - append = _testcapi.list_append + append = _testlimitedcapi.list_append lst = [1, 2, 3] append(lst, 10) self.assertEqual(lst, [1, 2, 3, 10]) @@ -186,7 +187,7 @@ def test_list_append(self): def test_list_getslice(self): # Test PyList_GetSlice() - getslice = _testcapi.list_getslice + getslice = _testlimitedcapi.list_getslice lst = [1, 2, 3] # empty @@ -210,7 +211,7 @@ def test_list_getslice(self): def test_list_setslice(self): # Test PyList_SetSlice() - list_setslice = _testcapi.list_setslice + list_setslice = _testlimitedcapi.list_setslice def set_slice(lst, low, high, value): lst = lst.copy() self.assertEqual(list_setslice(lst, low, high, value), 0) @@ -265,7 +266,7 @@ def set_slice(lst, low, high, value): def test_list_sort(self): # Test PyList_Sort() - sort = _testcapi.list_sort + sort = _testlimitedcapi.list_sort lst = [4, 6, 7, 3, 1, 5, 9, 2, 0, 8] sort(lst) self.assertEqual(lst, list(range(10))) @@ -281,7 +282,7 @@ def test_list_sort(self): def test_list_reverse(self): # Test PyList_Reverse() - reverse = _testcapi.list_reverse + reverse = _testlimitedcapi.list_reverse def list_reverse(lst): self.assertEqual(reverse(lst), 0) return lst @@ -295,7 +296,7 @@ def list_reverse(lst): def test_list_astuple(self): # Test PyList_AsTuple() - astuple = _testcapi.list_astuple + astuple = _testlimitedcapi.list_astuple self.assertEqual(astuple([]), ()) self.assertEqual(astuple([2, 5, 10]), (2, 5, 10)) diff --git a/Lib/test/test_capi/test_set.py b/Lib/test/test_capi/test_set.py index 5235f81543e0b6..499a5148d782ab 100644 --- a/Lib/test/test_capi/test_set.py +++ b/Lib/test/test_capi/test_set.py @@ -2,8 +2,10 @@ from test.support import import_helper -# Skip this test if the _testcapi or _testinternalcapi modules aren't available. +# Skip this test if the _testcapi, _testlimitedcapi or _testinternalcapi +# modules aren't available. _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') _testinternalcapi = import_helper.import_module('_testinternalcapi') class set_subclass(set): @@ -23,7 +25,7 @@ def assertImmutable(self, action, *args): class TestSetCAPI(BaseSetTests, unittest.TestCase): def test_set_check(self): - check = _testcapi.set_check + check = _testlimitedcapi.set_check self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertFalse(check(frozenset())) @@ -33,7 +35,7 @@ def test_set_check(self): # CRASHES: check(NULL) def test_set_check_exact(self): - check = _testcapi.set_checkexact + check = _testlimitedcapi.set_checkexact self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertFalse(check(frozenset())) @@ -43,7 +45,7 @@ def test_set_check_exact(self): # CRASHES: check(NULL) def test_frozenset_check(self): - check = _testcapi.frozenset_check + check = _testlimitedcapi.frozenset_check self.assertFalse(check(set())) self.assertTrue(check(frozenset())) self.assertTrue(check(frozenset({1, 2}))) @@ -53,7 +55,7 @@ def test_frozenset_check(self): # CRASHES: check(NULL) def test_frozenset_check_exact(self): - check = _testcapi.frozenset_checkexact + check = _testlimitedcapi.frozenset_checkexact self.assertFalse(check(set())) self.assertTrue(check(frozenset())) self.assertTrue(check(frozenset({1, 2}))) @@ -63,7 +65,7 @@ def test_frozenset_check_exact(self): # CRASHES: check(NULL) def test_anyset_check(self): - check = _testcapi.anyset_check + check = _testlimitedcapi.anyset_check self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertTrue(check(frozenset())) @@ -74,7 +76,7 @@ def test_anyset_check(self): # CRASHES: check(NULL) def test_anyset_check_exact(self): - check = _testcapi.anyset_checkexact + check = _testlimitedcapi.anyset_checkexact self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertTrue(check(frozenset())) @@ -85,7 +87,7 @@ def test_anyset_check_exact(self): # CRASHES: check(NULL) def test_set_new(self): - set_new = _testcapi.set_new + set_new = _testlimitedcapi.set_new self.assertEqual(set_new().__class__, set) self.assertEqual(set_new(), set()) self.assertEqual(set_new((1, 1, 2)), {1, 2}) @@ -98,7 +100,7 @@ def test_set_new(self): set_new((1, {})) def test_frozenset_new(self): - frozenset_new = _testcapi.frozenset_new + frozenset_new = _testlimitedcapi.frozenset_new self.assertEqual(frozenset_new().__class__, frozenset) self.assertEqual(frozenset_new(), frozenset()) self.assertEqual(frozenset_new((1, 1, 2)), frozenset({1, 2})) @@ -111,7 +113,7 @@ def test_frozenset_new(self): frozenset_new((1, {})) def test_set_size(self): - get_size = _testcapi.set_size + get_size = _testlimitedcapi.set_size self.assertEqual(get_size(set()), 0) self.assertEqual(get_size(frozenset()), 0) self.assertEqual(get_size({1, 1, 2}), 2) @@ -134,7 +136,7 @@ def test_set_get_size(self): # CRASHES: get_size(object()) def test_set_contains(self): - contains = _testcapi.set_contains + contains = _testlimitedcapi.set_contains for cls in (set, frozenset, set_subclass, frozenset_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) @@ -147,7 +149,7 @@ def test_set_contains(self): # CRASHES: contains(NULL, NULL) def test_add(self): - add = _testcapi.set_add + add = _testlimitedcapi.set_add for cls in (set, set_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) @@ -165,7 +167,7 @@ def test_add(self): # CRASHES: add(NULL, NULL) def test_discard(self): - discard = _testcapi.set_discard + discard = _testlimitedcapi.set_discard for cls in (set, set_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) @@ -187,7 +189,7 @@ def test_discard(self): # CRASHES: discard(NULL, NULL) def test_pop(self): - pop = _testcapi.set_pop + pop = _testlimitedcapi.set_pop orig = (1, 2) for cls in (set, set_subclass): with self.subTest(cls=cls): @@ -204,7 +206,7 @@ def test_pop(self): # CRASHES: pop(NULL) def test_clear(self): - clear = _testcapi.set_clear + clear = _testlimitedcapi.set_clear for cls in (set, set_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index deada66cf1a807..d4743d31b465a9 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -163,7 +163,7 @@ @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/pyos.c _testlimitedcapi/sys.c _testlimitedcapi/vectorcall_limited.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/list.c b/Modules/_testcapi/list.c index 2cb6499e28336d..09cec4c30c8c36 100644 --- a/Modules/_testcapi/list.c +++ b/Modules/_testcapi/list.c @@ -1,32 +1,6 @@ #include "parts.h" #include "util.h" -static PyObject * -list_check(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyList_Check(obj)); -} - -static PyObject * -list_check_exact(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyList_CheckExact(obj)); -} - -static PyObject * -list_new(PyObject* Py_UNUSED(module), PyObject *obj) -{ - return PyList_New(PyLong_AsSsize_t(obj)); -} - -static PyObject * -list_size(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PyList_Size(obj)); -} static PyObject * list_get_size(PyObject *Py_UNUSED(module), PyObject *obj) @@ -35,17 +9,6 @@ list_get_size(PyObject *Py_UNUSED(module), PyObject *obj) RETURN_SIZE(PyList_GET_SIZE(obj)); } -static PyObject * -list_getitem(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "On", &obj, &i)) { - return NULL; - } - NULLABLE(obj); - return Py_XNewRef(PyList_GetItem(obj, i)); -} static PyObject * list_get_item(PyObject *Py_UNUSED(module), PyObject *args) @@ -59,31 +22,6 @@ list_get_item(PyObject *Py_UNUSED(module), PyObject *args) return Py_XNewRef(PyList_GET_ITEM(obj, i)); } -static PyObject * -list_get_item_ref(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "On", &obj, &i)) { - return NULL; - } - NULLABLE(obj); - return PyList_GetItemRef(obj, i); -} - -static PyObject * -list_setitem(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj, *value; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "OnO", &obj, &i, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyList_SetItem(obj, i, Py_XNewRef(value))); - -} static PyObject * list_set_item(PyObject *Py_UNUSED(module), PyObject *args) @@ -100,79 +38,6 @@ list_set_item(PyObject *Py_UNUSED(module), PyObject *args) } -static PyObject * -list_insert(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj, *value; - Py_ssize_t where; - if (!PyArg_ParseTuple(args, "OnO", &obj, &where, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyList_Insert(obj, where, Py_XNewRef(value))); - -} - -static PyObject * -list_append(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj, *value; - if (!PyArg_ParseTuple(args, "OO", &obj, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyList_Append(obj, value)); -} - -static PyObject * -list_getslice(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - Py_ssize_t ilow, ihigh; - if (!PyArg_ParseTuple(args, "Onn", &obj, &ilow, &ihigh)) { - return NULL; - } - NULLABLE(obj); - return PyList_GetSlice(obj, ilow, ihigh); - -} - -static PyObject * -list_setslice(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj, *value; - Py_ssize_t ilow, ihigh; - if (!PyArg_ParseTuple(args, "OnnO", &obj, &ilow, &ihigh, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyList_SetSlice(obj, ilow, ihigh, value)); -} - -static PyObject * -list_sort(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyList_Sort(obj)); -} - -static PyObject * -list_reverse(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyList_Reverse(obj)); -} - -static PyObject * -list_astuple(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyList_AsTuple(obj); -} - static PyObject * list_clear(PyObject* Py_UNUSED(module), PyObject *obj) @@ -196,25 +61,12 @@ list_extend(PyObject* Py_UNUSED(module), PyObject *args) static PyMethodDef test_methods[] = { - {"list_check", list_check, METH_O}, - {"list_check_exact", list_check_exact, METH_O}, - {"list_new", list_new, METH_O}, - {"list_size", list_size, METH_O}, {"list_get_size", list_get_size, METH_O}, - {"list_getitem", list_getitem, METH_VARARGS}, {"list_get_item", list_get_item, METH_VARARGS}, - {"list_get_item_ref", list_get_item_ref, METH_VARARGS}, - {"list_setitem", list_setitem, METH_VARARGS}, {"list_set_item", list_set_item, METH_VARARGS}, - {"list_insert", list_insert, METH_VARARGS}, - {"list_append", list_append, METH_VARARGS}, - {"list_getslice", list_getslice, METH_VARARGS}, - {"list_setslice", list_setslice, METH_VARARGS}, - {"list_sort", list_sort, METH_O}, - {"list_reverse", list_reverse, METH_O}, - {"list_astuple", list_astuple, METH_O}, {"list_clear", list_clear, METH_O}, {"list_extend", list_extend, METH_VARARGS}, + {NULL}, }; diff --git a/Modules/_testcapi/set.c b/Modules/_testcapi/set.c index 2fbd0aeffcd9f9..31b52cee5e9623 100644 --- a/Modules/_testcapi/set.c +++ b/Modules/_testcapi/set.c @@ -1,75 +1,6 @@ #include "parts.h" #include "util.h" -static PyObject * -set_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PySet_Check(obj)); -} - -static PyObject * -set_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PySet_CheckExact(obj)); -} - -static PyObject * -frozenset_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyFrozenSet_Check(obj)); -} - -static PyObject * -frozenset_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyFrozenSet_CheckExact(obj)); -} - -static PyObject * -anyset_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyAnySet_Check(obj)); -} - -static PyObject * -anyset_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyAnySet_CheckExact(obj)); -} - -static PyObject * -set_new(PyObject *self, PyObject *args) -{ - PyObject *iterable = NULL; - if (!PyArg_ParseTuple(args, "|O", &iterable)) { - return NULL; - } - return PySet_New(iterable); -} - -static PyObject * -frozenset_new(PyObject *self, PyObject *args) -{ - PyObject *iterable = NULL; - if (!PyArg_ParseTuple(args, "|O", &iterable)) { - return NULL; - } - return PyFrozenSet_New(iterable); -} - -static PyObject * -set_size(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PySet_Size(obj)); -} - static PyObject * set_get_size(PyObject *self, PyObject *obj) { @@ -77,111 +8,8 @@ set_get_size(PyObject *self, PyObject *obj) RETURN_SIZE(PySet_GET_SIZE(obj)); } -static PyObject * -set_contains(PyObject *self, PyObject *args) -{ - PyObject *obj, *item; - if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(item); - RETURN_INT(PySet_Contains(obj, item)); -} - -static PyObject * -set_add(PyObject *self, PyObject *args) -{ - PyObject *obj, *item; - if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(item); - RETURN_INT(PySet_Add(obj, item)); -} - -static PyObject * -set_discard(PyObject *self, PyObject *args) -{ - PyObject *obj, *item; - if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(item); - RETURN_INT(PySet_Discard(obj, item)); -} - -static PyObject * -set_pop(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PySet_Pop(obj); -} - -static PyObject * -set_clear(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PySet_Clear(obj)); -} - -static PyObject * -test_frozenset_add_in_capi(PyObject *self, PyObject *Py_UNUSED(obj)) -{ - // Test that `frozenset` can be used with `PySet_Add`, - // when frozenset is just created in CAPI. - PyObject *fs = PyFrozenSet_New(NULL); - if (fs == NULL) { - return NULL; - } - PyObject *num = PyLong_FromLong(1); - if (num == NULL) { - goto error; - } - if (PySet_Add(fs, num) < 0) { - goto error; - } - int contains = PySet_Contains(fs, num); - if (contains < 0) { - goto error; - } - else if (contains == 0) { - goto unexpected; - } - Py_DECREF(fs); - Py_DECREF(num); - Py_RETURN_NONE; - -unexpected: - PyErr_SetString(PyExc_ValueError, "set does not contain expected value"); -error: - Py_DECREF(fs); - Py_XDECREF(num); - return NULL; -} - static PyMethodDef test_methods[] = { - {"set_check", set_check, METH_O}, - {"set_checkexact", set_checkexact, METH_O}, - {"frozenset_check", frozenset_check, METH_O}, - {"frozenset_checkexact", frozenset_checkexact, METH_O}, - {"anyset_check", anyset_check, METH_O}, - {"anyset_checkexact", anyset_checkexact, METH_O}, - - {"set_new", set_new, METH_VARARGS}, - {"frozenset_new", frozenset_new, METH_VARARGS}, - - {"set_size", set_size, METH_O}, {"set_get_size", set_get_size, METH_O}, - {"set_contains", set_contains, METH_VARARGS}, - {"set_add", set_add, METH_VARARGS}, - {"set_discard", set_discard, METH_VARARGS}, - {"set_pop", set_pop, METH_O}, - {"set_clear", set_clear, METH_O}, - - {"test_frozenset_add_in_capi", test_frozenset_add_in_capi, METH_NOARGS}, {NULL}, }; diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index 49bf6a3ea39613..a25b0aa2de7c99 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -35,9 +35,15 @@ PyInit__testlimitedcapi(void) if (_PyTestCapi_Init_HeaptypeRelative(mod) < 0) { return NULL; } + if (_PyTestCapi_Init_List(mod) < 0) { + return NULL; + } if (_PyTestCapi_Init_PyOS(mod) < 0) { return NULL; } + if (_PyTestCapi_Init_Set(mod) < 0) { + return NULL; + } if (_PyTestCapi_Init_Sys(mod) < 0) { return NULL; } diff --git a/Modules/_testlimitedcapi/list.c b/Modules/_testlimitedcapi/list.c new file mode 100644 index 00000000000000..0917900da390ce --- /dev/null +++ b/Modules/_testlimitedcapi/list.c @@ -0,0 +1,169 @@ +#include "parts.h" +#include "util.h" + +static PyObject * +list_check(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyList_Check(obj)); +} + +static PyObject * +list_check_exact(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyList_CheckExact(obj)); +} + +static PyObject * +list_new(PyObject* Py_UNUSED(module), PyObject *obj) +{ + return PyList_New(PyLong_AsSsize_t(obj)); +} + +static PyObject * +list_size(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyList_Size(obj)); +} + +static PyObject * +list_getitem(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return Py_XNewRef(PyList_GetItem(obj, i)); +} + +static PyObject * +list_get_item_ref(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return PyList_GetItemRef(obj, i); +} + +static PyObject * +list_setitem(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "OnO", &obj, &i, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_SetItem(obj, i, Py_XNewRef(value))); + +} + +static PyObject * +list_insert(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + Py_ssize_t where; + if (!PyArg_ParseTuple(args, "OnO", &obj, &where, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_Insert(obj, where, Py_XNewRef(value))); + +} + +static PyObject * +list_append(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + if (!PyArg_ParseTuple(args, "OO", &obj, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_Append(obj, value)); +} + +static PyObject * +list_getslice(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t ilow, ihigh; + if (!PyArg_ParseTuple(args, "Onn", &obj, &ilow, &ihigh)) { + return NULL; + } + NULLABLE(obj); + return PyList_GetSlice(obj, ilow, ihigh); + +} + +static PyObject * +list_setslice(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + Py_ssize_t ilow, ihigh; + if (!PyArg_ParseTuple(args, "OnnO", &obj, &ilow, &ihigh, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_SetSlice(obj, ilow, ihigh, value)); +} + +static PyObject * +list_sort(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyList_Sort(obj)); +} + +static PyObject * +list_reverse(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyList_Reverse(obj)); +} + +static PyObject * +list_astuple(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyList_AsTuple(obj); +} + + +static PyMethodDef test_methods[] = { + {"list_check", list_check, METH_O}, + {"list_check_exact", list_check_exact, METH_O}, + {"list_new", list_new, METH_O}, + {"list_size", list_size, METH_O}, + {"list_getitem", list_getitem, METH_VARARGS}, + {"list_get_item_ref", list_get_item_ref, METH_VARARGS}, + {"list_setitem", list_setitem, METH_VARARGS}, + {"list_insert", list_insert, METH_VARARGS}, + {"list_append", list_append, METH_VARARGS}, + {"list_getslice", list_getslice, METH_VARARGS}, + {"list_setslice", list_setslice, METH_VARARGS}, + {"list_sort", list_sort, METH_O}, + {"list_reverse", list_reverse, METH_O}, + {"list_astuple", list_astuple, METH_O}, + {NULL}, +}; + +int +_PyTestCapi_Init_List(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index 9bc52413382eb5..bb867eccbfe5c3 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -25,7 +25,9 @@ int _PyTestCapi_Init_ByteArray(PyObject *module); int _PyTestCapi_Init_Bytes(PyObject *module); int _PyTestCapi_Init_HeaptypeRelative(PyObject *module); +int _PyTestCapi_Init_List(PyObject *module); int _PyTestCapi_Init_PyOS(PyObject *module); +int _PyTestCapi_Init_Set(PyObject *module); int _PyTestCapi_Init_Sys(PyObject *module); int _PyTestCapi_Init_VectorcallLimited(PyObject *module); diff --git a/Modules/_testlimitedcapi/set.c b/Modules/_testlimitedcapi/set.c new file mode 100644 index 00000000000000..471b515f874473 --- /dev/null +++ b/Modules/_testlimitedcapi/set.c @@ -0,0 +1,189 @@ +#include "parts.h" +#include "util.h" + +static PyObject * +set_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PySet_Check(obj)); +} + +static PyObject * +set_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PySet_CheckExact(obj)); +} + +static PyObject * +frozenset_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyFrozenSet_Check(obj)); +} + +static PyObject * +frozenset_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyFrozenSet_CheckExact(obj)); +} + +static PyObject * +anyset_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyAnySet_Check(obj)); +} + +static PyObject * +anyset_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyAnySet_CheckExact(obj)); +} + +static PyObject * +set_new(PyObject *self, PyObject *args) +{ + PyObject *iterable = NULL; + if (!PyArg_ParseTuple(args, "|O", &iterable)) { + return NULL; + } + return PySet_New(iterable); +} + +static PyObject * +frozenset_new(PyObject *self, PyObject *args) +{ + PyObject *iterable = NULL; + if (!PyArg_ParseTuple(args, "|O", &iterable)) { + return NULL; + } + return PyFrozenSet_New(iterable); +} + +static PyObject * +set_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySet_Size(obj)); +} + +static PyObject * +set_contains(PyObject *self, PyObject *args) +{ + PyObject *obj, *item; + if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(item); + RETURN_INT(PySet_Contains(obj, item)); +} + +static PyObject * +set_add(PyObject *self, PyObject *args) +{ + PyObject *obj, *item; + if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(item); + RETURN_INT(PySet_Add(obj, item)); +} + +static PyObject * +set_discard(PyObject *self, PyObject *args) +{ + PyObject *obj, *item; + if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(item); + RETURN_INT(PySet_Discard(obj, item)); +} + +static PyObject * +set_pop(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySet_Pop(obj); +} + +static PyObject * +set_clear(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PySet_Clear(obj)); +} + +static PyObject * +test_frozenset_add_in_capi(PyObject *self, PyObject *Py_UNUSED(obj)) +{ + // Test that `frozenset` can be used with `PySet_Add`, + // when frozenset is just created in CAPI. + PyObject *fs = PyFrozenSet_New(NULL); + if (fs == NULL) { + return NULL; + } + PyObject *num = PyLong_FromLong(1); + if (num == NULL) { + goto error; + } + if (PySet_Add(fs, num) < 0) { + goto error; + } + int contains = PySet_Contains(fs, num); + if (contains < 0) { + goto error; + } + else if (contains == 0) { + goto unexpected; + } + Py_DECREF(fs); + Py_DECREF(num); + Py_RETURN_NONE; + +unexpected: + PyErr_SetString(PyExc_ValueError, "set does not contain expected value"); +error: + Py_DECREF(fs); + Py_XDECREF(num); + return NULL; +} + +static PyMethodDef test_methods[] = { + {"set_check", set_check, METH_O}, + {"set_checkexact", set_checkexact, METH_O}, + {"frozenset_check", frozenset_check, METH_O}, + {"frozenset_checkexact", frozenset_checkexact, METH_O}, + {"anyset_check", anyset_check, METH_O}, + {"anyset_checkexact", anyset_checkexact, METH_O}, + + {"set_new", set_new, METH_VARARGS}, + {"frozenset_new", frozenset_new, METH_VARARGS}, + + {"set_size", set_size, METH_O}, + {"set_contains", set_contains, METH_VARARGS}, + {"set_add", set_add, METH_VARARGS}, + {"set_discard", set_discard, METH_VARARGS}, + {"set_pop", set_pop, METH_O}, + {"set_clear", set_clear, METH_O}, + + {"test_frozenset_add_in_capi", test_frozenset_add_in_capi, METH_NOARGS}, + + {NULL}, +}; + +int +_PyTestCapi_Init_Set(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index 1afeacaa93396e..dc2ccca6bad963 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -97,7 +97,9 @@ + + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index b3eeb86185c372..33616be32c09b2 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -12,7 +12,9 @@ + + From 165cb4578c3cbd4d21faf1050193c297662fd031 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 18 Mar 2024 17:05:43 -0400 Subject: [PATCH 158/158] gh-116941: Fix pyatomic_std.h syntax errors (#116967) --- Include/cpython/pyatomic_std.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index f3970a45df24f9..ef34bb0b77dfe5 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -916,6 +916,7 @@ _Py_atomic_load_uint32_acquire(const uint32_t *obj) { _Py_USING_STD; return atomic_load_explicit((const _Atomic(uint32_t)*)obj, + memory_order_acquire); } static inline Py_ssize_t @@ -923,6 +924,7 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) { _Py_USING_STD; return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj, + memory_order_acquire); }

-

{% trans %}Reporting bugs{% endtrans %}

+

{% trans %}Reporting issues{% endtrans %}

{% trans %}Contributing to Docs{% endtrans %}

{% trans %}About the documentation{% endtrans %}

 -

{% trans %}History and License of Python{% endtrans %}

+

{% trans %}History and license of Python{% endtrans %}

{% trans %}Copyright{% endtrans %}

{% trans %}Download the documentation{% endtrans %}