diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index d3e842d9f31d01..7490dd947e1504 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -5,7 +5,7 @@ jobs: displayName: Pre-build checks pool: - vmImage: ubuntu-22.04 + vmImage: ubuntu-24.04 steps: - template: ./prebuild-checks.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9024639e25fcdf..669844854b2fe5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -16,6 +16,9 @@ configure* @erlend-aasland @corona10 Makefile.pre.in @erlend-aasland Modules/Setup* @erlend-aasland +# argparse +**/*argparse* @savannahostrowski + # asyncio **/*asyncio* @1st1 @asvetlov @kumaraditya303 @willingc @@ -23,7 +26,7 @@ Modules/Setup* @erlend-aasland **/*context* @1st1 **/*genobject* @markshannon **/*hamt* @1st1 -**/*jit* @brandtbucher +**/*jit* @brandtbucher @savannahostrowski Objects/set* @rhettinger Objects/dict* @methane @markshannon Objects/typevarobject.c @JelleZijlstra @@ -88,7 +91,6 @@ Objects/exceptions.c @iritkatriel **/sha* @gpshead @tiran Modules/md5* @gpshead @tiran **/*blake* @gpshead @tiran -Modules/_blake2/** @gpshead @tiran Modules/_hacl/** @gpshead # logging diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 083b07156674df..c854c13e12f922 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,7 +46,7 @@ jobs: # reproducible: to get the same tools versions (autoconf, aclocal, ...) runs-on: ubuntu-24.04 container: - image: ghcr.io/python/autoconf:2024.10.16.11360930377 + image: ghcr.io/python/autoconf:2024.11.11.11786316759 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' @@ -76,7 +76,7 @@ jobs: # Check for changes in regenerated files if test -n "$changes"; then echo "Generated files not up to date." - echo "Perhaps you forgot to run make regen-all or build.bat --regen. ;)" + echo "Perhaps you forgot to run make regen-configure ;)" echo "configure files must be regenerated with a specific version of autoconf." echo "$changes" echo "" @@ -88,7 +88,7 @@ jobs: name: 'Check if generated files are up to date' # Don't use ubuntu-latest but a specific version to make the job # reproducible: to get the same tools versions (autoconf, aclocal, ...) - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' @@ -237,7 +237,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04] + os: [ubuntu-24.04] openssl_ver: [3.0.15, 3.1.7, 3.2.3, 3.3.2] env: OPENSSL_VER: ${{ matrix.openssl_ver }} @@ -297,7 +297,7 @@ jobs: test_hypothesis: name: "Hypothesis tests on Ubuntu" - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' @@ -417,7 +417,7 @@ jobs: if: needs.check_source.outputs.run_tests == 'true' strategy: matrix: - os: [ubuntu-22.04] + os: [ubuntu-24.04] env: OPENSSL_VER: 3.0.15 PYTHONSTRICTEXTENSIONBUILD: 1 diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 5fb599b232dfb5..35d5d59b762660 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -28,7 +28,7 @@ concurrency: jobs: interpreter: name: Interpreter (Debug) - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 90 steps: - uses: actions/checkout@v4 @@ -85,19 +85,19 @@ jobs: compiler: clang - target: x86_64-unknown-linux-gnu/gcc architecture: x86_64 - runner: ubuntu-latest + runner: ubuntu-22.04 compiler: gcc - target: x86_64-unknown-linux-gnu/clang architecture: x86_64 - runner: ubuntu-latest + runner: ubuntu-22.04 compiler: clang - target: aarch64-unknown-linux-gnu/gcc architecture: aarch64 - runner: ubuntu-latest + runner: ubuntu-22.04 compiler: gcc - target: aarch64-unknown-linux-gnu/clang architecture: aarch64 - runner: ubuntu-latest + runner: ubuntu-22.04 compiler: clang env: CC: ${{ matrix.compiler }} @@ -131,8 +131,8 @@ jobs: brew update find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} - SDKROOT="$(xcrun --show-sdk-path)" \ - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + export SDKROOT="$(xcrun --show-sdk-path)" + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 @@ -169,7 +169,7 @@ jobs: jit-with-disabled-gil: name: Free-Threaded (Debug) needs: interpreter - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: llvm: diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index bfc5a0874281bd..d5538cd9367ec6 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -13,7 +13,6 @@ apt-get -yq install \ libgdbm-dev \ libgdbm-compat-dev \ liblzma-dev \ - libmpdec-dev \ libncurses5-dev \ libreadline6-dev \ libsqlite3-dev \ diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml index 65072efa8e9023..7a4d81f0bdcad1 100644 --- a/.github/workflows/reusable-tsan.yml +++ b/.github/workflows/reusable-tsan.yml @@ -21,7 +21,7 @@ on: jobs: build_tsan_reusable: name: 'Thread sanitizer' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index f0ca6a9e7ed793..ec39025504efd1 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04] + os: [ubuntu-24.04] env: FORCE_COLOR: 1 OPENSSL_VER: 3.0.15 diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index abc617a317cc0f..3f96c888e2dd30 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -11,13 +11,13 @@ jobs: build_wasi_reusable: name: 'build and test' timeout-minutes: 60 - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 env: WASMTIME_VERSION: 22.0.0 WASI_SDK_VERSION: 24 WASI_SDK_PATH: /opt/wasi-sdk CROSS_BUILD_PYTHON: cross-build/build - CROSS_BUILD_WASI: cross-build/wasm32-wasi + CROSS_BUILD_WASI: cross-build/wasm32-wasip1 steps: - uses: actions/checkout@v4 # No problem resolver registered as one doesn't currently exist for Clang. @@ -31,7 +31,7 @@ jobs: with: path: ${{ env.WASI_SDK_PATH }} key: ${{ runner.os }}-wasi-sdk-${{ env.WASI_SDK_VERSION }} - - name: "Install WASI SDK" + - name: "Install WASI SDK" # Hard-coded to x64. if: steps.cache-wasi-sdk.outputs.cache-hit != 'true' run: | mkdir ${{ env.WASI_SDK_PATH }} && \ diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 0d53b18ea87d5e..b7e0f22b52c57e 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -15,10 +15,8 @@ Allocating Objects on the Heap .. c:function:: PyObject* PyObject_Init(PyObject *op, PyTypeObject *type) Initialize a newly allocated object *op* with its type and initial - reference. Returns the initialized object. If *type* indicates that the - object participates in the cyclic garbage detector, it is added to the - detector's set of observed objects. Other fields of the object are not - affected. + reference. Returns the initialized object. Other fields of the object are + not affected. .. c:function:: PyVarObject* PyObject_InitVar(PyVarObject *op, PyTypeObject *type, Py_ssize_t size) diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index 4aaf3905e81c8a..c92ef4c653a675 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -105,7 +105,7 @@ The following functions provide locale-independent string to number conversions. If ``s`` represents a value that is too large to store in a float (for example, ``"1e500"`` is such a string on many platforms) then - if ``overflow_exception`` is ``NULL`` return ``Py_HUGE_VAL`` (with + if ``overflow_exception`` is ``NULL`` return ``Py_INFINITY`` (with an appropriate sign) and don't set any exception. Otherwise, ``overflow_exception`` must point to a Python exception object; raise that exception and return ``-1.0``. In both cases, set diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 9ff3e5265004a1..32bb451b08d413 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -582,6 +582,39 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.14 +.. c:function:: int PyLong_IsPositive(PyObject *obj) + + Check if the integer object *obj* is positive (``obj > 0``). + + If *obj* is an instance of :c:type:`PyLongObject` or its subtype, + return ``1`` when it's positive and ``0`` otherwise. Else set an + exception and return ``-1``. + + .. versionadded:: next + + +.. c:function:: int PyLong_IsNegative(PyObject *obj) + + Check if the integer object *obj* is negative (``obj < 0``). + + If *obj* is an instance of :c:type:`PyLongObject` or its subtype, + return ``1`` when it's negative and ``0`` otherwise. Else set an + exception and return ``-1``. + + .. versionadded:: next + + +.. c:function:: int PyLong_IsZero(PyObject *obj) + + Check if the integer object *obj* is zero. + + If *obj* is an instance of :c:type:`PyLongObject` or its subtype, + return ``1`` when it's zero and ``0`` otherwise. Else set an + exception and return ``-1``. + + .. versionadded:: next + + .. c:function:: PyObject* PyLong_GetInfo(void) On success, return a read only :term:`named tuple`, that holds diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index b9085ad3ec361d..61218a1bf6f171 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -13,11 +13,12 @@ binary mode. Numeric values are stored with the least significant byte first. -The module supports two versions of the data format: version 0 is the -historical version, version 1 shares interned strings in the file, and upon -unmarshalling. Version 2 uses a binary format for floating-point numbers. -``Py_MARSHAL_VERSION`` indicates the current file format (currently 2). +The module supports several versions of the data format; see +the :py:mod:`Python module documentation ` for details. +.. c:macro:: Py_MARSHAL_VERSION + + The current format version. See :py:data:`marshal.version`. .. c:function:: void PyMarshal_WriteLongToFile(long value, FILE *file, int version) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 630114a4339110..1e1cf6e6bfd7e9 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -575,3 +575,27 @@ Object Protocol has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set. .. versionadded:: 3.13 + +.. c:function:: int PyUnstable_Object_EnableDeferredRefcount(PyObject *obj) + + Enable `deferred reference counting `_ on *obj*, + if supported by the runtime. In the :term:`free-threaded ` build, + this allows the interpreter to avoid reference count adjustments to *obj*, + which may improve multi-threaded performance. The tradeoff is + that *obj* will only be deallocated by the tracing garbage collector. + + This function returns ``1`` if deferred reference counting is enabled on *obj* + (including when it was enabled before the call), + and ``0`` if deferred reference counting is not supported or if the hint was + ignored by the runtime. This function is thread-safe, and cannot fail. + + This function does nothing on builds with the :term:`GIL` enabled, which do + not support deferred reference counting. This also does nothing if *obj* is not + an object tracked by the garbage collector (see :func:`gc.is_tracked` and + :c:func:`PyObject_GC_IsTracked`). + + This function is intended to be used soon after *obj* is created, + by the code that creates it. + + .. versionadded:: next + diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 8a185486fe44f1..ba58cc1c26c70b 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -2230,7 +2230,7 @@ This is done by filling a :c:type:`PyType_Spec` structure and calling .. _number-structs: Number Object Structures -======================== +------------------------ .. sectionauthor:: Amaury Forgeot d'Arc @@ -2344,7 +2344,7 @@ Number Object Structures .. _mapping-structs: Mapping Object Structures -========================= +------------------------- .. sectionauthor:: Amaury Forgeot d'Arc @@ -2381,7 +2381,7 @@ Mapping Object Structures .. _sequence-structs: Sequence Object Structures -========================== +-------------------------- .. sectionauthor:: Amaury Forgeot d'Arc @@ -2461,7 +2461,7 @@ Sequence Object Structures .. _buffer-structs: Buffer Object Structures -======================== +------------------------ .. sectionauthor:: Greg J. Stein .. sectionauthor:: Benjamin Peterson @@ -2556,7 +2556,7 @@ Buffer Object Structures Async Object Structures -======================= +----------------------- .. sectionauthor:: Yury Selivanov @@ -2624,7 +2624,7 @@ Async Object Structures .. _slot-typedefs: Slot Type typedefs -================== +------------------ .. c:type:: PyObject *(*allocfunc)(PyTypeObject *cls, Py_ssize_t nitems) @@ -2733,7 +2733,7 @@ Slot Type typedefs .. _typedef-examples: Examples -======== +-------- The following are simple examples of Python type definitions. They include common usage you may encounter. Some demonstrate tricky corner diff --git a/Doc/conf.py b/Doc/conf.py index 73d7d5db26ff7b..738c9901eef06f 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -67,10 +67,7 @@ # General substitutions. project = 'Python' -if sphinx.version_info[:2] >= (8, 1): - copyright = "2001-%Y, Python Software Foundation" -else: - copyright = f"2001-{time.strftime('%Y')}, Python Software Foundation" +copyright = "2001 Python Software Foundation" # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. diff --git a/Doc/copyright.rst b/Doc/copyright.rst index 8629ed1fc38009..9210d5f50ed841 100644 --- a/Doc/copyright.rst +++ b/Doc/copyright.rst @@ -4,7 +4,7 @@ Copyright Python and this documentation is: -Copyright © 2001-2024 Python Software Foundation. All rights reserved. +Copyright © 2001 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 65d48f8bea7de8..6bfcc191b2270b 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -1284,6 +1284,19 @@ PyLong_FromUnsignedLong:unsignedlong:v:: PyLong_FromVoidPtr:PyObject*::+1: PyLong_FromVoidPtr:void*:p:: +PyLong_IsPositive:int::: +PyLong_IsPositive:PyObject*:obj:0: + +PyLong_IsNegative:int::: +PyLong_IsNegative:PyObject*:obj:0: + +PyLong_IsZero:int::: +PyLong_IsZero:PyObject*:obj:0: + +PyLong_GetSign:int::: +PyLong_GetSign:PyObject*:v:0: +PyLong_GetSign:int*:sign:: + PyMapping_Check:int::: PyMapping_Check:PyObject*:o:0: diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst index b8791b8d6c387e..6159fa48848285 100644 --- a/Doc/deprecations/pending-removal-in-3.14.rst +++ b/Doc/deprecations/pending-removal-in-3.14.rst @@ -1,13 +1,6 @@ Pending removal in Python 3.14 ------------------------------ -* The import system: - - * Setting :attr:`~module.__loader__` on a module while - failing to set :attr:`__spec__.loader ` - is deprecated. In Python 3.14, :attr:`!__loader__` will cease to be set or - taken into consideration by the import system or the standard library. - * :mod:`argparse`: The *type*, *choices*, and *metavar* parameters of :class:`!argparse.BooleanOptionalAction` are deprecated and will be removed in 3.14. @@ -85,7 +78,7 @@ Pending removal in Python 3.14 :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is deprecated. -* :mod:`pkgutil`: :func:`~pkgutil.find_loader` and :func:`~pkgutil.get_loader` +* :mod:`pkgutil`: :func:`!pkgutil.find_loader` and :func:!pkgutil.get_loader` now raise :exc:`DeprecationWarning`; use :func:`importlib.util.find_spec` instead. (Contributed by Nikita Sobolev in :gh:`97850`.) @@ -103,16 +96,6 @@ Pending removal in Python 3.14 if :ref:`named placeholders ` are used and *parameters* is a sequence instead of a :class:`dict`. - * date and datetime adapter, date and timestamp converter: - see the :mod:`sqlite3` documentation for suggested replacement recipes. - -* :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was - deprecated in :pep:`626` - since 3.10 and was planned to be removed in 3.12, - but it only got a proper :exc:`DeprecationWarning` in 3.12. - May be removed in 3.14. - (Contributed by Nikita Sobolev in :gh:`101866`.) - * :mod:`typing`: :class:`!typing.ByteString`, deprecated since Python 3.9, now causes a :exc:`DeprecationWarning` to be emitted when it is used. diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst index 17029b8d4773bd..3b03e1f49e6754 100644 --- a/Doc/deprecations/pending-removal-in-3.15.rst +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -59,6 +59,15 @@ Pending removal in Python 3.15 but the C version allows any number of positional or keyword arguments, ignoring every argument. +* :mod:`types`: + + * :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was + deprecated in :pep:`626` + since 3.10 and was planned to be removed in 3.12, + but it only got a proper :exc:`DeprecationWarning` in 3.12. + May be removed in 3.15. + (Contributed by Nikita Sobolev in :gh:`101866`.) + * :mod:`typing`: * The undocumented keyword argument syntax for creating diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index fac500d34742ca..6f6954b783a1ae 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -1,6 +1,13 @@ Pending removal in Python 3.16 ------------------------------ +* The import system: + + * Setting :attr:`~module.__loader__` on a module while + failing to set :attr:`__spec__.loader ` + is deprecated. In Python 3.16, :attr:`!__loader__` will cease to be set or + taken into consideration by the import system or the standard library. + * :mod:`array`: * The ``'u'`` format code (:c:type:`wchar_t`) diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 321ec0c0f73871..3cd2f1d96a7b34 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -1267,11 +1267,8 @@ to adapt in your own applications. You could also write your own handler which uses the :class:`~multiprocessing.Lock` class from the :mod:`multiprocessing` module to serialize access to the -file from your processes. The existing :class:`FileHandler` and subclasses do -not make use of :mod:`multiprocessing` at present, though they may do so in the -future. Note that at present, the :mod:`multiprocessing` module does not provide -working lock functionality on all platforms (see -https://bugs.python.org/issue3770). +file from your processes. The stdlib :class:`FileHandler` and subclasses do +not make use of :mod:`multiprocessing`. .. currentmodule:: logging.handlers diff --git a/Doc/library/aifc.rst b/Doc/library/aifc.rst new file mode 100644 index 00000000000000..a756d679036ecb --- /dev/null +++ b/Doc/library/aifc.rst @@ -0,0 +1,15 @@ +:mod:`!aifc` --- Read and write AIFF and AIFC files +=================================================== + +.. module:: aifc + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!aifc` module was +`Python 3.12 `_. diff --git a/Doc/library/asynchat.rst b/Doc/library/asynchat.rst new file mode 100644 index 00000000000000..5e5c3a99fe66f1 --- /dev/null +++ b/Doc/library/asynchat.rst @@ -0,0 +1,17 @@ +:mod:`!asynchat` --- Asynchronous socket command/response handler +================================================================= + +.. module:: asynchat + :synopsis: Removed in 3.12. + :deprecated: + +.. deprecated-removed:: 3.6 3.12 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.12 ` after +being deprecated in Python 3.6. The removal was decided in :pep:`594`. + +Applications should use the :mod:`asyncio` module instead. + +The last version of Python that provided the :mod:`!asynchat` module was +`Python 3.11 `_. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 14fd153f640f05..9f1aec148f8750 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -59,9 +59,8 @@ an event loop: instead of using these lower level functions to manually create and close an event loop. - .. deprecated:: 3.12 - Deprecation warning is emitted if there is no current event loop. - In some future Python release this will become an error. + .. versionchanged:: 3.14 + Raises a :exc:`RuntimeError` if there is no current event loop. .. function:: set_event_loop(loop) @@ -1798,7 +1797,7 @@ By default asyncio is configured to use :class:`EventLoop`. .. seealso:: `MSDN documentation on I/O Completion Ports - `_. + `_. .. class:: EventLoop diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index 837ccc6606786e..09b75762ff0272 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -97,11 +97,9 @@ asyncio ships with the following built-in policies: On Windows, :class:`ProactorEventLoop` is now used by default. - .. deprecated:: 3.12 - The :meth:`get_event_loop` method of the default asyncio policy now emits - a :exc:`DeprecationWarning` if there is no current event loop set and it - decides to create one. - In some future Python release this will become an error. + .. versionchanged:: 3.14 + The :meth:`get_event_loop` method of the default asyncio policy now + raises a :exc:`RuntimeError` if there is no set event loop. .. class:: WindowsSelectorEventLoopPolicy diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 3fdc79b3c6896c..48f2890c5eef8c 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -92,7 +92,8 @@ and work with streams: family=socket.AF_UNSPEC, \ flags=socket.AI_PASSIVE, sock=None, \ backlog=100, ssl=None, reuse_address=None, \ - reuse_port=None, ssl_handshake_timeout=None, \ + reuse_port=None, keep_alive=None, \ + ssl_handshake_timeout=None, \ ssl_shutdown_timeout=None, start_serving=True) Start a socket server. @@ -128,6 +129,9 @@ and work with streams: .. versionchanged:: 3.11 Added the *ssl_shutdown_timeout* parameter. + .. versionchanged:: 3.13 + Added the *keep_alive* parameter. + .. rubric:: Unix Sockets diff --git a/Doc/library/asyncore.rst b/Doc/library/asyncore.rst new file mode 100644 index 00000000000000..22c9881c3cca36 --- /dev/null +++ b/Doc/library/asyncore.rst @@ -0,0 +1,17 @@ +:mod:`!asyncore` --- Asynchronous socket handler +================================================ + +.. module:: asyncore + :synopsis: Removed in 3.12. + :deprecated: + +.. deprecated-removed:: 3.6 3.12 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.12 ` after +being deprecated in Python 3.6. The removal was decided in :pep:`594`. + +Applications should use the :mod:`asyncio` module instead. + +The last version of Python that provided the :mod:`!asyncore` module was +`Python 3.11 `_. diff --git a/Doc/library/audioop.rst b/Doc/library/audioop.rst new file mode 100644 index 00000000000000..3bc580b0bd3433 --- /dev/null +++ b/Doc/library/audioop.rst @@ -0,0 +1,15 @@ +:mod:`!audioop` --- Manipulate raw audio data +============================================= + +.. module:: audioop + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!audioop` module was +`Python 3.12 `_. diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst new file mode 100644 index 00000000000000..f9108fa954a906 --- /dev/null +++ b/Doc/library/cgi.rst @@ -0,0 +1,19 @@ +:mod:`!cgi` --- Common Gateway Interface support +================================================ + +.. module:: cgi + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +A fork of the module on PyPI can be used instead: :pypi:`legacy-cgi`. +This is a copy of the cgi module, no longer maintained or supported by the core +Python team. + +The last version of Python that provided the :mod:`!cgi` module was +`Python 3.12 `_. diff --git a/Doc/library/cgitb.rst b/Doc/library/cgitb.rst new file mode 100644 index 00000000000000..fc646aa4c48acd --- /dev/null +++ b/Doc/library/cgitb.rst @@ -0,0 +1,19 @@ +:mod:`!cgitb` --- Traceback manager for CGI scripts +=================================================== + +.. module:: cgitb + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +A fork of the module on PyPI can now be used instead: :pypi:`legacy-cgi`. +This is a copy of the cgi module, no longer maintained or supported by the core +Python team. + +The last version of Python that provided the :mod:`!cgitb` module was +`Python 3.12 `_. diff --git a/Doc/library/chunk.rst b/Doc/library/chunk.rst new file mode 100644 index 00000000000000..9950a0ea70649a --- /dev/null +++ b/Doc/library/chunk.rst @@ -0,0 +1,15 @@ +:mod:`!chunk` --- Read IFF chunked data +======================================= + +.. module:: chunk + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!chunk` module was +`Python 3.12 `_. diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index 98519a11c95eb6..e7c027dd4d0c22 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -221,19 +221,21 @@ Classification functions ``False`` otherwise. Whether or not two values are considered close is determined according to - given absolute and relative tolerances. + given absolute and relative tolerances. If no errors occur, the result will + be: ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``. *rel_tol* is the relative tolerance -- it is the maximum allowed difference between *a* and *b*, relative to the larger absolute value of *a* or *b*. For example, to set a tolerance of 5%, pass ``rel_tol=0.05``. The default tolerance is ``1e-09``, which assures that the two values are the same - within about 9 decimal digits. *rel_tol* must be greater than zero. - - *abs_tol* is the minimum absolute tolerance -- useful for comparisons near - zero. *abs_tol* must be at least zero. - - If no errors occur, the result will be: - ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``. + within about 9 decimal digits. *rel_tol* must be nonnegative and less + than ``1.0``. + + *abs_tol* is the absolute tolerance; it defaults to ``0.0`` and it must be + nonnegative. When comparing ``x`` to ``0.0``, ``isclose(x, 0)`` is computed + as ``abs(x) <= rel_tol * abs(x)``, which is ``False`` for any ``x`` and + rel_tol less than ``1.0``. So add an appropriate positive abs_tol argument + to the call. The IEEE 754 special values of ``NaN``, ``inf``, and ``-inf`` will be handled according to IEEE rules. Specifically, ``NaN`` is not considered diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 2cfd8a1eaee806..a129a26190ba99 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1042,6 +1042,10 @@ is meant to be exhaustive. Notice that spelling alternatives that only differ in case or use a hyphen instead of an underscore are also valid aliases; therefore, e.g. ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec. +On Windows, ``cpXXX`` codecs are available for all code pages. +But only codecs listed in the following table are guarantead to exist on +other platforms. + .. impl-detail:: Some common encodings can bypass the codecs lookup machinery to @@ -1307,6 +1311,9 @@ particular, the following variants typically exist: .. versionchanged:: 3.8 ``cp65001`` is now an alias to ``utf_8``. +.. versionchanged:: 3.14 + On Windows, ``cpXXX`` codecs are now available for all code pages. + Python Specific Encodings ------------------------- diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 3aad6f7b5d2d20..ac0f3fca3d72fd 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -942,7 +942,13 @@ interpolation if an option used is not defined elsewhere. :: ConfigParser Objects -------------------- -.. class:: ConfigParser(defaults=None, dict_type=dict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=configparser.DEFAULTSECT, interpolation=BasicInterpolation(), converters={}) +.. class:: ConfigParser(defaults=None, dict_type=dict, allow_no_value=False, *, \ + delimiters=('=', ':'), comment_prefixes=('#', ';'), \ + inline_comment_prefixes=None, strict=True, \ + empty_lines_in_values=True, \ + default_section=configparser.DEFAULTSECT, \ + interpolation=BasicInterpolation(), converters={}, \ + allow_unnamed_section=False) The main configuration parser. When *defaults* is given, it is initialized into the dictionary of intrinsic defaults. When *dict_type* is given, it @@ -990,6 +996,10 @@ ConfigParser Objects converter gets its own corresponding :meth:`!get*` method on the parser object and section proxies. + When *allow_unnamed_section* is ``True`` (default: ``False``), + the first section name can be omitted. See the + `"Unnamed Sections" section <#unnamed-sections>`_. + It is possible to read several configurations into a single :class:`ConfigParser`, where the most recently added configuration has the highest priority. Any conflicting keys are taken from the more recent @@ -1039,6 +1049,9 @@ ConfigParser Objects Raise a :exc:`MultilineContinuationError` when *allow_no_value* is ``True``, and a key without a value is continued with an indented line. + .. versionchanged:: 3.13 + The *allow_unnamed_section* argument was added. + .. method:: defaults() Return a dictionary containing the instance-wide defaults. @@ -1295,18 +1308,30 @@ RawConfigParser Objects comment_prefixes=('#', ';'), \ inline_comment_prefixes=None, strict=True, \ empty_lines_in_values=True, \ - default_section=configparser.DEFAULTSECT[, \ - interpolation]) + default_section=configparser.DEFAULTSECT, \ + interpolation=BasicInterpolation(), converters={}, \ + allow_unnamed_section=False) Legacy variant of the :class:`ConfigParser`. It has interpolation disabled by default and allows for non-string section names, option names, and values via its unsafe ``add_section`` and ``set`` methods, as well as the legacy ``defaults=`` keyword argument handling. + .. versionchanged:: 3.2 + *allow_no_value*, *delimiters*, *comment_prefixes*, *strict*, + *empty_lines_in_values*, *default_section* and *interpolation* were + added. + + .. versionchanged:: 3.5 + The *converters* argument was added. + .. versionchanged:: 3.8 The default *dict_type* is :class:`dict`, since it now preserves insertion order. + .. versionchanged:: 3.13 + The *allow_unnamed_section* argument was added. + .. note:: Consider using :class:`ConfigParser` instead which checks types of the values to be stored internally. If you don't want interpolation, you diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst new file mode 100644 index 00000000000000..9ff37196ccf69f --- /dev/null +++ b/Doc/library/crypt.rst @@ -0,0 +1,20 @@ +:mod:`!crypt` --- Function to check Unix passwords +================================================== + +.. module:: crypt + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +Applications can use the :mod:`hashlib` module from the standard library. +Other possible replacements are third-party libraries from PyPI: +:pypi:`legacycrypt`, :pypi:`bcrypt`, :pypi:`argon2-cffi`, or :pypi:`passlib`. +These are not supported or maintained by the Python core team. + +The last version of Python that provided the :mod:`!crypt` module was +`Python 3.12 `_. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 6c12d1b5e0dcea..e2926f2440af6d 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1395,6 +1395,13 @@ iterations of the loop. This opcode is now only used in situations where the local variable is guaranteed to be initialized. It cannot raise :exc:`UnboundLocalError`. +.. opcode:: LOAD_FAST_LOAD_FAST (var_nums) + + Pushes references to ``co_varnames[var_nums >> 4]`` and + ``co_varnames[var_nums & 15]`` onto the stack. + + .. versionadded:: 3.13 + .. opcode:: LOAD_FAST_CHECK (var_num) Pushes a reference to the local ``co_varnames[var_num]`` onto the stack, @@ -1415,6 +1422,20 @@ iterations of the loop. Stores ``STACK.pop()`` into the local ``co_varnames[var_num]``. +.. opcode:: STORE_FAST_STORE_FAST (var_nums) + + Stores ``STACK[-1]`` into ``co_varnames[var_nums >> 4]`` + and ``STACK[-2]`` into ``co_varnames[var_nums & 15]``. + + .. versionadded:: 3.13 + +.. opcode:: STORE_FAST_LOAD_FAST (var_nums) + + Stores ``STACK.pop()`` into the local ``co_varnames[var_nums >> 4]`` + and pushes a reference to the local ``co_varnames[var_nums & 15]`` + onto the stack. + + .. versionadded:: 3.13 .. opcode:: DELETE_FAST (var_num) @@ -1562,7 +1583,7 @@ iterations of the loop. .. opcode:: MAKE_FUNCTION - Pushes a new function object on the stack built from the code object at ``STACK[1]``. + Pushes a new function object on the stack built from the code object at ``STACK[-1]``. .. versionchanged:: 3.10 Flag value ``0x04`` is a tuple of strings instead of dictionary @@ -1647,7 +1668,7 @@ iterations of the loop. .. versionadded:: 3.13 -.. opcode:: FORMAT_SPEC +.. opcode:: FORMAT_WITH_SPEC Formats the given value with the given format spec:: diff --git a/Doc/library/distutils.rst b/Doc/library/distutils.rst new file mode 100644 index 00000000000000..af63e035bf3c4a --- /dev/null +++ b/Doc/library/distutils.rst @@ -0,0 +1,17 @@ +:mod:`!distutils` --- Building and installing Python modules +============================================================ + +.. module:: distutils + :synopsis: Removed in 3.12. + :deprecated: + +.. deprecated-removed:: 3.10 3.12 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.12 ` after +being deprecated in Python 3.10. The removal was decided in :pep:`632`, +which has `migration advice +`_. + +The last version of Python that provided the :mod:`!distutils` module was +`Python 3.11 `_. diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 242b2436439903..16a9b0326e9f3d 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -44,7 +44,7 @@ using function-call syntax:: ... BLUE = 3 >>> # functional syntax - >>> Color = Enum('Color', ['RED', 'GREEN', 'BLUE']) + >>> Color = Enum('Color', [('RED', 1), ('GREEN', 2), ('BLUE', 3)]) Even though we can use :keyword:`class` syntax to create Enums, Enums are not normal Python classes. See diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 03fc41fa793977..a7549b9bce76e2 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1205,14 +1205,19 @@ are always available. They are listed here in alphabetical order. unchanged from previous versions. -.. function:: map(function, iterable, *iterables) +.. function:: map(function, iterable, /, *iterables, strict=False) Return an iterator that applies *function* to every item of *iterable*, yielding the results. If additional *iterables* arguments are passed, *function* must take that many arguments and is applied to the items from all iterables in parallel. With multiple iterables, the iterator stops when the - shortest iterable is exhausted. For cases where the function inputs are - already arranged into argument tuples, see :func:`itertools.starmap`\. + shortest iterable is exhausted. If *strict* is ``True`` and one of the + iterables is exhausted before the others, a :exc:`ValueError` is raised. For + cases where the function inputs are already arranged into argument tuples, + see :func:`itertools.starmap`. + + .. versionchanged:: 3.14 + Added the *strict* parameter. .. function:: max(iterable, *, key=None) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index e26a2226aa947a..a9aceee4170004 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -453,7 +453,7 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.4 -.. function:: reduce(function, iterable[, initial], /) +.. function:: reduce(function, iterable, /[, initial]) Apply *function* of two arguments cumulatively to the items of *iterable*, from left to right, so as to reduce the iterable to a single value. For example, @@ -468,7 +468,7 @@ The :mod:`functools` module defines the following functions: initial_missing = object() - def reduce(function, iterable, initial=initial_missing, /): + def reduce(function, iterable, /, initial=initial_missing): it = iter(iterable) if initial is initial_missing: value = next(it) @@ -481,6 +481,9 @@ The :mod:`functools` module defines the following functions: See :func:`itertools.accumulate` for an iterator that yields all intermediate values. + .. versionchanged:: next + *initial* is now supported as a keyword argument. + .. decorator:: singledispatch Transform a function into a :term:`single-dispatch >> import getopt >>> args = '-a -b -cfoo -d bar a1 a2'.split() >>> args @@ -109,6 +127,8 @@ An example using only Unix style options: Using long option names is equally easy: +.. doctest:: + >>> s = '--condition=foo --testing --output-file abc.def -x a1 a2' >>> args = s.split() >>> args @@ -120,7 +140,37 @@ Using long option names is equally easy: >>> args ['a1', 'a2'] -In a script, typical usage is something like this:: +Optional arguments should be specified explicitly: + +.. doctest:: + + >>> s = '-Con -C --color=off --color a1 a2' + >>> args = s.split() + >>> args + ['-Con', '-C', '--color=off', '--color', 'a1', 'a2'] + >>> optlist, args = getopt.getopt(args, 'C::', ['color=?']) + >>> optlist + [('-C', 'on'), ('-C', ''), ('--color', 'off'), ('--color', '')] + >>> args + ['a1', 'a2'] + +The order of options and non-option arguments can be preserved: + +.. doctest:: + + >>> s = 'a1 -x a2 a3 a4 --long a5 a6' + >>> args = s.split() + >>> args + ['a1', '-x', 'a2', 'a3', 'a4', '--long', 'a5', 'a6'] + >>> optlist, args = getopt.gnu_getopt(args, '-x:', ['long=']) + >>> optlist + [(None, ['a1']), ('-x', 'a2'), (None, ['a3', 'a4']), ('--long', 'a5')] + >>> args + ['a6'] + +In a script, typical usage is something like this: + +.. testcode:: import getopt, sys @@ -150,7 +200,9 @@ In a script, typical usage is something like this:: main() Note that an equivalent command line interface could be produced with less code -and more informative help and error messages by using the :mod:`argparse` module:: +and more informative help and error messages by using the :mod:`argparse` module: + +.. testcode:: import argparse diff --git a/Doc/library/imghdr.rst b/Doc/library/imghdr.rst new file mode 100644 index 00000000000000..56f26355f42558 --- /dev/null +++ b/Doc/library/imghdr.rst @@ -0,0 +1,19 @@ +:mod:`!imghdr` --- Determine the type of an image +================================================= + +.. module:: imghdr + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +Possible replacements are third-party libraries from PyPI: +:pypi:`filetype`, :pypi:`puremagic`, or :pypi:`python-magic`. +These are not supported or maintained by the Python core team. + +The last version of Python that provided the :mod:`!imghdr` module was +`Python 3.12 `_. diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst new file mode 100644 index 00000000000000..3dc4c568b1ae2f --- /dev/null +++ b/Doc/library/imp.rst @@ -0,0 +1,18 @@ +:mod:`!imp` --- Access the import internals +=========================================== + +.. module:: imp + :synopsis: Removed in 3.12. + :deprecated: + +.. deprecated-removed:: 3.4 3.12 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.12 ` after +being deprecated in Python 3.4. + +The :ref:`removal notice ` includes guidance for +migrating code from :mod:`!imp` to :mod:`importlib`. + +The last version of Python that provided the :mod:`!imp` module was +`Python 3.11 `_. diff --git a/Doc/library/index.rst b/Doc/library/index.rst index 0b348ae6f5c8c0..951fbcf13fbb13 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -75,4 +75,5 @@ the `Python Package Index `_. unix.rst cmdline.rst superseded.rst + removed.rst security_warnings.rst diff --git a/Doc/library/mailcap.rst b/Doc/library/mailcap.rst new file mode 100644 index 00000000000000..4467da146a5a05 --- /dev/null +++ b/Doc/library/mailcap.rst @@ -0,0 +1,15 @@ +:mod:`!mailcap` --- Mailcap file handling +========================================= + +.. module:: mailcap + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!mailcap` module was +`Python 3.12 `_. diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index 9e4606df0f774e..8b14ee449d4631 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -38,23 +38,39 @@ supports a substantially wider range of objects than marshal. maliciously constructed data. Never unmarshal data received from an untrusted or unauthenticated source. +There are functions that read/write files as well as functions operating on +bytes-like objects. + .. index:: object; code, code object Not all Python object types are supported; in general, only objects whose value is independent from a particular invocation of Python can be written and read by -this module. The following types are supported: booleans, integers, floating-point -numbers, complex numbers, strings, bytes, bytearrays, tuples, lists, sets, -frozensets, dictionaries, and code objects (if *allow_code* is true), -where it should be understood that -tuples, lists, sets, frozensets and dictionaries are only supported as long as -the values contained therein are themselves supported. The -singletons :const:`None`, :const:`Ellipsis` and :exc:`StopIteration` can also be -marshalled and unmarshalled. -For format *version* lower than 3, recursive lists, sets and dictionaries cannot -be written (see below). +this module. The following types are supported: + +* Numeric types: :class:`int`, :class:`bool`, :class:`float`, :class:`complex`. +* Strings (:class:`str`) and :class:`bytes`. + :term:`Bytes-like objects ` like :class:`bytearray` are + marshalled as :class:`!bytes`. +* Containers: :class:`tuple`, :class:`list`, :class:`set`, :class:`frozenset`, + and (since :data:`version` 5), :class:`slice`. + It should be understood that these are supported only if the values contained + therein are themselves supported. + Recursive containers are supported since :data:`version` 3. +* The singletons :const:`None`, :const:`Ellipsis` and :exc:`StopIteration`. +* :class:`code` objects, if *allow_code* is true. See note above about + version dependence. + +.. versionchanged:: 3.4 + + * Added format version 3, which supports marshalling recursive lists, sets + and dictionaries. + * Added format version 4, which supports efficient representations + of short strings. + +.. versionchanged:: next + + Added format version 5, which allows marshalling slices. -There are functions that read/write files as well as functions operating on -bytes-like objects. The module defines these functions: @@ -140,11 +156,24 @@ In addition, the following constants are defined: .. data:: version - Indicates the format that the module uses. Version 0 is the historical - format, version 1 shares interned strings and version 2 uses a binary format - for floating-point numbers. - Version 3 adds support for object instancing and recursion. - The current version is 4. + Indicates the format that the module uses. + Version 0 is the historical first version; subsequent versions + add new features. + Generally, a new version becomes the default when it is introduced. + + ======= =============== ==================================================== + Version Available since New features + ======= =============== ==================================================== + 1 Python 2.4 Sharing interned strings + ------- --------------- ---------------------------------------------------- + 2 Python 2.5 Binary representation of floats + ------- --------------- ---------------------------------------------------- + 3 Python 3.4 Support for object instancing and recursion + ------- --------------- ---------------------------------------------------- + 4 Python 3.4 Efficient representation of short strings + ------- --------------- ---------------------------------------------------- + 5 Python 3.14 Support for :class:`slice` objects + ======= =============== ==================================================== .. rubric:: Footnotes @@ -154,4 +183,3 @@ In addition, the following constants are defined: around in a self-contained form. Strictly speaking, "to marshal" means to convert some data from internal to external form (in an RPC buffer for instance) and "unmarshalling" for the reverse process. - diff --git a/Doc/library/math.rst b/Doc/library/math.rst index dd2ba419b5bd12..bf79b23a72bbf9 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -26,15 +26,103 @@ The following functions are provided by this module. Except when explicitly noted otherwise, all return values are floats. -Number-theoretic and representation functions ---------------------------------------------- - -.. function:: ceil(x) - - Return the ceiling of *x*, the smallest integer greater than or equal to *x*. - If *x* is not a float, delegates to :meth:`x.__ceil__ `, - which should return an :class:`~numbers.Integral` value. - +==================================================== ============================================ +**Number-theoretic functions** +-------------------------------------------------------------------------------------------------- +:func:`comb(n, k) ` Number of ways to choose *k* items from *n* items without repetition and without order +:func:`factorial(n) ` *n* factorial +:func:`gcd(*integers) ` Greatest common divisor of the integer arguments +:func:`isqrt(n) ` Integer square root of a nonnegative integer *n* +:func:`lcm(*integers) ` Least common multiple of the integer arguments +:func:`perm(n, k) ` Number of ways to choose *k* items from *n* items without repetition and with order + +**Floating point arithmetic** +-------------------------------------------------------------------------------------------------- +:func:`ceil(x) ` Ceiling of *x*, the smallest integer greater than or equal to *x* +:func:`fabs(x) ` Absolute value of *x* +:func:`floor(x) ` Floor of *x*, the largest integer less than or equal to *x* +:func:`fma(x, y, z) ` Fused multiply-add operation: ``(x * y) + z`` +:func:`fmod(x, y) ` Remainder of division ``x / y`` +:func:`modf(x) ` Fractional and integer parts of *x* +:func:`remainder(x, y) ` Remainder of *x* with respect to *y* +:func:`trunc(x) ` Integer part of *x* + +**Floating point manipulation functions** +-------------------------------------------------------------------------------------------------- +:func:`copysign(x, y) ` Magnitude (absolute value) of *x* with the sign of *y* +:func:`frexp(x) ` Mantissa and exponent of *x* +:func:`isclose(a, b, rel_tol, abs_tol) ` Check if the values *a* and *b* are close to each other +:func:`isfinite(x) ` Check if *x* is neither an infinity nor a NaN +:func:`isinf(x) ` Check if *x* is a positive or negative infinity +:func:`isnan(x) ` Check if *x* is a NaN (not a number) +:func:`ldexp(x, i) ` ``x * (2**i)``, inverse of function :func:`frexp` +:func:`nextafter(x, y, steps) ` Floating-point value *steps* steps after *x* towards *y* +:func:`ulp(x) ` Value of the least significant bit of *x* + +**Power, exponential and logarithmic functions** +-------------------------------------------------------------------------------------------------- +:func:`cbrt(x) ` Cube root of *x* +:func:`exp(x) ` *e* raised to the power *x* +:func:`exp2(x) ` *2* raised to the power *x* +:func:`expm1(x) ` *e* raised to the power *x*, minus 1 +:func:`log(x, base) ` Logarithm of *x* to the given base (*e* by default) +:func:`log1p(x) ` Natural logarithm of *1+x* (base *e*) +:func:`log2(x) ` Base-2 logarithm of *x* +:func:`log10(x) ` Base-10 logarithm of *x* +:func:`pow(x, y) ` *x* raised to the power *y* +:func:`sqrt(x) ` Square root of *x* + +**Summation and product functions** +-------------------------------------------------------------------------------------------------- +:func:`dist(p, q) ` Euclidean distance between two points *p* and *q* given as an iterable of coordinates +:func:`fsum(iterable) ` Sum of values in the input *iterable* +:func:`hypot(*coordinates) ` Euclidean norm of an iterable of coordinates +:func:`prod(iterable, start) ` Product of elements in the input *iterable* with a *start* value +:func:`sumprod(p, q) ` Sum of products from two iterables *p* and *q* + +**Angular conversion** +-------------------------------------------------------------------------------------------------- +:func:`degrees(x) ` Convert angle *x* from radians to degrees +:func:`radians(x) ` Convert angle *x* from degrees to radians + +**Trigonometric functions** +-------------------------------------------------------------------------------------------------- +:func:`acos(x) ` Arc cosine of *x* +:func:`asin(x) ` Arc sine of *x* +:func:`atan(x) ` Arc tangent of *x* +:func:`atan2(y, x) ` ``atan(y / x)`` +:func:`cos(x) ` Cosine of *x* +:func:`sin(x) ` Sine of *x* +:func:`tan(x) ` Tangent of *x* + +**Hyperbolic functions** +-------------------------------------------------------------------------------------------------- +:func:`acosh(x) ` Inverse hyperbolic cosine of *x* +:func:`asinh(x) ` Inverse hyperbolic sine of *x* +:func:`atanh(x) ` Inverse hyperbolic tangent of *x* +:func:`cosh(x) ` Hyperbolic cosine of *x* +:func:`sinh(x) ` Hyperbolic sine of *x* +:func:`tanh(x) ` Hyperbolic tangent of *x* + +**Special functions** +-------------------------------------------------------------------------------------------------- +:func:`erf(x) ` `Error function `_ at *x* +:func:`erfc(x) ` `Complementary error function `_ at *x* +:func:`gamma(x) ` `Gamma function `_ at *x* +:func:`lgamma(x) ` Natural logarithm of the absolute value of the `Gamma function `_ at *x* + +**Constants** +-------------------------------------------------------------------------------------------------- +:data:`pi` *π* = 3.141592... +:data:`e` *e* = 2.718281... +:data:`tau` *τ* = 2\ *π* = 6.283185... +:data:`inf` Positive infinity +:data:`nan` "Not a number" (NaN) +==================================================== ============================================ + + +Number-theoretic functions +-------------------------- .. function:: comb(n, k) @@ -54,25 +142,85 @@ Number-theoretic and representation functions .. versionadded:: 3.8 -.. function:: copysign(x, y) +.. function:: factorial(n) - Return a float with the magnitude (absolute value) of *x* but the sign of - *y*. On platforms that support signed zeros, ``copysign(1.0, -0.0)`` - returns *-1.0*. + Return *n* factorial as an integer. Raises :exc:`ValueError` if *n* is not integral or + is negative. + .. versionchanged:: 3.10 + Floats with integral values (like ``5.0``) are no longer accepted. -.. function:: fabs(x) - Return the absolute value of *x*. +.. function:: gcd(*integers) + Return the greatest common divisor of the specified integer arguments. + If any of the arguments is nonzero, then the returned value is the largest + positive integer that is a divisor of all arguments. If all arguments + are zero, then the returned value is ``0``. ``gcd()`` without arguments + returns ``0``. -.. function:: factorial(n) + .. versionadded:: 3.5 - Return *n* factorial as an integer. Raises :exc:`ValueError` if *n* is not integral or - is negative. + .. versionchanged:: 3.9 + Added support for an arbitrary number of arguments. Formerly, only two + arguments were supported. - .. versionchanged:: 3.10 - Floats with integral values (like ``5.0``) are no longer accepted. + +.. function:: isqrt(n) + + Return the integer square root of the nonnegative integer *n*. This is the + floor of the exact square root of *n*, or equivalently the greatest integer + *a* such that *a*\ ² |nbsp| ≤ |nbsp| *n*. + + For some applications, it may be more convenient to have the least integer + *a* such that *n* |nbsp| ≤ |nbsp| *a*\ ², or in other words the ceiling of + the exact square root of *n*. For positive *n*, this can be computed using + ``a = 1 + isqrt(n - 1)``. + + .. versionadded:: 3.8 + + +.. function:: lcm(*integers) + + Return the least common multiple of the specified integer arguments. + If all arguments are nonzero, then the returned value is the smallest + positive integer that is a multiple of all arguments. If any of the arguments + is zero, then the returned value is ``0``. ``lcm()`` without arguments + returns ``1``. + + .. versionadded:: 3.9 + + +.. function:: perm(n, k=None) + + Return the number of ways to choose *k* items from *n* items + without repetition and with order. + + Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates + to zero when ``k > n``. + + If *k* is not specified or is ``None``, then *k* defaults to *n* + and the function returns ``n!``. + + Raises :exc:`TypeError` if either of the arguments are not integers. + Raises :exc:`ValueError` if either of the arguments are negative. + + .. versionadded:: 3.8 + + +Floating point arithmetic +------------------------- + +.. function:: ceil(x) + + Return the ceiling of *x*, the smallest integer greater than or equal to *x*. + If *x* is not a float, delegates to :meth:`x.__ceil__ `, + which should return an :class:`~numbers.Integral` value. + + +.. function:: fabs(x) + + Return the absolute value of *x*. .. function:: floor(x) @@ -113,64 +261,97 @@ Number-theoretic and representation functions floats, while Python's ``x % y`` is preferred when working with integers. -.. function:: frexp(x) +.. function:: modf(x) - Return the mantissa and exponent of *x* as the pair ``(m, e)``. *m* is a float - and *e* is an integer such that ``x == m * 2**e`` exactly. If *x* is zero, - returns ``(0.0, 0)``, otherwise ``0.5 <= abs(m) < 1``. This is used to "pick - apart" the internal representation of a float in a portable way. + Return the fractional and integer parts of *x*. Both results carry the sign + of *x* and are floats. + Note that :func:`modf` has a different call/return pattern + than its C equivalents: it takes a single argument and return a pair of + values, rather than returning its second return value through an 'output + parameter' (there is no such thing in Python). -.. function:: fsum(iterable) - Return an accurate floating-point sum of values in the iterable. Avoids - loss of precision by tracking multiple intermediate partial sums. +.. function:: remainder(x, y) - The algorithm's accuracy depends on IEEE-754 arithmetic guarantees and the - typical case where the rounding mode is half-even. On some non-Windows - builds, the underlying C library uses extended precision addition and may - occasionally double-round an intermediate sum causing it to be off in its - least significant bit. + Return the IEEE 754-style remainder of *x* with respect to *y*. For + finite *x* and finite nonzero *y*, this is the difference ``x - n*y``, + where ``n`` is the closest integer to the exact value of the quotient ``x / + y``. If ``x / y`` is exactly halfway between two consecutive integers, the + nearest *even* integer is used for ``n``. The remainder ``r = remainder(x, + y)`` thus always satisfies ``abs(r) <= 0.5 * abs(y)``. - For further discussion and two alternative approaches, see the `ASPN cookbook - recipes for accurate floating-point summation - `_\. + Special cases follow IEEE 754: in particular, ``remainder(x, math.inf)`` is + *x* for any finite *x*, and ``remainder(x, 0)`` and + ``remainder(math.inf, x)`` raise :exc:`ValueError` for any non-NaN *x*. + If the result of the remainder operation is zero, that zero will have + the same sign as *x*. + On platforms using IEEE 754 binary floating point, the result of this + operation is always exactly representable: no rounding error is introduced. -.. function:: gcd(*integers) + .. versionadded:: 3.7 - Return the greatest common divisor of the specified integer arguments. - If any of the arguments is nonzero, then the returned value is the largest - positive integer that is a divisor of all arguments. If all arguments - are zero, then the returned value is ``0``. ``gcd()`` without arguments - returns ``0``. - .. versionadded:: 3.5 +.. function:: trunc(x) - .. versionchanged:: 3.9 - Added support for an arbitrary number of arguments. Formerly, only two - arguments were supported. + Return *x* with the fractional part + removed, leaving the integer part. This rounds toward 0: ``trunc()`` is + equivalent to :func:`floor` for positive *x*, and equivalent to :func:`ceil` + for negative *x*. If *x* is not a float, delegates to :meth:`x.__trunc__ + `, which should return an :class:`~numbers.Integral` value. +For the :func:`ceil`, :func:`floor`, and :func:`modf` functions, note that *all* +floating-point numbers of sufficiently large magnitude are exact integers. +Python floats typically carry no more than 53 bits of precision (the same as the +platform C double type), in which case any float *x* with ``abs(x) >= 2**52`` +necessarily has no fractional bits. + + +Floating point manipulation functions +------------------------------------- + +.. function:: copysign(x, y) + + Return a float with the magnitude (absolute value) of *x* but the sign of + *y*. On platforms that support signed zeros, ``copysign(1.0, -0.0)`` + returns *-1.0*. + + +.. function:: frexp(x) + + Return the mantissa and exponent of *x* as the pair ``(m, e)``. *m* is a float + and *e* is an integer such that ``x == m * 2**e`` exactly. If *x* is zero, + returns ``(0.0, 0)``, otherwise ``0.5 <= abs(m) < 1``. This is used to "pick + apart" the internal representation of a float in a portable way. + + Note that :func:`frexp` has a different call/return pattern + than its C equivalents: it takes a single argument and return a pair of + values, rather than returning its second return value through an 'output + parameter' (there is no such thing in Python). + .. function:: isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0) Return ``True`` if the values *a* and *b* are close to each other and ``False`` otherwise. Whether or not two values are considered close is determined according to - given absolute and relative tolerances. + given absolute and relative tolerances. If no errors occur, the result will + be: ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``. *rel_tol* is the relative tolerance -- it is the maximum allowed difference between *a* and *b*, relative to the larger absolute value of *a* or *b*. For example, to set a tolerance of 5%, pass ``rel_tol=0.05``. The default tolerance is ``1e-09``, which assures that the two values are the same - within about 9 decimal digits. *rel_tol* must be greater than zero. - - *abs_tol* is the minimum absolute tolerance -- useful for comparisons near - zero. *abs_tol* must be at least zero. + within about 9 decimal digits. *rel_tol* must be nonnegative and less + than ``1.0``. - If no errors occur, the result will be: - ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``. + *abs_tol* is the absolute tolerance; it defaults to ``0.0`` and it must be + nonnegative. When comparing ``x`` to ``0.0``, ``isclose(x, 0)`` is computed + as ``abs(x) <= rel_tol * abs(x)``, which is ``False`` for any ``x`` and + rel_tol less than ``1.0``. So add an appropriate positive abs_tol argument + to the call. The IEEE 754 special values of ``NaN``, ``inf``, and ``-inf`` will be handled according to IEEE rules. Specifically, ``NaN`` is not considered @@ -203,43 +384,12 @@ Number-theoretic and representation functions Return ``True`` if *x* is a NaN (not a number), and ``False`` otherwise. -.. function:: isqrt(n) - - Return the integer square root of the nonnegative integer *n*. This is the - floor of the exact square root of *n*, or equivalently the greatest integer - *a* such that *a*\ ² |nbsp| ≤ |nbsp| *n*. - - For some applications, it may be more convenient to have the least integer - *a* such that *n* |nbsp| ≤ |nbsp| *a*\ ², or in other words the ceiling of - the exact square root of *n*. For positive *n*, this can be computed using - ``a = 1 + isqrt(n - 1)``. - - .. versionadded:: 3.8 - - -.. function:: lcm(*integers) - - Return the least common multiple of the specified integer arguments. - If all arguments are nonzero, then the returned value is the smallest - positive integer that is a multiple of all arguments. If any of the arguments - is zero, then the returned value is ``0``. ``lcm()`` without arguments - returns ``1``. - - .. versionadded:: 3.9 - - .. function:: ldexp(x, i) Return ``x * (2**i)``. This is essentially the inverse of function :func:`frexp`. -.. function:: modf(x) - - Return the fractional and integer parts of *x*. Both results carry the sign - of *x* and are floats. - - .. function:: nextafter(x, y, steps=1) Return the floating-point value *steps* steps after *x* towards *y*. @@ -260,79 +410,6 @@ Number-theoretic and representation functions .. versionchanged:: 3.12 Added the *steps* argument. -.. function:: perm(n, k=None) - - Return the number of ways to choose *k* items from *n* items - without repetition and with order. - - Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates - to zero when ``k > n``. - - If *k* is not specified or is ``None``, then *k* defaults to *n* - and the function returns ``n!``. - - Raises :exc:`TypeError` if either of the arguments are not integers. - Raises :exc:`ValueError` if either of the arguments are negative. - - .. versionadded:: 3.8 - - -.. function:: prod(iterable, *, start=1) - - Calculate the product of all the elements in the input *iterable*. - The default *start* value for the product is ``1``. - - When the iterable is empty, return the start value. This function is - intended specifically for use with numeric values and may reject - non-numeric types. - - .. versionadded:: 3.8 - - -.. function:: remainder(x, y) - - Return the IEEE 754-style remainder of *x* with respect to *y*. For - finite *x* and finite nonzero *y*, this is the difference ``x - n*y``, - where ``n`` is the closest integer to the exact value of the quotient ``x / - y``. If ``x / y`` is exactly halfway between two consecutive integers, the - nearest *even* integer is used for ``n``. The remainder ``r = remainder(x, - y)`` thus always satisfies ``abs(r) <= 0.5 * abs(y)``. - - Special cases follow IEEE 754: in particular, ``remainder(x, math.inf)`` is - *x* for any finite *x*, and ``remainder(x, 0)`` and - ``remainder(math.inf, x)`` raise :exc:`ValueError` for any non-NaN *x*. - If the result of the remainder operation is zero, that zero will have - the same sign as *x*. - - On platforms using IEEE 754 binary floating point, the result of this - operation is always exactly representable: no rounding error is introduced. - - .. versionadded:: 3.7 - - -.. function:: sumprod(p, q) - - Return the sum of products of values from two iterables *p* and *q*. - - Raises :exc:`ValueError` if the inputs do not have the same length. - - Roughly equivalent to:: - - sum(itertools.starmap(operator.mul, zip(p, q, strict=True))) - - For float and mixed int/float inputs, the intermediate products - and sums are computed with extended precision. - - .. versionadded:: 3.12 - - -.. function:: trunc(x) - - Return *x* with the fractional part - removed, leaving the integer part. This rounds toward 0: ``trunc()`` is - equivalent to :func:`floor` for positive *x*, and equivalent to :func:`ceil` - for negative *x*. If *x* is not a float, delegates to :meth:`x.__trunc__ - `, which should return an :class:`~numbers.Integral` value. .. function:: ulp(x) @@ -359,20 +436,8 @@ Number-theoretic and representation functions .. versionadded:: 3.9 -Note that :func:`frexp` and :func:`modf` have a different call/return pattern -than their C equivalents: they take a single argument and return a pair of -values, rather than returning their second return value through an 'output -parameter' (there is no such thing in Python). - -For the :func:`ceil`, :func:`floor`, and :func:`modf` functions, note that *all* -floating-point numbers of sufficiently large magnitude are exact integers. -Python floats typically carry no more than 53 bits of precision (the same as the -platform C double type), in which case any float *x* with ``abs(x) >= 2**52`` -necessarily has no fractional bits. - - -Power and logarithmic functions -------------------------------- +Power, exponential and logarithmic functions +-------------------------------------------- .. function:: cbrt(x) @@ -447,11 +512,11 @@ Power and logarithmic functions .. function:: pow(x, y) - Return ``x`` raised to the power ``y``. Exceptional cases follow + Return *x* raised to the power *y*. Exceptional cases follow the IEEE 754 standard as far as possible. In particular, ``pow(1.0, x)`` and ``pow(x, 0.0)`` always return ``1.0``, even - when ``x`` is a zero or a NaN. If both ``x`` and ``y`` are finite, - ``x`` is negative, and ``y`` is not an integer then ``pow(x, y)`` + when *x* is a zero or a NaN. If both *x* and *y* are finite, + *x* is negative, and *y* is not an integer then ``pow(x, y)`` is undefined, and raises :exc:`ValueError`. Unlike the built-in ``**`` operator, :func:`math.pow` converts both @@ -469,41 +534,8 @@ Power and logarithmic functions Return the square root of *x*. -Trigonometric functions ------------------------ - -.. function:: acos(x) - - Return the arc cosine of *x*, in radians. The result is between ``0`` and - ``pi``. - - -.. function:: asin(x) - - Return the arc sine of *x*, in radians. The result is between ``-pi/2`` and - ``pi/2``. - - -.. function:: atan(x) - - Return the arc tangent of *x*, in radians. The result is between ``-pi/2`` and - ``pi/2``. - - -.. function:: atan2(y, x) - - Return ``atan(y / x)``, in radians. The result is between ``-pi`` and ``pi``. - The vector in the plane from the origin to point ``(x, y)`` makes this angle - with the positive X axis. The point of :func:`atan2` is that the signs of both - inputs are known to it, so it can compute the correct quadrant for the angle. - For example, ``atan(1)`` and ``atan2(1, 1)`` are both ``pi/4``, but ``atan2(-1, - -1)`` is ``-3*pi/4``. - - -.. function:: cos(x) - - Return the cosine of *x* radians. - +Summation and product functions +------------------------------- .. function:: dist(p, q) @@ -518,6 +550,22 @@ Trigonometric functions .. versionadded:: 3.8 +.. function:: fsum(iterable) + + Return an accurate floating-point sum of values in the iterable. Avoids + loss of precision by tracking multiple intermediate partial sums. + + The algorithm's accuracy depends on IEEE-754 arithmetic guarantees and the + typical case where the rounding mode is half-even. On some non-Windows + builds, the underlying C library uses extended precision addition and may + occasionally double-round an intermediate sum causing it to be off in its + least significant bit. + + For further discussion and two alternative approaches, see the `ASPN cookbook + recipes for accurate floating-point summation + `_\. + + .. function:: hypot(*coordinates) Return the Euclidean norm, ``sqrt(sum(x**2 for x in coordinates))``. @@ -538,14 +586,32 @@ Trigonometric functions is almost always correctly rounded to within 1/2 ulp. -.. function:: sin(x) +.. function:: prod(iterable, *, start=1) - Return the sine of *x* radians. + Calculate the product of all the elements in the input *iterable*. + The default *start* value for the product is ``1``. + When the iterable is empty, return the start value. This function is + intended specifically for use with numeric values and may reject + non-numeric types. -.. function:: tan(x) + .. versionadded:: 3.8 - Return the tangent of *x* radians. + +.. function:: sumprod(p, q) + + Return the sum of products of values from two iterables *p* and *q*. + + Raises :exc:`ValueError` if the inputs do not have the same length. + + Roughly equivalent to:: + + sum(map(operator.mul, p, q, strict=True)) + + For float and mixed int/float inputs, the intermediate products + and sums are computed with extended precision. + + .. versionadded:: 3.12 Angular conversion @@ -561,6 +627,52 @@ Angular conversion Convert angle *x* from degrees to radians. +Trigonometric functions +----------------------- + +.. function:: acos(x) + + Return the arc cosine of *x*, in radians. The result is between ``0`` and + ``pi``. + + +.. function:: asin(x) + + Return the arc sine of *x*, in radians. The result is between ``-pi/2`` and + ``pi/2``. + + +.. function:: atan(x) + + Return the arc tangent of *x*, in radians. The result is between ``-pi/2`` and + ``pi/2``. + + +.. function:: atan2(y, x) + + Return ``atan(y / x)``, in radians. The result is between ``-pi`` and ``pi``. + The vector in the plane from the origin to point ``(x, y)`` makes this angle + with the positive X axis. The point of :func:`atan2` is that the signs of both + inputs are known to it, so it can compute the correct quadrant for the angle. + For example, ``atan(1)`` and ``atan2(1, 1)`` are both ``pi/4``, but ``atan2(-1, + -1)`` is ``-3*pi/4``. + + +.. function:: cos(x) + + Return the cosine of *x* radians. + + +.. function:: sin(x) + + Return the sine of *x* radians. + + +.. function:: tan(x) + + Return the tangent of *x* radians. + + Hyperbolic functions -------------------- diff --git a/Doc/library/msilib.rst b/Doc/library/msilib.rst new file mode 100644 index 00000000000000..eb1ac551ded456 --- /dev/null +++ b/Doc/library/msilib.rst @@ -0,0 +1,15 @@ +:mod:`!msilib` --- Read and write Microsoft Installer files +=========================================================== + +.. module:: msilib + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!msilib` module was +`Python 3.12 `_. diff --git a/Doc/library/nis.rst b/Doc/library/nis.rst new file mode 100644 index 00000000000000..dcc36dd43fc313 --- /dev/null +++ b/Doc/library/nis.rst @@ -0,0 +1,15 @@ +:mod:`!nis` --- Interface to Sun’s NIS (Yellow Pages) +===================================================== + +.. module:: nis + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!nis` module was +`Python 3.12 `_. diff --git a/Doc/library/nntplib.rst b/Doc/library/nntplib.rst new file mode 100644 index 00000000000000..8053fe8cb8b9e1 --- /dev/null +++ b/Doc/library/nntplib.rst @@ -0,0 +1,15 @@ +:mod:`!nntplib` --- NNTP protocol client +======================================== + +.. module:: nntplib + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!nntplib` module was +`Python 3.12 `_. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index f9cded40c2c755..61144256f47ddb 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -193,10 +193,6 @@ process and user. to the environment made after this time are not reflected in :data:`os.environ`, except for changes made by modifying :data:`os.environ` directly. - The :meth:`!os.environ.refresh` method updates :data:`os.environ` with - changes to the environment made by :func:`os.putenv`, by - :func:`os.unsetenv`, or made outside Python in the same process. - This mapping may be used to modify the environment as well as query the environment. :func:`putenv` will be called automatically when the mapping is modified. @@ -226,12 +222,13 @@ process and user. :data:`os.environ`, and when one of the :meth:`pop` or :meth:`clear` methods is called. + .. seealso:: + + The :func:`os.reload_environ` function. + .. versionchanged:: 3.9 Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators. - .. versionchanged:: 3.14 - Added the :meth:`!os.environ.refresh` method. - .. data:: environb @@ -249,6 +246,24 @@ process and user. Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators. +.. function:: reload_environ() + + The :data:`os.environ` and :data:`os.environb` mappings are a cache of + environment variables at the time that Python started. + As such, changes to the current process environment are not reflected + if made outside Python, or by :func:`os.putenv` or :func:`os.unsetenv`. + Use :func:`!os.reload_environ` to update :data:`os.environ` and :data:`os.environb` + with any such changes to the current process environment. + + .. warning:: + This function is not thread-safe. Calling it while the environment is + being modified in an other thread is an undefined behavior. Reading from + :data:`os.environ` or :data:`os.environb`, or calling :func:`os.getenv` + while reloading, may return an empty result. + + .. versionadded:: next + + .. function:: chdir(path) fchdir(fd) getcwd() @@ -568,7 +583,7 @@ process and user. of :data:`os.environ`. This also applies to :func:`getenv` and :func:`getenvb`, which respectively use :data:`os.environ` and :data:`os.environb` in their implementations. - See also the :data:`os.environ.refresh() ` method. + See also the :func:`os.reload_environ` function. .. note:: @@ -818,7 +833,7 @@ process and user. don't update :data:`os.environ`, so it is actually preferable to delete items of :data:`os.environ`. - See also the :data:`os.environ.refresh() ` method. + See also the :func:`os.reload_environ` function. .. audit-event:: os.unsetenv key os.unsetenv @@ -4562,8 +4577,7 @@ written in Python, such as a mail server's external command delivery program. only be sent to console processes which share a common console window, e.g., some subprocesses. Any other value for *sig* will cause the process to be unconditionally killed by the TerminateProcess API, and the exit code - will be set to *sig*. The Windows version of :func:`kill` additionally takes - process handles to be killed. + will be set to *sig*. See also :func:`signal.pthread_kill`. diff --git a/Doc/library/ossaudiodev.rst b/Doc/library/ossaudiodev.rst new file mode 100644 index 00000000000000..320adbeff82539 --- /dev/null +++ b/Doc/library/ossaudiodev.rst @@ -0,0 +1,15 @@ +:mod:`!ossaudiodev` --- Access to OSS-compatible audio devices +============================================================== + +.. module:: ossaudiodev + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!ossaudiodev` module was +`Python 3.12 `_. diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 4380122eb1be7d..a42ac1f8bcdf71 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1289,6 +1289,35 @@ Reading directories raised. +.. method:: Path.scandir() + + When the path points to a directory, return an iterator of + :class:`os.DirEntry` objects corresponding to entries in the directory. The + returned iterator supports the :term:`context manager` protocol. It is + implemented using :func:`os.scandir` and gives the same guarantees. + + Using :meth:`~Path.scandir` instead of :meth:`~Path.iterdir` can + significantly increase the performance of code that also needs file type or + file attribute information, because :class:`os.DirEntry` objects expose + this information if the operating system provides it when scanning a + directory. + + The following example displays the names of subdirectories. The + ``entry.is_dir()`` check will generally not make an additional system call:: + + >>> p = Path('docs') + >>> with p.scandir() as entries: + ... for entry in entries: + ... if entry.is_dir(): + ... entry.name + ... + '_templates' + '_build' + '_static' + + .. versionadded:: 3.14 + + .. method:: Path.glob(pattern, *, case_sensitive=None, recurse_symlinks=False) Glob the given relative *pattern* in the directory represented by this path, @@ -1563,6 +1592,11 @@ Copying, moving and deleting This argument has no effect when copying files on Windows (where metadata is always preserved). + .. note:: + Where supported by the operating system and file system, this method + performs a lightweight copy, where data blocks are only copied when + modified. This is known as copy-on-write. + .. versionadded:: 3.14 diff --git a/Doc/library/pipes.rst b/Doc/library/pipes.rst new file mode 100644 index 00000000000000..d9bcc3a5d99c9b --- /dev/null +++ b/Doc/library/pipes.rst @@ -0,0 +1,17 @@ +:mod:`!pipes` --- Interface to shell pipelines +============================================== + +.. module:: pipes + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +Applications should use the :mod:`subprocess` module instead. + +The last version of Python that provided the :mod:`!pipes` module was +`Python 3.12 `_. diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 4a39d53a5f1440..20b8f6bcf19117 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -49,25 +49,6 @@ support. this function to raise an exception (in line with :func:`os.path.isdir` behavior). -.. function:: find_loader(fullname) - - Retrieve a module :term:`loader` for the given *fullname*. - - This is a backwards compatibility wrapper around - :func:`importlib.util.find_spec` that converts most failures to - :exc:`ImportError` and only returns the loader rather than the full - :class:`importlib.machinery.ModuleSpec`. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal :pep:`302` import emulation. - - .. versionchanged:: 3.4 - Updated to be based on :pep:`451` - - .. deprecated-removed:: 3.12 3.14 - Use :func:`importlib.util.find_spec` instead. - .. function:: get_importer(path_item) @@ -84,27 +65,6 @@ support. on the package internal :pep:`302` import emulation. -.. function:: get_loader(module_or_name) - - Get a :term:`loader` object for *module_or_name*. - - If the module or package is accessible via the normal import mechanism, a - wrapper around the relevant part of that machinery is returned. Returns - ``None`` if the module cannot be found or imported. If the named module is - not already imported, its containing package (if any) is imported, in order - to establish the package ``__path__``. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal :pep:`302` import emulation. - - .. versionchanged:: 3.4 - Updated to be based on :pep:`451` - - .. deprecated-removed:: 3.12 3.14 - Use :func:`importlib.util.find_spec` instead. - - .. function:: iter_importers(fullname='') Yield :term:`finder` objects for the given module name. diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 1beb3b9eb89d22..cfe1e7ba48da32 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -359,3 +359,15 @@ Android Platform `__. .. versionadded:: 3.13 + + +Miscellaneous +------------- + +.. function:: invalidate_caches() + + Clear out the internal cache of information, such as the :func:`uname`. + This is typically useful when the platform's :func:`node` is changed + by an external process and one needs to retrieve the updated value. + + .. versionadded:: 3.14 diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index 1b3498e51f766d..2985f31bacb47a 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -267,7 +267,7 @@ let's fetch information about a project from `PyPI `_:: >>> import json >>> import pprint >>> from urllib.request import urlopen - >>> with urlopen('https://pypi.org/pypi/sampleproject/json') as resp: + >>> with urlopen('https://pypi.org/pypi/sampleproject/1.2.0/json') as resp: ... project_info = json.load(resp)['info'] In its basic form, :func:`~pprint.pp` shows the whole object:: diff --git a/Doc/library/removed.rst b/Doc/library/removed.rst new file mode 100644 index 00000000000000..4d75842eca1a03 --- /dev/null +++ b/Doc/library/removed.rst @@ -0,0 +1,39 @@ +:tocdepth: 1 + +.. _removed: + +*************** +Removed Modules +*************** + +The modules described in this chapter have been removed from the Python +standard library. They are documented here to help people find replacements. + + +.. toctree:: + :maxdepth: 1 + + aifc.rst + asynchat.rst + asyncore.rst + audioop.rst + cgi.rst + cgitb.rst + chunk.rst + crypt.rst + distutils.rst + imghdr.rst + imp.rst + mailcap.rst + msilib.rst + nis.rst + nntplib.rst + ossaudiodev.rst + pipes.rst + smtpd.rst + sndhdr.rst + spwd.rst + sunau.rst + telnetlib.rst + uu.rst + xdrlib.rst diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst new file mode 100644 index 00000000000000..c704f4a241b469 --- /dev/null +++ b/Doc/library/smtpd.rst @@ -0,0 +1,18 @@ +:mod:`!smtpd` --- SMTP Server +============================= + +.. module:: smtpd + :synopsis: Removed in 3.12. + :deprecated: + +.. deprecated-removed:: 3.6 3.12 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.12 ` after +being deprecated in Python 3.6. The removal was decided in :pep:`594`. + +A possible replacement is the third-party :pypi:`aiosmtpd` library. This +library is not maintained or supported by the Python core team. + +The last version of Python that provided the :mod:`!smtpd` module was +`Python 3.11 `_. diff --git a/Doc/library/sndhdr.rst b/Doc/library/sndhdr.rst new file mode 100644 index 00000000000000..6b71db4f6338a8 --- /dev/null +++ b/Doc/library/sndhdr.rst @@ -0,0 +1,19 @@ +:mod:`!sndhdr` --- Determine type of sound file +=============================================== + +.. module:: sndhdr + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +Possible replacements are third-party modules from PyPI: +:pypi:`filetype`, :pypi:`puremagic`, or :pypi:`python-magic`. +These are not supported or maintained by the Python core team. + +The last version of Python that provided the :mod:`!sndhdr` module was +`Python 3.12 `_. diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 0c7b9328648f66..6358d140484c78 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -928,7 +928,9 @@ The :mod:`socket` module also offers various network-related services: .. versionadded:: 3.7 -.. function:: getaddrinfo(host, port, family=0, type=0, proto=0, flags=0) +.. function:: getaddrinfo(host, port, family=AF_UNSPEC, type=0, proto=0, flags=0) + + This function wraps the C function ``getaddrinfo`` of the underlying system. Translate the *host*/*port* argument into a sequence of 5-tuples that contain all the necessary arguments for creating a socket connected to that service. @@ -938,8 +940,10 @@ The :mod:`socket` module also offers various network-related services: and *port*, you can pass ``NULL`` to the underlying C API. The *family*, *type* and *proto* arguments can be optionally specified - in order to narrow the list of addresses returned. Passing zero as a - value for each of these arguments selects the full range of results. + in order to provide options and limit the list of addresses returned. + Pass their default values (:data:`AF_UNSPEC`, 0, and 0, respectively) + to not limit the results. See the note below for details. + The *flags* argument can be one or several of the ``AI_*`` constants, and will influence how results are computed and returned. For example, :const:`AI_NUMERICHOST` will disable domain name resolution @@ -959,6 +963,29 @@ The :mod:`socket` module also offers various network-related services: :const:`AF_INET6`), and is meant to be passed to the :meth:`socket.connect` method. + .. note:: + + If you intend to use results from :func:`!getaddrinfo` to create a socket + (rather than, for example, retrieve *canonname*), + consider limiting the results by *type* (e.g. :data:`SOCK_STREAM` or + :data:`SOCK_DGRAM`) and/or *proto* (e.g. :data:`IPPROTO_TCP` or + :data:`IPPROTO_UDP`) that your application can handle. + + The behavior with default values of *family*, *type*, *proto* + and *flags* is system-specific. + + Many systems (for example, most Linux configurations) will return a sorted + list of all matching addresses. + These addresses should generally be tried in order until a connection succeeds + (possibly tried in parallel, for example, using a `Happy Eyeballs`_ algorithm). + In these cases, limiting the *type* and/or *proto* can help eliminate + unsuccessful or unusable connecton attempts. + + Some systems will, however, only return a single address. + (For example, this was reported on Solaris and AIX configurations.) + On these systems, limiting the *type* and/or *proto* helps ensure that + this address is usable. + .. audit-event:: socket.getaddrinfo host,port,family,type,protocol socket.getaddrinfo The following example fetches address information for a hypothetical TCP @@ -978,6 +1005,8 @@ The :mod:`socket` module also offers various network-related services: for IPv6 multicast addresses, string representing an address will not contain ``%scope_id`` part. +.. _Happy Eyeballs: https://en.wikipedia.org/wiki/Happy_Eyeballs + .. function:: getfqdn([name]) Return a fully qualified domain name for *name*. If *name* is omitted or empty, diff --git a/Doc/library/spwd.rst b/Doc/library/spwd.rst new file mode 100644 index 00000000000000..c16854bb380e52 --- /dev/null +++ b/Doc/library/spwd.rst @@ -0,0 +1,18 @@ +:mod:`!spwd` --- The shadow password database +============================================= + +.. module:: spwd + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +A possible replacement is the third-party library :pypi:`python-pam`. +This library is not supported or maintained by the Python core team. + +The last version of Python that provided the :mod:`!spwd` module was +`Python 3.12 `_. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index fc0383823a172b..096892b605b99c 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -2442,6 +2442,7 @@ Some useful URI tricks include: >>> con.execute("CREATE TABLE readonly(data)") Traceback (most recent call last): OperationalError: attempt to write a readonly database + >>> con.close() * Do not implicitly create a new database file if it does not already exist; will raise :exc:`~sqlite3.OperationalError` if unable to create a new file: diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 9e8e44a8abe770..a000bb49f14800 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -589,6 +589,11 @@ The available presentation types for :class:`float` and | | as altered by the other format modifiers. | +---------+----------------------------------------------------------+ +The result should be correctly rounded to a given precision ``p`` of digits +after the decimal point. The rounding mode for :class:`float` matches that +of the :func:`round` builtin. For :class:`~decimal.Decimal`, the rounding +mode of the current :ref:`context ` will be used. + The available presentation types for :class:`complex` are the same as those for :class:`float` (``'%'`` is not allowed). Both the real and imaginary components of a complex number are formatted as floating-point numbers, according to the diff --git a/Doc/library/sunau.rst b/Doc/library/sunau.rst new file mode 100644 index 00000000000000..feb7768f8bdd68 --- /dev/null +++ b/Doc/library/sunau.rst @@ -0,0 +1,15 @@ +:mod:`!sunau` --- Read and write Sun AU files +============================================= + +.. module:: sunau + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!sunau` module was +`Python 3.12 `_. diff --git a/Doc/library/telnetlib.rst b/Doc/library/telnetlib.rst new file mode 100644 index 00000000000000..6971ad33ff9751 --- /dev/null +++ b/Doc/library/telnetlib.rst @@ -0,0 +1,19 @@ +:mod:`!telnetlib` --- Telnet client +=================================== + +.. module:: telnetlib + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +Possible replacements are third-party libraries from PyPI: :pypi:`telnetlib3` +or :pypi:`Exscript`. These are not supported or maintained by the Python core +team. + +The last version of Python that provided the :mod:`!telnetlib` module was +`Python 3.12 `_. diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 8e29e57d00f9b2..6265c2214eaa0d 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -327,7 +327,7 @@ Functions .. impl-detail:: - On CPython, use the same clock than :func:`time.monotonic` and is a + On CPython, use the same clock as :func:`time.monotonic` and is a monotonic clock, i.e. a clock that cannot go backwards. Use :func:`perf_counter_ns` to avoid the precision loss caused by the @@ -339,7 +339,7 @@ Functions On Windows, the function is now system-wide. .. versionchanged:: 3.13 - Use the same clock than :func:`time.monotonic`. + Use the same clock as :func:`time.monotonic`. .. function:: perf_counter_ns() -> int @@ -390,7 +390,7 @@ Functions threads ready to run, the function returns immediately, and the thread continues execution. On Windows 8.1 and newer the implementation uses a `high-resolution timer - `_ + `_ which provides resolution of 100 nanoseconds. If *secs* is zero, ``Sleep(0)`` is used. Unix implementation: diff --git a/Doc/library/tomllib.rst b/Doc/library/tomllib.rst index 521a7a17fb3e8b..4b88b2e29e7822 100644 --- a/Doc/library/tomllib.rst +++ b/Doc/library/tomllib.rst @@ -60,9 +60,36 @@ This module defines the following functions: The following exceptions are available: -.. exception:: TOMLDecodeError +.. exception:: TOMLDecodeError(msg, doc, pos) - Subclass of :exc:`ValueError`. + Subclass of :exc:`ValueError` with the following additional attributes: + + .. attribute:: msg + + The unformatted error message. + + .. attribute:: doc + + The TOML document being parsed. + + .. attribute:: pos + + The index of *doc* where parsing failed. + + .. attribute:: lineno + + The line corresponding to *pos*. + + .. attribute:: colno + + The column corresponding to *pos*. + + .. versionchanged:: next + Added the *msg*, *doc* and *pos* parameters. + Added the :attr:`msg`, :attr:`doc`, :attr:`pos`, :attr:`lineno` and :attr:`colno` attributes. + + .. deprecated:: next + Passing free-form positional arguments is deprecated. Examples diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index cd8b90854b0e94..0fee782121b0af 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1726,11 +1726,11 @@ without the dedicated syntax, as documented below. class Sequence[T]: # T is a TypeVar ... - This syntax can also be used to create bound and constrained type + This syntax can also be used to create bounded and constrained type variables:: - class StrSequence[S: str]: # S is a TypeVar bound to str - ... + class StrSequence[S: str]: # S is a TypeVar with a `str` upper bound; + ... # we can say that S is "bounded by `str`" class StrOrBytesSequence[A: (str, bytes)]: # A is a TypeVar constrained to str or bytes @@ -1763,8 +1763,8 @@ without the dedicated syntax, as documented below. """Add two strings or bytes objects together.""" return x + y - Note that type variables can be *bound*, *constrained*, or neither, but - cannot be both bound *and* constrained. + Note that type variables can be *bounded*, *constrained*, or neither, but + cannot be both bounded *and* constrained. The variance of type variables is inferred by type checkers when they are created through the :ref:`type parameter syntax ` or when @@ -1774,8 +1774,8 @@ without the dedicated syntax, as documented below. By default, manually created type variables are invariant. See :pep:`484` and :pep:`695` for more details. - Bound type variables and constrained type variables have different - semantics in several important ways. Using a *bound* type variable means + Bounded type variables and constrained type variables have different + semantics in several important ways. Using a *bounded* type variable means that the ``TypeVar`` will be solved using the most specific type possible:: x = print_capitalized('a string') @@ -1789,8 +1789,8 @@ without the dedicated syntax, as documented below. z = print_capitalized(45) # error: int is not a subtype of str - Type variables can be bound to concrete types, abstract types (ABCs or - protocols), and even unions of types:: + The upper bound of a type variable can be a concrete type, abstract type + (ABC or Protocol), or even a union of types:: # Can be anything with an __abs__ method def print_abs[T: SupportsAbs](arg: T) -> None: @@ -1834,7 +1834,7 @@ without the dedicated syntax, as documented below. .. attribute:: __bound__ - The bound of the type variable, if any. + The upper bound of the type variable, if any. .. versionchanged:: 3.12 @@ -2100,7 +2100,7 @@ without the dedicated syntax, as documented below. return x + y Without ``ParamSpec``, the simplest way to annotate this previously was to - use a :class:`TypeVar` with bound ``Callable[..., Any]``. However this + use a :class:`TypeVar` with upper bound ``Callable[..., Any]``. However this causes two problems: 1. The type checker can't type check the ``inner`` function because diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index fb5353e1895bf9..44a9c79cba2216 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -239,6 +239,10 @@ or on combining URL components into a URL string. query parameter separator. This has been changed to allow only a single separator key, with ``&`` as the default separator. + .. deprecated:: 3.14 + Accepting objects with false values (like ``0`` and ``[]``) except empty + strings and byte-like objects and ``None`` is now deprecated. + .. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator='&') @@ -403,6 +407,15 @@ or on combining URL components into a URL string. If you do not want that behavior, preprocess the *url* with :func:`urlsplit` and :func:`urlunsplit`, removing possible *scheme* and *netloc* parts. + .. warning:: + + Because an absolute URL may be passed as the ``url`` parameter, it is + generally **not secure** to use ``urljoin`` with an attacker-controlled + ``url``. For example in, + ``urljoin("https://website.com/users/", username)``, if ``username`` can + contain an absolute URL, the result of ``urljoin`` will be the absolute + URL. + .. versionchanged:: 3.5 @@ -745,6 +758,10 @@ task isn't already covered by the URL parsing functions above. .. versionchanged:: 3.5 Added the *quote_via* parameter. + .. deprecated:: 3.14 + Accepting objects with false values (like ``0`` and ``[]``) except empty + strings and byte-like objects and ``None`` is now deprecated. + .. seealso:: diff --git a/Doc/library/uu.rst b/Doc/library/uu.rst new file mode 100644 index 00000000000000..0636d180294d47 --- /dev/null +++ b/Doc/library/uu.rst @@ -0,0 +1,15 @@ +:mod:`!uu` --- Encode and decode uuencode files +=============================================== + +.. module:: uu + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!uu` module was +`Python 3.12 `_. diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index 0f2d7820cb25c8..9be12edd36b9a8 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -1,8 +1,8 @@ -:mod:`!uuid` --- UUID objects according to :rfc:`4122` +:mod:`!uuid` --- UUID objects according to :rfc:`9562` ====================================================== .. module:: uuid - :synopsis: UUID objects (universally unique identifiers) according to RFC 4122 + :synopsis: UUID objects (universally unique identifiers) according to RFC 9562 .. moduleauthor:: Ka-Ping Yee .. sectionauthor:: George Yoshida @@ -11,8 +11,9 @@ -------------- This module provides immutable :class:`UUID` objects (the :class:`UUID` class) -and the functions :func:`uuid1`, :func:`uuid3`, :func:`uuid4`, :func:`uuid5` for -generating version 1, 3, 4, and 5 UUIDs as specified in :rfc:`4122`. +and the functions :func:`uuid1`, :func:`uuid3`, :func:`uuid4`, :func:`uuid5`, +and :func:`uuid.uuid8` for generating version 1, 3, 4, 5, and 8 UUIDs as +specified in :rfc:`9562` (which supersedes :rfc:`4122`). If all you want is a unique ID, you should probably call :func:`uuid1` or :func:`uuid4`. Note that :func:`uuid1` may compromise privacy since it creates @@ -65,7 +66,7 @@ which relays any information about the UUID's safety, using this enumeration: Exactly one of *hex*, *bytes*, *bytes_le*, *fields*, or *int* must be given. The *version* argument is optional; if given, the resulting UUID will have its - variant and version number set according to :rfc:`4122`, overriding bits in the + variant and version number set according to :rfc:`9562`, overriding bits in the given *hex*, *bytes*, *bytes_le*, *fields*, or *int*. Comparison of UUID objects are made by way of comparing their @@ -137,7 +138,7 @@ which relays any information about the UUID's safety, using this enumeration: .. attribute:: UUID.urn - The UUID as a URN as specified in :rfc:`4122`. + The UUID as a URN as specified in :rfc:`9562`. .. attribute:: UUID.variant @@ -149,9 +150,13 @@ which relays any information about the UUID's safety, using this enumeration: .. attribute:: UUID.version - The UUID version number (1 through 5, meaningful only when the variant is + The UUID version number (1 through 8, meaningful only when the variant is :const:`RFC_4122`). + .. versionchanged:: next + Added UUID version 8. + + .. attribute:: UUID.is_safe An enumeration of :class:`SafeUUID` which indicates whether the platform @@ -216,6 +221,23 @@ The :mod:`uuid` module defines the following functions: .. index:: single: uuid5 + +.. function:: uuid8(a=None, b=None, c=None) + + Generate a pseudo-random UUID according to + :rfc:`RFC 9562, §5.8 <9562#section-5.8>`. + + When specified, the parameters *a*, *b* and *c* are expected to be + positive integers of 48, 12 and 62 bits respectively. If they exceed + their expected bit count, only their least significant bits are kept; + non-specified arguments are substituted for a pseudo-random integer of + appropriate size. + + .. versionadded:: next + +.. index:: single: uuid8 + + The :mod:`uuid` module defines the following namespace identifiers for use with :func:`uuid3` or :func:`uuid5`. @@ -252,7 +274,9 @@ of the :attr:`~UUID.variant` attribute: .. data:: RFC_4122 - Specifies the UUID layout given in :rfc:`4122`. + Specifies the UUID layout given in :rfc:`4122`. This constant is kept + for backward compatibility even though :rfc:`4122` has been superseded + by :rfc:`9562`. .. data:: RESERVED_MICROSOFT @@ -267,7 +291,7 @@ of the :attr:`~UUID.variant` attribute: .. seealso:: - :rfc:`4122` - A Universally Unique IDentifier (UUID) URN Namespace + :rfc:`9562` - A Universally Unique IDentifier (UUID) URN Namespace This specification defines a Uniform Resource Name namespace for UUIDs, the internal format of UUIDs, and methods of generating UUIDs. @@ -283,7 +307,7 @@ The :mod:`uuid` module can be executed as a script from the command line. .. code-block:: sh - python -m uuid [-h] [-u {uuid1,uuid3,uuid4,uuid5}] [-n NAMESPACE] [-N NAME] + python -m uuid [-h] [-u {uuid1,uuid3,uuid4,uuid5,uuid8}] [-n NAMESPACE] [-N NAME] The following options are accepted: @@ -299,6 +323,9 @@ The following options are accepted: Specify the function name to use to generate the uuid. By default :func:`uuid4` is used. + .. versionadded:: next + Allow generating UUID version 8. + .. option:: -n --namespace diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 1e7c4c8b915e7e..0c7e8543f331db 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -180,6 +180,19 @@ If a warning is reported and doesn't match any registered filter then the "default" action is applied (hence its name). + +.. _repeated-warning-suppression-criteria: + +Repeated Warning Suppression Criteria +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The filters that suppress repeated warnings apply the following criteria to determine if a warning is considered a repeat: + +- ``"default"``: A warning is considered a repeat only if the (*message*, *category*, *module*, *lineno*) are all the same. +- ``"module"``: A warning is considered a repeat if the (*message*, *category*, *module*) are the same, ignoring the line number. +- ``"once"``: A warning is considered a repeat if the (*message*, *category*) are the same, ignoring the module and line number. + + .. _describing-warning-filters: Describing Warning Filters diff --git a/Doc/library/xdrlib.rst b/Doc/library/xdrlib.rst new file mode 100644 index 00000000000000..59b801c8e4072e --- /dev/null +++ b/Doc/library/xdrlib.rst @@ -0,0 +1,15 @@ +:mod:`!xdrlib` --- Encode and decode XDR data +============================================= + +.. module:: xdrlib + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!xdrlib` module was +`Python 3.12 `_. diff --git a/Doc/license.rst b/Doc/license.rst index 674ac5f56e6f97..428dc22b817ebe 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -100,7 +100,7 @@ PSF LICENSE AGREEMENT FOR PYTHON |release| analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python |release| alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of - copyright, i.e., "Copyright © 2001-2024 Python Software Foundation; All Rights + copyright, i.e., "Copyright © 2001 Python Software Foundation; All Rights Reserved" are retained in Python |release| alone or in any derivative version prepared by Licensee. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 1b1e9f479cbe08..e73ce44270b082 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -534,18 +534,15 @@ is semantically equivalent to:: enter = type(manager).__enter__ exit = type(manager).__exit__ value = enter(manager) - hit_except = False try: TARGET = value SUITE except: - hit_except = True if not exit(manager, *sys.exc_info()): raise - finally: - if not hit_except: - exit(manager, None, None, None) + else: + exit(manager, None, None, None) With more than one item, the context managers are processed as if multiple :keyword:`with` statements were nested:: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index dfd1addf656a85..66b836eaf0008a 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1028,9 +1028,9 @@ this approach. using the :class:`types.ModuleType` constructor. Previously the attribute was optional. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.16 Setting :attr:`!__loader__` on a module while failing to set - :attr:`!__spec__.loader` is deprecated. In Python 3.14, + :attr:`!__spec__.loader` is deprecated. In Python 3.16, :attr:`!__loader__` will cease to be set or taken into consideration by the import system or the standard library. @@ -1507,7 +1507,7 @@ Special read-only attributes .. deprecated:: 3.12 This attribute of code objects is deprecated, and may be removed in - Python 3.14. + Python 3.15. * - .. attribute:: codeobject.co_stacksize - The required stack size of the code object diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 63b8dd73127984..7c95b207b1aed2 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1155,7 +1155,8 @@ a user-defined function: first thing the code block will do is bind the formal parameters to the arguments; this is described in section :ref:`function`. When the code block executes a :keyword:`return` statement, this specifies the return value of the - function call. + function call. If execution reaches the end of the code block without + executing a :keyword:`return` statement, the return value is ``None``. a built-in function or method: .. index:: diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 24df4a6ba7b678..2a72af4e9a3299 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -408,9 +408,9 @@ The extended form, ``assert expression1, expression2``, is equivalent to :: These equivalences assume that :const:`__debug__` and :exc:`AssertionError` refer to the built-in variables with those names. In the current implementation, the -built-in variable :const:`__debug__` is ``True`` under normal circumstances, +built-in variable ``__debug__`` is ``True`` under normal circumstances, ``False`` when optimization is requested (command line option :option:`-O`). The current -code generator emits no code for an assert statement when optimization is +code generator emits no code for an :keyword:`assert` statement when optimization is requested at compile time. Note that it is unnecessary to include the source code for the expression that failed in the error message; it will be displayed as part of the stack trace. @@ -533,8 +533,8 @@ The :keyword:`!yield` statement yield_stmt: `yield_expression` A :keyword:`yield` statement is semantically equivalent to a :ref:`yield -expression `. The yield statement can be used to omit the parentheses -that would otherwise be required in the equivalent yield expression +expression `. The ``yield`` statement can be used to omit the +parentheses that would otherwise be required in the equivalent yield expression statement. For example, the yield statements :: yield @@ -546,7 +546,7 @@ are equivalent to the yield expression statements :: (yield from ) Yield expressions and statements are only used when defining a :term:`generator` -function, and are only used in the body of the generator function. Using yield +function, and are only used in the body of the generator function. Using :keyword:`yield` in a function definition is sufficient to cause that definition to create a generator function instead of a normal function. @@ -966,25 +966,14 @@ The :keyword:`!global` statement .. productionlist:: python-grammar global_stmt: "global" `identifier` ("," `identifier`)* -The :keyword:`global` statement is a declaration which holds for the entire -current code block. It means that the listed identifiers are to be interpreted -as globals. It would be impossible to assign to a global variable without +The :keyword:`global` statement causes the listed identifiers to be interpreted +as globals. It would be impossible to assign to a global variable without :keyword:`!global`, although free variables may refer to globals without being declared global. -Names listed in a :keyword:`global` statement must not be used in the same code -block textually preceding that :keyword:`!global` statement. - -Names listed in a :keyword:`global` statement must not be defined as formal -parameters, or as targets in :keyword:`with` statements or :keyword:`except` clauses, or in a :keyword:`for` target list, :keyword:`class` -definition, function definition, :keyword:`import` statement, or -:term:`variable annotations `. - -.. impl-detail:: - - The current implementation does not enforce some of these restrictions, but - programs should not abuse this freedom, as future implementations may enforce - them or silently change the meaning of the program. +The :keyword:`global` statement applies to the entire scope of a function or +class body. A :exc:`SyntaxError` is raised if a variable is used or +assigned to prior to its global declaration in the scope. .. index:: pair: built-in function; exec @@ -1020,7 +1009,7 @@ identifiers. If a name is bound in more than one nonlocal scope, the nearest binding is used. If a name is not bound in any nonlocal scope, or if there is no nonlocal scope, a :exc:`SyntaxError` is raised. -The nonlocal statement applies to the entire scope of a function or +The :keyword:`nonlocal` statement applies to the entire scope of a function or class body. A :exc:`SyntaxError` is raised if a variable is used or assigned to prior to its nonlocal declaration in the scope. diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 83994af795e3fc..e7733a6dc11451 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -297,6 +297,19 @@ General Options .. versionadded:: 3.13 +.. option:: --enable-experimental-jit=[no|yes|yes-off|interpreter] + + Indicate how to integrate the :ref:`JIT compiler `. + + * ``no`` - build the interpreter without the JIT. + * ``yes`` - build the interpreter with the JIT. + * ``yes-off`` - build the interpreter with the JIT but disable it by default. + * ``interpreter`` - build the interpreter without the JIT, but with the tier 2 enabled interpreter. + + By convention, ``--enable-experimental-jit`` is a shorthand for ``--enable-experimental-jit=yes``. + + .. versionadded:: 3.13 + .. option:: PKG_CONFIG Path to ``pkg-config`` utility. @@ -441,15 +454,6 @@ Options for third-party dependencies WebAssembly Options ------------------- -.. option:: --with-emscripten-target=[browser|node] - - Set build flavor for ``wasm32-emscripten``. - - * ``browser`` (default): preload minimal stdlib, default MEMFS. - * ``node``: NODERAWFS and pthread support. - - .. versionadded:: 3.11 - .. option:: --enable-wasm-dynamic-linking Turn on dynamic linking support for WASM. @@ -1084,7 +1088,8 @@ CPython project) this is usually the ``all`` target. The all`` will build. The three choices are: * ``profile-opt`` (configured with ``--enable-optimizations``) -* ``build_wasm`` (configured with ``--with-emscripten-target``) +* ``build_wasm`` (chosen if the host platform matches ``wasm32-wasi*`` or + ``wasm32-emscripten``) * ``build_all`` (configured without explicitly using either of the others) Depending on the most recent source file changes, Make will rebuild diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index daaf8822af1161..1a6322d72341ff 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -435,7 +435,7 @@ When writing to the Windows Registry, the following behaviors exist: For more detail on the technical basis for these limitations, please consult Microsoft's documentation on packaged full-trust apps, currently available at `docs.microsoft.com/en-us/windows/msix/desktop/desktop-to-uwp-behind-the-scenes -`_ +`_ .. _windows-nuget: @@ -536,7 +536,7 @@ dependents, such as Idle), pip and the Python documentation are not included. .. note:: The embedded distribution does not include the `Microsoft C Runtime - `_ and it is + `_ and it is the responsibility of the application installer to provide this. The runtime may have already been installed on a user's system previously or automatically via Windows Update, and can be detected by finding @@ -679,13 +679,13 @@ System variables, you need non-restricted access to your machine .. seealso:: - https://docs.microsoft.com/en-us/windows/win32/procthread/environment-variables + https://learn.microsoft.com/windows/win32/procthread/environment-variables Overview of environment variables on Windows - https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/set_1 + https://learn.microsoft.com/windows-server/administration/windows-commands/set_1 The ``set`` command, for temporarily modifying environment variables - https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/setx + https://learn.microsoft.com/windows-server/administration/windows-commands/setx The ``setx`` command, for permanently modifying environment variables @@ -1291,13 +1291,13 @@ is a collection of modules for advanced Windows-specific support. This includes utilities for: * `Component Object Model - `_ + `_ (COM) * Win32 API calls * Registry * Event log * `Microsoft Foundation Classes - `_ + `_ (MFC) user interfaces `PythonWin `. -* :mod:`pkgutil`: :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` +* :mod:`pkgutil`: :func:`!pkgutil.find_loader` and :func:`!pkgutil.get_loader` are deprecated and will be removed in Python 3.14; use :func:`importlib.util.find_spec` instead. (Contributed by Nikita Sobolev in :gh:`97850`.) @@ -1327,8 +1327,8 @@ Deprecated * Accessing :attr:`~codeobject.co_lnotab` on code objects was deprecated in Python 3.10 via :pep:`626`, - but it only got a proper :exc:`DeprecationWarning` in 3.12, - therefore it will be removed in 3.14. + but it only got a proper :exc:`DeprecationWarning` in 3.12. + May be removed in 3.15. (Contributed by Nikita Sobolev in :gh:`101866`.) .. include:: ../deprecations/pending-removal-in-3.13.rst @@ -1341,6 +1341,8 @@ Deprecated .. include:: ../deprecations/pending-removal-in-future.rst +.. _whatsnew312-removed: + Removed ======= @@ -1366,6 +1368,8 @@ configparser * :class:`configparser.ConfigParser` no longer has a ``readfp`` method. Use :meth:`~configparser.ConfigParser.read_file` instead. +.. _whatsnew312-removed-distutils: + distutils --------- @@ -1447,6 +1451,8 @@ importlib * ``importlib.abc.Finder``, ``pkgutil.ImpImporter``, and ``pkgutil.ImpLoader`` have been removed. (Contributed by Barry Warsaw in :gh:`98040`.) +.. _whatsnew312-removed-imp: + imp --- diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index de4c7fd4c0486b..664b1866172378 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1568,6 +1568,8 @@ and are now removed: For audio playback, use the :pypi:`pygame` library from PyPI instead. * :mod:`!pipes`: Use the :mod:`subprocess` module instead. + Use :func:`shlex.quote` to replace the undocumented ``pipes.quote`` + function. * :mod:`!sndhdr`: The :pypi:`filetype`, :pypi:`puremagic`, or :pypi:`python-magic` libraries should be used as replacements. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index e04cc77c16a820..8e1cb05c2c8198 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -175,6 +175,10 @@ Improved error messages Other language changes ====================== +* The :func:`map` built-in now has an optional keyword-only *strict* flag + like :func:`zip` to check that all the iterables are of equal length. + (Contributed by Wannes Boeykens in :gh:`119793`.) + * Incorrect usage of :keyword:`await` and asynchronous comprehensions is now detected even if the code is optimized away by the :option:`-O` command-line option. For example, ``python -O -c 'assert await 1'`` @@ -194,6 +198,9 @@ Other language changes specified by C standards since C99. (Contributed by Sergey B Kirpichev in :gh:`69639`.) +* All Windows code pages are now supported as "cpXXX" codecs on Windows. + (Contributed by Serhiy Storchaka in :gh:`123803`.) + * :class:`super` objects are now :mod:`pickleable ` and :mod:`copyable `. (Contributed by Serhiy Storchaka in :gh:`125767`.) @@ -250,6 +257,12 @@ concurrent.futures same process) to Python code. This is separate from the proposed API in :pep:`734`. (Contributed by Eric Snow in :gh:`124548`.) +* The default ``ProcessPoolExecutor`` start method (see + :ref:`multiprocessing-start-methods`) changed from *fork* to *forkserver* on + platforms other than macOS & Windows. If you require the threading + incompatible *fork* start method you must explicitly request it by + supplying a *mp_context* to :class:`concurrent.futures.ProcessPoolExecutor`. + (Contributed by Gregory P. Smith in :gh:`84559`.) ctypes ------ @@ -314,6 +327,19 @@ functools to reserve a place for positional arguments. (Contributed by Dominykas Grigonis in :gh:`119127`.) +* Allow the *initial* parameter of :func:`functools.reduce` to be passed + as a keyword argument. + (Contributed by Sayandip Dutta in :gh:`125916`.) + + +getopt +------ + +* Add support for options with optional arguments. + (Contributed by Serhiy Storchaka in :gh:`126374`.) + +* Add support for returning intermixed options and non-option arguments in order. + (Contributed by Serhiy Storchaka in :gh:`126390`.) http ---- @@ -352,6 +378,60 @@ json (Contributed by Trey Hunner in :gh:`122873`.) +mimetypes +--------- + +* Add MS and :rfc:`8081` MIME types for fonts: + + * Embedded OpenType: ``application/vnd.ms-fontobject`` + * OpenType Layout (OTF) ``font/otf`` + * TrueType: ``font/ttf`` + * WOFF 1.0 ``font/woff`` + * WOFF 2.0 ``font/woff2`` + + (Contributed by Sahil Prajapati and Hugo van Kemenade in :gh:`84852`.) + +* Add :rfc:`9559` MIME types for Matroska audiovisual data container + structures, containing: + + * audio with no video: ``audio/matroska`` (``.mka``) + * video: ``video/matroska`` (``.mkv``) + * stereoscopic video: ``video/matroska-3d`` (``.mk3d``) + + (Contributed by Hugo van Kemenade in :gh:`89416`.) + +* Add MIME types for images with RFCs: + + * :rfc:`1494`: CCITT Group 3 (``.g3``) + * :rfc:`3362`: Real-time Facsimile, T.38 (``.t38``) + * :rfc:`3745`: JPEG 2000 (``.jp2``), extension (``.jpx``) and compound (``.jpm``) + * :rfc:`3950`: Tag Image File Format Fax eXtended, TIFF-FX (``.tfx``) + * :rfc:`4047`: Flexible Image Transport System (``.fits``) + * :rfc:`7903`: Enhanced Metafile (``.emf``) and Windows Metafile (``.wmf``) + + (Contributed by Hugo van Kemenade in :gh:`85957`.) + + +multiprocessing +--------------- + +* The default start method (see :ref:`multiprocessing-start-methods`) changed + from *fork* to *forkserver* on platforms other than macOS & Windows where + it was already *spawn*. If you require the threading incompatible *fork* + start method you must explicitly request it using a context from + :func:`multiprocessing.get_context` (preferred) or change the default via + :func:`multiprocessing.set_start_method`. + (Contributed by Gregory P. Smith in :gh:`84559`.) +* The :ref:`multiprocessing proxy objects ` + for *list* and *dict* types gain previously overlooked missing methods: + + * :meth:`!clear` and :meth:`!copy` for proxies of :class:`list`. + * :meth:`~dict.fromkeys`, ``reversed(d)``, ``d | {}``, ``{} | d``, + ``d |= {'b': 2}`` for proxies of :class:`dict`. + + (Contributed by Roy Hyunjin Han for :gh:`103134`) + + operator -------- @@ -365,9 +445,10 @@ operator os -- -* Add the :data:`os.environ.refresh() ` method to update - :data:`os.environ` with changes to the environment made by :func:`os.putenv`, - by :func:`os.unsetenv`, or made outside Python in the same process. +* Add the :func:`os.reload_environ` function to update :data:`os.environ` and + :data:`os.environb` with changes to the environment made by + :func:`os.putenv`, by :func:`os.unsetenv`, or made outside Python in the + same process. (Contributed by Victor Stinner in :gh:`120057`.) @@ -384,6 +465,20 @@ pathlib (Contributed by Barney Gale in :gh:`73991`.) +* Add :meth:`pathlib.Path.scandir` to scan a directory and return an iterator + of :class:`os.DirEntry` objects. This is exactly equivalent to calling + :func:`os.scandir` on a path object. + + (Contributed by Barney Gale in :gh:`125413`.) + + +platform +-------- + +* Add :func:`platform.invalidate_caches` to invalidate the cached results. + + (Contributed by Bénédikt Tran in :gh:`122549`.) + pdb --- @@ -451,6 +546,14 @@ unittest (Contributed by Jacob Walls in :gh:`80958`.) +uuid +---- + +* Add support for UUID version 8 via :func:`uuid.uuid8` as specified + in :rfc:`9562`. + (Contributed by Bénédikt Tran in :gh:`89083`.) + + .. Add improved modules above alphabetically, not here at the end. Optimizations @@ -464,6 +567,15 @@ asyncio reduces memory usage. (Contributed by Kumar Aditya in :gh:`107803`.) +io +--- +* :mod:`io` which provides the built-in :func:`open` makes less system calls + when opening regular files as well as reading whole files. Reading a small + operating system cached file in full is up to 15% faster. + :func:`pathlib.Path.read_bytes` has the most optimizations for reading a + file's bytes in full. (Contributed by Cody Maloney and Victor Stinner in + :gh:`120754` and :gh:`90102`.) + Deprecated ========== @@ -490,14 +602,6 @@ Deprecated as a single positional argument. (Contributed by Serhiy Storchaka in :gh:`109218`.) -* :mod:`multiprocessing` and :mod:`concurrent.futures`: - The default start method (see :ref:`multiprocessing-start-methods`) changed - away from *fork* to *forkserver* on platforms where it was not already - *spawn* (Windows & macOS). If you require the threading incompatible *fork* - start method you must explicitly specify it when using :mod:`multiprocessing` - or :mod:`concurrent.futures` APIs. - (Contributed by Gregory P. Smith in :gh:`84559`.) - * :mod:`os`: :term:`Soft deprecate ` :func:`os.popen` and :func:`os.spawn* ` functions. They should no longer be used to @@ -508,6 +612,13 @@ Deprecated Deprecate :meth:`symtable.Class.get_methods` due to the lack of interest. (Contributed by Bénédikt Tran in :gh:`119698`.) +* :mod:`urllib.parse`: + Accepting objects with false values (like ``0`` and ``[]``) except empty + strings, byte-like objects and ``None`` in :mod:`urllib.parse` functions + :func:`~urllib.parse.parse_qsl` and :func:`~urllib.parse.parse_qs` is now + deprecated. + (Contributed by Serhiy Storchaka in :gh:`116897`.) + .. Add deprecations above alphabetically, not here at the end. .. include:: ../deprecations/pending-removal-in-3.15.rst @@ -574,6 +685,11 @@ asyncio (Contributed by Kumar Aditya in :gh:`120804`.) +* Removed implicit creation of event loop by :func:`asyncio.get_event_loop`. + It now raises a :exc:`RuntimeError` if there is no current event loop. + (Contributed by Kumar Aditya in :gh:`126353`.) + + collections.abc --------------- @@ -619,6 +735,13 @@ pathlib :meth:`~pathlib.PurePath.is_relative_to`. In previous versions, any such arguments are joined onto *other*. +pkgutil +------- + +* Remove deprecated :func:`!pkgutil.get_loader` and :func:`!pkgutil.find_loader`. + These had previously raised a :exc:`DeprecationWarning` since Python 3.12. + (Contributed by Bénédikt Tran in :gh:`97850`.) + pty --- @@ -686,6 +809,16 @@ Changes in the Python API Build changes ============= +PEP 761: Discontinuation of PGP signatures +------------------------------------------ + +PGP signatures will not be available for CPython 3.14 and onwards. +Users verifying artifacts must use `Sigstore verification materials`_ for +verifying CPython artifacts. This change in release process is specified +in :pep:`761`. + +.. _Sigstore verification materials: https://www.python.org/downloads/metadata/sigstore/ + C API changes ============= @@ -728,6 +861,11 @@ New features an interned string and deallocate it during module shutdown. (Contributed by Eddie Elizondo in :gh:`113601`.) +* Add :c:func:`PyLong_IsPositive`, :c:func:`PyLong_IsNegative` + and :c:func:`PyLong_IsZero` for checking if :c:type:`PyLongObject` + is positive, negative, or zero, respectively. + (Contribued by James Roy and Sergey B Kirpichev in :gh:`126061`.) + * Add new functions to convert C ```` numbers from/to Python :class:`int`: @@ -791,6 +929,9 @@ New features * Add :c:func:`PyType_Freeze` function to make a type immutable. (Contributed by Victor Stinner in :gh:`121654`.) +* Add :c:func:`PyUnstable_Object_EnableDeferredRefcount` for enabling + deferred reference counting, as outlined in :pep:`703`. + Porting to Python 3.14 ---------------------- @@ -803,6 +944,10 @@ Porting to Python 3.14 Deprecated ---------- +* The :c:macro:`!Py_HUGE_VAL` macro is :term:`soft deprecated`, + use :c:macro:`!Py_INFINITY` instead. + (Contributed by Sergey B Kirpichev in :gh:`120026`.) + * Macros :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY` and :c:macro:`!Py_IS_FINITE` are :term:`soft deprecated`, use instead :c:macro:`!isnan`, :c:macro:`!isinf` and diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index fc9f49e65af847..bdc4ca5cab5245 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -936,7 +936,7 @@ Add option ``--json-lines`` to parse every input line as a separate JSON object. logging ------- -Added a *force* keyword argument to :func:`logging.basicConfig` +Added a *force* keyword argument to :func:`logging.basicConfig`. When set to true, any existing handlers attached to the root logger are removed and closed before carrying out the configuration specified by the other arguments. diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 2561b2b88baacc..370f1d259abe0f 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -72,6 +72,24 @@ typedef struct { uint8_t *per_instruction_tools; } _PyCoMonitoringData; +#ifdef Py_GIL_DISABLED + +/* Each thread specializes a thread-local copy of the bytecode in free-threaded + * builds. These copies are stored on the code object in a `_PyCodeArray`. The + * first entry in the array always points to the "main" copy of the bytecode + * that is stored at the end of the code object. + */ +typedef struct { + Py_ssize_t size; + char *entries[1]; +} _PyCodeArray; + +#define _PyCode_DEF_THREAD_LOCAL_BYTECODE() \ + _PyCodeArray *co_tlbc; +#else +#define _PyCode_DEF_THREAD_LOCAL_BYTECODE() +#endif + // To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are // defined in this macro: #define _PyCode_DEF(SIZE) { \ @@ -138,6 +156,7 @@ typedef struct { Type is a void* to keep the format private in codeobject.c to force \ people to go through the proper APIs. */ \ void *co_extra; \ + _PyCode_DEF_THREAD_LOCAL_BYTECODE() \ char co_code_adaptive[(SIZE)]; \ } diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index c2cb4e3cdd92fb..f69c586a4f96f3 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -183,6 +183,7 @@ typedef struct PyConfig { int cpu_count; #ifdef Py_GIL_DISABLED int enable_gil; + int tlbc_enabled; #endif /* --- Path configuration inputs ------------ */ diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index c1214d5e3714ea..4d6e618f831ad8 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -61,6 +61,24 @@ PyAPI_FUNC(PyObject*) PyLong_FromUnsignedNativeBytes(const void* buffer, PyAPI_FUNC(int) PyUnstable_Long_IsCompact(const PyLongObject* op); PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op); +/* PyLong_IsPositive. Check if the integer object is positive. + + - On success, return 1 if *obj is positive, and 0 otherwise. + - On failure, set an exception, and return -1. */ +PyAPI_FUNC(int) PyLong_IsPositive(PyObject *obj); + +/* PyLong_IsNegative. Check if the integer object is negative. + + - On success, return 1 if *obj is negative, and 0 otherwise. + - On failure, set an exception, and return -1. */ +PyAPI_FUNC(int) PyLong_IsNegative(PyObject *obj); + +/* PyLong_IsZero. Check if the integer object is zero. + + - On success, return 1 if *obj is zero, and 0 if it is non-zero. + - On failure, set an exception, and return -1. */ +PyAPI_FUNC(int) PyLong_IsZero(PyObject *obj); + /* PyLong_GetSign. Get the sign of an integer object: 0, -1 or +1 for zero, negative or positive integer, respectively. diff --git a/Include/cpython/object.h b/Include/cpython/object.h index f0f61796cd3ec8..e4797029da431e 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -527,3 +527,10 @@ typedef enum { typedef int (*PyRefTracer)(PyObject *, PyRefTracerEvent event, void *); PyAPI_FUNC(int) PyRefTracer_SetTracer(PyRefTracer tracer, void *data); PyAPI_FUNC(PyRefTracer) PyRefTracer_GetTracer(void**); + +/* Enable PEP-703 deferred reference counting on the object. + * + * Returns 1 if deferred reference counting was successfully enabled, and + * 0 if the runtime ignored it. This function cannot fail. + */ +PyAPI_FUNC(int) PyUnstable_Object_EnableDeferredRefcount(PyObject *); diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index f1ca54839fbc38..2ae48002d720e9 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -99,6 +99,8 @@ typedef struct _gc_stats { uint64_t collections; uint64_t object_visits; uint64_t objects_collected; + uint64_t objects_transitively_reachable; + uint64_t objects_not_transitively_reachable; } GCStats; typedef struct _uop_stats { diff --git a/Include/floatobject.h b/Include/floatobject.h index 8963c16832a4bc..4d24a76edd5de1 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -21,10 +21,10 @@ PyAPI_DATA(PyTypeObject) PyFloat_Type; #define Py_RETURN_INF(sign) \ do { \ if (copysign(1., sign) == 1.) { \ - return PyFloat_FromDouble(Py_HUGE_VAL); \ + return PyFloat_FromDouble(Py_INFINITY); \ } \ else { \ - return PyFloat_FromDouble(-Py_HUGE_VAL); \ + return PyFloat_FromDouble(-Py_INFINITY); \ } \ } while(0) diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 20436a68b69677..3e02728522828e 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -102,10 +102,9 @@ backoff_counter_triggers(_Py_BackoffCounter counter) } /* Initial JUMP_BACKWARD counter. - * This determines when we create a trace for a loop. -* Backoff sequence 16, 32, 64, 128, 256, 512, 1024, 2048, 4096. */ -#define JUMP_BACKWARD_INITIAL_VALUE 15 -#define JUMP_BACKWARD_INITIAL_BACKOFF 4 + * This determines when we create a trace for a loop. */ +#define JUMP_BACKWARD_INITIAL_VALUE 4095 +#define JUMP_BACKWARD_INITIAL_BACKOFF 12 static inline _Py_BackoffCounter initial_jump_backoff_counter(void) { diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 411bbff106dd69..80bd19a887871c 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -174,6 +174,18 @@ _PyEval_IsGILEnabled(PyThreadState *tstate) extern int _PyEval_EnableGILTransient(PyThreadState *tstate); extern int _PyEval_EnableGILPermanent(PyThreadState *tstate); extern int _PyEval_DisableGIL(PyThreadState *state); + + +static inline _Py_CODEUNIT * +_PyEval_GetExecutableCode(PyThreadState *tstate, PyCodeObject *co) +{ + _Py_CODEUNIT *bc = _PyCode_GetTLBCFast(tstate, co); + if (bc != NULL) { + return bc; + } + return _PyCode_GetTLBC(co); +} + #endif extern void _PyEval_DeactivateOpCache(void); diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 57e0a14bb9b5bd..a0acf76db6f04d 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -11,6 +11,7 @@ extern "C" { #include "pycore_stackref.h" // _PyStackRef #include "pycore_lock.h" // PyMutex #include "pycore_backoff.h" // _Py_BackoffCounter +#include "pycore_tstate.h" // _PyThreadStateImpl /* Each instruction in a code object is a fixed-width value, @@ -313,11 +314,17 @@ extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); /** API for executors */ extern void _PyCode_Clear_Executors(PyCodeObject *code); + #ifdef Py_GIL_DISABLED // gh-115999 tracks progress on addressing this. #define ENABLE_SPECIALIZATION 0 +// Use this to enable specialization families once they are thread-safe. All +// uses will be replaced with ENABLE_SPECIALIZATION once all families are +// thread-safe. +#define ENABLE_SPECIALIZATION_FT 1 #else #define ENABLE_SPECIALIZATION 1 +#define ENABLE_SPECIALIZATION_FT ENABLE_SPECIALIZATION #endif /* Specialization functions */ @@ -600,6 +607,40 @@ struct _PyCode8 _PyCode_DEF(8); PyAPI_DATA(const struct _PyCode8) _Py_InitCleanup; +#ifdef Py_GIL_DISABLED + +// Return a pointer to the thread-local bytecode for the current thread, if it +// exists. +static inline _Py_CODEUNIT * +_PyCode_GetTLBCFast(PyThreadState *tstate, PyCodeObject *co) +{ + _PyCodeArray *code = _Py_atomic_load_ptr_acquire(&co->co_tlbc); + int32_t idx = ((_PyThreadStateImpl*) tstate)->tlbc_index; + if (idx < code->size && code->entries[idx] != NULL) { + return (_Py_CODEUNIT *) code->entries[idx]; + } + return NULL; +} + +// Return a pointer to the thread-local bytecode for the current thread, +// creating it if necessary. +extern _Py_CODEUNIT *_PyCode_GetTLBC(PyCodeObject *co); + +// Reserve an index for the current thread into thread-local bytecode +// arrays +// +// Returns the reserved index or -1 on error. +extern int32_t _Py_ReserveTLBCIndex(PyInterpreterState *interp); + +// Release the current thread's index into thread-local bytecode arrays +extern void _Py_ClearTLBCIndex(_PyThreadStateImpl *tstate); + +// Free all TLBC copies not associated with live threads. +// +// Returns 0 on success or -1 on error. +extern int _Py_ClearUnusedTLBC(PyInterpreterState *interp); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 2dd165eae74850..69a60d73e05c26 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -11,6 +11,7 @@ extern "C" { #include "pycore_lock.h" // PyMutex #include "pycore_pyerrors.h" + /**************/ /* exceptions */ /**************/ @@ -38,28 +39,28 @@ extern int _Py_CallInInterpreterAndRawFree( /* cross-interpreter data */ /**************************/ -typedef struct _xid _PyCrossInterpreterData; -typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); +typedef struct _xidata _PyXIData_t; +typedef PyObject *(*xid_newobjfunc)(_PyXIData_t *); typedef void (*xid_freefunc)(void *); -// _PyCrossInterpreterData is similar to Py_buffer as an effectively +// _PyXIData_t is similar to Py_buffer as an effectively // opaque struct that holds data outside the object machinery. This // is necessary to pass safely between interpreters in the same process. -struct _xid { +struct _xidata { // data is the cross-interpreter-safe derivation of a Python object - // (see _PyObject_GetCrossInterpreterData). It will be NULL if the + // (see _PyObject_GetXIData). It will be NULL if the // new_object func (below) encodes the data. void *data; // obj is the Python object from which the data was derived. This // is non-NULL only if the data remains bound to the object in some // way, such that the object must be "released" (via a decref) when // the data is released. In that case the code that sets the field, - // likely a registered "crossinterpdatafunc", is responsible for + // likely a registered "xidatafunc", is responsible for // ensuring it owns the reference (i.e. incref). PyObject *obj; // interp is the ID of the owning interpreter of the original // object. It corresponds to the active interpreter when - // _PyObject_GetCrossInterpreterData() was called. This should only + // _PyObject_GetXIData() was called. This should only // be set by the cross-interpreter machinery. // // We use the ID rather than the PyInterpreterState to avoid issues @@ -71,130 +72,138 @@ struct _xid { // interpreter given the data. The resulting object (a new // reference) will be equivalent to the original object. This field // is required. - xid_newobjectfunc new_object; + xid_newobjfunc new_object; // free is called when the data is released. If it is NULL then // nothing will be done to free the data. For some types this is // okay (e.g. bytes) and for those types this field should be set // to NULL. However, for most the data was allocated just for // cross-interpreter use, so it must be freed when - // _PyCrossInterpreterData_Release is called or the memory will + // _PyXIData_Release is called or the memory will // leak. In that case, at the very least this field should be set // to PyMem_RawFree (the default if not explicitly set to NULL). // The call will happen with the original interpreter activated. xid_freefunc free; }; -PyAPI_FUNC(_PyCrossInterpreterData *) _PyCrossInterpreterData_New(void); -PyAPI_FUNC(void) _PyCrossInterpreterData_Free(_PyCrossInterpreterData *data); +PyAPI_FUNC(_PyXIData_t *) _PyXIData_New(void); +PyAPI_FUNC(void) _PyXIData_Free(_PyXIData_t *data); -#define _PyCrossInterpreterData_DATA(DATA) ((DATA)->data) -#define _PyCrossInterpreterData_OBJ(DATA) ((DATA)->obj) -#define _PyCrossInterpreterData_INTERPID(DATA) ((DATA)->interpid) +#define _PyXIData_DATA(DATA) ((DATA)->data) +#define _PyXIData_OBJ(DATA) ((DATA)->obj) +#define _PyXIData_INTERPID(DATA) ((DATA)->interpid) // Users should not need getters for "new_object" or "free". +/* getting cross-interpreter data */ + +typedef int (*xidatafunc)(PyThreadState *tstate, PyObject *, _PyXIData_t *); + +typedef struct _xid_lookup_state _PyXIData_lookup_t; + +typedef struct { + _PyXIData_lookup_t *global; + _PyXIData_lookup_t *local; + PyObject *PyExc_NotShareableError; +} _PyXIData_lookup_context_t; + +PyAPI_FUNC(int) _PyXIData_GetLookupContext( + PyInterpreterState *, + _PyXIData_lookup_context_t *); + +PyAPI_FUNC(xidatafunc) _PyXIData_Lookup( + _PyXIData_lookup_context_t *, + PyObject *); +PyAPI_FUNC(int) _PyObject_CheckXIData( + _PyXIData_lookup_context_t *, + PyObject *); +PyAPI_FUNC(int) _PyObject_GetXIData( + _PyXIData_lookup_context_t *, + PyObject *, + _PyXIData_t *); + + +/* using cross-interpreter data */ + +PyAPI_FUNC(PyObject *) _PyXIData_NewObject(_PyXIData_t *); +PyAPI_FUNC(int) _PyXIData_Release(_PyXIData_t *); +PyAPI_FUNC(int) _PyXIData_ReleaseAndRawFree(_PyXIData_t *); + + /* defining cross-interpreter data */ -PyAPI_FUNC(void) _PyCrossInterpreterData_Init( - _PyCrossInterpreterData *data, +PyAPI_FUNC(void) _PyXIData_Init( + _PyXIData_t *data, PyInterpreterState *interp, void *shared, PyObject *obj, - xid_newobjectfunc new_object); -PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( - _PyCrossInterpreterData *, + xid_newobjfunc new_object); +PyAPI_FUNC(int) _PyXIData_InitWithSize( + _PyXIData_t *, PyInterpreterState *interp, const size_t, PyObject *, - xid_newobjectfunc); -PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( - PyInterpreterState *, _PyCrossInterpreterData *); + xid_newobjfunc); +PyAPI_FUNC(void) _PyXIData_Clear( PyInterpreterState *, _PyXIData_t *); // Normally the Init* functions are sufficient. The only time // additional initialization might be needed is to set the "free" func, // though that should be infrequent. -#define _PyCrossInterpreterData_SET_FREE(DATA, FUNC) \ +#define _PyXIData_SET_FREE(DATA, FUNC) \ do { \ (DATA)->free = (FUNC); \ } while (0) // Additionally, some shareable types are essentially light wrappers -// around other shareable types. The crossinterpdatafunc of the wrapper +// around other shareable types. The xidatafunc of the wrapper // can often be implemented by calling the wrapped object's -// crossinterpdatafunc and then changing the "new_object" function. -// We have _PyCrossInterpreterData_SET_NEW_OBJECT() here for that, +// xidatafunc and then changing the "new_object" function. +// We have _PyXIData_SET_NEW_OBJECT() here for that, // but might be better to have a function like -// _PyCrossInterpreterData_AdaptToWrapper() instead. -#define _PyCrossInterpreterData_SET_NEW_OBJECT(DATA, FUNC) \ +// _PyXIData_AdaptToWrapper() instead. +#define _PyXIData_SET_NEW_OBJECT(DATA, FUNC) \ do { \ (DATA)->new_object = (FUNC); \ } while (0) -/* using cross-interpreter data */ - -PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); -PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); -PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *); - - /* cross-interpreter data registry */ -// For now we use a global registry of shareable classes. An -// alternative would be to add a tp_* slot for a class's -// crossinterpdatafunc. It would be simpler and more efficient. - -typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *, - _PyCrossInterpreterData *); - -struct _xidregitem; - -struct _xidregitem { - struct _xidregitem *prev; - struct _xidregitem *next; - /* This can be a dangling pointer, but only if weakref is set. */ - PyTypeObject *cls; - /* This is NULL for builtin types. */ - PyObject *weakref; - size_t refcount; - crossinterpdatafunc getdata; -}; - -struct _xidregistry { - int global; /* builtin types or heap types */ - int initialized; - PyMutex mutex; - struct _xidregitem *head; -}; - -PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); -PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); -PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); +#define Py_CORE_CROSSINTERP_DATA_REGISTRY_H +#include "pycore_crossinterp_data_registry.h" +#undef Py_CORE_CROSSINTERP_DATA_REGISTRY_H /*****************************/ /* runtime state & lifecycle */ /*****************************/ -struct _xi_runtime_state { +typedef struct { // builtin types - // XXX Remove this field once we have a tp_* slot. - struct _xidregistry registry; -}; + _PyXIData_lookup_t data_lookup; +} _PyXI_global_state_t; -struct _xi_state { +typedef struct { // heap types - // XXX Remove this field once we have a tp_* slot. - struct _xidregistry registry; + _PyXIData_lookup_t data_lookup; - // heap types - PyObject *PyExc_NotShareableError; -}; + struct xi_exceptions { + // static types + PyObject *PyExc_InterpreterError; + PyObject *PyExc_InterpreterNotFoundError; + // heap types + PyObject *PyExc_NotShareableError; + } exceptions; +} _PyXI_state_t; + +#define _PyXI_GET_GLOBAL_STATE(interp) (&(interp)->runtime->xi) +#define _PyXI_GET_STATE(interp) (&(interp)->xi) +#ifndef Py_BUILD_CORE_MODULE extern PyStatus _PyXI_Init(PyInterpreterState *interp); extern void _PyXI_Fini(PyInterpreterState *interp); - extern PyStatus _PyXI_InitTypes(PyInterpreterState *interp); extern void _PyXI_FiniTypes(PyInterpreterState *interp); +#endif // Py_BUILD_CORE_MODULE -#define _PyInterpreterState_GetXIState(interp) (&(interp)->xi) +int _Py_xi_global_state_init(_PyXI_global_state_t *); +void _Py_xi_global_state_fini(_PyXI_global_state_t *); +int _Py_xi_state_init(_PyXI_state_t *, PyInterpreterState *); +void _Py_xi_state_fini(_PyXI_state_t *, PyInterpreterState *); /***************************/ diff --git a/Include/internal/pycore_crossinterp_data_registry.h b/Include/internal/pycore_crossinterp_data_registry.h new file mode 100644 index 00000000000000..bbad4de770857f --- /dev/null +++ b/Include/internal/pycore_crossinterp_data_registry.h @@ -0,0 +1,41 @@ +#ifndef Py_CORE_CROSSINTERP_DATA_REGISTRY_H +# error "this header must not be included directly" +#endif + + +// For now we use a global registry of shareable classes. An +// alternative would be to add a tp_* slot for a class's +// xidatafunc. It would be simpler and more efficient. + +struct _xid_regitem; + +typedef struct _xid_regitem { + struct _xid_regitem *prev; + struct _xid_regitem *next; + /* This can be a dangling pointer, but only if weakref is set. */ + PyTypeObject *cls; + /* This is NULL for builtin types. */ + PyObject *weakref; + size_t refcount; + xidatafunc getdata; +} _PyXIData_regitem_t; + +typedef struct { + int global; /* builtin types or heap types */ + int initialized; + PyMutex mutex; + _PyXIData_regitem_t *head; +} _PyXIData_registry_t; + +PyAPI_FUNC(int) _PyXIData_RegisterClass( + _PyXIData_lookup_context_t *, + PyTypeObject *, + xidatafunc); +PyAPI_FUNC(int) _PyXIData_UnregisterClass( + _PyXIData_lookup_context_t *, + PyTypeObject *); + +struct _xid_lookup_state { + // XXX Remove this field once we have a tp_* slot. + _PyXIData_registry_t registry; +}; diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index c5399ad8e0497f..f7d747cc6c62a1 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -43,8 +43,6 @@ extern int _PyDict_Next( extern int _PyDict_HasOnlyStringKeys(PyObject *mp); -extern void _PyDict_MaybeUntrack(PyObject *mp); - // Export for '_ctypes' shared extension PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index c9ac3819d0390b..b786c5f49e9831 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -68,15 +68,20 @@ typedef struct _PyInterpreterFrame { PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */ _Py_CODEUNIT *instr_ptr; /* Instruction currently executing (or about to begin) */ +#ifdef Py_GIL_DISABLED + /* Index of thread-local bytecode containing instr_ptr. */ + int32_t tlbc_index; +#endif _PyStackRef *stackpointer; uint16_t return_offset; /* Only relevant during a function call */ char owner; + char visited; /* Locals and stack */ _PyStackRef localsplus[1]; } _PyInterpreterFrame; #define _PyInterpreterFrame_LASTI(IF) \ - ((int)((IF)->instr_ptr - _PyCode_CODE(_PyFrame_GetCode(IF)))) + ((int)((IF)->instr_ptr - _PyFrame_GetBytecode((IF)))) static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable); @@ -84,6 +89,19 @@ static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { return (PyCodeObject *)executable; } +static inline _Py_CODEUNIT * +_PyFrame_GetBytecode(_PyInterpreterFrame *f) +{ +#ifdef Py_GIL_DISABLED + PyCodeObject *co = _PyFrame_GetCode(f); + _PyCodeArray *tlbc = _Py_atomic_load_ptr_acquire(&co->co_tlbc); + assert(f->tlbc_index >= 0 && f->tlbc_index < tlbc->size); + return (_Py_CODEUNIT *)tlbc->entries[f->tlbc_index]; +#else + return _PyCode_CODE(_PyFrame_GetCode(f)); +#endif +} + static inline PyFunctionObject *_PyFrame_GetFunction(_PyInterpreterFrame *f) { PyObject *func = PyStackRef_AsPyObjectBorrow(f->f_funcobj); assert(PyFunction_Check(func)); @@ -144,13 +162,33 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * #endif } +#ifdef Py_GIL_DISABLED +static inline void +_PyFrame_InitializeTLBC(PyThreadState *tstate, _PyInterpreterFrame *frame, + PyCodeObject *code) +{ + _Py_CODEUNIT *tlbc = _PyCode_GetTLBCFast(tstate, code); + if (tlbc == NULL) { + // No thread-local bytecode exists for this thread yet; use the main + // thread's copy, deferring thread-local bytecode creation to the + // execution of RESUME. + frame->instr_ptr = _PyCode_CODE(code); + frame->tlbc_index = 0; + } + else { + frame->instr_ptr = tlbc; + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + } +} +#endif + /* Consumes reference to func and locals. Does not initialize frame->previous, which happens when frame is linked into the frame stack. */ static inline void _PyFrame_Initialize( - _PyInterpreterFrame *frame, _PyStackRef func, + PyThreadState *tstate, _PyInterpreterFrame *frame, _PyStackRef func, PyObject *locals, PyCodeObject *code, int null_locals_from, _PyInterpreterFrame *previous) { frame->previous = previous; @@ -162,9 +200,15 @@ _PyFrame_Initialize( frame->f_locals = locals; frame->stackpointer = frame->localsplus + code->co_nlocalsplus; frame->frame_obj = NULL; +#ifdef Py_GIL_DISABLED + _PyFrame_InitializeTLBC(tstate, frame, code); +#else + (void)tstate; frame->instr_ptr = _PyCode_CODE(code); +#endif frame->return_offset = 0; frame->owner = FRAME_OWNED_BY_THREAD; + frame->visited = 0; for (int i = null_locals_from; i < code->co_nlocalsplus; i++) { frame->localsplus[i] = PyStackRef_NULL; @@ -224,7 +268,8 @@ _PyFrame_IsIncomplete(_PyInterpreterFrame *frame) return true; } return frame->owner != FRAME_OWNED_BY_GENERATOR && - frame->instr_ptr < _PyCode_CODE(_PyFrame_GetCode(frame)) + _PyFrame_GetCode(frame)->_co_firsttraceable; + frame->instr_ptr < _PyFrame_GetBytecode(frame) + + _PyFrame_GetCode(frame)->_co_firsttraceable; } static inline _PyInterpreterFrame * @@ -315,7 +360,8 @@ _PyFrame_PushUnchecked(PyThreadState *tstate, _PyStackRef func, int null_locals_ _PyInterpreterFrame *new_frame = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); - _PyFrame_Initialize(new_frame, func, NULL, code, null_locals_from, previous); + _PyFrame_Initialize(tstate, new_frame, func, NULL, code, null_locals_from, + previous); return new_frame; } @@ -339,8 +385,13 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int assert(stackdepth <= code->co_stacksize); frame->stackpointer = frame->localsplus + code->co_nlocalsplus + stackdepth; frame->frame_obj = NULL; +#ifdef Py_GIL_DISABLED + _PyFrame_InitializeTLBC(tstate, frame, code); +#else frame->instr_ptr = _PyCode_CODE(code); +#endif frame->owner = FRAME_OWNED_BY_THREAD; + frame->visited = 0; frame->return_offset = 0; #ifdef Py_GIL_DISABLED diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index b85957df5a6b9f..c81e7a1de4b727 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -10,11 +10,11 @@ extern "C" { /* GC information is stored BEFORE the object structure. */ typedef struct { - // Pointer to next object in the list. + // Tagged pointer to next object in the list. // 0 means the object is not tracked uintptr_t _gc_next; - // Pointer to previous object in the list. + // Tagged pointer to previous object in the list. // Lowest two bits are used for flags documented later. uintptr_t _gc_prev; } PyGC_Head; @@ -302,6 +302,11 @@ struct gc_generation_stats { Py_ssize_t uncollectable; }; +enum _GCPhase { + GC_PHASE_MARK = 0, + GC_PHASE_COLLECT = 1 +}; + struct _gc_runtime_state { /* List of objects that still need to be cleaned up, singly linked * via their gc headers' gc_prev pointers. */ @@ -325,10 +330,12 @@ struct _gc_runtime_state { /* a list of callbacks to be invoked when collection is performed */ PyObject *callbacks; + Py_ssize_t prior_heap_size; Py_ssize_t heap_size; Py_ssize_t work_to_do; /* Which of the old spaces is the visited space */ int visited_space; + int phase; #ifdef Py_GIL_DISABLED /* This is the number of objects that survived the last full @@ -389,6 +396,10 @@ extern int _PyGC_VisitStackRef(union _PyStackRef *ref, visitproc visit, void *ar } \ } while (0) +#ifdef Py_GIL_DISABLED +extern void _PyGC_VisitObjectsWorldStopped(PyInterpreterState *interp, + gcvisitobjects_t callback, void *arg); +#endif #ifdef __cplusplus } diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 2fd7d5d13a98b2..e4f0138e17edfa 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1231,6 +1231,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(strict_mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(string)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sub_key)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(subcalls)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(symmetric_difference_update)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tabsize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tag)); @@ -1248,8 +1249,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(threading)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(throw)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timeout)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timer)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(times)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timetuple)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timeunit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(top)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trace_callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(traceback)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index fc3871570cc49d..e70f11e2a26cd5 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -720,6 +720,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(strict_mode) STRUCT_FOR_ID(string) STRUCT_FOR_ID(sub_key) + STRUCT_FOR_ID(subcalls) STRUCT_FOR_ID(symmetric_difference_update) STRUCT_FOR_ID(tabsize) STRUCT_FOR_ID(tag) @@ -737,8 +738,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(threading) STRUCT_FOR_ID(throw) STRUCT_FOR_ID(timeout) + STRUCT_FOR_ID(timer) STRUCT_FOR_ID(times) STRUCT_FOR_ID(timetuple) + STRUCT_FOR_ID(timeunit) STRUCT_FOR_ID(top) STRUCT_FOR_ID(trace_callback) STRUCT_FOR_ID(traceback) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 290ba95e1a0ad7..318c712bdfa174 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -21,6 +21,7 @@ extern int _PyImport_SetModuleString(const char *name, PyObject* module); extern void _PyImport_AcquireLock(PyInterpreterState *interp); extern void _PyImport_ReleaseLock(PyInterpreterState *interp); +extern void _PyImport_ReInitLock(PyInterpreterState *interp); // This is used exclusively for the sys and builtins modules: extern int _PyImport_FixupBuiltin( diff --git a/Include/internal/pycore_index_pool.h b/Include/internal/pycore_index_pool.h new file mode 100644 index 00000000000000..e81bfd4d6ed03d --- /dev/null +++ b/Include/internal/pycore_index_pool.h @@ -0,0 +1,56 @@ +#ifndef Py_INTERNAL_INDEX_POOL_H +#define Py_INTERNAL_INDEX_POOL_H + +#include "Python.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_GIL_DISABLED + +// This contains code for allocating unique indices in an array. It is used by +// the free-threaded build to assign each thread a globally unique index into +// each code object's thread-local bytecode array. + +// A min-heap of indices +typedef struct _PyIndexHeap { + int32_t *values; + + // Number of items stored in values + Py_ssize_t size; + + // Maximum number of items that can be stored in values + Py_ssize_t capacity; +} _PyIndexHeap; + +// An unbounded pool of indices. Indices are allocated starting from 0. They +// may be released back to the pool once they are no longer in use. +typedef struct _PyIndexPool { + PyMutex mutex; + + // Min heap of indices available for allocation + _PyIndexHeap free_indices; + + // Next index to allocate if no free indices are available + int32_t next_index; +} _PyIndexPool; + +// Allocate the smallest available index. Returns -1 on error. +extern int32_t _PyIndexPool_AllocIndex(_PyIndexPool *indices); + +// Release `index` back to the pool +extern void _PyIndexPool_FreeIndex(_PyIndexPool *indices, int32_t index); + +extern void _PyIndexPool_Fini(_PyIndexPool *indices); + +#endif // Py_GIL_DISABLED + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_INDEX_POOL_H diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 36cd71e5a007d5..824b865eda60df 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -16,7 +16,7 @@ extern "C" { #include "pycore_code.h" // struct callable_cache #include "pycore_codecs.h" // struct codecs_state #include "pycore_context.h" // struct _Py_context_state -#include "pycore_crossinterp.h" // struct _xidregistry +#include "pycore_crossinterp.h" // _PyXI_state_t #include "pycore_dict_state.h" // struct _Py_dict_state #include "pycore_dtoa.h" // struct _dtoa_state #include "pycore_exceptions.h" // struct _Py_exc_state @@ -26,6 +26,7 @@ extern "C" { #include "pycore_genobject.h" // _PyGen_FetchStopIterationValue #include "pycore_global_objects.h"// struct _Py_interp_cached_objects #include "pycore_import.h" // struct _import_state +#include "pycore_index_pool.h" // _PyIndexPool #include "pycore_instruments.h" // _PY_MONITORING_EVENTS #include "pycore_list.h" // struct _Py_list_state #include "pycore_mimalloc.h" // struct _mimalloc_interp_state @@ -204,7 +205,7 @@ struct _is { freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS]; /* cross-interpreter data and utils */ - struct _xi_state xi; + _PyXI_state_t xi; #ifdef HAVE_FORK PyObject *before_forkers; @@ -222,6 +223,7 @@ struct _is { struct _brc_state brc; // biased reference counting state struct _Py_unique_id_pool unique_ids; // object ids for per-thread refcounts PyMutex weakref_locks[NUM_WEAKREF_LIST_LOCKS]; + _PyIndexPool tlbc_indices; #endif // Per-interpreter state for the obmalloc allocator. For the main diff --git a/Include/internal/pycore_modsupport.h b/Include/internal/pycore_modsupport.h index 11fde814875938..c661f1d82a84f6 100644 --- a/Include/internal/pycore_modsupport.h +++ b/Include/internal/pycore_modsupport.h @@ -85,20 +85,14 @@ PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywords( int minpos, int maxpos, int minkw, + int varpos, PyObject **buf); -#define _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, parser, minpos, maxpos, minkw, buf) \ +#define _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, parser, minpos, maxpos, minkw, varpos, buf) \ (((minkw) == 0 && (kwargs) == NULL && (kwnames) == NULL && \ - (minpos) <= (nargs) && (nargs) <= (maxpos) && (args) != NULL) ? (args) : \ + (minpos) <= (nargs) && ((varpos) || (nargs) <= (maxpos)) && (args) != NULL) ? \ + (args) : \ _PyArg_UnpackKeywords((args), (nargs), (kwargs), (kwnames), (parser), \ - (minpos), (maxpos), (minkw), (buf))) - -// Export for '_testclinic' shared extension -PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywordsWithVararg( - PyObject *const *args, Py_ssize_t nargs, - PyObject *kwargs, PyObject *kwnames, - struct _PyArg_Parser *parser, - int minpos, int maxpos, int minkw, - int vararg, PyObject **buf); + (minpos), (maxpos), (minkw), (varpos), (buf))) #ifdef __cplusplus } diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index c7af720b1ce43d..0cba7bc083087e 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -466,8 +466,8 @@ static inline void _PyObject_GC_TRACK( PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); - /* Young objects will be moved into the visited space during GC, so set the bit here */ - gc->_gc_next = ((uintptr_t)generation0) | (uintptr_t)interp->gc.visited_space; + uintptr_t not_visited = 1 ^ interp->gc.visited_space; + gc->_gc_next = ((uintptr_t)generation0) | not_visited; generation0->_gc_prev = (uintptr_t)gc; #endif } diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index f92c0a0cddf906..6d70b42f708854 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -58,7 +58,8 @@ typedef struct { uint16_t error_target; }; }; - uint64_t operand; // A cache entry + uint64_t operand0; // A cache entry + uint64_t operand1; } _PyUOpInstruction; typedef struct { diff --git a/Include/internal/pycore_pymath.h b/Include/internal/pycore_pymath.h index 7a4e1c1eb714f7..eea8996ba68ca0 100644 --- a/Include/internal/pycore_pymath.h +++ b/Include/internal/pycore_pymath.h @@ -33,7 +33,7 @@ extern "C" { static inline void _Py_ADJUST_ERANGE1(double x) { if (errno == 0) { - if (x == Py_HUGE_VAL || x == -Py_HUGE_VAL) { + if (x == Py_INFINITY || x == -Py_INFINITY) { errno = ERANGE; } } @@ -44,8 +44,8 @@ static inline void _Py_ADJUST_ERANGE1(double x) static inline void _Py_ADJUST_ERANGE2(double x, double y) { - if (x == Py_HUGE_VAL || x == -Py_HUGE_VAL || - y == Py_HUGE_VAL || y == -Py_HUGE_VAL) + if (x == Py_INFINITY || x == -Py_INFINITY || + y == Py_INFINITY || y == -Py_INFINITY) { if (errno == 0) { errno = ERANGE; diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index fade55945b7dbf..f4fbf3734e2d44 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -82,7 +82,7 @@ PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_SetRunningMain(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_SetNotRunningMain(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IsRunningMain(PyInterpreterState *); -PyAPI_FUNC(int) _PyInterpreterState_FailIfRunningMain(PyInterpreterState *); +PyAPI_FUNC(void) _PyErr_SetInterpreterAlreadyRunning(void); extern int _PyThreadState_IsRunningMain(PyThreadState *); extern void _PyInterpreterState_ReinitRunningMain(PyThreadState *); @@ -141,6 +141,12 @@ _PyThreadState_GET(void) #endif } +static inline int +_PyThreadState_IsAttached(PyThreadState *tstate) +{ + return (_Py_atomic_load_int_relaxed(&tstate->state) == _Py_THREAD_ATTACHED); +} + // Attaches the current thread to the interpreter. // // This may block while acquiring the GIL (if the GIL is enabled) or while diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 7f592aa6cf9f05..2f2cec22cf1589 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -11,7 +11,7 @@ extern "C" { #include "pycore_atexit.h" // struct _atexit_runtime_state #include "pycore_audit.h" // _Py_AuditHookEntry #include "pycore_ceval_state.h" // struct _ceval_runtime_state -#include "pycore_crossinterp.h" // struct _xidregistry +#include "pycore_crossinterp.h" // _PyXI_global_state_t #include "pycore_debug_offsets.h" // _Py_DebugOffsets #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state @@ -106,7 +106,7 @@ typedef struct pyruntimestate { tools. */ /* cross-interpreter data and utils */ - struct _xi_runtime_state xi; + _PyXI_global_state_t xi; struct _pymem_allocators allocators; struct _obmalloc_global_state obmalloc; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index e99febab2f3d57..ce116ad609b4bb 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -50,8 +50,10 @@ extern PyTypeObject _PyExc_MemoryError; .next_id = -1, \ }, \ .xi = { \ - .registry = { \ - .global = 1, \ + .data_lookup = { \ + .registry = { \ + .global = 1, \ + }, \ }, \ }, \ /* A TSS key must be initialized with Py_tss_NEEDS_INIT \ @@ -85,7 +87,7 @@ extern PyTypeObject _PyExc_MemoryError; .double_format = _py_float_format_unknown, \ }, \ .types = { \ - .next_version_tag = 1, \ + .next_version_tag = _Py_TYPE_VERSION_NEXT, \ }, \ .static_objects = { \ .singletons = { \ @@ -132,6 +134,7 @@ extern PyTypeObject _PyExc_MemoryError; { .threshold = 0, }, \ }, \ .work_to_do = -5000, \ + .phase = GC_PHASE_MARK, \ }, \ .qsbr = { \ .wr_seq = QSBR_INITIAL, \ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 3b80e265b0ca50..5d404c8fd91ca6 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1229,6 +1229,7 @@ extern "C" { INIT_ID(strict_mode), \ INIT_ID(string), \ INIT_ID(sub_key), \ + INIT_ID(subcalls), \ INIT_ID(symmetric_difference_update), \ INIT_ID(tabsize), \ INIT_ID(tag), \ @@ -1246,8 +1247,10 @@ extern "C" { INIT_ID(threading), \ INIT_ID(throw), \ INIT_ID(timeout), \ + INIT_ID(timer), \ INIT_ID(times), \ INIT_ID(timetuple), \ + INIT_ID(timeunit), \ INIT_ID(top), \ INIT_ID(trace_callback), \ INIT_ID(traceback), \ diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index e0e7d5ebf0912c..b8bea72baeaaf5 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -42,6 +42,9 @@ typedef struct _PyThreadStateImpl { int is_finalized; } refcounts; + // Index to use to retrieve thread-local bytecode for this thread + int32_t tlbc_index; + // When >1, code objects do not immortalize their non-string constants. int suppress_co_const_immortalization; #endif @@ -52,7 +55,6 @@ typedef struct _PyThreadStateImpl { } _PyThreadStateImpl; - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index dfbbd6fd0c7de5..82b875241f4116 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -20,7 +20,7 @@ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *); #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item) -extern PyObject *_PyTuple_FromArray(PyObject *const *, Py_ssize_t); +PyAPI_FUNC(PyObject *)_PyTuple_FromArray(PyObject *const *, Py_ssize_t); PyAPI_FUNC(PyObject *)_PyTuple_FromStackRefSteal(const union _PyStackRef *, Py_ssize_t); PyAPI_FUNC(PyObject *)_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 118bc98b35d5e3..5debdd68fe94ca 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -14,6 +14,21 @@ extern "C" { /* state */ +#define _Py_TYPE_VERSION_INT 1 +#define _Py_TYPE_VERSION_FLOAT 2 +#define _Py_TYPE_VERSION_LIST 3 +#define _Py_TYPE_VERSION_TUPLE 4 +#define _Py_TYPE_VERSION_STR 5 +#define _Py_TYPE_VERSION_SET 6 +#define _Py_TYPE_VERSION_FROZEN_SET 7 +#define _Py_TYPE_VERSION_DICT 8 +#define _Py_TYPE_VERSION_BYTEARRAY 9 +#define _Py_TYPE_VERSION_BYTES 10 +#define _Py_TYPE_VERSION_COMPLEX 11 + +#define _Py_TYPE_VERSION_NEXT 16 + + #define _Py_TYPE_BASE_VERSION_TAG (2<<16) #define _Py_MAX_GLOBAL_TYPE_VERSION_TAG (_Py_TYPE_BASE_VERSION_TAG - 1) @@ -243,6 +258,7 @@ extern PyObject* _PyType_GetFullyQualifiedName(PyTypeObject *type, char sep); // self->tp_flags = (self->tp_flags & ~mask) | flags; extern void _PyType_SetFlags(PyTypeObject *self, unsigned long mask, unsigned long flags); +extern int _PyType_AddMethod(PyTypeObject *, PyMethodDef *); // Like _PyType_SetFlags(), but apply the operation to self and any of its // subclasses without Py_TPFLAGS_IMMUTABLETYPE set. diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index eb2eca06ec4d4f..d0bc8d7186c053 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -2676,6 +2676,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(subcalls); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(symmetric_difference_update); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2744,6 +2748,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(timer); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(times); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2752,6 +2760,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(timeunit); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(top); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index de628d240d1c07..fab4ce6a25b347 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -63,90 +63,91 @@ extern "C" { #define _CHECK_FUNCTION 333 #define _CHECK_FUNCTION_EXACT_ARGS 334 #define _CHECK_FUNCTION_VERSION 335 -#define _CHECK_FUNCTION_VERSION_KW 336 -#define _CHECK_IS_NOT_PY_CALLABLE 337 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 338 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 339 -#define _CHECK_METHOD_VERSION 340 -#define _CHECK_METHOD_VERSION_KW 341 -#define _CHECK_PEP_523 342 -#define _CHECK_PERIODIC 343 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 344 -#define _CHECK_STACK_SPACE 345 -#define _CHECK_STACK_SPACE_OPERAND 346 -#define _CHECK_VALIDITY 347 -#define _CHECK_VALIDITY_AND_SET_IP 348 -#define _COMPARE_OP 349 -#define _COMPARE_OP_FLOAT 350 -#define _COMPARE_OP_INT 351 -#define _COMPARE_OP_STR 352 -#define _CONTAINS_OP 353 +#define _CHECK_FUNCTION_VERSION_INLINE 336 +#define _CHECK_FUNCTION_VERSION_KW 337 +#define _CHECK_IS_NOT_PY_CALLABLE 338 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 339 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 340 +#define _CHECK_METHOD_VERSION 341 +#define _CHECK_METHOD_VERSION_KW 342 +#define _CHECK_PEP_523 343 +#define _CHECK_PERIODIC 344 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 345 +#define _CHECK_STACK_SPACE 346 +#define _CHECK_STACK_SPACE_OPERAND 347 +#define _CHECK_VALIDITY 348 +#define _CHECK_VALIDITY_AND_SET_IP 349 +#define _COMPARE_OP 350 +#define _COMPARE_OP_FLOAT 351 +#define _COMPARE_OP_INT 352 +#define _COMPARE_OP_STR 353 +#define _CONTAINS_OP 354 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT #define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE #define _COPY COPY #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 354 +#define _CREATE_INIT_FRAME 355 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 355 +#define _DEOPT 356 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 356 -#define _DO_CALL_FUNCTION_EX 357 -#define _DO_CALL_KW 358 -#define _DYNAMIC_EXIT 359 +#define _DO_CALL 357 +#define _DO_CALL_FUNCTION_EX 358 +#define _DO_CALL_KW 359 +#define _DYNAMIC_EXIT 360 #define _END_SEND END_SEND -#define _ERROR_POP_N 360 +#define _ERROR_POP_N 361 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 361 -#define _EXPAND_METHOD_KW 362 -#define _FATAL_ERROR 363 +#define _EXPAND_METHOD 362 +#define _EXPAND_METHOD_KW 363 +#define _FATAL_ERROR 364 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 364 -#define _FOR_ITER_GEN_FRAME 365 -#define _FOR_ITER_TIER_TWO 366 +#define _FOR_ITER 365 +#define _FOR_ITER_GEN_FRAME 366 +#define _FOR_ITER_TIER_TWO 367 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 367 -#define _GUARD_BOTH_INT 368 -#define _GUARD_BOTH_UNICODE 369 -#define _GUARD_BUILTINS_VERSION_PUSH_KEYS 370 -#define _GUARD_DORV_NO_DICT 371 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 372 -#define _GUARD_GLOBALS_VERSION 373 -#define _GUARD_GLOBALS_VERSION_PUSH_KEYS 374 -#define _GUARD_IS_FALSE_POP 375 -#define _GUARD_IS_NONE_POP 376 -#define _GUARD_IS_NOT_NONE_POP 377 -#define _GUARD_IS_TRUE_POP 378 -#define _GUARD_KEYS_VERSION 379 -#define _GUARD_NOS_FLOAT 380 -#define _GUARD_NOS_INT 381 -#define _GUARD_NOT_EXHAUSTED_LIST 382 -#define _GUARD_NOT_EXHAUSTED_RANGE 383 -#define _GUARD_NOT_EXHAUSTED_TUPLE 384 -#define _GUARD_TOS_FLOAT 385 -#define _GUARD_TOS_INT 386 -#define _GUARD_TYPE_VERSION 387 +#define _GUARD_BOTH_FLOAT 368 +#define _GUARD_BOTH_INT 369 +#define _GUARD_BOTH_UNICODE 370 +#define _GUARD_BUILTINS_VERSION_PUSH_KEYS 371 +#define _GUARD_DORV_NO_DICT 372 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 373 +#define _GUARD_GLOBALS_VERSION 374 +#define _GUARD_GLOBALS_VERSION_PUSH_KEYS 375 +#define _GUARD_IS_FALSE_POP 376 +#define _GUARD_IS_NONE_POP 377 +#define _GUARD_IS_NOT_NONE_POP 378 +#define _GUARD_IS_TRUE_POP 379 +#define _GUARD_KEYS_VERSION 380 +#define _GUARD_NOS_FLOAT 381 +#define _GUARD_NOS_INT 382 +#define _GUARD_NOT_EXHAUSTED_LIST 383 +#define _GUARD_NOT_EXHAUSTED_RANGE 384 +#define _GUARD_NOT_EXHAUSTED_TUPLE 385 +#define _GUARD_TOS_FLOAT 386 +#define _GUARD_TOS_INT 387 +#define _GUARD_TYPE_VERSION 388 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 388 -#define _INIT_CALL_PY_EXACT_ARGS 389 -#define _INIT_CALL_PY_EXACT_ARGS_0 390 -#define _INIT_CALL_PY_EXACT_ARGS_1 391 -#define _INIT_CALL_PY_EXACT_ARGS_2 392 -#define _INIT_CALL_PY_EXACT_ARGS_3 393 -#define _INIT_CALL_PY_EXACT_ARGS_4 394 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 389 +#define _INIT_CALL_PY_EXACT_ARGS 390 +#define _INIT_CALL_PY_EXACT_ARGS_0 391 +#define _INIT_CALL_PY_EXACT_ARGS_1 392 +#define _INIT_CALL_PY_EXACT_ARGS_2 393 +#define _INIT_CALL_PY_EXACT_ARGS_3 394 +#define _INIT_CALL_PY_EXACT_ARGS_4 395 #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER @@ -158,141 +159,142 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 395 -#define _IS_NONE 396 +#define _INTERNAL_INCREMENT_OPT_COUNTER 396 +#define _IS_NONE 397 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 397 -#define _ITER_CHECK_RANGE 398 -#define _ITER_CHECK_TUPLE 399 -#define _ITER_JUMP_LIST 400 -#define _ITER_JUMP_RANGE 401 -#define _ITER_JUMP_TUPLE 402 -#define _ITER_NEXT_LIST 403 -#define _ITER_NEXT_RANGE 404 -#define _ITER_NEXT_TUPLE 405 -#define _JUMP_TO_TOP 406 +#define _ITER_CHECK_LIST 398 +#define _ITER_CHECK_RANGE 399 +#define _ITER_CHECK_TUPLE 400 +#define _ITER_JUMP_LIST 401 +#define _ITER_JUMP_RANGE 402 +#define _ITER_JUMP_TUPLE 403 +#define _ITER_NEXT_LIST 404 +#define _ITER_NEXT_RANGE 405 +#define _ITER_NEXT_TUPLE 406 +#define _JUMP_TO_TOP 407 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 407 -#define _LOAD_ATTR_CLASS 408 -#define _LOAD_ATTR_CLASS_0 409 -#define _LOAD_ATTR_CLASS_1 410 +#define _LOAD_ATTR 408 +#define _LOAD_ATTR_CLASS 409 +#define _LOAD_ATTR_CLASS_0 410 +#define _LOAD_ATTR_CLASS_1 411 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 411 -#define _LOAD_ATTR_INSTANCE_VALUE_0 412 -#define _LOAD_ATTR_INSTANCE_VALUE_1 413 -#define _LOAD_ATTR_METHOD_LAZY_DICT 414 -#define _LOAD_ATTR_METHOD_NO_DICT 415 -#define _LOAD_ATTR_METHOD_WITH_VALUES 416 -#define _LOAD_ATTR_MODULE 417 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 418 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 419 -#define _LOAD_ATTR_PROPERTY_FRAME 420 -#define _LOAD_ATTR_SLOT 421 -#define _LOAD_ATTR_SLOT_0 422 -#define _LOAD_ATTR_SLOT_1 423 -#define _LOAD_ATTR_WITH_HINT 424 +#define _LOAD_ATTR_INSTANCE_VALUE 412 +#define _LOAD_ATTR_INSTANCE_VALUE_0 413 +#define _LOAD_ATTR_INSTANCE_VALUE_1 414 +#define _LOAD_ATTR_METHOD_LAZY_DICT 415 +#define _LOAD_ATTR_METHOD_NO_DICT 416 +#define _LOAD_ATTR_METHOD_WITH_VALUES 417 +#define _LOAD_ATTR_MODULE 418 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 419 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 420 +#define _LOAD_ATTR_PROPERTY_FRAME 421 +#define _LOAD_ATTR_SLOT 422 +#define _LOAD_ATTR_SLOT_0 423 +#define _LOAD_ATTR_SLOT_1 424 +#define _LOAD_ATTR_WITH_HINT 425 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS +#define _LOAD_BYTECODE 426 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST #define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL -#define _LOAD_CONST_INLINE 425 -#define _LOAD_CONST_INLINE_BORROW 426 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 427 -#define _LOAD_CONST_INLINE_WITH_NULL 428 +#define _LOAD_CONST_INLINE 427 +#define _LOAD_CONST_INLINE_BORROW 428 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 429 +#define _LOAD_CONST_INLINE_WITH_NULL 430 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 429 -#define _LOAD_FAST_0 430 -#define _LOAD_FAST_1 431 -#define _LOAD_FAST_2 432 -#define _LOAD_FAST_3 433 -#define _LOAD_FAST_4 434 -#define _LOAD_FAST_5 435 -#define _LOAD_FAST_6 436 -#define _LOAD_FAST_7 437 +#define _LOAD_FAST 431 +#define _LOAD_FAST_0 432 +#define _LOAD_FAST_1 433 +#define _LOAD_FAST_2 434 +#define _LOAD_FAST_3 435 +#define _LOAD_FAST_4 436 +#define _LOAD_FAST_5 437 +#define _LOAD_FAST_6 438 +#define _LOAD_FAST_7 439 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 438 -#define _LOAD_GLOBAL_BUILTINS 439 -#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 440 -#define _LOAD_GLOBAL_MODULE 441 -#define _LOAD_GLOBAL_MODULE_FROM_KEYS 442 +#define _LOAD_GLOBAL 440 +#define _LOAD_GLOBAL_BUILTINS 441 +#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 442 +#define _LOAD_GLOBAL_MODULE 443 +#define _LOAD_GLOBAL_MODULE_FROM_KEYS 444 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 443 -#define _LOAD_SMALL_INT_0 444 -#define _LOAD_SMALL_INT_1 445 -#define _LOAD_SMALL_INT_2 446 -#define _LOAD_SMALL_INT_3 447 +#define _LOAD_SMALL_INT 445 +#define _LOAD_SMALL_INT_0 446 +#define _LOAD_SMALL_INT_1 447 +#define _LOAD_SMALL_INT_2 448 +#define _LOAD_SMALL_INT_3 449 #define _LOAD_SPECIAL LOAD_SPECIAL #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 448 +#define _MAKE_CALLARGS_A_TUPLE 450 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 449 +#define _MAKE_WARM 451 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 450 -#define _MAYBE_EXPAND_METHOD_KW 451 -#define _MONITOR_CALL 452 -#define _MONITOR_JUMP_BACKWARD 453 -#define _MONITOR_RESUME 454 +#define _MAYBE_EXPAND_METHOD 452 +#define _MAYBE_EXPAND_METHOD_KW 453 +#define _MONITOR_CALL 454 +#define _MONITOR_JUMP_BACKWARD 455 +#define _MONITOR_RESUME 456 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 455 -#define _POP_JUMP_IF_TRUE 456 +#define _POP_JUMP_IF_FALSE 457 +#define _POP_JUMP_IF_TRUE 458 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 457 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 459 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 458 +#define _PUSH_FRAME 460 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 459 -#define _PY_FRAME_KW 460 -#define _QUICKEN_RESUME 461 -#define _REPLACE_WITH_TRUE 462 +#define _PY_FRAME_GENERAL 461 +#define _PY_FRAME_KW 462 +#define _QUICKEN_RESUME 463 +#define _REPLACE_WITH_TRUE 464 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 463 -#define _SEND 464 -#define _SEND_GEN_FRAME 465 +#define _SAVE_RETURN_OFFSET 465 +#define _SEND 466 +#define _SEND_GEN_FRAME 467 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 466 -#define _STORE_ATTR 467 -#define _STORE_ATTR_INSTANCE_VALUE 468 -#define _STORE_ATTR_SLOT 469 -#define _STORE_ATTR_WITH_HINT 470 +#define _START_EXECUTOR 468 +#define _STORE_ATTR 469 +#define _STORE_ATTR_INSTANCE_VALUE 470 +#define _STORE_ATTR_SLOT 471 +#define _STORE_ATTR_WITH_HINT 472 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 471 -#define _STORE_FAST_0 472 -#define _STORE_FAST_1 473 -#define _STORE_FAST_2 474 -#define _STORE_FAST_3 475 -#define _STORE_FAST_4 476 -#define _STORE_FAST_5 477 -#define _STORE_FAST_6 478 -#define _STORE_FAST_7 479 +#define _STORE_FAST 473 +#define _STORE_FAST_0 474 +#define _STORE_FAST_1 475 +#define _STORE_FAST_2 476 +#define _STORE_FAST_3 477 +#define _STORE_FAST_4 478 +#define _STORE_FAST_5 479 +#define _STORE_FAST_6 480 +#define _STORE_FAST_7 481 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 480 -#define _STORE_SUBSCR 481 +#define _STORE_SLICE 482 +#define _STORE_SUBSCR 483 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 482 -#define _TO_BOOL 483 +#define _TIER2_RESUME_CHECK 484 +#define _TO_BOOL 485 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -302,13 +304,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 484 +#define _UNPACK_SEQUENCE 486 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 484 +#define MAX_UOP_ID 486 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 4cfdecec78b0db..1b2880cb6bb67e 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -213,6 +213,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG, [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CHECK_FUNCTION_VERSION_INLINE] = HAS_EXIT_FLAG, [_CHECK_METHOD_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_EXPAND_METHOD] = HAS_ARG_FLAG, [_CHECK_IS_NOT_PY_CALLABLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -289,7 +290,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_FATAL_ERROR] = 0, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, [_DEOPT] = 0, - [_ERROR_POP_N] = HAS_ARG_FLAG, + [_ERROR_POP_N] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_TIER2_RESUME_CHECK] = HAS_DEOPT_FLAG, }; @@ -353,6 +354,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_FUNCTION] = "_CHECK_FUNCTION", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", [_CHECK_FUNCTION_VERSION] = "_CHECK_FUNCTION_VERSION", + [_CHECK_FUNCTION_VERSION_INLINE] = "_CHECK_FUNCTION_VERSION_INLINE", [_CHECK_FUNCTION_VERSION_KW] = "_CHECK_FUNCTION_VERSION_KW", [_CHECK_IS_NOT_PY_CALLABLE] = "_CHECK_IS_NOT_PY_CALLABLE", [_CHECK_IS_NOT_PY_CALLABLE_KW] = "_CHECK_IS_NOT_PY_CALLABLE_KW", @@ -658,7 +660,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _TO_BOOL: return 1; case _TO_BOOL_BOOL: - return 1; + return 0; case _TO_BOOL_INT: return 1; case _TO_BOOL_LIST: @@ -672,11 +674,11 @@ int _PyUop_num_popped(int opcode, int oparg) case _UNARY_INVERT: return 1; case _GUARD_BOTH_INT: - return 2; + return 0; case _GUARD_NOS_INT: - return 2; + return 0; case _GUARD_TOS_INT: - return 1; + return 0; case _BINARY_OP_MULTIPLY_INT: return 2; case _BINARY_OP_ADD_INT: @@ -684,11 +686,11 @@ int _PyUop_num_popped(int opcode, int oparg) case _BINARY_OP_SUBTRACT_INT: return 2; case _GUARD_BOTH_FLOAT: - return 2; + return 0; case _GUARD_NOS_FLOAT: - return 2; + return 0; case _GUARD_TOS_FLOAT: - return 1; + return 0; case _BINARY_OP_MULTIPLY_FLOAT: return 2; case _BINARY_OP_ADD_FLOAT: @@ -696,7 +698,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _BINARY_OP_SUBTRACT_FLOAT: return 2; case _GUARD_BOTH_UNICODE: - return 2; + return 0; case _BINARY_OP_ADD_UNICODE: return 2; case _BINARY_OP_INPLACE_ADD_UNICODE: @@ -716,13 +718,13 @@ int _PyUop_num_popped(int opcode, int oparg) case _BINARY_SUBSCR_DICT: return 2; case _BINARY_SUBSCR_CHECK_FUNC: - return 2; + return 0; case _BINARY_SUBSCR_INIT_CALL: return 2; case _LIST_APPEND: - return 2 + (oparg-1); + return 1; case _SET_ADD: - return 2 + (oparg-1); + return 1; case _STORE_SUBSCR: return 3; case _STORE_SUBSCR_LIST_INT: @@ -740,11 +742,11 @@ int _PyUop_num_popped(int opcode, int oparg) case _GET_AITER: return 1; case _GET_ANEXT: - return 1; + return 0; case _GET_AWAITABLE: return 1; case _SEND_GEN_FRAME: - return 2; + return 1; case _YIELD_VALUE: return 1; case _POP_EXCEPT: @@ -812,9 +814,9 @@ int _PyUop_num_popped(int opcode, int oparg) case _BUILD_LIST: return oparg; case _LIST_EXTEND: - return 2 + (oparg-1); + return 1; case _SET_UPDATE: - return 2 + (oparg-1); + return 1; case _BUILD_SET: return oparg; case _BUILD_MAP: @@ -822,11 +824,11 @@ int _PyUop_num_popped(int opcode, int oparg) case _SETUP_ANNOTATIONS: return 0; case _DICT_UPDATE: - return 2 + (oparg - 1); + return 1; case _DICT_MERGE: - return 5 + (oparg - 1); + return 1; case _MAP_ADD: - return 3 + (oparg - 1); + return 2; case _LOAD_SUPER_ATTR_ATTR: return 3; case _LOAD_SUPER_ATTR_METHOD: @@ -834,9 +836,9 @@ int _PyUop_num_popped(int opcode, int oparg) case _LOAD_ATTR: return 1; case _GUARD_TYPE_VERSION: - return 1; + return 0; case _CHECK_MANAGED_OBJECT_HAS_VALUES: - return 1; + return 0; case _LOAD_ATTR_INSTANCE_VALUE_0: return 1; case _LOAD_ATTR_INSTANCE_VALUE_1: @@ -844,11 +846,11 @@ int _PyUop_num_popped(int opcode, int oparg) case _LOAD_ATTR_INSTANCE_VALUE: return 1; case _CHECK_ATTR_MODULE: - return 1; + return 0; case _LOAD_ATTR_MODULE: return 1; case _CHECK_ATTR_WITH_HINT: - return 1; + return 0; case _LOAD_ATTR_WITH_HINT: return 1; case _LOAD_ATTR_SLOT_0: @@ -858,7 +860,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _LOAD_ATTR_SLOT: return 1; case _CHECK_ATTR_CLASS: - return 1; + return 0; case _LOAD_ATTR_CLASS_0: return 1; case _LOAD_ATTR_CLASS_1: @@ -868,7 +870,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _LOAD_ATTR_PROPERTY_FRAME: return 1; case _GUARD_DORV_NO_DICT: - return 1; + return 0; case _STORE_ATTR_INSTANCE_VALUE: return 2; case _STORE_ATTR_WITH_HINT: @@ -894,59 +896,59 @@ int _PyUop_num_popped(int opcode, int oparg) case _CHECK_EG_MATCH: return 2; case _CHECK_EXC_MATCH: - return 2; + return 1; case _IMPORT_NAME: return 2; case _IMPORT_FROM: - return 1; + return 0; case _IS_NONE: return 1; case _GET_LEN: - return 1; + return 0; case _MATCH_CLASS: return 3; case _MATCH_MAPPING: - return 1; + return 0; case _MATCH_SEQUENCE: - return 1; + return 0; case _MATCH_KEYS: - return 2; + return 0; case _GET_ITER: return 1; case _GET_YIELD_FROM_ITER: return 1; case _FOR_ITER_TIER_TWO: - return 1; + return 0; case _ITER_CHECK_LIST: - return 1; + return 0; case _GUARD_NOT_EXHAUSTED_LIST: - return 1; + return 0; case _ITER_NEXT_LIST: - return 1; + return 0; case _ITER_CHECK_TUPLE: - return 1; + return 0; case _GUARD_NOT_EXHAUSTED_TUPLE: - return 1; + return 0; case _ITER_NEXT_TUPLE: - return 1; + return 0; case _ITER_CHECK_RANGE: - return 1; + return 0; case _GUARD_NOT_EXHAUSTED_RANGE: - return 1; + return 0; case _ITER_NEXT_RANGE: - return 1; + return 0; case _FOR_ITER_GEN_FRAME: - return 1; + return 0; case _LOAD_SPECIAL: return 1; case _WITH_EXCEPT_START: - return 5; + return 0; case _PUSH_EXC_INFO: return 1; case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: - return 1; + return 0; case _GUARD_KEYS_VERSION: - return 1; + return 0; case _LOAD_ATTR_METHOD_WITH_VALUES: return 1; case _LOAD_ATTR_METHOD_NO_DICT: @@ -956,7 +958,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: return 1; case _CHECK_ATTR_METHOD_LAZY_DICT: - return 1; + return 0; case _LOAD_ATTR_METHOD_LAZY_DICT: return 1; case _MAYBE_EXPAND_METHOD: @@ -964,25 +966,27 @@ int _PyUop_num_popped(int opcode, int oparg) case _PY_FRAME_GENERAL: return 2 + oparg; case _CHECK_FUNCTION_VERSION: - return 2 + oparg; + return 0; + case _CHECK_FUNCTION_VERSION_INLINE: + return 0; case _CHECK_METHOD_VERSION: - return 2 + oparg; + return 0; case _EXPAND_METHOD: return 2 + oparg; case _CHECK_IS_NOT_PY_CALLABLE: - return 2 + oparg; + return 0; case _CALL_NON_PY_GENERAL: return 2 + oparg; case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: - return 2 + oparg; + return 0; case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: return 2 + oparg; case _CHECK_PEP_523: return 0; case _CHECK_FUNCTION_EXACT_ARGS: - return 2 + oparg; + return 0; case _CHECK_STACK_SPACE: - return 2 + oparg; + return 0; case _INIT_CALL_PY_EXACT_ARGS_0: return 2 + oparg; case _INIT_CALL_PY_EXACT_ARGS_1: @@ -1036,17 +1040,17 @@ int _PyUop_num_popped(int opcode, int oparg) case _PY_FRAME_KW: return 3 + oparg; case _CHECK_FUNCTION_VERSION_KW: - return 3 + oparg; + return 0; case _CHECK_METHOD_VERSION_KW: - return 3 + oparg; + return 0; case _EXPAND_METHOD_KW: return 3 + oparg; case _CHECK_IS_NOT_PY_CALLABLE_KW: - return 3 + oparg; + return 0; case _CALL_KW_NON_PY: return 3 + oparg; case _MAKE_CALLARGS_A_TUPLE: - return 3 + (oparg & 1); + return 1 + (oparg & 1); case _MAKE_FUNCTION: return 1; case _SET_FUNCTION_ATTRIBUTE: @@ -1062,7 +1066,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _FORMAT_WITH_SPEC: return 2; case _COPY: - return 1 + (oparg-1); + return 0; case _BINARY_OP: return 2; case _SWAP: diff --git a/Include/marshal.h b/Include/marshal.h index f8b0de80cfc38d..f773587bdd0429 100644 --- a/Include/marshal.h +++ b/Include/marshal.h @@ -13,7 +13,7 @@ PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, Py_ssize_t); PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int); -#define Py_MARSHAL_VERSION 4 +#define Py_MARSHAL_VERSION 5 PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *); PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *); diff --git a/Include/pymath.h b/Include/pymath.h index d8f763f808d662..0ead1f95670fde 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -49,7 +49,7 @@ /* Py_HUGE_VAL should always be the same as Py_INFINITY. But historically * this was not reliable and Python did not require IEEE floats and C99 - * conformity. Prefer Py_INFINITY for new code. + * conformity. The macro was soft deprecated in Python 3.14, use Py_INFINITY instead. */ #ifndef Py_HUGE_VAL # define Py_HUGE_VAL HUGE_VAL diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md index 0da4670c792cb5..37964bd99428df 100644 --- a/InternalDocs/compiler.md +++ b/InternalDocs/compiler.md @@ -42,10 +42,10 @@ The definitions for literal tokens (such as `:`, numbers, etc.) can be found in See Also: -* [Guide to the parser](https://devguide.python.org/internals/parser/index.html) +* [Guide to the parser](parser.md) for a detailed description of the parser. -* [Changing CPython’s grammar](https://devguide.python.org/developer-workflow/grammar/#grammar) +* [Changing CPython’s grammar](changing_grammar.md) for a detailed description of the grammar. diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index d624cf4befd31a..ea21928b50e34a 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -108,7 +108,7 @@ As is explained later in the [Optimization: reusing fields to save memory](#optimization-reusing-fields-to-save-memory) section, these two extra fields are normally used to keep doubly linked lists of all the objects tracked by the garbage collector (these lists are the GC generations, more on -that in the [Optimization: generations](#Optimization-generations) section), but +that in the [Optimization: incremental collection](#Optimization-incremental-collection) section), but they are also reused to fulfill other purposes when the full doubly linked list structure is not needed as a memory optimization. @@ -351,38 +351,91 @@ follows these steps in order: the reference counts fall to 0, triggering the destruction of all unreachable objects. -Optimization: generations -========================= -In order to limit the time each garbage collection takes, the GC -implementation for the default build uses a popular optimization: -generations. The main idea behind this concept is the assumption that most -objects have a very short lifespan and can thus be collected soon after their -creation. This has proven to be very close to the reality of many Python +Optimization: incremental collection +==================================== + +In order to bound the length of each garbage collection pause, the GC implementation +for the default build uses incremental collection with two generations. + +Generational garbage collection takes advantage of what is known as the weak +generational hypothesis: Most objects die young. +This has proven to be very close to the reality of many Python programs as many temporary objects are created and destroyed very quickly. To take advantage of this fact, all container objects are segregated into -three spaces/generations. Every new -object starts in the first generation (generation 0). The previous algorithm is -executed only over the objects of a particular generation and if an object -survives a collection of its generation it will be moved to the next one -(generation 1), where it will be surveyed for collection less often. If -the same object survives another GC round in this new generation (generation 1) -it will be moved to the last generation (generation 2) where it will be -surveyed the least often. - -The GC implementation for the free-threaded build does not use multiple -generations. Every collection operates on the entire heap. +two generations: young and old. Every new object starts in the young generation. +Each garbage collection scans the entire young generation and part of the old generation. + +The time taken to scan the young generation can be controlled by controlling its +size, but the size of the old generation cannot be controlled. +In order to keep pause times down, scanning of the old generation of the heap +occurs in increments. + +To keep track of what has been scanned, the old generation contains two lists: + +* Those objects that have not yet been scanned, referred to as the `pending` list. +* Those objects that have been scanned, referred to as the `visited` list. + +To detect and collect all unreachable objects in the heap, the garbage collector +must scan the whole heap. This whole heap scan is called a full scavenge. + +Increments +---------- + +Each full scavenge is performed in a series of increments. +For each full scavenge, the combined increments will cover the whole heap. + +Each increment is made up of: + +* The young generation +* The old generation's least recently scanned objects +* All objects reachable from those objects that have not yet been scanned this full scavenge + +The surviving objects (those that are not collected) are moved to the back of the +`visited` list in the old generation. + +When a full scavenge starts, no objects in the heap are considered to have been scanned, +so all objects in the old generation must be in the `pending` space. +When all objects in the heap have been scanned a cycle ends, and all objects are moved +to the `pending` list again. To avoid having to traverse the entire list, which list is +`pending` and which is `visited` is determined by a field in the `GCState` struct. +The `visited` and `pending` lists can be swapped by toggling this bit. + +Correctness +----------- + +The [algorithm for identifying cycles](#Identifying-reference-cycles) will find all +unreachable cycles in a list of objects, but will not find any cycles that are +even partly outside of that list. +Therefore, to be guaranteed that a full scavenge will find all unreachable cycles, +each cycle must be fully contained within a single increment. + +To make sure that no partial cycles are included in the increment we perform a +[transitive closure](https://en.wikipedia.org/wiki/Transitive_closure) +over reachable, unscanned objects from the initial increment. +Since the transitive closure of objects reachable from an object must be a (non-strict) +superset of any unreachable cycle including that object, we are guaranteed that a +transitive closure cannot contain any partial cycles. +We can exclude scanned objects, as they must have been reachable when scanned. +If a scanned object becomes part of an unreachable cycle after being scanned, it will +not be collected this at this time, but it will be collected in the next full scavenge. + +> [!NOTE] +> The GC implementation for the free-threaded build does not use incremental collection. +> Every collection operates on the entire heap. In order to decide when to run, the collector keeps track of the number of object allocations and deallocations since the last collection. When the number of -allocations minus the number of deallocations exceeds `threshold_0`, -collection starts. Initially only generation 0 is examined. If generation 0 has -been examined more than `threshold_1` times since generation 1 has been -examined, then generation 1 is examined as well. With generation 2, -things are a bit more complicated; see -[Collecting the oldest generation](#Collecting-the-oldest-generation) for -more information. These thresholds can be examined using the +allocations minus the number of deallocations exceeds `threshold0`, +collection starts. `threshold1` determines the fraction of the old +collection that is included in the increment. +The fraction is inversely proportional to `threshold1`, +as historically a larger `threshold1` meant that old generation +collections were performed less frequently. +`threshold2` is ignored. + +These thresholds can be examined using the [`gc.get_threshold()`](https://docs.python.org/3/library/gc.html#gc.get_threshold) function: @@ -402,8 +455,8 @@ specifically in a generation by calling `gc.collect(generation=NUM)`. ... pass ... - # Move everything to the last generation so it's easier to inspect - # the younger generations. + # Move everything to the old generation so it's easier to inspect + # the young generation. >>> gc.collect() 0 @@ -413,40 +466,64 @@ specifically in a generation by calling `gc.collect(generation=NUM)`. >>> x = MyObj() >>> x.self = x - # Initially the object is in the youngest generation. + # Initially the object is in the young generation. >>> gc.get_objects(generation=0) [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] # After a collection of the youngest generation the object - # moves to the next generation. + # moves to the old generation. >>> gc.collect(generation=0) 0 >>> gc.get_objects(generation=0) [] >>> gc.get_objects(generation=1) + [] + >>> gc.get_objects(generation=2) [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] ``` -Collecting the oldest generation --------------------------------- - -In addition to the various configurable thresholds, the GC only triggers a full -collection of the oldest generation if the ratio `long_lived_pending / long_lived_total` -is above a given value (hardwired to 25%). The reason is that, while "non-full" -collections (that is, collections of the young and middle generations) will always -examine roughly the same number of objects (determined by the aforementioned -thresholds) the cost of a full collection is proportional to the total -number of long-lived objects, which is virtually unbounded. Indeed, it has -been remarked that doing a full collection every of object -creations entails a dramatic performance degradation in workloads which consist -of creating and storing lots of long-lived objects (for example, building a large list -of GC-tracked objects would show quadratic performance, instead of linear as -expected). Using the above ratio, instead, yields amortized linear performance -in the total number of objects (the effect of which can be summarized thusly: -"each full garbage collection is more and more costly as the number of objects -grows, but we do fewer and fewer of them"). + +Optimization: visiting reachable objects +======================================== + +An object cannot be garbage if it can be reached. + +To avoid having to identify reference cycles across the whole heap, we can +reduce the amount of work done considerably by first moving most reachable objects +to the `visited` space. Empirically, most reachable objects can be reached from a +small set of global objects and local variables. +This step does much less work per object, so reduces the time spent +performing garbage collection by at least half. + +> [!NOTE] +> Objects that are not determined to be reachable by this pass are not necessarily +> unreachable. We still need to perform the main algorithm to determine which objects +> are actually unreachable. + +We use the same technique of forming a transitive closure as the incremental +collector does to find reachable objects, seeding the list with some global +objects and the currently executing frames. + +This phase moves objects to the `visited` space, as follows: + +1. All objects directly referred to by any builtin class, the `sys` module, the `builtins` +module and all objects directly referred to from stack frames are added to a working +set of reachable objects. +2. Until this working set is empty: + 1. Pop an object from the set and move it to the `visited` space + 2. For each object directly reachable from that object: + * If it is not already in `visited` space and it is a GC object, + add it to the working set + + +Before each increment of collection is performed, the stacks are scanned +to check for any new stack frames that have been created since the last +increment. All objects directly referred to from those stack frames are +added to the working set. +Then the above algorithm is repeated, starting from step 2. + Optimization: reusing fields to save memory =========================================== @@ -496,8 +573,8 @@ of `PyGC_Head` discussed in the `Memory layout and object structure`_ section: currently in. Instead, when that's needed, ad hoc tricks (like the `NEXT_MASK_UNREACHABLE` flag) are employed. -Optimization: delay tracking containers -======================================= +Optimization: delayed untracking of containers +============================================== Certain types of containers cannot participate in a reference cycle, and so do not need to be tracked by the garbage collector. Untracking these objects @@ -512,8 +589,8 @@ a container: As a general rule, instances of atomic types aren't tracked and instances of non-atomic types (containers, user-defined objects...) are. However, some type-specific optimizations can be present in order to suppress the garbage -collector footprint of simple instances. Some examples of native types that -benefit from delayed tracking: +collector footprint of simple instances. Historically, both dictionaries and +tuples were untracked during garbage collection. Now it is only tuples: - Tuples containing only immutable objects (integers, strings etc, and recursively, tuples of immutable objects) do not need to be tracked. The @@ -522,14 +599,8 @@ benefit from delayed tracking: tuples at creation time. Instead, all tuples except the empty tuple are tracked when created. During garbage collection it is determined whether any surviving tuples can be untracked. A tuple can be untracked if all of its contents are - already not tracked. Tuples are examined for untracking in all garbage collection - cycles. It may take more than one cycle to untrack a tuple. - -- Dictionaries containing only immutable objects also do not need to be tracked. - Dictionaries are untracked when created. If a tracked item is inserted into a - dictionary (either as a key or value), the dictionary becomes tracked. During a - full garbage collection (all generations), the collector will untrack any dictionaries - whose contents are not tracked. + already not tracked. Tuples are examined for untracking when moved from the + young to the old generation. The garbage collector module provides the Python function `is_tracked(obj)`, which returns the current tracking status of the object. Subsequent garbage collections may change the @@ -542,11 +613,9 @@ tracking status of the object. False >>> gc.is_tracked([]) True - >>> gc.is_tracked({}) + >>> gc.is_tracked(("a", 1)) False >>> gc.is_tracked({"a": 1}) - False - >>> gc.is_tracked({"a": []}) True ``` @@ -588,9 +657,9 @@ heap. be more difficult. -> [!NOTE] +> [!NOTE] > **Document history** -> +> > Pablo Galindo Salgado - Original author -> +> > Irit Katriel - Convert to Markdown diff --git a/InternalDocs/parser.md b/InternalDocs/parser.md index 6398ba6cd2838f..348988b7c2f003 100644 --- a/InternalDocs/parser.md +++ b/InternalDocs/parser.md @@ -17,7 +17,7 @@ Therefore, changes to the Python language are made by modifying the [grammar file](../Grammar/python.gram). Developers rarely need to modify the generator itself. -See the devguide's [Changing CPython's grammar](https://devguide.python.org/developer-workflow/grammar/#grammar) +See [Changing CPython's grammar](changing_grammar.md) for a detailed description of the grammar and the process for changing it. How PEG parsers work diff --git a/LICENSE b/LICENSE index 14603b95c2e23b..20cf39097c68ba 100644 --- a/LICENSE +++ b/LICENSE @@ -83,7 +83,7 @@ grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, -i.e., "Copyright (c) 2001-2024 Python Software Foundation; All Rights Reserved" +i.e., "Copyright (c) 2001 Python Software Foundation; All Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee. 3. In the event Licensee prepares a derivative work that is based on diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 7b6d10c008d3cb..42b0aea4e2eb2e 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -1480,6 +1480,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): """ if self._fd >= 0: # Have to close the existing file first. + self._stat_atopen = None try: if self._closefd: os.close(self._fd) @@ -1583,6 +1584,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if e.errno != errno.ESPIPE: raise except: + self._stat_atopen = None if owned_fd is not None: os.close(owned_fd) raise @@ -1756,6 +1758,7 @@ def close(self): called more than once without error. """ if not self.closed: + self._stat_atopen = None try: if self._closefd: os.close(self._fd) diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py index 849617bf7585e4..d884f880f50ac7 100644 --- a/Lib/_pyrepl/_minimal_curses.py +++ b/Lib/_pyrepl/_minimal_curses.py @@ -34,7 +34,7 @@ def _find_clib() -> str: clib.setupterm.restype = ctypes.c_int clib.tigetstr.argtypes = [ctypes.c_char_p] -clib.tigetstr.restype = ctypes.POINTER(ctypes.c_char) +clib.tigetstr.restype = ctypes.c_ssize_t clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] # type: ignore[operator] clib.tparm.restype = ctypes.c_char_p @@ -56,7 +56,7 @@ def tigetstr(cap): if not isinstance(cap, bytes): cap = cap.encode("ascii") result = clib.tigetstr(cap) - if ctypes.cast(result, ctypes.c_void_p).value == ERR: + if result == ERR: return None return ctypes.cast(result, ctypes.c_char_p).value diff --git a/Lib/argparse.py b/Lib/argparse.py index 072cd5e7dc0d06..5ecfdca17175e3 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -863,6 +863,9 @@ def __init__(self, _option_strings.append(option_string) if option_string.startswith('--'): + if option_string.startswith('--no-'): + raise ValueError(f'invalid option name {option_string!r} ' + f'for BooleanOptionalAction') option_string = '--no-' + option_string[2:] _option_strings.append(option_string) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index b63fe6aa79604b..ca0a4f2fee5840 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -668,7 +668,6 @@ class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): class _Local(threading.local): _loop = None - _set_called = False def __init__(self): self._local = self._Local() @@ -678,28 +677,6 @@ def get_event_loop(self): Returns an instance of EventLoop or raises an exception. """ - if (self._local._loop is None and - not self._local._set_called and - threading.current_thread() is threading.main_thread()): - stacklevel = 2 - try: - f = sys._getframe(1) - except AttributeError: - pass - else: - # Move up the call stack so that the warning is attached - # to the line outside asyncio itself. - while f: - module = f.f_globals.get('__name__') - if not (module == 'asyncio' or module.startswith('asyncio.')): - break - f = f.f_back - stacklevel += 1 - import warnings - warnings.warn('There is no current event loop', - DeprecationWarning, stacklevel=stacklevel) - self.set_event_loop(self.new_event_loop()) - if self._local._loop is None: raise RuntimeError('There is no current event loop in thread %r.' % threading.current_thread().name) @@ -708,7 +685,6 @@ def get_event_loop(self): def set_event_loop(self, loop): """Set the event loop.""" - self._local._set_called = True if loop is not None and not isinstance(loop, AbstractEventLoop): raise TypeError(f"loop must be an instance of AbstractEventLoop or None, not '{type(loop).__name__}'") self._local._loop = loop diff --git a/Lib/calendar.py b/Lib/calendar.py index 069dd5174112ae..8c1c646da46a98 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -27,7 +27,9 @@ error = ValueError # Exceptions raised for bad input -class IllegalMonthError(ValueError): +# This is trick for backward compatibility. Since 3.13, we will raise IllegalMonthError instead of +# IndexError for bad month number(out of 1-12). But we can't remove IndexError for backward compatibility. +class IllegalMonthError(ValueError, IndexError): def __init__(self, month): self.month = month def __str__(self): @@ -158,11 +160,14 @@ def weekday(year, month, day): return Day(datetime.date(year, month, day).weekday()) +def _validate_month(month): + if not 1 <= month <= 12: + raise IllegalMonthError(month) + def monthrange(year, month): """Return weekday of first day of month (0-6 ~ Mon-Sun) and number of days (28-31) for year, month.""" - if not 1 <= month <= 12: - raise IllegalMonthError(month) + _validate_month(month) day1 = weekday(year, month, 1) ndays = mdays[month] + (month == FEBRUARY and isleap(year)) return day1, ndays @@ -370,6 +375,8 @@ def formatmonthname(self, theyear, themonth, width, withyear=True): """ Return a formatted month name. """ + _validate_month(themonth) + s = month_name[themonth] if withyear: s = "%s %r" % (s, theyear) @@ -500,6 +507,7 @@ def formatmonthname(self, theyear, themonth, withyear=True): """ Return a month name as a table row. """ + _validate_month(themonth) if withyear: s = '%s %s' % (month_name[themonth], theyear) else: @@ -786,6 +794,8 @@ def main(args=None): if options.month is None: optdict["c"] = options.spacing optdict["m"] = options.months + if options.month is not None: + _validate_month(options.month) if options.year is None: result = cal.formatyear(datetime.date.today().year, **optdict) elif options.month is None: diff --git a/Lib/contextvars.py b/Lib/contextvars.py index d78c80dfe6f99c..14514f185e069d 100644 --- a/Lib/contextvars.py +++ b/Lib/contextvars.py @@ -1,4 +1,8 @@ +import _collections_abc from _contextvars import Context, ContextVar, Token, copy_context __all__ = ('Context', 'ContextVar', 'Token', 'copy_context') + + +_collections_abc.Mapping.register(Context) diff --git a/Lib/email/__init__.py b/Lib/email/__init__.py index 9fa47783004185..6d597006e5eefe 100644 --- a/Lib/email/__init__.py +++ b/Lib/email/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2007 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index 36625e35ffb6a7..84917038874ba1 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2007 Python Software Foundation +# Copyright (C) 2002 Python Software Foundation # Contact: email-sig@python.org """Email address parsing code. diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index c7694a44e26639..4b63b97217a835 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -302,12 +302,12 @@ def header_source_parse(self, sourcelines): """+ The name is parsed as everything up to the ':' and returned unmodified. The value is determined by stripping leading whitespace off the - remainder of the first line, joining all subsequent lines together, and + remainder of the first line joined with all subsequent lines, and stripping any trailing carriage return or linefeed characters. """ name, value = sourcelines[0].split(':', 1) - value = value.lstrip(' \t') + ''.join(sourcelines[1:]) + value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n') return (name, value.rstrip('\r\n')) def header_store_parse(self, name, value): diff --git a/Lib/email/base64mime.py b/Lib/email/base64mime.py index d440de95255bf1..a5a3f737a97b51 100644 --- a/Lib/email/base64mime.py +++ b/Lib/email/base64mime.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2007 Python Software Foundation +# Copyright (C) 2002 Python Software Foundation # Author: Ben Gertzfield # Contact: email-sig@python.org diff --git a/Lib/email/charset.py b/Lib/email/charset.py index cfd5a0c456e497..5036c3f58a5633 100644 --- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2007 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Ben Gertzfield, Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/encoders.py b/Lib/email/encoders.py index 17bd1ab7b19f32..55741a22a07b20 100644 --- a/Lib/email/encoders.py +++ b/Lib/email/encoders.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/errors.py b/Lib/email/errors.py index 02aa5eced6ae46..6bc744bd59c5bb 100644 --- a/Lib/email/errors.py +++ b/Lib/email/errors.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 06d6b4a3afcd07..b2bc4afc1cc26f 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -1,4 +1,4 @@ -# Copyright (C) 2004-2006 Python Software Foundation +# Copyright (C) 2004 Python Software Foundation # Authors: Baxter, Wouters and Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/generator.py b/Lib/email/generator.py index 205caf0fe9e81d..ab5bd0653e440c 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2010 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/header.py b/Lib/email/header.py index 66a1d46db50c45..113a81f41314ec 100644 --- a/Lib/email/header.py +++ b/Lib/email/header.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2007 Python Software Foundation +# Copyright (C) 2002 Python Software Foundation # Author: Ben Gertzfield, Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/iterators.py b/Lib/email/iterators.py index 2f436aefc2300b..08ede3ec679613 100644 --- a/Lib/email/iterators.py +++ b/Lib/email/iterators.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/message.py b/Lib/email/message.py index 08192c50a8ff5c..a58afc5fe5f68e 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2007 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/mime/application.py b/Lib/email/mime/application.py index f67cbad3f03407..9a9d213d2a940d 100644 --- a/Lib/email/mime/application.py +++ b/Lib/email/mime/application.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Keith Dart # Contact: email-sig@python.org diff --git a/Lib/email/mime/audio.py b/Lib/email/mime/audio.py index aa0c4905cbb2b4..85f4a955238c52 100644 --- a/Lib/email/mime/audio.py +++ b/Lib/email/mime/audio.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2007 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Anthony Baxter # Contact: email-sig@python.org diff --git a/Lib/email/mime/base.py b/Lib/email/mime/base.py index f601f621cec393..da4c6e591a5cb8 100644 --- a/Lib/email/mime/base.py +++ b/Lib/email/mime/base.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/mime/image.py b/Lib/email/mime/image.py index 4b7f2f9cbad425..dab9685848172b 100644 --- a/Lib/email/mime/image.py +++ b/Lib/email/mime/image.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/mime/message.py b/Lib/email/mime/message.py index 61836b5a7861fc..13d9ff599f86db 100644 --- a/Lib/email/mime/message.py +++ b/Lib/email/mime/message.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/mime/multipart.py b/Lib/email/mime/multipart.py index 47fc218e1ae032..1abb84d5fed0bb 100644 --- a/Lib/email/mime/multipart.py +++ b/Lib/email/mime/multipart.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2006 Python Software Foundation +# Copyright (C) 2002 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/mime/nonmultipart.py b/Lib/email/mime/nonmultipart.py index a41386eb148c0c..5beab3a441e2bc 100644 --- a/Lib/email/mime/nonmultipart.py +++ b/Lib/email/mime/nonmultipart.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2006 Python Software Foundation +# Copyright (C) 2002 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/mime/text.py b/Lib/email/mime/text.py index 7672b789138600..aa4da7f8217e43 100644 --- a/Lib/email/mime/text.py +++ b/Lib/email/mime/text.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/parser.py b/Lib/email/parser.py index 475aa2b1a66680..039f03cba74fa0 100644 --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2007 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw, Thomas Wouters, Anthony Baxter # Contact: email-sig@python.org diff --git a/Lib/email/policy.py b/Lib/email/policy.py index 46b7de5bb6d8ae..6e109b65011a44 100644 --- a/Lib/email/policy.py +++ b/Lib/email/policy.py @@ -119,13 +119,13 @@ def header_source_parse(self, sourcelines): """+ The name is parsed as everything up to the ':' and returned unmodified. The value is determined by stripping leading whitespace off the - remainder of the first line, joining all subsequent lines together, and + remainder of the first line joined with all subsequent lines, and stripping any trailing carriage return or linefeed characters. (This is the same as Compat32). """ name, value = sourcelines[0].split(':', 1) - value = value.lstrip(' \t') + ''.join(sourcelines[1:]) + value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n') return (name, value.rstrip('\r\n')) def header_store_parse(self, name, value): diff --git a/Lib/email/quoprimime.py b/Lib/email/quoprimime.py index 500bbc5151769d..27c7ea55c7871f 100644 --- a/Lib/email/quoprimime.py +++ b/Lib/email/quoprimime.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Ben Gertzfield # Contact: email-sig@python.org diff --git a/Lib/email/utils.py b/Lib/email/utils.py index f276303197396b..7eab74dc0db9df 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2010 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/encodings/__init__.py b/Lib/encodings/__init__.py index f9075b8f0d98ac..298177eb8003a7 100644 --- a/Lib/encodings/__init__.py +++ b/Lib/encodings/__init__.py @@ -156,19 +156,22 @@ def search_function(encoding): codecs.register(search_function) if sys.platform == 'win32': - # bpo-671666, bpo-46668: If Python does not implement a codec for current - # Windows ANSI code page, use the "mbcs" codec instead: - # WideCharToMultiByte() and MultiByteToWideChar() functions with CP_ACP. - # Python does not support custom code pages. - def _alias_mbcs(encoding): + from ._win_cp_codecs import create_win32_code_page_codec + + def win32_code_page_search_function(encoding): + encoding = encoding.lower() + if not encoding.startswith('cp'): + return None try: - import _winapi - ansi_code_page = "cp%s" % _winapi.GetACP() - if encoding == ansi_code_page: - import encodings.mbcs - return encodings.mbcs.getregentry() - except ImportError: - # Imports may fail while we are shutting down - pass + cp = int(encoding[2:]) + except ValueError: + return None + # Test if the code page is supported + try: + codecs.code_page_encode(cp, 'x') + except (OverflowError, OSError): + return None + + return create_win32_code_page_codec(cp) - codecs.register(_alias_mbcs) + codecs.register(win32_code_page_search_function) diff --git a/Lib/encodings/_win_cp_codecs.py b/Lib/encodings/_win_cp_codecs.py new file mode 100644 index 00000000000000..4f8eb886794404 --- /dev/null +++ b/Lib/encodings/_win_cp_codecs.py @@ -0,0 +1,36 @@ +import codecs + +def create_win32_code_page_codec(cp): + from codecs import code_page_encode, code_page_decode + + def encode(input, errors='strict'): + return code_page_encode(cp, input, errors) + + def decode(input, errors='strict'): + return code_page_decode(cp, input, errors, True) + + class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + return code_page_encode(cp, input, self.errors)[0] + + class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input, errors, final): + return code_page_decode(cp, input, errors, final) + + class StreamWriter(codecs.StreamWriter): + def encode(self, input, errors='strict'): + return code_page_encode(cp, input, errors) + + class StreamReader(codecs.StreamReader): + def decode(self, input, errors, final): + return code_page_decode(cp, input, errors, final) + + return codecs.CodecInfo( + name=f'cp{cp}', + encode=encode, + decode=decode, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + streamreader=StreamReader, + streamwriter=StreamWriter, + ) diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 585afc85836c06..645ad998129348 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] -_PIP_VERSION = "24.2" +_PIP_VERSION = "24.3.1" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora diff --git a/Lib/ensurepip/_bundled/pip-24.2-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-24.3.1-py3-none-any.whl similarity index 84% rename from Lib/ensurepip/_bundled/pip-24.2-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-24.3.1-py3-none-any.whl index 542cdd1e7284ae..5f1d35be6dd56b 100644 Binary files a/Lib/ensurepip/_bundled/pip-24.2-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-24.3.1-py3-none-any.whl differ diff --git a/Lib/functools.py b/Lib/functools.py index 27abd622a8cff1..eff6540c7f606e 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -6,7 +6,7 @@ # Written by Nick Coghlan , # Raymond Hettinger , # and Łukasz Langa . -# Copyright (C) 2006-2024 Python Software Foundation. +# Copyright (C) 2006 Python Software Foundation. # See C source code for _functools credits/copyright __all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', @@ -236,7 +236,7 @@ def __ge__(self, other): def reduce(function, sequence, initial=_initial_missing): """ - reduce(function, iterable[, initial], /) -> value + reduce(function, iterable, /[, initial]) -> value Apply a function of two arguments cumulatively to the items of an iterable, from left to right. diff --git a/Lib/getopt.py b/Lib/getopt.py index e5fd04fe12a7ee..a9c452a601ee81 100644 --- a/Lib/getopt.py +++ b/Lib/getopt.py @@ -24,21 +24,14 @@ # TODO for gnu_getopt(): # # - GNU getopt_long_only mechanism -# - allow the caller to specify ordering -# - RETURN_IN_ORDER option -# - GNU extension with '-' as first character of option string -# - optional arguments, specified by double colons # - an option string with a W followed by semicolon should # treat "-W foo" as "--foo" __all__ = ["GetoptError","error","getopt","gnu_getopt"] import os -try: - from gettext import gettext as _ -except ImportError: - # Bootstrapping Python: gettext's dependencies not built yet - def _(s): return s +from gettext import gettext as _ + class GetoptError(Exception): opt = '' @@ -61,12 +54,14 @@ def getopt(args, shortopts, longopts = []): running program. Typically, this means "sys.argv[1:]". shortopts is the string of option letters that the script wants to recognize, with options that require an argument followed by a - colon (i.e., the same format that Unix getopt() uses). If + colon and options that accept an optional argument followed by + two colons (i.e., the same format that Unix getopt() uses). If specified, longopts is a list of strings with the names of the long options which should be supported. The leading '--' characters should not be included in the option name. Options which require an argument should be followed by an equal sign - ('='). + ('='). Options which accept an optional argument should be + followed by an equal sign and question mark ('=?'). The return value consists of two elements: the first is a list of (option, value) pairs; the second is the list of program arguments @@ -118,8 +113,13 @@ def gnu_getopt(args, shortopts, longopts = []): else: longopts = list(longopts) + return_in_order = False + if shortopts.startswith('-'): + shortopts = shortopts[1:] + all_options_first = False + return_in_order = True # Allow options after non-option arguments? - if shortopts.startswith('+'): + elif shortopts.startswith('+'): shortopts = shortopts[1:] all_options_first = True elif os.environ.get("POSIXLY_CORRECT"): @@ -133,8 +133,14 @@ def gnu_getopt(args, shortopts, longopts = []): break if args[0][:2] == '--': + if return_in_order and prog_args: + opts.append((None, prog_args)) + prog_args = [] opts, args = do_longs(opts, args[0][2:], longopts, args[1:]) elif args[0][:1] == '-' and args[0] != '-': + if return_in_order and prog_args: + opts.append((None, prog_args)) + prog_args = [] opts, args = do_shorts(opts, args[0][1:], shortopts, args[1:]) else: if all_options_first: @@ -156,7 +162,7 @@ def do_longs(opts, opt, longopts, args): has_arg, opt = long_has_args(opt, longopts) if has_arg: - if optarg is None: + if optarg is None and has_arg != '?': if not args: raise GetoptError(_('option --%s requires argument') % opt, opt) optarg, args = args[0], args[1:] @@ -177,6 +183,8 @@ def long_has_args(opt, longopts): return False, opt elif opt + '=' in possibilities: return True, opt + elif opt + '=?' in possibilities: + return '?', opt # No exact match, so better be unique. if len(possibilities) > 1: # XXX since possibilities contains all valid continuations, might be @@ -184,6 +192,8 @@ def long_has_args(opt, longopts): raise GetoptError(_('option --%s not a unique prefix') % opt, opt) assert len(possibilities) == 1 unique_match = possibilities[0] + if unique_match.endswith('=?'): + return '?', unique_match[:-2] has_arg = unique_match.endswith('=') if has_arg: unique_match = unique_match[:-1] @@ -192,8 +202,9 @@ def long_has_args(opt, longopts): def do_shorts(opts, optstring, shortopts, args): while optstring != '': opt, optstring = optstring[0], optstring[1:] - if short_has_arg(opt, shortopts): - if optstring == '': + has_arg = short_has_arg(opt, shortopts) + if has_arg: + if optstring == '' and has_arg != '?': if not args: raise GetoptError(_('option -%s requires argument') % opt, opt) @@ -207,7 +218,11 @@ def do_shorts(opts, optstring, shortopts, args): def short_has_arg(opt, shortopts): for i in range(len(shortopts)): if opt == shortopts[i] != ':': - return shortopts.startswith(':', i+1) + if not shortopts.startswith(':', i+1): + return False + if shortopts.startswith('::', i+1): + return '?' + return True raise GetoptError(_('option -%s not recognized') % opt, opt) if __name__ == '__main__': diff --git a/Lib/glob.py b/Lib/glob.py index 574e5ad51b601d..ce9b3698888dd9 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -364,12 +364,6 @@ def concat_path(path, text): """ raise NotImplementedError - @staticmethod - def parse_entry(entry): - """Returns the path of an entry yielded from scandir(). - """ - raise NotImplementedError - # High-level methods def compile(self, pat): @@ -438,6 +432,7 @@ def select_wildcard(path, exists=False): except OSError: pass else: + prefix = self.add_slash(path) for entry in entries: if match is None or match(entry.name): if dir_only: @@ -446,7 +441,7 @@ def select_wildcard(path, exists=False): continue except OSError: continue - entry_path = self.parse_entry(entry) + entry_path = self.concat_path(prefix, entry.name) if dir_only: yield from select_next(entry_path, exists=True) else: @@ -495,6 +490,7 @@ def select_recursive_step(stack, match_pos): except OSError: pass else: + prefix = self.add_slash(path) for entry in entries: is_dir = False try: @@ -504,7 +500,7 @@ def select_recursive_step(stack, match_pos): pass if is_dir or not dir_only: - entry_path = self.parse_entry(entry) + entry_path = self.concat_path(prefix, entry.name) if match is None or match(str(entry_path), match_pos): if dir_only: yield from select_next(entry_path, exists=True) @@ -533,7 +529,6 @@ class _StringGlobber(_GlobberBase): """ lexists = staticmethod(os.path.lexists) scandir = staticmethod(os.scandir) - parse_entry = operator.attrgetter('path') concat_path = operator.add if os.name == 'nt': diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 6b9ed24ad8ec78..d7e8d08b2d92c1 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -266,6 +266,8 @@ class Morsel(dict): "samesite" : "SameSite", } + _reserved_defaults = dict.fromkeys(_reserved, "") + _flags = {'secure', 'httponly'} def __init__(self): @@ -273,8 +275,7 @@ def __init__(self): self._key = self._value = self._coded_value = None # Set default attributes - for key in self._reserved: - dict.__setitem__(self, key, "") + dict.update(self, self._reserved_defaults) @property def key(self): diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 1b76328429f63a..fa36159711846f 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -209,7 +209,11 @@ def _write_atomic(path, data, mode=0o666): # We first write data to a temporary file, and then use os.replace() to # perform an atomic rename. with _io.FileIO(fd, 'wb') as file: - file.write(data) + bytes_written = file.write(data) + if bytes_written != len(data): + # Raise an OSError so the 'except' below cleans up the partially + # written file. + raise OSError("os.write() didn't write the full pyc file") _os.replace(path_tmp, path) except OSError: try: diff --git a/Lib/inspect.py b/Lib/inspect.py index 08718d82e91582..e3f74e9f047eaf 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1507,11 +1507,15 @@ def getclosurevars(func): global_vars = {} builtin_vars = {} unbound_names = set() - for name in code.co_names: - if name in ("None", "True", "False"): - # Because these used to be builtins instead of keywords, they - # may still show up as name references. We ignore them. - continue + global_names = set() + for instruction in dis.get_instructions(code): + opname = instruction.opname + name = instruction.argval + if opname == "LOAD_ATTR": + unbound_names.add(name) + elif opname == "LOAD_GLOBAL": + global_names.add(name) + for name in global_names: try: global_vars[name] = global_ns[name] except KeyError: diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index d7c4e8444f8dec..61cba1ac4932d0 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -479,6 +479,7 @@ def _default_mime_types(): '.m3u8' : 'application/vnd.apple.mpegurl', '.xls' : 'application/vnd.ms-excel', '.xlb' : 'application/vnd.ms-excel', + '.eot' : 'application/vnd.ms-fontobject', '.ppt' : 'application/vnd.ms-powerpoint', '.pot' : 'application/vnd.ms-powerpoint', '.ppa' : 'application/vnd.ms-powerpoint', @@ -534,6 +535,7 @@ def _default_mime_types(): '.ass' : 'audio/aac', '.au' : 'audio/basic', '.snd' : 'audio/basic', + '.mka' : 'audio/matroska', '.mp3' : 'audio/mpeg', '.mp2' : 'audio/mpeg', '.opus' : 'audio/opus', @@ -542,21 +544,34 @@ def _default_mime_types(): '.aiff' : 'audio/x-aiff', '.ra' : 'audio/x-pn-realaudio', '.wav' : 'audio/x-wav', + '.otf' : 'font/otf', + '.ttf' : 'font/ttf', + '.woff' : 'font/woff', + '.woff2' : 'font/woff2', '.avif' : 'image/avif', '.bmp' : 'image/bmp', + '.emf' : 'image/emf', + '.fits' : 'image/fits', + '.g3' : 'image/g3fax', '.gif' : 'image/gif', '.ief' : 'image/ief', + '.jp2' : 'image/jp2', '.jpg' : 'image/jpeg', '.jpe' : 'image/jpeg', '.jpeg' : 'image/jpeg', + '.jpm' : 'image/jpm', + '.jpx' : 'image/jpx', '.heic' : 'image/heic', '.heif' : 'image/heif', '.png' : 'image/png', '.svg' : 'image/svg+xml', + '.t38' : 'image/t38', '.tiff' : 'image/tiff', '.tif' : 'image/tiff', + '.tfx' : 'image/tiff-fx', '.ico' : 'image/vnd.microsoft.icon', '.webp' : 'image/webp', + '.wmf' : 'image/wmf', '.ras' : 'image/x-cmu-raster', '.pnm' : 'image/x-portable-anymap', '.pbm' : 'image/x-portable-bitmap', @@ -595,6 +610,8 @@ def _default_mime_types(): '.sgml' : 'text/x-sgml', '.vcf' : 'text/x-vcard', '.xml' : 'text/xml', + '.mkv' : 'video/matroska', + '.mk3d' : 'video/matroska-3d', '.mp4' : 'video/mp4', '.mpeg' : 'video/mpeg', '.m1v' : 'video/mpeg', diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 53b8c492675878..bff7fb91d974b3 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -168,6 +168,8 @@ def ensure_running(self): def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): '''Run forkserver.''' if preload: + if sys_path is not None: + sys.path[:] = sys_path if '__main__' in preload and main_path is not None: process.current_process()._inheriting = True try: diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 0f5f9f64c2de9e..040f4674d735c0 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -18,6 +18,7 @@ import threading import signal import array +import collections.abc import queue import time import types @@ -758,22 +759,29 @@ class BaseProxy(object): _address_to_local = {} _mutex = util.ForkAwareThreadLock() + # Each instance gets a `_serial` number. Unlike `id(...)`, this number + # is never reused. + _next_serial = 1 + def __init__(self, token, serializer, manager=None, authkey=None, exposed=None, incref=True, manager_owned=False): with BaseProxy._mutex: - tls_idset = BaseProxy._address_to_local.get(token.address, None) - if tls_idset is None: - tls_idset = util.ForkAwareLocal(), ProcessLocalSet() - BaseProxy._address_to_local[token.address] = tls_idset + tls_serials = BaseProxy._address_to_local.get(token.address, None) + if tls_serials is None: + tls_serials = util.ForkAwareLocal(), ProcessLocalSet() + BaseProxy._address_to_local[token.address] = tls_serials + + self._serial = BaseProxy._next_serial + BaseProxy._next_serial += 1 # self._tls is used to record the connection used by this # thread to communicate with the manager at token.address - self._tls = tls_idset[0] + self._tls = tls_serials[0] - # self._idset is used to record the identities of all shared - # objects for which the current process owns references and + # self._all_serials is a set used to record the identities of all + # shared objects for which the current process owns references and # which are in the manager at token.address - self._idset = tls_idset[1] + self._all_serials = tls_serials[1] self._token = token self._id = self._token.id @@ -856,20 +864,20 @@ def _incref(self): dispatch(conn, None, 'incref', (self._id,)) util.debug('INCREF %r', self._token.id) - self._idset.add(self._id) + self._all_serials.add(self._serial) state = self._manager and self._manager._state self._close = util.Finalize( self, BaseProxy._decref, - args=(self._token, self._authkey, state, - self._tls, self._idset, self._Client), + args=(self._token, self._serial, self._authkey, state, + self._tls, self._all_serials, self._Client), exitpriority=10 ) @staticmethod - def _decref(token, authkey, state, tls, idset, _Client): - idset.discard(token.id) + def _decref(token, serial, authkey, state, tls, idset, _Client): + idset.discard(serial) # check whether manager is still alive if state is None or state.value == State.STARTED: @@ -1167,8 +1175,9 @@ def __imul__(self, value): __class_getitem__ = classmethod(types.GenericAlias) +collections.abc.MutableSequence.register(BaseListProxy) -_BaseDictProxy = MakeProxyType('DictProxy', ( +_BaseDictProxy = MakeProxyType('_BaseDictProxy', ( '__contains__', '__delitem__', '__getitem__', '__ior__', '__iter__', '__len__', '__or__', '__reversed__', '__ror__', '__setitem__', 'clear', 'copy', 'fromkeys', 'get', 'items', @@ -1184,6 +1193,8 @@ def __ior__(self, value): __class_getitem__ = classmethod(types.GenericAlias) +collections.abc.MutableMapping.register(_BaseDictProxy) + ArrayProxy = MakeProxyType('ArrayProxy', ( '__len__', '__getitem__', '__setitem__' )) diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 3ccbfe311c71f3..4f72373c951abc 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -21,22 +21,21 @@ from . import process from . import util -# Try to import the mp.synchronize module cleanly, if it fails -# raise ImportError for platforms lacking a working sem_open implementation. -# See issue 3770 +# TODO: Do any platforms still lack a functioning sem_open? try: from _multiprocessing import SemLock, sem_unlink -except (ImportError): +except ImportError: raise ImportError("This platform lacks a functioning sem_open" + - " implementation, therefore, the required" + - " synchronization primitives needed will not" + - " function, see issue 3770.") + " implementation. https://github.com/python/cpython/issues/48020.") # # Constants # -RECURSIVE_MUTEX, SEMAPHORE = list(range(2)) +# These match the enum in Modules/_multiprocessing/semaphore.c +RECURSIVE_MUTEX = 0 +SEMAPHORE = 1 + SEM_VALUE_MAX = _multiprocessing.SemLock.SEM_VALUE_MAX # @@ -174,7 +173,7 @@ def __repr__(self): name = process.current_process().name if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name - elif self._semlock._get_value() == 1: + elif not self._semlock._is_zero(): name = 'None' elif self._semlock._count() > 0: name = 'SomeOtherThread' @@ -200,7 +199,7 @@ def __repr__(self): if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name count = self._semlock._count() - elif self._semlock._get_value() == 1: + elif not self._semlock._is_zero(): name, count = 'None', 0 elif self._semlock._count() > 0: name, count = 'SomeOtherThread', 'nonzero' diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 1b1873f08b608b..5481bb8888ef59 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -553,28 +553,21 @@ def normpath(path): return prefix + sep.join(comps) -def _abspath_fallback(path): - """Return the absolute version of a path as a fallback function in case - `nt._getfullpathname` is not available or raises OSError. See bpo-31047 for - more. - - """ - - path = os.fspath(path) - if not isabs(path): - if isinstance(path, bytes): - cwd = os.getcwdb() - else: - cwd = os.getcwd() - path = join(cwd, path) - return normpath(path) - # Return an absolute path. try: from nt import _getfullpathname except ImportError: # not running on Windows - mock up something sensible - abspath = _abspath_fallback + def abspath(path): + """Return the absolute version of a path.""" + path = os.fspath(path) + if not isabs(path): + if isinstance(path, bytes): + cwd = os.getcwdb() + else: + cwd = os.getcwd() + path = join(cwd, path) + return normpath(path) else: # use native Windows method on Windows def abspath(path): @@ -582,7 +575,27 @@ def abspath(path): try: return _getfullpathname(normpath(path)) except (OSError, ValueError): - return _abspath_fallback(path) + # See gh-75230, handle outside for cleaner traceback + pass + path = os.fspath(path) + if not isabs(path): + if isinstance(path, bytes): + sep = b'\\' + getcwd = os.getcwdb + else: + sep = '\\' + getcwd = os.getcwd + drive, root, path = splitroot(path) + # Either drive or root can be nonempty, but not both. + if drive or root: + try: + path = join(_getfullpathname(drive + root), path) + except (OSError, ValueError): + # Drive "\0:" cannot exist; use the root directory. + path = drive + sep + path + else: + path = join(getcwd(), path) + return normpath(path) try: from nt import _findfirstfile, _getfinalpathname, readlink as _nt_readlink diff --git a/Lib/nturl2path.py b/Lib/nturl2path.py index 6453f202c26d14..255eb2f547c2ce 100644 --- a/Lib/nturl2path.py +++ b/Lib/nturl2path.py @@ -19,28 +19,19 @@ def url2pathname(url): url = url.replace(':', '|') if not '|' in url: # No drive specifier, just convert slashes - if url[:4] == '////': - # path is something like ////host/path/on/remote/host - # convert this to \\host\path\on\remote\host - # (notice halving of slashes at the start of the path) + if url[:3] == '///': + # URL has an empty authority section, so the path begins on the + # third character. url = url[2:] - components = url.split('/') # make sure not to convert quoted slashes :-) - return urllib.parse.unquote('\\'.join(components)) + return urllib.parse.unquote(url.replace('/', '\\')) comp = url.split('|') if len(comp) != 2 or comp[0][-1] not in string.ascii_letters: error = 'Bad URL: ' + url raise OSError(error) drive = comp[0][-1].upper() - components = comp[1].split('/') - path = drive + ':' - for comp in components: - if comp: - path = path + '\\' + urllib.parse.unquote(comp) - # Issue #11474 - handing url such as |c/| - if path.endswith(':') and url.endswith('/'): - path += '\\' - return path + tail = urllib.parse.unquote(comp[1].replace('/', '\\')) + return drive + ':' + tail def pathname2url(p): """OS-specific conversion from a file system path to a relative URL @@ -52,25 +43,21 @@ def pathname2url(p): import urllib.parse # First, clean up some special forms. We are going to sacrifice # the additional information anyway - if p[:4] == '\\\\?\\': + p = p.replace('\\', '/') + if p[:4] == '//?/': p = p[4:] - if p[:4].upper() == 'UNC\\': - p = '\\\\' + p[4:] + if p[:4].upper() == 'UNC/': + p = '//' + p[4:] elif p[1:2] != ':': raise OSError('Bad path: ' + p) if not ':' in p: - # No drive specifier, just convert slashes and quote the name - components = p.split('\\') - return urllib.parse.quote('/'.join(components)) + # No DOS drive specified, just quote the pathname + return urllib.parse.quote(p) comp = p.split(':', maxsplit=2) if len(comp) != 2 or len(comp[0]) > 1: error = 'Bad path: ' + p raise OSError(error) drive = urllib.parse.quote(comp[0].upper()) - components = comp[1].split('\\') - path = '///' + drive + ':' - for comp in components: - if comp: - path = path + '/' + urllib.parse.quote(comp) - return path + tail = urllib.parse.quote(comp[1]) + return '///' + drive + ':' + tail diff --git a/Lib/optparse.py b/Lib/optparse.py index 1c450c6fcbe3b6..cbe3451ced8bc3 100644 --- a/Lib/optparse.py +++ b/Lib/optparse.py @@ -43,7 +43,7 @@ __copyright__ = """ Copyright (c) 2001-2006 Gregory P. Ward. All rights reserved. -Copyright (c) 2002-2006 Python Software Foundation. All rights reserved. +Copyright (c) 2002 Python Software Foundation. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -75,6 +75,8 @@ import sys, os import textwrap +from gettext import gettext as _, ngettext + def _repr(self): return "<%s at 0x%x: %s>" % (self.__class__.__name__, id(self), self) @@ -86,19 +88,6 @@ def _repr(self): # Id: help.py 527 2006-07-23 15:21:30Z greg # Id: errors.py 509 2006-04-20 00:58:24Z gward -try: - from gettext import gettext, ngettext -except ImportError: - def gettext(message): - return message - - def ngettext(singular, plural, n): - if n == 1: - return singular - return plural - -_ = gettext - class OptParseError (Exception): def __init__(self, msg): diff --git a/Lib/os.py b/Lib/os.py index aaa758d955fe4c..9c2258e6ccf5ba 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -765,17 +765,6 @@ def __ror__(self, other): new.update(self) return new - if _exists("_create_environ"): - def refresh(self): - data = _create_environ() - if name == 'nt': - data = {self.encodekey(key): value - for key, value in data.items()} - - # modify in-place to keep os.environb in sync - self._data.clear() - self._data.update(data) - def _create_environ_mapping(): if name == 'nt': # Where Env Var Names Must Be UPPERCASE @@ -810,6 +799,20 @@ def decode(value): del _create_environ_mapping +if _exists("_create_environ"): + def reload_environ(): + data = _create_environ() + if name == 'nt': + encodekey = environ.encodekey + data = {encodekey(key): value + for key, value in data.items()} + + # modify in-place to keep os.environb in sync + env_data = environ._data + env_data.clear() + env_data.update(data) + + def getenv(key, default=None): """Get an environment variable, return None if it doesn't exist. The optional second argument can specify an alternate default. diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 11c8018b28f26b..2c243d470d4eda 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -94,24 +94,12 @@ class PathGlobber(_GlobberBase): lexists = operator.methodcaller('exists', follow_symlinks=False) add_slash = operator.methodcaller('joinpath', '') - - @staticmethod - def scandir(path): - """Emulates os.scandir(), which returns an object that can be used as - a context manager. This method is called by walk() and glob(). - """ - import contextlib - return contextlib.nullcontext(path.iterdir()) + scandir = operator.methodcaller('scandir') @staticmethod def concat_path(path, text): """Appends text to the given path.""" - return path.with_segments(path._raw_path + text) - - @staticmethod - def parse_entry(entry): - """Returns the path of an entry yielded from scandir().""" - return entry + return path.with_segments(str(path) + text) class PurePathBase: @@ -124,9 +112,9 @@ class PurePathBase: """ __slots__ = ( - # The `_raw_path` slot store a joined string path. This is set in the - # `__init__()` method. - '_raw_path', + # The `_raw_paths` slot stores unjoined string paths. This is set in + # the `__init__()` method. + '_raw_paths', # The '_resolving' slot stores a boolean indicating whether the path # is being processed by `PathBase.resolve()`. This prevents duplicate @@ -136,11 +124,12 @@ class PurePathBase: parser = ParserBase() _globber = PathGlobber - def __init__(self, path, *paths): - self._raw_path = self.parser.join(path, *paths) if paths else path - if not isinstance(self._raw_path, str): - raise TypeError( - f"path should be a str, not {type(self._raw_path).__name__!r}") + def __init__(self, *args): + for arg in args: + if not isinstance(arg, str): + raise TypeError( + f"argument should be a str, not {type(arg).__name__!r}") + self._raw_paths = list(args) self._resolving = False def with_segments(self, *pathsegments): @@ -153,7 +142,19 @@ def with_segments(self, *pathsegments): def __str__(self): """Return the string representation of the path, suitable for passing to system calls.""" - return self._raw_path + paths = self._raw_paths + if len(paths) == 1: + return paths[0] + elif paths: + # Join path segments from the initializer. + path = self.parser.join(*paths) + # Cache the joined path. + paths.clear() + paths.append(path) + return path + else: + paths.append('') + return '' def as_posix(self): """Return the string representation of the path with forward (/) @@ -178,7 +179,7 @@ def anchor(self): @property def name(self): """The final path component, if any.""" - return self.parser.split(self._raw_path)[1] + return self.parser.split(str(self))[1] @property def suffix(self): @@ -214,7 +215,7 @@ def with_name(self, name): split = self.parser.split if split(name)[0]: raise ValueError(f"Invalid name {name!r}") - return self.with_segments(split(self._raw_path)[0], name) + return self.with_segments(split(str(self))[0], name) def with_stem(self, stem): """Return a new path with the stem changed.""" @@ -254,7 +255,7 @@ def relative_to(self, other, *, walk_up=False): anchor0, parts0 = self._stack anchor1, parts1 = other._stack if anchor0 != anchor1: - raise ValueError(f"{self._raw_path!r} and {other._raw_path!r} have different anchors") + raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") while parts0 and parts1 and parts0[-1] == parts1[-1]: parts0.pop() parts1.pop() @@ -262,12 +263,12 @@ def relative_to(self, other, *, walk_up=False): if not part or part == '.': pass elif not walk_up: - raise ValueError(f"{self._raw_path!r} is not in the subpath of {other._raw_path!r}") + raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") elif part == '..': - raise ValueError(f"'..' segment in {other._raw_path!r} cannot be walked") + raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") else: parts0.append('..') - return self.with_segments('', *reversed(parts0)) + return self.with_segments(*reversed(parts0)) def is_relative_to(self, other): """Return True if the path is relative to another path or False. @@ -301,17 +302,17 @@ def joinpath(self, *pathsegments): paths) or a totally different path (if one of the arguments is anchored). """ - return self.with_segments(self._raw_path, *pathsegments) + return self.with_segments(*self._raw_paths, *pathsegments) def __truediv__(self, key): try: - return self.with_segments(self._raw_path, key) + return self.with_segments(*self._raw_paths, key) except TypeError: return NotImplemented def __rtruediv__(self, key): try: - return self.with_segments(key, self._raw_path) + return self.with_segments(key, *self._raw_paths) except TypeError: return NotImplemented @@ -323,7 +324,7 @@ def _stack(self): *parts* is a reversed list of parts following the anchor. """ split = self.parser.split - path = self._raw_path + path = str(self) parent, name = split(path) names = [] while path != parent: @@ -335,7 +336,7 @@ def _stack(self): @property def parent(self): """The logical parent of the path.""" - path = self._raw_path + path = str(self) parent = self.parser.split(path)[0] if path != parent: parent = self.with_segments(parent) @@ -347,7 +348,7 @@ def parent(self): def parents(self): """A sequence of this path's logical parents.""" split = self.parser.split - path = self._raw_path + path = str(self) parent = split(path)[0] parents = [] while path != parent: @@ -359,7 +360,7 @@ def parents(self): def is_absolute(self): """True if the path is absolute (has both a root and, if applicable, a drive).""" - return self.parser.isabs(self._raw_path) + return self.parser.isabs(str(self)) @property def _pattern_str(self): @@ -639,13 +640,23 @@ def write_text(self, data, encoding=None, errors=None, newline=None): with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: return f.write(data) + def scandir(self): + """Yield os.DirEntry objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + raise UnsupportedOperation(self._unsupported_msg('scandir()')) + def iterdir(self): """Yield path objects of the directory contents. The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. """ - raise UnsupportedOperation(self._unsupported_msg('iterdir()')) + with self.scandir() as entries: + names = [entry.name for entry in entries] + return map(self.joinpath, names) def _glob_selector(self, parts, case_sensitive, recurse_symlinks): if case_sensitive is None: @@ -695,16 +706,18 @@ def walk(self, top_down=True, on_error=None, follow_symlinks=False): if not top_down: paths.append((path, dirnames, filenames)) try: - for child in path.iterdir(): - try: - if child.is_dir(follow_symlinks=follow_symlinks): - if not top_down: - paths.append(child) - dirnames.append(child.name) - else: - filenames.append(child.name) - except OSError: - filenames.append(child.name) + with path.scandir() as entries: + for entry in entries: + name = entry.name + try: + if entry.is_dir(follow_symlinks=follow_symlinks): + if not top_down: + paths.append(path.joinpath(name)) + dirnames.append(name) + else: + filenames.append(name) + except OSError: + filenames.append(name) except OSError as error: if on_error is not None: on_error(error) @@ -722,7 +735,13 @@ def absolute(self): Use resolve() to resolve symlinks and remove '..' segments. """ - raise UnsupportedOperation(self._unsupported_msg('absolute()')) + if self.is_absolute(): + return self + elif self.parser is not posixpath: + raise UnsupportedOperation(self._unsupported_msg('absolute()')) + else: + # Treat the root directory as the current working directory. + return self.with_segments('/', *self._raw_paths) @classmethod def cwd(cls): @@ -731,7 +750,7 @@ def cwd(cls): # enable users to replace the implementation of 'absolute()' in a # subclass and benefit from the new behaviour here. This works because # os.path.abspath('.') == os.getcwd(). - return cls('').absolute() + return cls().absolute() def expanduser(self): """ Return a new path with expanded ~ and ~user constructs @@ -759,10 +778,13 @@ def resolve(self, strict=False): """ if self._resolving: return self + elif self.parser is not posixpath: + raise UnsupportedOperation(self._unsupported_msg('resolve()')) - def getcwd(): - return str(self.with_segments().absolute()) + def raise_error(*args): + raise OSError("Unsupported operation.") + getcwd = raise_error if strict or getattr(self.readlink, '_supported', True): def lstat(path_str): path = self.with_segments(path_str) @@ -777,14 +799,10 @@ def readlink(path_str): # If the user has *not* overridden the `readlink()` method, then # symlinks are unsupported and (in non-strict mode) we can improve # performance by not calling `path.lstat()`. - def skip(path_str): - # This exception will be internally consumed by `_realpath()`. - raise OSError("Operation skipped.") - - lstat = readlink = skip + lstat = readlink = raise_error return self.with_segments(posixpath._realpath( - str(self), strict, self.parser.sep, + str(self.absolute()), strict, self.parser.sep, getcwd=getcwd, lstat=lstat, readlink=readlink, maxlinks=self._max_symlinks)) diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index a78997179820b1..b27f456d375225 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -68,10 +68,6 @@ class PurePath(PurePathBase): """ __slots__ = ( - # The `_raw_paths` slot stores unnormalized string paths. This is set - # in the `__init__()` method. - '_raw_paths', - # The `_drv`, `_root` and `_tail_cached` slots store parsed and # normalized parts of the path. They are set when any of the `drive`, # `root` or `_tail` properties are accessed for the first time. The @@ -274,17 +270,30 @@ def _parse_path(cls, path): root = sep return drv, root, [x for x in rel.split(sep) if x and x != '.'] - @property - def _raw_path(self): - """The joined but unnormalized path.""" - paths = self._raw_paths - if len(paths) == 0: - path = '' - elif len(paths) == 1: - path = paths[0] - else: - path = self.parser.join(*paths) - return path + @classmethod + def _parse_pattern(cls, pattern): + """Parse a glob pattern to a list of parts. This is much like + _parse_path, except: + + - Rather than normalizing and returning the drive and root, we raise + NotImplementedError if either are present. + - If the path has no real parts, we raise ValueError. + - If the path ends in a slash, then a final empty part is added. + """ + drv, root, rel = cls.parser.splitroot(pattern) + if root or drv: + raise NotImplementedError("Non-relative patterns are unsupported") + sep = cls.parser.sep + altsep = cls.parser.altsep + if altsep: + rel = rel.replace(altsep, sep) + parts = [x for x in rel.split(sep) if x and x != '.'] + if not parts: + raise ValueError(f"Unacceptable pattern: {str(pattern)!r}") + elif rel.endswith(sep): + # GH-65238: preserve trailing slash in glob patterns. + parts.append('') + return parts @property def drive(self): @@ -292,7 +301,8 @@ def drive(self): try: return self._drv except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + raw_path = PurePathBase.__str__(self) + self._drv, self._root, self._tail_cached = self._parse_path(raw_path) return self._drv @property @@ -301,7 +311,8 @@ def root(self): try: return self._root except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + raw_path = PurePathBase.__str__(self) + self._drv, self._root, self._tail_cached = self._parse_path(raw_path) return self._root @property @@ -309,7 +320,8 @@ def _tail(self): try: return self._tail_cached except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + raw_path = PurePathBase.__str__(self) + self._drv, self._root, self._tail_cached = self._parse_path(raw_path) return self._tail_cached @property @@ -615,6 +627,14 @@ def _filter_trailing_slash(self, paths): path_str = path_str[:-1] yield path_str + def scandir(self): + """Yield os.DirEntry objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + return os.scandir(self) + def iterdir(self): """Yield path objects of the directory contents. @@ -633,17 +653,7 @@ def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): kind, including directories) matching the given relative pattern. """ sys.audit("pathlib.Path.glob", self, pattern) - if not isinstance(pattern, PurePath): - pattern = self.with_segments(pattern) - if pattern.anchor: - raise NotImplementedError("Non-relative patterns are unsupported") - parts = pattern._tail.copy() - if not parts: - raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - raw = pattern._raw_path - if raw[-1] in (self.parser.sep, self.parser.altsep): - # GH-65238: pathlib doesn't preserve trailing slash. Add it back. - parts.append('') + parts = self._parse_pattern(pattern) select = self._glob_selector(parts[::-1], case_sensitive, recurse_symlinks) root = str(self) paths = select(root) @@ -664,9 +674,7 @@ def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): this subtree. """ sys.audit("pathlib.Path.rglob", self, pattern) - if not isinstance(pattern, PurePath): - pattern = self.with_segments(pattern) - pattern = '**' / pattern + pattern = self.parser.join('**', pattern) return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) def walk(self, top_down=True, on_error=None, follow_symlinks=False): diff --git a/Lib/pickle.py b/Lib/pickle.py index ed8138beb908ee..25dadb3f75a573 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -26,7 +26,7 @@ from types import FunctionType from copyreg import dispatch_table from copyreg import _extension_registry, _inverted_registry, _extension_cache -from itertools import islice +from itertools import batched from functools import partial import sys from sys import maxsize @@ -548,10 +548,11 @@ def save(self, obj, save_persistent_id=True): self.framer.commit_frame() # Check for persistent id (defined by a subclass) - pid = self.persistent_id(obj) - if pid is not None and save_persistent_id: - self.save_pers(pid) - return + if save_persistent_id: + pid = self.persistent_id(obj) + if pid is not None: + self.save_pers(pid) + return # Check the memo x = self.memo.get(id(obj)) @@ -1033,31 +1034,26 @@ def _batch_appends(self, items, obj): write(APPEND) return - it = iter(items) start = 0 - while True: - tmp = list(islice(it, self._BATCHSIZE)) - n = len(tmp) - if n > 1: + for batch in batched(items, self._BATCHSIZE): + batch_len = len(batch) + if batch_len != 1: write(MARK) - for i, x in enumerate(tmp, start): + for i, x in enumerate(batch, start): try: save(x) except BaseException as exc: exc.add_note(f'when serializing {_T(obj)} item {i}') raise write(APPENDS) - elif n: + else: try: - save(tmp[0]) + save(batch[0]) except BaseException as exc: exc.add_note(f'when serializing {_T(obj)} item {start}') raise write(APPEND) - # else tmp is empty, and we're done - if n < self._BATCHSIZE: - return - start += n + start += batch_len def save_dict(self, obj): if self.bin: @@ -1086,13 +1082,10 @@ def _batch_setitems(self, items, obj): write(SETITEM) return - it = iter(items) - while True: - tmp = list(islice(it, self._BATCHSIZE)) - n = len(tmp) - if n > 1: + for batch in batched(items, self._BATCHSIZE): + if len(batch) != 1: write(MARK) - for k, v in tmp: + for k, v in batch: save(k) try: save(v) @@ -1100,8 +1093,8 @@ def _batch_setitems(self, items, obj): exc.add_note(f'when serializing {_T(obj)} item {k!r}') raise write(SETITEMS) - elif n: - k, v = tmp[0] + else: + k, v = batch[0] save(k) try: save(v) @@ -1109,9 +1102,6 @@ def _batch_setitems(self, items, obj): exc.add_note(f'when serializing {_T(obj)} item {k!r}') raise write(SETITEM) - # else tmp is empty, and we're done - if n < self._BATCHSIZE: - return def save_set(self, obj): save = self.save @@ -1124,21 +1114,15 @@ def save_set(self, obj): write(EMPTY_SET) self.memoize(obj) - it = iter(obj) - while True: - batch = list(islice(it, self._BATCHSIZE)) - n = len(batch) - if n > 0: - write(MARK) - try: - for item in batch: - save(item) - except BaseException as exc: - exc.add_note(f'when serializing {_T(obj)} element') - raise - write(ADDITEMS) - if n < self._BATCHSIZE: - return + for batch in batched(obj, self._BATCHSIZE): + write(MARK) + try: + for item in batch: + save(item) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} element') + raise + write(ADDITEMS) dispatch[set] = save_set def save_frozenset(self, obj): diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index dccbec52aa731e..b84d72f2395d45 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -12,7 +12,7 @@ import warnings __all__ = [ - 'get_importer', 'iter_importers', 'get_loader', 'find_loader', + 'get_importer', 'iter_importers', 'walk_packages', 'iter_modules', 'get_data', 'read_code', 'extend_path', 'ModuleInfo', @@ -263,59 +263,6 @@ def iter_importers(fullname=""): yield get_importer(item) -def get_loader(module_or_name): - """Get a "loader" object for module_or_name - - Returns None if the module cannot be found or imported. - If the named module is not already imported, its containing package - (if any) is imported, in order to establish the package __path__. - """ - warnings._deprecated("pkgutil.get_loader", - f"{warnings._DEPRECATED_MSG}; " - "use importlib.util.find_spec() instead", - remove=(3, 14)) - if module_or_name in sys.modules: - module_or_name = sys.modules[module_or_name] - if module_or_name is None: - return None - if isinstance(module_or_name, ModuleType): - module = module_or_name - loader = getattr(module, '__loader__', None) - if loader is not None: - return loader - if getattr(module, '__spec__', None) is None: - return None - fullname = module.__name__ - else: - fullname = module_or_name - return find_loader(fullname) - - -def find_loader(fullname): - """Find a "loader" object for fullname - - This is a backwards compatibility wrapper around - importlib.util.find_spec that converts most failures to ImportError - and only returns the loader rather than the full spec - """ - warnings._deprecated("pkgutil.find_loader", - f"{warnings._DEPRECATED_MSG}; " - "use importlib.util.find_spec() instead", - remove=(3, 14)) - if fullname.startswith('.'): - msg = "Relative module name {!r} not supported".format(fullname) - raise ImportError(msg) - try: - spec = importlib.util.find_spec(fullname) - except (ImportError, AttributeError, TypeError, ValueError) as ex: - # This hack fixes an impedance mismatch between pkgutil and - # importlib, where the latter raises other errors for cases where - # pkgutil previously raised ImportError - msg = "Error while finding loader for {!r} ({}: {})" - raise ImportError(msg.format(fullname, type(ex), ex)) from ex - return spec.loader if spec is not None else None - - def extend_path(path, name): """Extend a package's path. diff --git a/Lib/platform.py b/Lib/platform.py index d6322c9d99d2f3..239e660cd1621d 100644 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -31,6 +31,7 @@ # # # +# 1.0.9 - added invalidate_caches() function to invalidate cached values # 1.0.8 - changed Windows support to read version from kernel32.dll # 1.0.7 - added DEV_NULL # 1.0.6 - added linux_distribution() @@ -109,7 +110,7 @@ """ -__version__ = '1.0.8' +__version__ = '1.0.9' import collections import os @@ -1441,6 +1442,18 @@ def freedesktop_os_release(): return _os_release_cache.copy() +def invalidate_caches(): + """Invalidate the cached results.""" + global _uname_cache + _uname_cache = None + + global _os_release_cache + _os_release_cache = None + + _sys_version_cache.clear() + _platform_cache.clear() + + ### Command line interface if __name__ == '__main__': diff --git a/Lib/posixpath.py b/Lib/posixpath.py index fccca4e066b76f..db72ded8826056 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -412,6 +412,10 @@ def _realpath(filename, strict=False, sep=sep, curdir=curdir, pardir=pardir, # very fast way of spelling list(reversed(...)). rest = filename.split(sep)[::-1] + # Number of unprocessed parts in 'rest'. This can differ from len(rest) + # later, because 'rest' might contain markers for unresolved symlinks. + part_count = len(rest) + # The resolved path, which is absolute throughout this function. # Note: getcwd() returns a normalized and symlink-free path. path = sep if filename.startswith(sep) else getcwd() @@ -426,12 +430,13 @@ def _realpath(filename, strict=False, sep=sep, curdir=curdir, pardir=pardir, # by *maxlinks*, this is used instead of *seen* to detect symlink loops. link_count = 0 - while rest: + while part_count: name = rest.pop() if name is None: # resolved symlink target seen[rest.pop()] = path continue + part_count -= 1 if not name or name == curdir: # current dir continue @@ -444,8 +449,11 @@ def _realpath(filename, strict=False, sep=sep, curdir=curdir, pardir=pardir, else: newpath = path + sep + name try: - st = lstat(newpath) - if not stat.S_ISLNK(st.st_mode): + st_mode = lstat(newpath).st_mode + if not stat.S_ISLNK(st_mode): + if strict and part_count and not stat.S_ISDIR(st_mode): + raise OSError(errno.ENOTDIR, os.strerror(errno.ENOTDIR), + newpath) path = newpath continue elif maxlinks is not None: @@ -487,7 +495,9 @@ def _realpath(filename, strict=False, sep=sep, curdir=curdir, pardir=pardir, rest.append(newpath) rest.append(None) # Push the unresolved symlink target parts onto the stack. - rest.extend(target.split(sep)[::-1]) + target_parts = target.split(sep)[::-1] + rest.extend(target_parts) + part_count += len(target_parts) return path diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index 29109f8812ee7b..20dd561d1c1520 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -255,11 +255,11 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): while True: try: if op is LITERAL: - if fixup: - lo = fixup(av) - charmap[lo] = 1 - if fixes and lo in fixes: - for k in fixes[lo]: + if fixup: # IGNORECASE and not LOCALE + av = fixup(av) + charmap[av] = 1 + if fixes and av in fixes: + for k in fixes[av]: charmap[k] = 1 if not hascased and iscased(av): hascased = True @@ -267,7 +267,7 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): charmap[av] = 1 elif op is RANGE: r = range(av[0], av[1]+1) - if fixup: + if fixup: # IGNORECASE and not LOCALE if fixes: for i in map(fixup, r): charmap[i] = 1 @@ -298,8 +298,7 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): # Character set contains non-BMP character codes. # For range, all BMP characters in the range are already # proceeded. - if fixup: - hascased = True + if fixup: # IGNORECASE and not LOCALE # For now, IN_UNI_IGNORE+LITERAL and # IN_UNI_IGNORE+RANGE_UNI_IGNORE work for all non-BMP # characters, because two characters (at least one of @@ -310,7 +309,13 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): # Also, both c.lower() and c.lower().upper() are single # characters for every non-BMP character. if op is RANGE: - op = RANGE_UNI_IGNORE + if fixes: # not ASCII + op = RANGE_UNI_IGNORE + hascased = True + else: + assert op is LITERAL + if not hascased and iscased(av): + hascased = True tail.append((op, av)) break diff --git a/Lib/site.py b/Lib/site.py index 07a6361fad44e5..54f07ab5b4e6d5 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -442,8 +442,9 @@ def setcopyright(): """Set 'copyright' and 'credits' in builtins""" builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright) builtins.credits = _sitebuiltins._Printer("credits", """\ - Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands - for supporting Python development. See www.python.org for more information.""") + Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software + Foundation, and a cast of thousands for supporting Python + development. See www.python.org for more information.""") files, dirs = [], [] # Not all modules are required to have a __file__ attribute. See # PEP 420 for more details. diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 43f9276799b848..67a071963d8c7d 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -173,9 +173,7 @@ def joinuser(*args): _PY_VERSION = sys.version.split()[0] _PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}' _PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}' -_PREFIX = os.path.normpath(sys.prefix) _BASE_PREFIX = os.path.normpath(sys.base_prefix) -_EXEC_PREFIX = os.path.normpath(sys.exec_prefix) _BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) # Mutex guarding initialization of _CONFIG_VARS. _CONFIG_VARS_LOCK = threading.RLock() @@ -355,7 +353,8 @@ def _init_posix(vars): else: _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) build_time_vars = _temp.build_time_vars - vars.update(build_time_vars) + # GH-126920: Make sure we don't overwrite any of the keys already set + vars.update(build_time_vars | vars) def _init_non_posix(vars): """Initialize the module as appropriate for NT""" @@ -466,8 +465,10 @@ def _init_config_vars(): # Normalized versions of prefix and exec_prefix are handy to have; # in fact, these are the standard versions used most places in the # Distutils. - _CONFIG_VARS['prefix'] = _PREFIX - _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX + _PREFIX = os.path.normpath(sys.prefix) + _EXEC_PREFIX = os.path.normpath(sys.exec_prefix) + _CONFIG_VARS['prefix'] = _PREFIX # FIXME: This gets overwriten by _init_posix. + _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX # FIXME: This gets overwriten by _init_posix. _CONFIG_VARS['py_version'] = _PY_VERSION _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT @@ -540,6 +541,7 @@ def get_config_vars(*args): With arguments, return a list of values that result from looking up each argument in the configuration variable dictionary. """ + global _CONFIG_VARS_INITIALIZED # Avoid claiming the lock once initialization is complete. if not _CONFIG_VARS_INITIALIZED: @@ -550,6 +552,15 @@ def get_config_vars(*args): # don't re-enter init_config_vars(). if _CONFIG_VARS is None: _init_config_vars() + else: + # If the site module initialization happened after _CONFIG_VARS was + # initialized, a virtual environment might have been activated, resulting in + # variables like sys.prefix changing their value, so we need to re-init the + # config vars (see GH-126789). + if _CONFIG_VARS['base'] != os.path.normpath(sys.prefix): + with _CONFIG_VARS_LOCK: + _CONFIG_VARS_INITIALIZED = False + _init_config_vars() if args: vals = [] diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 065fc27b770438..8329a848a90088 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -12,15 +12,19 @@ import sys import os import gc +import importlib import errno import functools import signal import array +import collections.abc import socket import random import logging +import shutil import subprocess import struct +import tempfile import operator import pickle import weakref @@ -254,6 +258,9 @@ def __call__(self, *args, **kwds): class BaseTestCase(object): ALLOWED_TYPES = ('processes', 'manager', 'threads') + # If not empty, limit which start method suites run this class. + START_METHODS: set[str] = set() + start_method = None # set by install_tests_in_module_dict() def assertTimingAlmostEqual(self, a, b): if CHECK_TIMINGS: @@ -1362,6 +1369,66 @@ def test_closed_queue_put_get_exceptions(self): class _TestLock(BaseTestCase): + @staticmethod + def _acquire(lock, l=None): + lock.acquire() + if l is not None: + l.append(repr(lock)) + + @staticmethod + def _acquire_event(lock, event): + lock.acquire() + event.set() + time.sleep(1.0) + + def test_repr_lock(self): + if self.TYPE != 'processes': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + lock = self.Lock() + self.assertEqual(f'', repr(lock)) + + lock.acquire() + self.assertEqual(f'', repr(lock)) + lock.release() + + tname = 'T1' + l = [] + t = threading.Thread(target=self._acquire, + args=(lock, l), + name=tname) + t.start() + time.sleep(0.1) + self.assertEqual(f'', l[0]) + lock.release() + + t = threading.Thread(target=self._acquire, + args=(lock,), + name=tname) + t.start() + time.sleep(0.1) + self.assertEqual('', repr(lock)) + lock.release() + + pname = 'P1' + l = multiprocessing.Manager().list() + p = self.Process(target=self._acquire, + args=(lock, l), + name=pname) + p.start() + p.join() + self.assertEqual(f'', l[0]) + + lock = self.Lock() + event = self.Event() + p = self.Process(target=self._acquire_event, + args=(lock, event), + name='P2') + p.start() + event.wait() + self.assertEqual(f'', repr(lock)) + p.terminate() + def test_lock(self): lock = self.Lock() self.assertEqual(lock.acquire(), True) @@ -1369,6 +1436,68 @@ def test_lock(self): self.assertEqual(lock.release(), None) self.assertRaises((ValueError, threading.ThreadError), lock.release) + @staticmethod + def _acquire_release(lock, timeout, l=None, n=1): + for _ in range(n): + lock.acquire() + if l is not None: + l.append(repr(lock)) + time.sleep(timeout) + for _ in range(n): + lock.release() + + def test_repr_rlock(self): + if self.TYPE != 'processes': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + lock = self.RLock() + self.assertEqual('', repr(lock)) + + n = 3 + for _ in range(n): + lock.acquire() + self.assertEqual(f'', repr(lock)) + for _ in range(n): + lock.release() + + t, l = [], [] + for i in range(n): + t.append(threading.Thread(target=self._acquire_release, + args=(lock, 0.1, l, i+1), + name=f'T{i+1}')) + t[-1].start() + for t_ in t: + t_.join() + for i in range(n): + self.assertIn(f'', l) + + + t = threading.Thread(target=self._acquire_release, + args=(lock, 0.2), + name=f'T1') + t.start() + time.sleep(0.1) + self.assertEqual('', repr(lock)) + time.sleep(0.2) + + pname = 'P1' + l = multiprocessing.Manager().list() + p = self.Process(target=self._acquire_release, + args=(lock, 0.1, l), + name=pname) + p.start() + p.join() + self.assertEqual(f'', l[0]) + + event = self.Event() + lock = self.RLock() + p = self.Process(target=self._acquire_event, + args=(lock, event)) + p.start() + event.wait() + self.assertEqual('', repr(lock)) + p.join() + def test_rlock(self): lock = self.RLock() self.assertEqual(lock.acquire(), True) @@ -2331,6 +2460,23 @@ def test_list(self): a.append('hello') self.assertEqual(f[0][:], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 'hello']) + def test_list_isinstance(self): + a = self.list() + self.assertIsInstance(a, collections.abc.MutableSequence) + + # MutableSequence also has __iter__, but we can iterate over + # ListProxy using __getitem__ instead. Adding __iter__ to ListProxy + # would change the behavior of a list modified during iteration. + mutable_sequence_methods = ( + '__contains__', '__delitem__', '__getitem__', '__iadd__', + '__len__', '__reversed__', '__setitem__', 'append', + 'clear', 'count', 'extend', 'index', 'insert', 'pop', 'remove', + 'reverse', + ) + for name in mutable_sequence_methods: + with self.subTest(name=name): + self.assertTrue(callable(getattr(a, name))) + def test_list_iter(self): a = self.list(list(range(10))) it = iter(a) @@ -2371,6 +2517,19 @@ def test_dict(self): self.assertEqual(sorted(d.values()), [chr(i) for i in indices]) self.assertEqual(sorted(d.items()), [(i, chr(i)) for i in indices]) + def test_dict_isinstance(self): + a = self.dict() + self.assertIsInstance(a, collections.abc.MutableMapping) + + mutable_mapping_methods = ( + '__contains__', '__delitem__', '__eq__', '__getitem__', '__iter__', + '__len__', '__ne__', '__setitem__', 'clear', 'get', 'items', + 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values', + ) + for name in mutable_mapping_methods: + with self.subTest(name=name): + self.assertTrue(callable(getattr(a, name))) + def test_dict_iter(self): d = self.dict() indices = list(range(65, 70)) @@ -6266,6 +6425,76 @@ def test_atexit(self): self.assertEqual(f.read(), 'deadbeef') +class _TestSpawnedSysPath(BaseTestCase): + """Test that sys.path is setup in forkserver and spawn processes.""" + + ALLOWED_TYPES = {'processes'} + # Not applicable to fork which inherits everything from the process as is. + START_METHODS = {"forkserver", "spawn"} + + def setUp(self): + self._orig_sys_path = list(sys.path) + self._temp_dir = tempfile.mkdtemp(prefix="test_sys_path-") + self._mod_name = "unique_test_mod" + module_path = os.path.join(self._temp_dir, f"{self._mod_name}.py") + with open(module_path, "w", encoding="utf-8") as mod: + mod.write("# A simple test module\n") + sys.path[:] = [p for p in sys.path if p] # remove any existing ""s + sys.path.insert(0, self._temp_dir) + sys.path.insert(0, "") # Replaced with an abspath in child. + self.assertIn(self.start_method, self.START_METHODS) + self._ctx = multiprocessing.get_context(self.start_method) + + def tearDown(self): + sys.path[:] = self._orig_sys_path + shutil.rmtree(self._temp_dir, ignore_errors=True) + + @staticmethod + def enq_imported_module_names(queue): + queue.put(tuple(sys.modules)) + + def test_forkserver_preload_imports_sys_path(self): + if self._ctx.get_start_method() != "forkserver": + self.skipTest("forkserver specific test.") + self.assertNotIn(self._mod_name, sys.modules) + multiprocessing.forkserver._forkserver._stop() # Must be fresh. + self._ctx.set_forkserver_preload( + ["test.test_multiprocessing_forkserver", self._mod_name]) + q = self._ctx.Queue() + proc = self._ctx.Process( + target=self.enq_imported_module_names, args=(q,)) + proc.start() + proc.join() + child_imported_modules = q.get() + q.close() + self.assertIn(self._mod_name, child_imported_modules) + + @staticmethod + def enq_sys_path_and_import(queue, mod_name): + queue.put(sys.path) + try: + importlib.import_module(mod_name) + except ImportError as exc: + queue.put(exc) + else: + queue.put(None) + + def test_child_sys_path(self): + q = self._ctx.Queue() + proc = self._ctx.Process( + target=self.enq_sys_path_and_import, args=(q, self._mod_name)) + proc.start() + proc.join() + child_sys_path = q.get() + import_error = q.get() + q.close() + self.assertNotIn("", child_sys_path) # replaced by an abspath + self.assertIn(self._temp_dir, child_sys_path) # our addition + # ignore the first element, it is the absolute "" replacement + self.assertEqual(child_sys_path[1:], sys.path[1:]) + self.assertIsNone(import_error, msg=f"child could not import {self._mod_name}") + + class MiscTestCase(unittest.TestCase): def test__all__(self): # Just make sure names in not_exported are excluded @@ -6460,6 +6689,8 @@ def install_tests_in_module_dict(remote_globs, start_method, if base is BaseTestCase: continue assert set(base.ALLOWED_TYPES) <= ALL_TYPES, base.ALLOWED_TYPES + if base.START_METHODS and start_method not in base.START_METHODS: + continue # class not intended for this start method. for type_ in base.ALLOWED_TYPES: if only_type and type_ != only_type: continue @@ -6473,6 +6704,7 @@ class Temp(base, Mixin, unittest.TestCase): Temp = hashlib_helper.requires_hashdigest('sha256')(Temp) Temp.__name__ = Temp.__qualname__ = newname Temp.__module__ = __module__ + Temp.start_method = start_method remote_globs[newname] = Temp elif issubclass(base, unittest.TestCase): if only_type: diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index e22324efc490be..4ad0b8b0910bbe 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -2199,7 +2199,8 @@ test_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec PyObject *a; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2213,7 +2214,7 @@ test_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec static PyObject * test_keywords_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=13ba007e1c842a37 input=0d3484844749c05b]*/ +/*[clinic end generated code: output=5411fdee0a85d60d input=0d3484844749c05b]*/ /*[clinic input] @@ -2269,7 +2270,8 @@ test_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *a; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2283,7 +2285,7 @@ test_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, static PyObject * test_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=789799a6d2d6eb4d input=384adc78bfa0bff7]*/ +/*[clinic end generated code: output=38b11b4dd0a94399 input=384adc78bfa0bff7]*/ /*[clinic input] @@ -2342,7 +2344,8 @@ test_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO PyObject *b = Py_None; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2367,7 +2370,7 @@ test_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO static PyObject * test_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=42430dd8ea5afde6 input=eda7964f784f4607]*/ +/*[clinic end generated code: output=c59ecb8eca9880cf input=eda7964f784f4607]*/ /*[clinic input] @@ -2429,7 +2432,8 @@ test_keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2464,7 +2468,7 @@ test_keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nar static PyObject * test_keywords_opt_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=f312c35c380d2bf9 input=209387a4815e5082]*/ +/*[clinic end generated code: output=b9572975a3e5d857 input=209387a4815e5082]*/ /*[clinic input] @@ -2524,7 +2528,8 @@ test_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *b = Py_None; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2549,7 +2554,7 @@ test_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nar static PyObject * test_keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=3937da2a8233ebe0 input=18393cc64fa000f4]*/ +/*[clinic end generated code: output=94114334841ff55a input=18393cc64fa000f4]*/ /*[clinic input] @@ -2605,7 +2610,8 @@ test_posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *a; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2619,7 +2625,7 @@ test_posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, static PyObject * test_posonly_keywords_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=6b4f6dd5f4db3877 input=1767b0ebdf06060e]*/ +/*[clinic end generated code: output=4abfbb43b77fd29a input=1767b0ebdf06060e]*/ /*[clinic input] @@ -2676,7 +2682,8 @@ test_posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *a; PyObject *c; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2690,7 +2697,7 @@ test_posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P static PyObject * test_posonly_kwonly_impl(PyObject *module, PyObject *a, PyObject *c) -/*[clinic end generated code: output=8bef2a8198e70b26 input=9042f2818f664839]*/ +/*[clinic end generated code: output=9fe05c76d80d63a9 input=9042f2818f664839]*/ /*[clinic input] @@ -2750,7 +2757,8 @@ test_posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *b; PyObject *c; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2766,7 +2774,7 @@ test_posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * test_posonly_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=a44b8ae8300955e1 input=29546ebdca492fea]*/ +/*[clinic end generated code: output=de2b8c631dc1b5d3 input=29546ebdca492fea]*/ /*[clinic input] @@ -2828,7 +2836,8 @@ test_posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2854,7 +2863,7 @@ test_posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t na static PyObject * test_posonly_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=cae6647c9e8e0238 input=cdf5a9625e554e9b]*/ +/*[clinic end generated code: output=70002e6beef31864 input=cdf5a9625e554e9b]*/ /*[clinic input] @@ -2914,7 +2923,8 @@ test_posonly_keywords_opt2(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *b = Py_None; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2939,7 +2949,7 @@ test_posonly_keywords_opt2(PyObject *module, PyObject *const *args, Py_ssize_t n static PyObject * test_posonly_keywords_opt2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=6526fd08aafa2149 input=1581299d21d16f14]*/ +/*[clinic end generated code: output=b0f78abf242565cf input=1581299d21d16f14]*/ /*[clinic input] @@ -3001,7 +3011,8 @@ test_posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_ PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3032,7 +3043,7 @@ test_posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_ static PyObject * test_posonly_opt_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=b8d01e98443738c2 input=408798ec3d42949f]*/ +/*[clinic end generated code: output=75708b03d2837419 input=408798ec3d42949f]*/ /*[clinic input] @@ -3095,7 +3106,8 @@ test_posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3121,7 +3133,7 @@ test_posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg static PyObject * test_posonly_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=81d71c288f13d4dc input=8d8e5643bbbc2309]*/ +/*[clinic end generated code: output=3aab93ed7b9cc713 input=8d8e5643bbbc2309]*/ /*[clinic input] @@ -3182,7 +3194,8 @@ test_posonly_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *b = Py_None; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3207,7 +3220,7 @@ test_posonly_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ssize_t nar static PyObject * test_posonly_kwonly_opt2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=a717d2a1a3310289 input=f7e5eed94f75fff0]*/ +/*[clinic end generated code: output=ac73a9a84a83e0c7 input=f7e5eed94f75fff0]*/ /*[clinic input] @@ -3270,7 +3283,8 @@ test_posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3301,7 +3315,7 @@ test_posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * test_posonly_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=0f50b4b8d45cf2de input=1e557dc979d120fd]*/ +/*[clinic end generated code: output=21a1bfee0c50ec98 input=1e557dc979d120fd]*/ /*[clinic input] @@ -3367,7 +3381,8 @@ test_posonly_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssi PyObject *d = Py_None; PyObject *e = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3395,7 +3410,7 @@ static PyObject * test_posonly_keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=8dac8d2a4e6105fa input=c3884a4f956fdc89]*/ +/*[clinic end generated code: output=60ac8589c5331fc9 input=c3884a4f956fdc89]*/ /*[clinic input] @@ -3458,7 +3473,8 @@ test_posonly_keywords_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ss PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3484,7 +3500,7 @@ test_posonly_keywords_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ss static PyObject * test_posonly_keywords_kwonly_opt2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=5a96d521e6414f5d input=68d01d7c0f6dafb0]*/ +/*[clinic end generated code: output=349e1a19c507956a input=68d01d7c0f6dafb0]*/ /*[clinic input] @@ -3551,7 +3567,8 @@ test_posonly_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py PyObject *d = Py_None; PyObject *e = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3588,7 +3605,7 @@ static PyObject * test_posonly_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=d5a474dcd5dc3e9f input=d0883d45876f186c]*/ +/*[clinic end generated code: output=94787c9e20861a56 input=d0883d45876f186c]*/ /*[clinic input] @@ -3655,7 +3672,8 @@ test_posonly_keywords_opt2_kwonly_opt(PyObject *module, PyObject *const *args, P PyObject *d = Py_None; PyObject *e = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3697,7 +3715,7 @@ static PyObject * test_posonly_keywords_opt2_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=ac239c5ee8a74408 input=c95e2e1ec93035ad]*/ +/*[clinic end generated code: output=ea8664ae9fdb4769 input=c95e2e1ec93035ad]*/ /*[clinic input] @@ -3767,7 +3785,8 @@ test_posonly_opt_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args PyObject *e = Py_None; PyObject *f = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3816,7 +3835,7 @@ test_posonly_opt_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e, PyObject *f) -/*[clinic end generated code: output=638bbd0005639342 input=9914857713c5bbf8]*/ +/*[clinic end generated code: output=6e0ee12cf458acb1 input=9914857713c5bbf8]*/ /*[clinic input] test_keyword_only_parameter @@ -3871,7 +3890,8 @@ test_keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyBytesObject *co_lnotab = (PyBytesObject *)self->co_lnotab; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3892,7 +3912,7 @@ test_keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * test_keyword_only_parameter_impl(PyObject *module, PyBytesObject *co_lnotab) -/*[clinic end generated code: output=b12fe2e515a62603 input=303df5046c7e37a3]*/ +/*[clinic end generated code: output=18b5b3c80412bc4a input=303df5046c7e37a3]*/ /*[clinic input] @@ -4135,7 +4155,7 @@ test_vararg_and_posonly a: object / - *args: object + *args: tuple [clinic start generated code]*/ @@ -4144,6 +4164,45 @@ PyDoc_STRVAR(test_vararg_and_posonly__doc__, "--\n" "\n"); +#define TEST_VARARG_AND_POSONLY_METHODDEF \ + {"test_vararg_and_posonly", _PyCFunction_CAST(test_vararg_and_posonly), METH_FASTCALL, test_vararg_and_posonly__doc__}, + +static PyObject * +test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args); + +static PyObject * +test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("test_vararg_and_posonly", nargs, 1, PY_SSIZE_T_MAX)) { + goto exit; + } + a = args[0]; + __clinic_args = _PyTuple_FromArray(args + 1, nargs - 1); + if (__clinic_args == NULL) { + goto exit; + } + return_value = test_vararg_and_posonly_impl(module, a, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +static PyObject * +test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) +/*[clinic end generated code: output=0c11c475e240869e input=2c49a482f68545c0]*/ + +PyDoc_STRVAR(test_vararg_and_posonly__doc__, +"test_vararg_and_posonly($module, a, /, *args)\n" +"--\n" +"\n"); + #define TEST_VARARG_AND_POSONLY_METHODDEF \ {"test_vararg_and_posonly", _PyCFunction_CAST(test_vararg_and_posonly), METH_FASTCALL, test_vararg_and_posonly__doc__}, @@ -4163,24 +4222,26 @@ test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t narg goto exit; } a = args[0]; - __clinic_args = args + 1; + __clinic_args = _PyTuple_FromArray(args + 1, nargs - 1); + if (__clinic_args == NULL) { + goto exit; + } return_value = test_vararg_and_posonly_impl(module, a, nvararg, __clinic_args); exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + return return_value; } -static PyObject * -test_vararg_and_posonly_impl(PyObject *module, PyObject *a, Py_ssize_t nargs, - PyObject *const *args) -/*[clinic end generated code: output=dc2dd9483cc0459e input=9cfa748bbff09877]*/ /*[clinic input] test_vararg a: object - *args: object + *args: tuple [clinic start generated code]*/ @@ -4224,33 +4285,42 @@ test_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[1]; + PyObject * const *fastargs; PyObject *a; PyObject *__clinic_args = NULL; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + __clinic_args = nargs > 1 + ? _PyTuple_FromArray(args + 1, nargs - 1) + : PyTuple_New(0); + if (__clinic_args == NULL) { goto exit; } - a = args[0]; - __clinic_args = args[1]; return_value = test_vararg_impl(module, a, __clinic_args); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } static PyObject * test_vararg_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=1411e464f358a7ba input=81d33815ad1bae6e]*/ +/*[clinic end generated code: output=5fa7dc68243a5211 input=7448995636d9186a]*/ /*[clinic input] test_vararg_with_default a: object - *args: object + *args: tuple b: bool = False [clinic start generated code]*/ @@ -4296,43 +4366,52 @@ test_vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nar .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[2]; + PyObject * const *fastargs; Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *a; PyObject *__clinic_args = NULL; int b = 0; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - a = args[0]; - __clinic_args = args[1]; + a = fastargs[0]; if (!noptargs) { goto skip_optional_kwonly; } - b = PyObject_IsTrue(args[2]); + b = PyObject_IsTrue(fastargs[1]); if (b < 0) { goto exit; } skip_optional_kwonly: + __clinic_args = nargs > 1 + ? _PyTuple_FromArray(args + 1, nargs - 1) + : PyTuple_New(0); + if (__clinic_args == NULL) { + goto exit; + } return_value = test_vararg_with_default_impl(module, a, __clinic_args, b); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } static PyObject * test_vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, int b) -/*[clinic end generated code: output=f09d4b917063ca41 input=6e110b54acd9b22d]*/ +/*[clinic end generated code: output=603ae52bfa307337 input=3a0f9f557ce1f712]*/ /*[clinic input] test_vararg_with_only_defaults - *args: object + *args: tuple b: bool = False c: object = ' ' @@ -4379,22 +4458,23 @@ test_vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[2]; + PyObject * const *fastargs; Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *__clinic_args = NULL; int b = 0; PyObject *c = " "; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - __clinic_args = args[0]; if (!noptargs) { goto skip_optional_kwonly; } - if (args[1]) { - b = PyObject_IsTrue(args[1]); + if (fastargs[0]) { + b = PyObject_IsTrue(fastargs[0]); if (b < 0) { goto exit; } @@ -4402,19 +4482,25 @@ test_vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize goto skip_optional_kwonly; } } - c = args[2]; + c = fastargs[1]; skip_optional_kwonly: + __clinic_args = _PyTuple_FromArray(args, nargs); + if (__clinic_args == NULL) { + goto exit; + } return_value = test_vararg_with_only_defaults_impl(module, __clinic_args, b, c); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } static PyObject * test_vararg_with_only_defaults_impl(PyObject *module, PyObject *args, int b, PyObject *c) -/*[clinic end generated code: output=cc6590b8805d5433 input=fa56a709a035666e]*/ +/*[clinic end generated code: output=195fe286176f4ee8 input=6983e66817f82924]*/ /*[clinic input] test_paramname_module @@ -4465,7 +4551,8 @@ test_paramname_module(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *argsbuf[1]; PyObject *mod; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -4478,7 +4565,7 @@ test_paramname_module(PyObject *module, PyObject *const *args, Py_ssize_t nargs, static PyObject * test_paramname_module_impl(PyObject *module, PyObject *mod) -/*[clinic end generated code: output=4a2a849ecbcc8b53 input=afefe259667f13ba]*/ +/*[clinic end generated code: output=7ab63a2adaea2873 input=afefe259667f13ba]*/ /*[clinic input] mangle1 @@ -4550,7 +4637,8 @@ mangle1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn PyObject *__clinic_nargs; PyObject *__clinic_noptargs; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 9, 9, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 9, /*maxpos*/ 9, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -4574,7 +4662,7 @@ mangle1_impl(PyObject *module, PyObject *args, PyObject *kwnames, PyObject *return_value, PyObject *_keywords, PyObject *_parser, PyObject *argsbuf, PyObject *fastargs, PyObject *nargs, PyObject *noptargs) -/*[clinic end generated code: output=083e5076be9987c3 input=a3ed51bdedf8a3c7]*/ +/*[clinic end generated code: output=f9127cea2f508da3 input=a3ed51bdedf8a3c7]*/ /*[clinic input] mangle2 @@ -4631,7 +4719,8 @@ mangle2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn PyObject *__clinic_kwargs; PyObject *__clinic_return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -4647,7 +4736,7 @@ mangle2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn static PyObject * mangle2_impl(PyObject *module, PyObject *args, PyObject *kwargs, PyObject *return_value) -/*[clinic end generated code: output=2ebb62aaefe7590a input=391766fee51bad7a]*/ +/*[clinic end generated code: output=3afbc3260971e5d3 input=391766fee51bad7a]*/ /*[clinic input] @@ -4700,7 +4789,8 @@ Test_cls_with_param(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ PyObject *argsbuf[1]; int a; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -4716,7 +4806,7 @@ Test_cls_with_param(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ static PyObject * Test_cls_with_param_impl(TestObj *self, PyTypeObject *cls, int a) -/*[clinic end generated code: output=a3a968137b0f320a input=af158077bd237ef9]*/ +/*[clinic end generated code: output=83a391eea66d08f8 input=af158077bd237ef9]*/ /*[clinic input] @@ -4915,7 +5005,7 @@ Test_an_metho_arg_named_arg_impl(TestObj *self, int arg) /*[clinic input] Test.__init__ - *args: object + *args: tuple Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE. [clinic start generated code]*/ @@ -4927,39 +5017,71 @@ PyDoc_STRVAR(Test___init____doc__, "Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE."); static int -Test___init___impl(TestObj *self, Py_ssize_t nargs, PyObject *const *args); +Test___init___impl(TestObj *self, PyObject *args); static int Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) { int return_value = -1; PyTypeObject *base_tp = TestType; - PyObject *const *__clinic_args = NULL; + PyObject *__clinic_args = NULL; if ((Py_IS_TYPE(self, base_tp) || Py_TYPE(self)->tp_new == base_tp->tp_new) && !_PyArg_NoKeywords("Test", kwargs)) { goto exit; } - if (!_PyArg_CheckPositional("Test", PyTuple_GET_SIZE(args), 0, PY_SSIZE_T_MAX)) { + __clinic_args = Py_NewRef(args); + return_value = Test___init___impl((TestObj *)self, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +static int +Test___init___impl(TestObj *self, PyObject *args) +/*[clinic end generated code: output=f172425cec373cd6 input=4b8388c4e6baab6f]*/ + +PyDoc_STRVAR(Test___init____doc__, +"Test(*args)\n" +"--\n" +"\n" +"Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE."); + +static int +Test___init___impl(TestObj *self, Py_ssize_t nargs, PyObject *const *args); + +static int +Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyTypeObject *base_tp = TestType; + PyObject *const *__clinic_args = NULL; + + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoKeywords("Test", kwargs)) { goto exit; } - __clinic_args = _PyTuple_CAST(args)->ob_item; + __clinic_args = Py_NewRef(args); return_value = Test___init___impl((TestObj *)self, nvararg, __clinic_args); exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + return return_value; } -static int -Test___init___impl(TestObj *self, Py_ssize_t nargs, PyObject *const *args) -/*[clinic end generated code: output=6a64b417c9080a73 input=2a8bd0033c9ac772]*/ /*[clinic input] @classmethod Test.__new__ - *args: object + *args: tuple Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE. [clinic start generated code]*/ @@ -4971,32 +5093,63 @@ PyDoc_STRVAR(Test__doc__, "Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE."); static PyObject * -Test_impl(PyTypeObject *type, Py_ssize_t nargs, PyObject *const *args); +Test_impl(PyTypeObject *type, PyObject *args); static PyObject * Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; PyTypeObject *base_tp = TestType; - PyObject *const *__clinic_args = NULL; + PyObject *__clinic_args = NULL; if ((type == base_tp || type->tp_init == base_tp->tp_init) && !_PyArg_NoKeywords("Test", kwargs)) { goto exit; } - if (!_PyArg_CheckPositional("Test", PyTuple_GET_SIZE(args), 0, PY_SSIZE_T_MAX)) { + __clinic_args = Py_NewRef(args); + return_value = Test_impl(type, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +static PyObject * +Test_impl(PyTypeObject *type, PyObject *args) +/*[clinic end generated code: output=ee1e8892a67abd4a input=a8259521129cad20]*/ + +PyDoc_STRVAR(Test__doc__, +"Test(*args)\n" +"--\n" +"\n" +"Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE."); + +static PyObject * +Test_impl(PyTypeObject *type, Py_ssize_t nargs, PyObject *const *args); + +static PyObject * +Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = TestType; + PyObject *const *__clinic_args = NULL; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("Test", kwargs)) { goto exit; } - __clinic_args = _PyTuple_CAST(args)->ob_item; + __clinic_args = Py_NewRef(args); return_value = Test_impl(type, nvararg, __clinic_args); exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + return return_value; } -static PyObject * -Test_impl(PyTypeObject *type, Py_ssize_t nargs, PyObject *const *args) -/*[clinic end generated code: output=bf22f942407383a5 input=70ad829df3dd9b84]*/ /*[clinic input] @@ -5048,7 +5201,8 @@ Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *a; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -5061,7 +5215,7 @@ Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) static int Test___init___impl(TestObj *self, PyObject *a) -/*[clinic end generated code: output=0e1239b9bc247bc1 input=a8f9222a6ab35c59]*/ +/*[clinic end generated code: output=aa1f3d73e611e438 input=a8f9222a6ab35c59]*/ /*[clinic input] @@ -5314,7 +5468,8 @@ mangled_c_keyword_identifier(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *argsbuf[1]; int int_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -5330,7 +5485,7 @@ mangled_c_keyword_identifier(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * mangled_c_keyword_identifier_impl(PyObject *module, int int_value) -/*[clinic end generated code: output=01a8088b57632916 input=060876448ab567a2]*/ +/*[clinic end generated code: output=19ec1a02f0819cf7 input=060876448ab567a2]*/ /*[clinic input] @@ -5573,7 +5728,8 @@ fn_with_default_binop_expr(PyObject *module, PyObject *const *args, Py_ssize_t n Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *arg = CONST_A + CONST_B; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -5590,7 +5746,7 @@ fn_with_default_binop_expr(PyObject *module, PyObject *const *args, Py_ssize_t n static PyObject * fn_with_default_binop_expr_impl(PyObject *module, PyObject *arg) -/*[clinic end generated code: output=018672772e4092ff input=1b55c8ae68d89453]*/ +/*[clinic end generated code: output=e15a6f5d9f7bea0c input=1b55c8ae68d89453]*/ /*[python input] @@ -5661,7 +5817,8 @@ docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py PyObject *argsbuf[1]; str a; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -5676,7 +5833,7 @@ docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py static PyObject * docstr_fallback_to_converter_default_impl(PyObject *module, str a) -/*[clinic end generated code: output=3fff7b702f0065cb input=0cbe6a4d24bc2274]*/ +/*[clinic end generated code: output=15bc9f32945ed0da input=0cbe6a4d24bc2274]*/ /*[clinic input] diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 8bef04cba81138..0c94fcc1907071 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -148,7 +148,7 @@ def __init__(self, **kwargs) -> None: self.randomize = False self.fromfile = None self.fail_env_changed = False - self.use_resources = None + self.use_resources: list[str] = [] self.trace = False self.coverdir = 'coverage' self.runleaks = False @@ -403,8 +403,6 @@ def _parse_args(args, **kwargs): raise TypeError('%r is an invalid keyword argument ' 'for this function' % k) setattr(ns, k, v) - if ns.use_resources is None: - ns.use_resources = [] parser = _create_parser() # Issue #14191: argparse doesn't support "intermixed" positional and diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py index 4ac95e23a56b8f..f01c1240774707 100644 --- a/Lib/test/libregrtest/findtests.py +++ b/Lib/test/libregrtest/findtests.py @@ -1,6 +1,7 @@ import os import sys import unittest +from collections.abc import Container from test import support @@ -34,7 +35,7 @@ def findtestdir(path: StrPath | None = None) -> StrPath: return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir -def findtests(*, testdir: StrPath | None = None, exclude=(), +def findtests(*, testdir: StrPath | None = None, exclude: Container[str] = (), split_test_dirs: set[TestName] = SPLITTESTDIRS, base_mod: str = "") -> TestList: """Return a list of all applicable test modules.""" @@ -60,8 +61,9 @@ def findtests(*, testdir: StrPath | None = None, exclude=(), return sorted(tests) -def split_test_packages(tests, *, testdir: StrPath | None = None, exclude=(), - split_test_dirs=SPLITTESTDIRS): +def split_test_packages(tests, *, testdir: StrPath | None = None, + exclude: Container[str] = (), + split_test_dirs=SPLITTESTDIRS) -> list[TestName]: testdir = findtestdir(testdir) splitted = [] for name in tests: @@ -75,9 +77,9 @@ def split_test_packages(tests, *, testdir: StrPath | None = None, exclude=(), return splitted -def _list_cases(suite): +def _list_cases(suite: unittest.TestSuite) -> None: for test in suite: - if isinstance(test, unittest.loader._FailedTest): + if isinstance(test, unittest.loader._FailedTest): # type: ignore[attr-defined] continue if isinstance(test, unittest.TestSuite): _list_cases(test) @@ -87,7 +89,7 @@ def _list_cases(suite): def list_cases(tests: TestTuple, *, match_tests: TestFilter | None = None, - test_dir: StrPath | None = None): + test_dir: StrPath | None = None) -> None: support.verbose = False set_match_tests(match_tests) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 11ebb09fbb4745..49209b0cec756e 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -6,6 +6,7 @@ import sysconfig import time import trace +from typing import NoReturn from test.support import os_helper, MS_WINDOWS, flush_std_streams @@ -122,7 +123,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.python_cmd = None self.coverage: bool = ns.trace self.coverage_dir: StrPath | None = ns.coverdir - self.tmp_dir: StrPath | None = ns.tempdir + self._tmp_dir: StrPath | None = ns.tempdir # Randomize self.randomize: bool = ns.randomize @@ -154,10 +155,12 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.next_single_test: TestName | None = None self.next_single_filename: StrPath | None = None - def log(self, line=''): + def log(self, line: str = '') -> None: self.logger.log(line) def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList | None]: + if tests is None: + tests = [] if self.single_test_run: self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest') try: @@ -230,11 +233,11 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList return (tuple(selected), tests) @staticmethod - def list_tests(tests: TestTuple): + def list_tests(tests: TestTuple) -> None: for name in tests: print(name) - def _rerun_failed_tests(self, runtests: RunTests): + def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: # Configure the runner to re-run tests if self.num_workers == 0 and not self.single_process: # Always run tests in fresh processes to have more deterministic @@ -266,7 +269,7 @@ def _rerun_failed_tests(self, runtests: RunTests): self.run_tests_sequentially(runtests) return runtests - def rerun_failed_tests(self, runtests: RunTests): + def rerun_failed_tests(self, runtests: RunTests) -> None: if self.python_cmd: # Temp patch for https://github.com/python/cpython/issues/94052 self.log( @@ -335,7 +338,7 @@ def run_bisect(self, runtests: RunTests) -> None: if not self._run_bisect(runtests, name, progress): return - def display_result(self, runtests): + def display_result(self, runtests: RunTests) -> None: # If running the test suite for PGO then no one cares about results. if runtests.pgo: return @@ -365,7 +368,7 @@ def run_test( return result - def run_tests_sequentially(self, runtests) -> None: + def run_tests_sequentially(self, runtests: RunTests) -> None: if self.coverage: tracer = trace.Trace(trace=False, count=True) else: @@ -422,7 +425,7 @@ def run_tests_sequentially(self, runtests) -> None: if previous_test: print(previous_test) - def get_state(self): + def get_state(self) -> str: state = self.results.get_state(self.fail_env_changed) if self.first_state: state = f'{self.first_state} then {state}' @@ -452,7 +455,12 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.junit_filename: self.results.write_junit(self.junit_filename) - def display_summary(self): + def display_summary(self) -> None: + if self.first_runtests is None: + raise ValueError( + "Should never call `display_summary()` before calling `_run_test()`" + ) + duration = time.perf_counter() - self.logger.start_time filtered = bool(self.match_tests) @@ -466,7 +474,7 @@ def display_summary(self): state = self.get_state() print(f"Result: {state}") - def create_run_tests(self, tests: TestTuple): + def create_run_tests(self, tests: TestTuple) -> RunTests: return RunTests( tests, fail_fast=self.fail_fast, @@ -674,9 +682,9 @@ def _execute_python(self, cmd, environ): f"Command: {cmd_text}") # continue executing main() - def _add_python_opts(self): - python_opts = [] - regrtest_opts = [] + def _add_python_opts(self) -> None: + python_opts: list[str] = [] + regrtest_opts: list[str] = [] environ, keep_environ = self._add_cross_compile_opts(regrtest_opts) if self.ci_mode: @@ -707,9 +715,17 @@ def _init(self): strip_py_suffix(self.cmdline_args) - self.tmp_dir = get_temp_dir(self.tmp_dir) + self._tmp_dir = get_temp_dir(self._tmp_dir) + + @property + def tmp_dir(self) -> StrPath: + if self._tmp_dir is None: + raise ValueError( + "Should never use `.tmp_dir` before calling `.main()`" + ) + return self._tmp_dir - def main(self, tests: TestList | None = None): + def main(self, tests: TestList | None = None) -> NoReturn: if self.want_add_python_opts: self._add_python_opts() @@ -738,7 +754,7 @@ def main(self, tests: TestList | None = None): sys.exit(exitcode) -def main(tests=None, _add_python_opts=False, **kwargs): +def main(tests=None, _add_python_opts=False, **kwargs) -> NoReturn: """Run the Python suite.""" ns = _parse_args(sys.argv[1:], **kwargs) Regrtest(ns, _add_python_opts=_add_python_opts).main(tests=tests) diff --git a/Lib/test/libregrtest/mypy.ini b/Lib/test/libregrtest/mypy.ini index 22c7c7a9acef14..905341cc04b8f1 100644 --- a/Lib/test/libregrtest/mypy.ini +++ b/Lib/test/libregrtest/mypy.ini @@ -22,12 +22,6 @@ disallow_untyped_defs = False check_untyped_defs = False warn_return_any = False -disable_error_code = return - -# Enable --strict-optional for these ASAP: -[mypy-Lib.test.libregrtest.main.*,Lib.test.libregrtest.run_workers.*] -strict_optional = False - # Various internal modules that typeshed deliberately doesn't have stubs for: [mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*] ignore_missing_imports = True diff --git a/Lib/test/libregrtest/pgo.py b/Lib/test/libregrtest/pgo.py index e3a6927be5db1d..f762345c88cde3 100644 --- a/Lib/test/libregrtest/pgo.py +++ b/Lib/test/libregrtest/pgo.py @@ -50,7 +50,7 @@ 'test_xml_etree_c', ] -def setup_pgo_tests(cmdline_args, pgo_extended: bool): +def setup_pgo_tests(cmdline_args, pgo_extended: bool) -> None: if not cmdline_args and not pgo_extended: # run default set of tests for PGO training cmdline_args[:] = PGO_TESTS[:] diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index fa447a4336a399..e783475cc7a36b 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -262,7 +262,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): sys._clear_internal_caches() -def warm_caches(): +def warm_caches() -> None: # char cache s = bytes(range(256)) for i in range(256): diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 74eae40440435d..7553efe5e8abeb 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -149,6 +149,7 @@ def __str__(self) -> str: case State.DID_NOT_RUN: return f"{self.test_name} ran no tests" case State.TIMEOUT: + assert self.duration is not None, "self.duration is None" return f"{self.test_name} timed out ({format_duration(self.duration)})" case _: raise ValueError("unknown result state: {state!r}") diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 0e28435bc7d629..4f3e84282dc5dc 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -71,7 +71,7 @@ def get_state(self, fail_env_changed: bool) -> str: return ', '.join(state) - def get_exitcode(self, fail_env_changed, fail_rerun): + def get_exitcode(self, fail_env_changed: bool, fail_rerun: bool) -> int: exitcode = 0 if self.bad: exitcode = EXITCODE_BAD_TEST @@ -87,7 +87,7 @@ def get_exitcode(self, fail_env_changed, fail_rerun): exitcode = EXITCODE_BAD_TEST return exitcode - def accumulate_result(self, result: TestResult, runtests: RunTests): + def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: test_name = result.test_name rerun = runtests.rerun fail_env_changed = runtests.fail_env_changed @@ -135,7 +135,7 @@ def get_coverage_results(self) -> trace.CoverageResults: counts = {loc: 1 for loc in self.covered_lines} return trace.CoverageResults(counts=counts) - def need_rerun(self): + def need_rerun(self) -> bool: return bool(self.rerun_results) def prepare_rerun(self, *, clear: bool = True) -> tuple[TestTuple, FilterDict]: @@ -158,7 +158,7 @@ def prepare_rerun(self, *, clear: bool = True) -> tuple[TestTuple, FilterDict]: return (tuple(tests), match_tests_dict) - def add_junit(self, xml_data: list[str]): + def add_junit(self, xml_data: list[str]) -> None: import xml.etree.ElementTree as ET for e in xml_data: try: @@ -167,7 +167,7 @@ def add_junit(self, xml_data: list[str]): print(xml_data, file=sys.__stderr__) raise - def write_junit(self, filename: StrPath): + def write_junit(self, filename: StrPath) -> None: if not self.testsuite_xml: # Don't create empty XML file return @@ -192,7 +192,7 @@ def write_junit(self, filename: StrPath): for s in ET.tostringlist(root): f.write(s) - def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool): + def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) -> None: if print_slowest: self.test_times.sort(reverse=True) print() @@ -234,7 +234,7 @@ def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool): print() print("Test suite interrupted by signal SIGINT.") - def display_summary(self, first_runtests: RunTests, filtered: bool): + def display_summary(self, first_runtests: RunTests, filtered: bool) -> None: # Total tests stats = self.stats text = f'run={stats.tests_run:,}' diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 387ddf9614cf79..0ca86a986ea436 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -102,6 +102,9 @@ def __init__(self, super().__init__() +_NOT_RUNNING = "" + + class WorkerThread(threading.Thread): def __init__(self, worker_id: int, runner: "RunWorkers") -> None: super().__init__() @@ -111,8 +114,8 @@ def __init__(self, worker_id: int, runner: "RunWorkers") -> None: self.output = runner.output self.timeout = runner.worker_timeout self.log = runner.log - self.test_name: TestName | None = None - self.start_time: float | None = None + self.test_name = _NOT_RUNNING + self.start_time = time.monotonic() self._popen: subprocess.Popen[str] | None = None self._killed = False self._stopped = False @@ -129,7 +132,7 @@ def __repr__(self) -> str: popen = self._popen if popen is not None: dt = time.monotonic() - self.start_time - info.extend((f'pid={self._popen.pid}', + info.extend((f'pid={popen.pid}', f'time={format_duration(dt)}')) return '<%s>' % ' '.join(info) @@ -211,6 +214,7 @@ def _run_process(self, runtests: WorkerRunTests, output_fd: int, # on reading closed stdout raise ExitThread raise + return None except: self._kill() raise @@ -400,7 +404,7 @@ def run(self) -> None: except WorkerError as exc: mp_result = exc.mp_result finally: - self.test_name = None + self.test_name = _NOT_RUNNING mp_result.result.duration = time.monotonic() - self.start_time self.output.put((False, mp_result)) @@ -415,6 +419,9 @@ def run(self) -> None: def _wait_completed(self) -> None: popen = self._popen + # only needed for mypy: + if popen is None: + raise ValueError("Should never access `._popen` before calling `.run()`") try: popen.wait(WAIT_COMPLETED_TIMEOUT) @@ -482,7 +489,7 @@ def __init__(self, num_workers: int, runtests: RunTests, self.worker_timeout: float | None = min(self.timeout * 1.5, self.timeout + 5 * 60) else: self.worker_timeout = None - self.workers: list[WorkerThread] | None = None + self.workers: list[WorkerThread] = [] jobs = self.runtests.get_jobs() if jobs is not None: @@ -502,7 +509,7 @@ def start_workers(self) -> None: processes = plural(nworkers, "process", "processes") msg = (f"Run {tests} in parallel using " f"{nworkers} worker {processes}") - if self.timeout: + if self.timeout and self.worker_timeout is not None: msg += (" (timeout: %s, worker timeout: %s)" % (format_duration(self.timeout), format_duration(self.worker_timeout))) @@ -544,6 +551,7 @@ def _get_result(self) -> QueueOutput | None: running = get_running(self.workers) if running: self.log(running) + return None def display_result(self, mp_result: MultiprocessResult) -> None: result = mp_result.result @@ -553,7 +561,7 @@ def display_result(self, mp_result: MultiprocessResult) -> None: if mp_result.err_msg: # WORKER_BUG text += ' (%s)' % mp_result.err_msg - elif (result.duration >= PROGRESS_MIN_TIME and not pgo): + elif (result.duration and result.duration >= PROGRESS_MIN_TIME and not pgo): text += ' (%s)' % format_duration(result.duration) if not pgo: running = get_running(self.workers) diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 8e9779524c56a2..cd1ce8080a04df 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -5,12 +5,12 @@ import shlex import subprocess import sys -from typing import Any +from typing import Any, Iterator from test import support from .utils import ( - StrPath, StrJSON, TestTuple, TestFilter, FilterTuple, FilterDict) + StrPath, StrJSON, TestTuple, TestName, TestFilter, FilterTuple, FilterDict) class JsonFileType: @@ -41,8 +41,8 @@ def configure_subprocess(self, popen_kwargs: dict) -> None: popen_kwargs['startupinfo'] = startupinfo @contextlib.contextmanager - def inherit_subprocess(self): - if self.file_type == JsonFileType.WINDOWS_HANDLE: + def inherit_subprocess(self) -> Iterator[None]: + if sys.platform == 'win32' and self.file_type == JsonFileType.WINDOWS_HANDLE: os.set_handle_inheritable(self.file, True) try: yield @@ -106,25 +106,25 @@ def copy(self, **override) -> 'RunTests': state.update(override) return RunTests(**state) - def create_worker_runtests(self, **override): + def create_worker_runtests(self, **override) -> WorkerRunTests: state = dataclasses.asdict(self) state.update(override) return WorkerRunTests(**state) - def get_match_tests(self, test_name) -> FilterTuple | None: + def get_match_tests(self, test_name: TestName) -> FilterTuple | None: if self.match_tests_dict is not None: return self.match_tests_dict.get(test_name, None) else: return None - def get_jobs(self): + def get_jobs(self) -> int | None: # Number of run_single_test() calls needed to run all tests. # None means that there is not bound limit (--forever option). if self.forever: return None return len(self.tests) - def iter_tests(self): + def iter_tests(self) -> Iterator[TestName]: if self.forever: while True: yield from self.tests diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 9e9741493e9a5b..ba57f06b4841d4 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -25,9 +25,10 @@ def setup_test_dir(testdir: str | None) -> None: sys.path.insert(0, os.path.abspath(testdir)) -def setup_process(): +def setup_process() -> None: fix_umask() + assert sys.__stderr__ is not None, "sys.__stderr__ is None" try: stderr_fd = sys.__stderr__.fileno() except (ValueError, AttributeError): @@ -35,7 +36,7 @@ def setup_process(): # and ValueError on a closed stream. # # Catch AttributeError for stderr being None. - stderr_fd = None + pass else: # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True, file=stderr_fd) @@ -68,7 +69,7 @@ def setup_process(): for index, path in enumerate(module.__path__): module.__path__[index] = os.path.abspath(path) if getattr(module, '__file__', None): - module.__file__ = os.path.abspath(module.__file__) + module.__file__ = os.path.abspath(module.__file__) # type: ignore[type-var] if hasattr(sys, 'addaudithook'): # Add an auditing hook for all tests to ensure PySys_Audit is tested @@ -87,7 +88,7 @@ def _test_audit_hook(name, args): os.environ.setdefault(UNICODE_GUARD_ENV, FS_NONASCII) -def setup_tests(runtests: RunTests): +def setup_tests(runtests: RunTests) -> None: support.verbose = runtests.verbose support.failfast = runtests.fail_fast support.PGO = runtests.pgo diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index dd18ae2584f5d8..822ac0f4044d9e 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -28,6 +28,6 @@ ] -def setup_tsan_tests(cmdline_args): +def setup_tsan_tests(cmdline_args) -> None: if not cmdline_args: cmdline_args[:] = TSAN_TESTS[:] diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index d6be4ad049d14a..1ecfc61af9e39d 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -58,7 +58,7 @@ FilterDict = dict[TestName, FilterTuple] -def format_duration(seconds): +def format_duration(seconds: float) -> str: ms = math.ceil(seconds * 1e3) seconds, ms = divmod(ms, 1000) minutes, seconds = divmod(seconds, 60) @@ -92,7 +92,7 @@ def strip_py_suffix(names: list[str] | None) -> None: names[idx] = basename -def plural(n, singular, plural=None): +def plural(n: int, singular: str, plural: str | None = None) -> str: if n == 1: return singular elif plural is not None: @@ -101,7 +101,7 @@ def plural(n, singular, plural=None): return singular + 's' -def count(n, word): +def count(n: int, word: str) -> str: if n == 1: return f"{n} {word}" else: @@ -123,14 +123,14 @@ def printlist(x, width=70, indent=4, file=None): file=file) -def print_warning(msg): +def print_warning(msg: str) -> None: support.print_warning(msg) -orig_unraisablehook = None +orig_unraisablehook: Callable[..., None] | None = None -def regrtest_unraisable_hook(unraisable): +def regrtest_unraisable_hook(unraisable) -> None: global orig_unraisablehook support.environment_altered = True support.print_warning("Unraisable exception") @@ -138,22 +138,23 @@ def regrtest_unraisable_hook(unraisable): try: support.flush_std_streams() sys.stderr = support.print_warning.orig_stderr + assert orig_unraisablehook is not None, "orig_unraisablehook not set" orig_unraisablehook(unraisable) sys.stderr.flush() finally: sys.stderr = old_stderr -def setup_unraisable_hook(): +def setup_unraisable_hook() -> None: global orig_unraisablehook orig_unraisablehook = sys.unraisablehook sys.unraisablehook = regrtest_unraisable_hook -orig_threading_excepthook = None +orig_threading_excepthook: Callable[..., None] | None = None -def regrtest_threading_excepthook(args): +def regrtest_threading_excepthook(args) -> None: global orig_threading_excepthook support.environment_altered = True support.print_warning(f"Uncaught thread exception: {args.exc_type.__name__}") @@ -161,13 +162,14 @@ def regrtest_threading_excepthook(args): try: support.flush_std_streams() sys.stderr = support.print_warning.orig_stderr + assert orig_threading_excepthook is not None, "orig_threading_excepthook not set" orig_threading_excepthook(args) sys.stderr.flush() finally: sys.stderr = old_stderr -def setup_threading_excepthook(): +def setup_threading_excepthook() -> None: global orig_threading_excepthook import threading orig_threading_excepthook = threading.excepthook @@ -476,7 +478,7 @@ def get_temp_dir(tmp_dir: StrPath | None = None) -> StrPath: return os.path.abspath(tmp_dir) -def fix_umask(): +def fix_umask() -> None: if support.is_emscripten: # Emscripten has default umask 0o777, which breaks some tests. # see https://github.com/emscripten-core/emscripten/issues/17269 @@ -572,7 +574,8 @@ def abs_module_name(test_name: TestName, test_dir: StrPath | None) -> TestName: 'setUpModule', 'tearDownModule', )) -def normalize_test_name(test_full_name, *, is_error=False): +def normalize_test_name(test_full_name: str, *, + is_error: bool = False) -> str | None: short_name = test_full_name.split(" ")[0] if is_error and short_name in _TEST_LIFECYCLE_HOOKS: if test_full_name.startswith(('setUpModule (', 'tearDownModule (')): @@ -593,7 +596,7 @@ def normalize_test_name(test_full_name, *, is_error=False): return short_name -def adjust_rlimit_nofile(): +def adjust_rlimit_nofile() -> None: """ On macOS the default fd limit (RLIMIT_NOFILE) is sometimes too low (256) for our test suite to succeed. Raise it to something more reasonable. 1024 @@ -619,17 +622,17 @@ def adjust_rlimit_nofile(): f"{new_fd_limit}: {err}.") -def get_host_runner(): +def get_host_runner() -> str: if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None: hostrunner = sysconfig.get_config_var("HOSTRUNNER") return hostrunner -def is_cross_compiled(): +def is_cross_compiled() -> bool: return ('_PYTHON_HOST_PLATFORM' in os.environ) -def format_resources(use_resources: Iterable[str]): +def format_resources(use_resources: Iterable[str]) -> str: use_resources = set(use_resources) all_resources = set(ALL_RESOURCES) @@ -654,7 +657,7 @@ def format_resources(use_resources: Iterable[str]): def display_header(use_resources: tuple[str, ...], - python_cmd: tuple[str, ...] | None): + python_cmd: tuple[str, ...] | None) -> None: # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) print("==", platform.platform(aliased=True), @@ -732,7 +735,7 @@ def display_header(use_resources: tuple[str, ...], print(flush=True) -def cleanup_temp_dir(tmp_dir: StrPath): +def cleanup_temp_dir(tmp_dir: StrPath) -> None: import glob path = os.path.join(glob.escape(tmp_dir), TMP_PREFIX + '*') @@ -763,5 +766,5 @@ def _sanitize_xml_replace(regs): return ''.join(f'\\x{ord(ch):02x}' if ch <= '\xff' else ascii(ch)[1:-1] for ch in text) -def sanitize_xml(text): +def sanitize_xml(text: str) -> str: return ILLEGAL_XML_CHARS_RE.sub(_sanitize_xml_replace, text) diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 86cc30835fdbda..da24760a82c6c6 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -104,7 +104,7 @@ def worker_process(worker_json: StrJSON) -> NoReturn: sys.exit(0) -def main(): +def main() -> NoReturn: if len(sys.argv) != 2: print("usage: python -m test.libregrtest.worker JSON") sys.exit(1) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 7c1ef42a4970d7..2ad267e3e08f0f 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1274,6 +1274,11 @@ def requires_specialization(test): _opcode.ENABLE_SPECIALIZATION, "requires specialization")(test) +def requires_specialization_ft(test): + return unittest.skipUnless( + _opcode.ENABLE_SPECIALIZATION_FT, "requires specialization")(test) + + #======================================================================= # Check for the presence of docstrings. diff --git a/Lib/test/support/i18n_helper.py b/Lib/test/support/i18n_helper.py new file mode 100644 index 00000000000000..2e304f29e8ba7f --- /dev/null +++ b/Lib/test/support/i18n_helper.py @@ -0,0 +1,63 @@ +import re +import subprocess +import sys +import unittest +from pathlib import Path +from test.support import REPO_ROOT, TEST_HOME_DIR, requires_subprocess +from test.test_tools import skip_if_missing + + +pygettext = Path(REPO_ROOT) / 'Tools' / 'i18n' / 'pygettext.py' + +msgid_pattern = re.compile(r'msgid(.*?)(?:msgid_plural|msgctxt|msgstr)', + re.DOTALL) +msgid_string_pattern = re.compile(r'"((?:\\"|[^"])*)"') + + +def _generate_po_file(path, *, stdout_only=True): + res = subprocess.run([sys.executable, pygettext, + '--no-location', '-o', '-', path], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True) + if stdout_only: + return res.stdout + return res + + +def _extract_msgids(po): + msgids = [] + for msgid in msgid_pattern.findall(po): + msgid_string = ''.join(msgid_string_pattern.findall(msgid)) + msgid_string = msgid_string.replace(r'\"', '"') + if msgid_string: + msgids.append(msgid_string) + return sorted(msgids) + + +def _get_snapshot_path(module_name): + return Path(TEST_HOME_DIR) / 'translationdata' / module_name / 'msgids.txt' + + +@requires_subprocess() +class TestTranslationsBase(unittest.TestCase): + + def assertMsgidsEqual(self, module): + '''Assert that msgids extracted from a given module match a + snapshot. + + ''' + skip_if_missing('i18n') + res = _generate_po_file(module.__file__, stdout_only=False) + self.assertEqual(res.returncode, 0) + self.assertEqual(res.stderr, '') + msgids = _extract_msgids(res.stdout) + snapshot_path = _get_snapshot_path(module.__name__) + snapshot = snapshot_path.read_text().splitlines() + self.assertListEqual(msgids, snapshot) + + +def update_translation_snapshots(module): + contents = _generate_po_file(module.__file__) + msgids = _extract_msgids(contents) + snapshot_path = _get_snapshot_path(module.__name__) + snapshot_path.write_text('\n'.join(msgids)) diff --git a/Lib/test/support/strace_helper.py b/Lib/test/support/strace_helper.py new file mode 100644 index 00000000000000..90d4b5bccb6fa3 --- /dev/null +++ b/Lib/test/support/strace_helper.py @@ -0,0 +1,170 @@ +import re +import sys +import textwrap +import unittest +from dataclasses import dataclass +from functools import cache +from test import support +from test.support.script_helper import run_python_until_end + +_strace_binary = "/usr/bin/strace" +_syscall_regex = re.compile( + r"(?P[^(]*)\((?P[^)]*)\)\s*[=]\s*(?P.+)") +_returncode_regex = re.compile( + br"\+\+\+ exited with (?P\d+) \+\+\+") + + +@dataclass +class StraceEvent: + syscall: str + args: list[str] + returncode: str + + +@dataclass +class StraceResult: + strace_returncode: int + python_returncode: int + + """The event messages generated by strace. This is very similar to the + stderr strace produces with returncode marker section removed.""" + event_bytes: bytes + stdout: bytes + stderr: bytes + + def events(self): + """Parse event_bytes data into system calls for easier processing. + + This assumes the program under inspection doesn't print any non-utf8 + strings which would mix into the strace output.""" + decoded_events = self.event_bytes.decode('utf-8') + matches = [ + _syscall_regex.match(event) + for event in decoded_events.splitlines() + ] + return [ + StraceEvent(match["syscall"], + [arg.strip() for arg in (match["args"].split(","))], + match["returncode"]) for match in matches if match + ] + + def sections(self): + """Find all "MARK " writes and use them to make groups of events. + + This is useful to avoid variable / overhead events, like those at + interpreter startup or when opening a file so a test can verify just + the small case under study.""" + current_section = "__startup" + sections = {current_section: []} + for event in self.events(): + if event.syscall == 'write' and len( + event.args) > 2 and event.args[1].startswith("\"MARK "): + # Found a new section, don't include the write in the section + # but all events until next mark should be in that section + current_section = event.args[1].split( + " ", 1)[1].removesuffix('\\n"') + if current_section not in sections: + sections[current_section] = list() + else: + sections[current_section].append(event) + + return sections + + +@support.requires_subprocess() +def strace_python(code, strace_flags, check=True): + """Run strace and return the trace. + + Sets strace_returncode and python_returncode to `-1` on error.""" + res = None + + def _make_error(reason, details): + return StraceResult( + strace_returncode=-1, + python_returncode=-1, + event_bytes=f"error({reason},details={details}) = -1".encode('utf-8'), + stdout=res.out if res else b"", + stderr=res.err if res else b"") + + # Run strace, and get out the raw text + try: + res, cmd_line = run_python_until_end( + "-c", + textwrap.dedent(code), + __run_using_command=[_strace_binary] + strace_flags) + except OSError as err: + return _make_error("Caught OSError", err) + + if check and res.rc: + res.fail(cmd_line) + + # Get out program returncode + stripped = res.err.strip() + output = stripped.rsplit(b"\n", 1) + if len(output) != 2: + return _make_error("Expected strace events and exit code line", + stripped[-50:]) + + returncode_match = _returncode_regex.match(output[1]) + if not returncode_match: + return _make_error("Expected to find returncode in last line.", + output[1][:50]) + + python_returncode = int(returncode_match["returncode"]) + if check and python_returncode: + res.fail(cmd_line) + + return StraceResult(strace_returncode=res.rc, + python_returncode=python_returncode, + event_bytes=output[0], + stdout=res.out, + stderr=res.err) + + +def get_events(code, strace_flags, prelude, cleanup): + # NOTE: The flush is currently required to prevent the prints from getting + # buffered and done all at once at exit + prelude = textwrap.dedent(prelude) + code = textwrap.dedent(code) + cleanup = textwrap.dedent(cleanup) + to_run = f""" +print("MARK prelude", flush=True) +{prelude} +print("MARK code", flush=True) +{code} +print("MARK cleanup", flush=True) +{cleanup} +print("MARK __shutdown", flush=True) + """ + trace = strace_python(to_run, strace_flags) + all_sections = trace.sections() + return all_sections['code'] + + +def get_syscalls(code, strace_flags, prelude="", cleanup=""): + """Get the syscalls which a given chunk of python code generates""" + events = get_events(code, strace_flags, prelude=prelude, cleanup=cleanup) + return [ev.syscall for ev in events] + + +# Moderately expensive (spawns a subprocess), so share results when possible. +@cache +def _can_strace(): + res = strace_python("import sys; sys.exit(0)", [], check=False) + assert res.events(), "Should have parsed multiple calls" + + return res.strace_returncode == 0 and res.python_returncode == 0 + + +def requires_strace(): + if sys.platform != "linux": + return unittest.skip("Linux only, requires strace.") + + if support.check_sanitizer(address=True, memory=True): + return unittest.skip("LeakSanitizer does not work under ptrace (strace, gdb, etc)") + + return unittest.skipUnless(_can_strace(), "Requires working strace") + + +__all__ = ["get_events", "get_syscalls", "requires_strace", "strace_python", + "StraceEvent", "StraceResult"] diff --git a/Lib/test/support/venv.py b/Lib/test/support/venv.py new file mode 100644 index 00000000000000..78e6a51ec1815e --- /dev/null +++ b/Lib/test/support/venv.py @@ -0,0 +1,70 @@ +import contextlib +import logging +import os +import subprocess +import shlex +import sys +import sysconfig +import tempfile +import venv + + +class VirtualEnvironment: + def __init__(self, prefix, **venv_create_args): + self._logger = logging.getLogger(self.__class__.__name__) + venv.create(prefix, **venv_create_args) + self._prefix = prefix + self._paths = sysconfig.get_paths( + scheme='venv', + vars={'base': self.prefix}, + expand=True, + ) + + @classmethod + @contextlib.contextmanager + def from_tmpdir(cls, *, prefix=None, dir=None, **venv_create_args): + delete = not bool(os.environ.get('PYTHON_TESTS_KEEP_VENV')) + with tempfile.TemporaryDirectory(prefix=prefix, dir=dir, delete=delete) as tmpdir: + yield cls(tmpdir, **venv_create_args) + + @property + def prefix(self): + return self._prefix + + @property + def paths(self): + return self._paths + + @property + def interpreter(self): + return os.path.join(self.paths['scripts'], os.path.basename(sys.executable)) + + def _format_output(self, name, data, indent='\t'): + if not data: + return indent + f'{name}: (none)' + if len(data.splitlines()) == 1: + return indent + f'{name}: {data}' + else: + prefixed_lines = '\n'.join(indent + '> ' + line for line in data.splitlines()) + return indent + f'{name}:\n' + prefixed_lines + + def run(self, *args, **subprocess_args): + if subprocess_args.get('shell'): + raise ValueError('Running the subprocess in shell mode is not supported.') + default_args = { + 'capture_output': True, + 'check': True, + } + try: + result = subprocess.run([self.interpreter, *args], **default_args | subprocess_args) + except subprocess.CalledProcessError as e: + if e.returncode != 0: + self._logger.error( + f'Interpreter returned non-zero exit status {e.returncode}.\n' + + self._format_output('COMMAND', shlex.join(e.cmd)) + '\n' + + self._format_output('STDOUT', e.stdout.decode()) + '\n' + + self._format_output('STDERR', e.stderr.decode()) + '\n' + ) + raise + else: + return result diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 14cd50bd30502c..bf3165e2341949 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -551,6 +551,24 @@ def test_still_running(self): self.assertTrue(_interpreters.is_running(interp)) +class CommonTests(TestBase): + def setUp(self): + super().setUp() + self.id = _interpreters.create() + + def test_signatures(self): + # for method in ['exec', 'run_string', 'run_func']: + msg = "expected 'shared' to be a dict" + with self.assertRaisesRegex(TypeError, msg): + _interpreters.exec(self.id, 'a', 1) + with self.assertRaisesRegex(TypeError, msg): + _interpreters.exec(self.id, 'a', shared=1) + with self.assertRaisesRegex(TypeError, msg): + _interpreters.run_string(self.id, 'a', shared=1) + with self.assertRaisesRegex(TypeError, msg): + _interpreters.run_func(self.id, lambda: None, shared=1) + + class RunStringTests(TestBase): def setUp(self): diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index ba9876570385d3..358cfb1c56aae4 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -7,10 +7,8 @@ import operator import os import py_compile -import re import shutil import stat -import subprocess import sys import textwrap import tempfile @@ -19,15 +17,11 @@ import warnings from enum import StrEnum -from pathlib import Path -from test.support import REPO_ROOT -from test.support import TEST_HOME_DIR from test.support import captured_stderr from test.support import import_helper from test.support import os_helper -from test.support import requires_subprocess from test.support import script_helper -from test.test_tools import skip_if_missing +from test.support.i18n_helper import TestTranslationsBase, update_translation_snapshots from unittest import mock @@ -789,6 +783,13 @@ def test_const(self): self.assertIn("got an unexpected keyword argument 'const'", str(cm.exception)) + def test_invalid_name(self): + parser = argparse.ArgumentParser() + with self.assertRaises(ValueError) as cm: + parser.add_argument('--no-foo', action=argparse.BooleanOptionalAction) + self.assertEqual(str(cm.exception), + "invalid option name '--no-foo' for BooleanOptionalAction") + class TestBooleanOptionalActionRequired(ParserTestCase): """Tests BooleanOptionalAction required""" @@ -7049,50 +7050,10 @@ def test_directory_in_zipfile_compiled(self): # Translation tests # ================= -pygettext = Path(REPO_ROOT) / 'Tools' / 'i18n' / 'pygettext.py' -snapshot_path = Path(TEST_HOME_DIR) / 'translationdata' / 'argparse' / 'msgids.txt' - -msgid_pattern = re.compile(r'msgid(.*?)(?:msgid_plural|msgctxt|msgstr)', re.DOTALL) -msgid_string_pattern = re.compile(r'"((?:\\"|[^"])*)"') - - -@requires_subprocess() -class TestTranslations(unittest.TestCase): +class TestTranslations(TestTranslationsBase): def test_translations(self): - # Test messages extracted from the argparse module against a snapshot - skip_if_missing('i18n') - res = generate_po_file(stdout_only=False) - self.assertEqual(res.returncode, 0) - self.assertEqual(res.stderr, '') - msgids = extract_msgids(res.stdout) - snapshot = snapshot_path.read_text().splitlines() - self.assertListEqual(msgids, snapshot) - - -def generate_po_file(*, stdout_only=True): - res = subprocess.run([sys.executable, pygettext, - '--no-location', '-o', '-', argparse.__file__], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - if stdout_only: - return res.stdout - return res - - -def extract_msgids(po): - msgids = [] - for msgid in msgid_pattern.findall(po): - msgid_string = ''.join(msgid_string_pattern.findall(msgid)) - msgid_string = msgid_string.replace(r'\"', '"') - if msgid_string: - msgids.append(msgid_string) - return sorted(msgids) - - -def update_translation_snapshots(): - contents = generate_po_file() - msgids = extract_msgids(contents) - snapshot_path.write_text('\n'.join(msgids)) + self.assertMsgidsEqual(argparse) def tearDownModule(): @@ -7104,6 +7065,6 @@ def tearDownModule(): if __name__ == '__main__': # To regenerate translation snapshots if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': - update_translation_snapshots() + update_translation_snapshots(argparse) sys.exit(0) unittest.main() diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 4dcf9f0e4037b6..2ab638dc527aec 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2704,33 +2704,22 @@ def test_event_loop_policy(self): def test_get_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() self.assertIsNone(policy._local._loop) - with self.assertWarns(DeprecationWarning) as cm: - loop = policy.get_event_loop() - self.assertEqual(cm.filename, __file__) - self.assertIsInstance(loop, asyncio.AbstractEventLoop) - self.assertIs(policy._local._loop, loop) - self.assertIs(loop, policy.get_event_loop()) - loop.close() + with self.assertRaises(RuntimeError): + loop = policy.get_event_loop() + self.assertIsNone(policy._local._loop) - def test_get_event_loop_calls_set_event_loop(self): + def test_get_event_loop_does_not_call_set_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() with mock.patch.object( policy, "set_event_loop", wraps=policy.set_event_loop) as m_set_event_loop: - with self.assertWarns(DeprecationWarning) as cm: + with self.assertRaises(RuntimeError): loop = policy.get_event_loop() - self.addCleanup(loop.close) - self.assertEqual(cm.filename, __file__) - # policy._local._loop must be set through .set_event_loop() - # (the unix DefaultEventLoopPolicy needs this call to attach - # the child watcher correctly) - m_set_event_loop.assert_called_with(loop) - - loop.close() + m_set_event_loop.assert_not_called() def test_get_event_loop_after_set_none(self): policy = asyncio.DefaultEventLoopPolicy() @@ -2912,17 +2901,12 @@ def test_get_event_loop_returns_running_loop2(self): loop = asyncio.new_event_loop() self.addCleanup(loop.close) - with self.assertWarns(DeprecationWarning) as cm: - loop2 = asyncio.get_event_loop() - self.addCleanup(loop2.close) - self.assertEqual(cm.filename, __file__) - asyncio.set_event_loop(None) with self.assertRaisesRegex(RuntimeError, 'no current'): asyncio.get_event_loop() - with self.assertRaisesRegex(RuntimeError, 'no running'): - asyncio.get_running_loop() - self.assertIs(asyncio._get_running_loop(), None) + asyncio.set_event_loop(None) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() async def func(): self.assertIs(asyncio.get_event_loop(), loop) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 9ae54b6887010b..021f45478d6f48 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1195,8 +1195,7 @@ async def test_fork_not_share_event_loop(self): if pid == 0: # child try: - with self.assertWarns(DeprecationWarning): - loop = asyncio.get_event_loop_policy().get_event_loop() + loop = asyncio.get_event_loop() os.write(w, b'LOOP:' + str(id(loop)).encode()) except RuntimeError: os.write(w, b'NO LOOP') @@ -1207,8 +1206,7 @@ async def test_fork_not_share_event_loop(self): else: # parent result = os.read(r, 100) - self.assertEqual(result[:5], b'LOOP:', result) - self.assertNotEqual(int(result[5:]), id(loop)) + self.assertEqual(result, b'NO LOOP') wait_process(pid, exitcode=0) @hashlib_helper.requires_hashdigest('md5') diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index ae938d12c9401b..cb38a69e390f3a 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -3910,6 +3910,8 @@ def test_memoryview_check_released(self): self.assertRaises(ValueError, memoryview, m) # memoryview.cast() self.assertRaises(ValueError, m.cast, 'c') + # memoryview.__iter__() + self.assertRaises(ValueError, m.__iter__) # getbuffer() self.assertRaises(ValueError, ndarray, m) # memoryview.tolist() diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index eb5906f8944c8e..f8e6f05cd607c8 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -148,6 +148,9 @@ def filter_char(arg): def map_char(arg): return chr(ord(arg)+1) +def pack(*args): + return args + class BuiltinTest(unittest.TestCase): # Helper to check picklability def check_iter_pickle(self, it, seq, proto): @@ -1269,6 +1272,108 @@ def test_map_pickle(self): m2 = map(map_char, "Is this the real life?") self.check_iter_pickle(m1, list(m2), proto) + # strict map tests based on strict zip tests + + def test_map_pickle_strict(self): + a = (1, 2, 3) + b = (4, 5, 6) + t = [(1, 4), (2, 5), (3, 6)] + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + m1 = map(pack, a, b, strict=True) + self.check_iter_pickle(m1, t, proto) + + def test_map_pickle_strict_fail(self): + a = (1, 2, 3) + b = (4, 5, 6, 7) + t = [(1, 4), (2, 5), (3, 6)] + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + m1 = map(pack, a, b, strict=True) + m2 = pickle.loads(pickle.dumps(m1, proto)) + self.assertEqual(self.iter_error(m1, ValueError), t) + self.assertEqual(self.iter_error(m2, ValueError), t) + + def test_map_strict(self): + self.assertEqual(tuple(map(pack, (1, 2, 3), 'abc', strict=True)), + ((1, 'a'), (2, 'b'), (3, 'c'))) + self.assertRaises(ValueError, tuple, + map(pack, (1, 2, 3, 4), 'abc', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, (1, 2), 'abc', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, (1, 2), (1, 2), 'abc', strict=True)) + + def test_map_strict_iterators(self): + x = iter(range(5)) + y = [0] + z = iter(range(5)) + self.assertRaises(ValueError, list, + (map(pack, x, y, z, strict=True))) + self.assertEqual(next(x), 2) + self.assertEqual(next(z), 1) + + def test_map_strict_error_handling(self): + + class Error(Exception): + pass + + class Iter: + def __init__(self, size): + self.size = size + def __iter__(self): + return self + def __next__(self): + self.size -= 1 + if self.size < 0: + raise Error + return self.size + + l1 = self.iter_error(map(pack, "AB", Iter(1), strict=True), Error) + self.assertEqual(l1, [("A", 0)]) + l2 = self.iter_error(map(pack, "AB", Iter(2), "A", strict=True), ValueError) + self.assertEqual(l2, [("A", 1, "A")]) + l3 = self.iter_error(map(pack, "AB", Iter(2), "ABC", strict=True), Error) + self.assertEqual(l3, [("A", 1, "A"), ("B", 0, "B")]) + l4 = self.iter_error(map(pack, "AB", Iter(3), strict=True), ValueError) + self.assertEqual(l4, [("A", 2), ("B", 1)]) + l5 = self.iter_error(map(pack, Iter(1), "AB", strict=True), Error) + self.assertEqual(l5, [(0, "A")]) + l6 = self.iter_error(map(pack, Iter(2), "A", strict=True), ValueError) + self.assertEqual(l6, [(1, "A")]) + l7 = self.iter_error(map(pack, Iter(2), "ABC", strict=True), Error) + self.assertEqual(l7, [(1, "A"), (0, "B")]) + l8 = self.iter_error(map(pack, Iter(3), "AB", strict=True), ValueError) + self.assertEqual(l8, [(2, "A"), (1, "B")]) + + def test_map_strict_error_handling_stopiteration(self): + + class Iter: + def __init__(self, size): + self.size = size + def __iter__(self): + return self + def __next__(self): + self.size -= 1 + if self.size < 0: + raise StopIteration + return self.size + + l1 = self.iter_error(map(pack, "AB", Iter(1), strict=True), ValueError) + self.assertEqual(l1, [("A", 0)]) + l2 = self.iter_error(map(pack, "AB", Iter(2), "A", strict=True), ValueError) + self.assertEqual(l2, [("A", 1, "A")]) + l3 = self.iter_error(map(pack, "AB", Iter(2), "ABC", strict=True), ValueError) + self.assertEqual(l3, [("A", 1, "A"), ("B", 0, "B")]) + l4 = self.iter_error(map(pack, "AB", Iter(3), strict=True), ValueError) + self.assertEqual(l4, [("A", 2), ("B", 1)]) + l5 = self.iter_error(map(pack, Iter(1), "AB", strict=True), ValueError) + self.assertEqual(l5, [(0, "A")]) + l6 = self.iter_error(map(pack, Iter(2), "A", strict=True), ValueError) + self.assertEqual(l6, [(1, "A")]) + l7 = self.iter_error(map(pack, Iter(2), "ABC", strict=True), ValueError) + self.assertEqual(l7, [(1, "A"), (0, "B")]) + l8 = self.iter_error(map(pack, Iter(3), "AB", strict=True), ValueError) + self.assertEqual(l8, [(2, "A"), (1, "B")]) + def test_max(self): self.assertEqual(max('123123'), '3') self.assertEqual(max(1, 2, 3), 3) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index f119d89c0ec39a..073df310bb49eb 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -457,6 +457,11 @@ def test_formatmonth(self): calendar.TextCalendar().formatmonth(0, 2), result_0_02_text ) + def test_formatmonth_with_invalid_month(self): + with self.assertRaises(calendar.IllegalMonthError): + calendar.TextCalendar().formatmonth(2017, 13) + with self.assertRaises(calendar.IllegalMonthError): + calendar.TextCalendar().formatmonth(2017, -1) def test_formatmonthname_with_year(self): self.assertEqual( @@ -1121,7 +1126,7 @@ def test__all__(self): not_exported = { 'mdays', 'January', 'February', 'EPOCH', 'different_locale', 'c', 'prweek', 'week', 'format', - 'formatstring', 'main', 'monthlen', 'prevmonth', 'nextmonth'} + 'formatstring', 'main', 'monthlen', 'prevmonth', 'nextmonth', ""} support.check__all__(self, calendar, not_exported=not_exported) @@ -1149,6 +1154,13 @@ def test_formatmonth(self): self.assertIn('class="text-center month"', self.cal.formatmonth(2017, 5)) + def test_formatmonth_with_invalid_month(self): + with self.assertRaises(calendar.IllegalMonthError): + self.cal.formatmonth(2017, 13) + with self.assertRaises(calendar.IllegalMonthError): + self.cal.formatmonth(2017, -1) + + def test_formatweek(self): weeks = self.cal.monthdays2calendar(2017, 5) self.assertIn('class="wed text-nowrap"', self.cal.formatweek(weeks[0])) diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 68e3b2a0d4d932..9d5256b566b8af 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -168,7 +168,7 @@ def test_varargs17_kw(self): print, 0, sep=1, end=2, file=3, flush=4, foo=5) def test_varargs18_kw(self): - # _PyArg_UnpackKeywordsWithVararg() + # _PyArg_UnpackKeywords() with varpos msg = r"invalid keyword argument for print\(\)$" with self.assertRaisesRegex(TypeError, msg): print(0, 1, **{BadStr('foo'): ','}) diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index 85491a89947318..a557e35e68915d 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -747,6 +747,49 @@ def test_codec_stream_writer(self): class CAPICodecErrors(unittest.TestCase): + @classmethod + def _generate_exception_args(cls): + for objlen in range(5): + maxind = 2 * max(2, objlen) + for start in range(-maxind, maxind + 1): + for end in range(-maxind, maxind + 1): + yield objlen, start, end + + @classmethod + def generate_encode_errors(cls): + return tuple( + UnicodeEncodeError('utf-8', '0' * objlen, start, end, 'why') + for objlen, start, end in cls._generate_exception_args() + ) + + @classmethod + def generate_decode_errors(cls): + return tuple( + UnicodeDecodeError('utf-8', b'0' * objlen, start, end, 'why') + for objlen, start, end in cls._generate_exception_args() + ) + + @classmethod + def generate_translate_errors(cls): + return tuple( + UnicodeTranslateError('0' * objlen, start, end, 'why') + for objlen, start, end in cls._generate_exception_args() + ) + + @classmethod + def setUpClass(cls): + cls.unicode_encode_errors = cls.generate_encode_errors() + cls.unicode_decode_errors = cls.generate_decode_errors() + cls.unicode_translate_errors = cls.generate_translate_errors() + cls.all_unicode_errors = ( + cls.unicode_encode_errors + + cls.unicode_decode_errors + + cls.unicode_translate_errors + ) + cls.bad_unicode_errors = ( + ValueError(), + ) + def test_codec_register_error(self): # for cleaning up between tests from _codecs import _unregister_error as _codecs_unregister_error @@ -780,33 +823,82 @@ def test_codec_lookup_error(self): self.assertIs(codec_lookup_error('ignore'), codecs.ignore_errors) self.assertIs(codec_lookup_error('replace'), codecs.replace_errors) self.assertIs(codec_lookup_error('xmlcharrefreplace'), codecs.xmlcharrefreplace_errors) + self.assertIs(codec_lookup_error('backslashreplace'), codecs.backslashreplace_errors) self.assertIs(codec_lookup_error('namereplace'), codecs.namereplace_errors) self.assertRaises(LookupError, codec_lookup_error, 'unknown') - def test_codec_error_handlers(self): - exceptions = [ - # A UnicodeError with an empty message currently crashes: - # See: https://github.com/python/cpython/issues/123378 - # UnicodeEncodeError('bad', '', 0, 1, 'reason'), - UnicodeEncodeError('bad', 'x', 0, 1, 'reason'), - UnicodeEncodeError('bad', 'xyz123', 0, 1, 'reason'), - UnicodeEncodeError('bad', 'xyz123', 1, 4, 'reason'), - ] - - strict_handler = _testcapi.codec_strict_errors + def test_codec_strict_errors_handler(self): + handler = _testcapi.codec_strict_errors + for exc in self.all_unicode_errors + self.bad_unicode_errors: + with self.subTest(handler=handler, exc=exc): + self.assertRaises(type(exc), handler, exc) + + def test_codec_ignore_errors_handler(self): + handler = _testcapi.codec_ignore_errors + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + + def test_codec_replace_errors_handler(self): + handler = _testcapi.codec_replace_errors + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + + def test_codec_xmlcharrefreplace_errors_handler(self): + handler = _testcapi.codec_xmlcharrefreplace_errors + self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) + + def test_codec_backslashreplace_errors_handler(self): + handler = _testcapi.codec_backslashreplace_errors + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + + def test_codec_namereplace_errors_handler(self): + handler = _testlimitedcapi.codec_namereplace_errors + self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) + + def do_test_codec_errors_handler(self, handler, exceptions): + at_least_one = False for exc in exceptions: - with self.subTest(handler=strict_handler, exc=exc): - self.assertRaises(UnicodeEncodeError, strict_handler, exc) - - for handler in [ - _testcapi.codec_ignore_errors, - _testcapi.codec_replace_errors, - _testcapi.codec_xmlcharrefreplace_errors, - _testlimitedcapi.codec_namereplace_errors, - ]: - for exc in exceptions: - with self.subTest(handler=handler, exc=exc): - self.assertIsInstance(handler(exc), tuple) + # See https://github.com/python/cpython/issues/123378 and related + # discussion and issues for details. + if self._exception_may_crash(exc): + continue + + at_least_one = True + with self.subTest(handler=handler, exc=exc): + # test that the handler does not crash + self.assertIsInstance(handler(exc), tuple) + + if exceptions: + self.assertTrue(at_least_one, "all exceptions are crashing") + + for bad_exc in ( + self.bad_unicode_errors + + tuple(e for e in self.all_unicode_errors if e not in exceptions) + ): + with self.subTest('bad type', handler=handler, exc=bad_exc): + self.assertRaises(TypeError, handler, bad_exc) + + @classmethod + def _exception_may_crash(cls, exc): + """Indicate whether a Unicode exception might currently crash + the interpreter when used by a built-in codecs error handler. + + Until gh-123378 is fixed, we skip the tests for these exceptions. + + This should only be used by "do_test_codec_errors_handler". + """ + message, start, end = exc.object, exc.start, exc.end + match exc: + case UnicodeEncodeError(): + return end < start or (end - start) >= len(message) + case UnicodeDecodeError(): + # The case "end - start >= len(message)" does not crash. + return end < start + case UnicodeTranslateError(): + # Test "end <= start" because PyCodec_ReplaceErrors checks + # the Unicode kind of a 0-length string which by convention + # is PyUnicode_1BYTE_KIND and not PyUnicode_2BYTE_KIND as + # the handler currently expects. + return end <= start or (end - start) >= len(message) + return False if __name__ == "__main__": diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py index 71fb9ae45c7c30..77730ad2f32085 100644 --- a/Lib/test/test_capi/test_config.py +++ b/Lib/test/test_capi/test_config.py @@ -100,6 +100,7 @@ def test_config_get(self): options.append(("run_presite", str | None, None)) if sysconfig.get_config_var('Py_GIL_DISABLED'): options.append(("enable_gil", int, None)) + options.append(("tlbc_enabled", int, None)) if support.MS_WINDOWS: options.extend(( ("legacy_windows_stdio", bool, None), diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 925fccd660bde3..a77094588a0edf 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -643,6 +643,51 @@ def test_long_getsign(self): # CRASHES getsign(NULL) + def test_long_ispositive(self): + # Test PyLong_IsPositive() + ispositive = _testcapi.pylong_ispositive + self.assertEqual(ispositive(1), 1) + self.assertEqual(ispositive(123), 1) + self.assertEqual(ispositive(-1), 0) + self.assertEqual(ispositive(0), 0) + self.assertEqual(ispositive(True), 1) + self.assertEqual(ispositive(False), 0) + self.assertEqual(ispositive(IntSubclass(-1)), 0) + self.assertRaises(TypeError, ispositive, 1.0) + self.assertRaises(TypeError, ispositive, Index(123)) + + # CRASHES ispositive(NULL) + + def test_long_isnegative(self): + # Test PyLong_IsNegative() + isnegative = _testcapi.pylong_isnegative + self.assertEqual(isnegative(1), 0) + self.assertEqual(isnegative(123), 0) + self.assertEqual(isnegative(-1), 1) + self.assertEqual(isnegative(0), 0) + self.assertEqual(isnegative(True), 0) + self.assertEqual(isnegative(False), 0) + self.assertEqual(isnegative(IntSubclass(-1)), 1) + self.assertRaises(TypeError, isnegative, 1.0) + self.assertRaises(TypeError, isnegative, Index(123)) + + # CRASHES isnegative(NULL) + + def test_long_iszero(self): + # Test PyLong_IsZero() + iszero = _testcapi.pylong_iszero + self.assertEqual(iszero(1), 0) + self.assertEqual(iszero(-1), 0) + self.assertEqual(iszero(0), 1) + self.assertEqual(iszero(True), 0) + self.assertEqual(iszero(False), 1) + self.assertEqual(iszero(IntSubclass(-1)), 0) + self.assertEqual(iszero(IntSubclass(0)), 1) + self.assertRaises(TypeError, iszero, 1.0) + self.assertRaises(TypeError, iszero, Index(123)) + + # CRASHES iszero(NULL) + def test_long_asint32(self): # Test PyLong_AsInt32() and PyLong_FromInt32() to_int32 = _testlimitedcapi.pylong_asint32 diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py index cc9c9b688f00e2..b0d39937fd865f 100644 --- a/Lib/test/test_capi/test_object.py +++ b/Lib/test/test_capi/test_object.py @@ -1,10 +1,13 @@ import enum import unittest +from test import support from test.support import import_helper from test.support import os_helper +from test.support import threading_helper _testlimitedcapi = import_helper.import_module('_testlimitedcapi') _testcapi = import_helper.import_module('_testcapi') +_testinternalcapi = import_helper.import_module('_testinternalcapi') class Constant(enum.IntEnum): @@ -131,5 +134,41 @@ def test_ClearWeakRefsNoCallbacks_no_weakref_support(self): _testcapi.pyobject_clear_weakrefs_no_callbacks(obj) +@threading_helper.requires_working_threading() +class EnableDeferredRefcountingTest(unittest.TestCase): + """Test PyUnstable_Object_EnableDeferredRefcount""" + @support.requires_resource("cpu") + def test_enable_deferred_refcount(self): + from threading import Thread + + self.assertEqual(_testcapi.pyobject_enable_deferred_refcount("not tracked"), 0) + foo = [] + self.assertEqual(_testcapi.pyobject_enable_deferred_refcount(foo), int(support.Py_GIL_DISABLED)) + + # Make sure reference counting works on foo now + self.assertEqual(foo, []) + if support.Py_GIL_DISABLED: + self.assertTrue(_testinternalcapi.has_deferred_refcount(foo)) + + # Make sure that PyUnstable_Object_EnableDeferredRefcount is thread safe + def silly_func(obj): + self.assertIn( + _testcapi.pyobject_enable_deferred_refcount(obj), + (0, 1) + ) + + silly_list = [1, 2, 3] + threads = [ + Thread(target=silly_func, args=(silly_list,)) for _ in range(4) + ] + + with threading_helper.start_threads(threads): + for i in range(10): + silly_list.append(i) + + if support.Py_GIL_DISABLED: + self.assertTrue(_testinternalcapi.has_deferred_refcount(silly_list)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index f1ab72180d714d..edd83872e573e4 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -7,7 +7,8 @@ import _opcode -from test.support import script_helper, requires_specialization, import_helper +from test.support import (script_helper, requires_specialization, + import_helper, Py_GIL_DISABLED) _testinternalcapi = import_helper.import_module("_testinternalcapi") @@ -34,6 +35,7 @@ def clear_executors(func): @requires_specialization +@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") @unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), "Requires optimizer infrastructure") class TestOptimizerAPI(unittest.TestCase): @@ -60,9 +62,9 @@ def test_get_set_optimizer(self): def test_counter_optimizer(self): # Generate a new function at each call ns = {} - exec(textwrap.dedent(""" + exec(textwrap.dedent(f""" def loop(): - for _ in range(1000): + for _ in range({TIER2_THRESHOLD + 1000}): pass """), ns, ns) loop = ns['loop'] @@ -73,20 +75,19 @@ def loop(): self.assertEqual(opt.get_count(), 0) with clear_executors(loop): loop() - # Subtract because optimizer doesn't kick in sooner - self.assertEqual(opt.get_count(), 1000 - TIER2_THRESHOLD) + self.assertEqual(opt.get_count(), 1001) def test_long_loop(self): "Check that we aren't confused by EXTENDED_ARG" # Generate a new function at each call ns = {} - exec(textwrap.dedent(""" + exec(textwrap.dedent(f""" def nop(): pass def long_loop(): - for _ in range(20): + for _ in range({TIER2_THRESHOLD + 20}): nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); @@ -101,7 +102,7 @@ def long_loop(): with temporary_optimizer(opt): self.assertEqual(opt.get_count(), 0) long_loop() - self.assertEqual(opt.get_count(), 20 - TIER2_THRESHOLD) # Need iterations to warm up + self.assertEqual(opt.get_count(), 21) # Need iterations to warm up def test_code_restore_for_ENTER_EXECUTOR(self): def testfunc(x): @@ -138,6 +139,7 @@ def get_opnames(ex): @requires_specialization +@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") @unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), "Requires optimizer infrastructure") class TestExecutorInvalidation(unittest.TestCase): @@ -156,7 +158,7 @@ def test_invalidate_object(self): func_src = "\n".join( f""" def f{n}(): - for _ in range(1000): + for _ in range({TIER2_THRESHOLD}): pass """ for n in range(5) ) @@ -187,9 +189,9 @@ def f{n}(): def test_uop_optimizer_invalidation(self): # Generate a new function at each call ns = {} - exec(textwrap.dedent(""" + exec(textwrap.dedent(f""" def f(): - for i in range(1000): + for i in range({TIER2_THRESHOLD}): pass """), ns, ns) f = ns['f'] @@ -204,7 +206,7 @@ def f(): def test_sys__clear_internal_caches(self): def f(): - for _ in range(1000): + for _ in range(TIER2_THRESHOLD): pass opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): @@ -219,6 +221,7 @@ def f(): @requires_specialization +@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") @unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), "Requires optimizer infrastructure") @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") @@ -232,7 +235,7 @@ def testfunc(x): opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): - testfunc(1000) + testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -243,7 +246,7 @@ def testfunc(x): def test_extended_arg(self): "Check EXTENDED_ARG handling in superblock creation" ns = {} - exec(textwrap.dedent(""" + exec(textwrap.dedent(f""" def many_vars(): # 260 vars, so z9 should have index 259 a0 = a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = 42 @@ -271,7 +274,7 @@ def many_vars(): w0 = w1 = w2 = w3 = w4 = w5 = w6 = w7 = w8 = w9 = 42 x0 = x1 = x2 = x3 = x4 = x5 = x6 = x7 = x8 = x9 = 42 y0 = y1 = y2 = y3 = y4 = y5 = y6 = y7 = y8 = y9 = 42 - z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = 42 + z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = {TIER2_THRESHOLD} while z9 > 0: z9 = z9 - 1 +z9 @@ -304,7 +307,7 @@ def testfunc(x): opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): - testfunc(20) + testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -319,7 +322,7 @@ def testfunc(n): opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): - testfunc(20) + testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -334,7 +337,7 @@ def testfunc(a): opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): - testfunc(range(20)) + testfunc(range(TIER2_THRESHOLD)) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -351,7 +354,7 @@ def testfunc(a): opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): - testfunc(range(20)) + testfunc(range(TIER2_THRESHOLD)) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -367,7 +370,7 @@ def testfunc(n): opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): - testfunc(20) + testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -382,7 +385,7 @@ def testfunc(n): opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): - testfunc(20) + testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -402,7 +405,7 @@ def testfunc(n): opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): - testfunc(20) + testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -420,8 +423,8 @@ def testfunc(n): opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): - total = testfunc(20) - self.assertEqual(total, 190) + total = testfunc(TIER2_THRESHOLD) + self.assertEqual(total, sum(range(TIER2_THRESHOLD))) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -441,9 +444,9 @@ def testfunc(a): opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): - a = list(range(20)) + a = list(range(TIER2_THRESHOLD)) total = testfunc(a) - self.assertEqual(total, 190) + self.assertEqual(total, sum(a)) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -463,9 +466,9 @@ def testfunc(a): opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): - a = tuple(range(20)) + a = tuple(range(TIER2_THRESHOLD)) total = testfunc(a) - self.assertEqual(total, 190) + self.assertEqual(total, sum(a)) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -499,7 +502,7 @@ def dummy(x): opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): - testfunc(20) + testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -517,7 +520,7 @@ def testfunc(n): opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): - testfunc(20) + testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -545,9 +548,9 @@ def testfunc(n, m): opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): - x = testfunc(10, 10) + x = testfunc(TIER2_THRESHOLD, TIER2_THRESHOLD) - self.assertEqual(x, sum(range(10)) * 10010) + self.assertEqual(x, sum(range(TIER2_THRESHOLD)) * TIER2_THRESHOLD * 1001) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -568,15 +571,13 @@ def testfunc(n): bits += 1 if i&0x10: bits += 1 - if i&0x20: - bits += 1 return bits opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): - x = testfunc(20) + x = testfunc(TIER2_THRESHOLD * 2) - self.assertEqual(x, 40) + self.assertEqual(x, TIER2_THRESHOLD * 5) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) ops = list(iter_opnames(ex)) @@ -586,6 +587,7 @@ def testfunc(n): @requires_specialization +@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") @unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), "Requires optimizer infrastructure") @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") @@ -610,9 +612,9 @@ def testfunc(loops): num += 1 return a - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertIsNotNone(ex) - self.assertEqual(res, 63) + self.assertEqual(res, (TIER2_THRESHOLD - 1) * 2 + 1) binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] self.assertGreaterEqual(len(binop_count), 3) @@ -632,11 +634,11 @@ def testfunc(loops): opt = _testinternalcapi.new_uop_optimizer() res = None with temporary_optimizer(opt): - res = testfunc(32) + res = testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - self.assertEqual(res, 124) + self.assertEqual(res, (TIER2_THRESHOLD - 1) * 4) binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] self.assertGreaterEqual(len(binop_count), 3) @@ -656,11 +658,11 @@ def testfunc(loops): opt = _testinternalcapi.new_uop_optimizer() res = None with temporary_optimizer(opt): - res = testfunc(32) + res = testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - self.assertEqual(res, 124) + self.assertEqual(res, (TIER2_THRESHOLD - 1) * 4) binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] self.assertGreaterEqual(len(binop_count), 3) @@ -677,7 +679,7 @@ def testfunc(loops): num += 1 return a - res, ex = self._run_with_optimizer(testfunc, 64) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertIsNotNone(ex) binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] self.assertGreaterEqual(len(binop_count), 3) @@ -689,7 +691,7 @@ def dummy(x): for i in range(n): dummy(i) - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertIn("_PUSH_FRAME", uops) @@ -703,8 +705,8 @@ def testfunc(n): x = i + i return x - res, ex = self._run_with_optimizer(testfunc, 32) - self.assertEqual(res, 62) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, (TIER2_THRESHOLD - 1) * 2) self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertNotIn("_GUARD_BOTH_INT", uops) @@ -721,7 +723,7 @@ def testfunc(n): res = x + z + a + b return res - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertEqual(res, 4) self.assertIsNotNone(ex) uops = get_opnames(ex) @@ -734,8 +736,8 @@ def testfunc(n): for _ in range(n): return [i for i in range(n)] - res, ex = self._run_with_optimizer(testfunc, 32) - self.assertEqual(res, list(range(32))) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, list(range(TIER2_THRESHOLD))) self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertNotIn("_BINARY_OP_ADD_INT", uops) @@ -792,7 +794,7 @@ def testfunc(n): opt = _testinternalcapi.new_uop_optimizer() _testinternalcapi.set_optimizer(opt) - testfunc(64) + testfunc(_testinternalcapi.TIER2_THRESHOLD) ex = get_first_executor(testfunc) assert ex is not None @@ -812,8 +814,8 @@ def testfunc(n): a = a + 0.25 return a - res, ex = self._run_with_optimizer(testfunc, 32) - self.assertAlmostEqual(res, 33.0) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertAlmostEqual(res, TIER2_THRESHOLD + 1) self.assertIsNotNone(ex) uops = get_opnames(ex) guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] @@ -832,8 +834,8 @@ def testfunc(n): a = a - 0.25 return a - res, ex = self._run_with_optimizer(testfunc, 32) - self.assertAlmostEqual(res, -31.0) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertAlmostEqual(res, -TIER2_THRESHOLD + 1) self.assertIsNotNone(ex) uops = get_opnames(ex) guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] @@ -852,7 +854,7 @@ def testfunc(n): a = a * 1.0 return a - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertAlmostEqual(res, 1.0) self.assertIsNotNone(ex) uops = get_opnames(ex) @@ -872,7 +874,7 @@ def testfunc(n): a + a return a - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertEqual(res, "") self.assertIsNotNone(ex) uops = get_opnames(ex) @@ -890,7 +892,7 @@ def testfunc(n): x = a == a return x - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertTrue(res) self.assertIsNotNone(ex) uops = get_opnames(ex) @@ -908,7 +910,7 @@ def testfunc(n): x = a == a return x - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertTrue(res) self.assertIsNotNone(ex) uops = get_opnames(ex) @@ -926,7 +928,7 @@ def testfunc(n): x = 1 return x - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertEqual(res, 1) self.assertIsNotNone(ex) uops = get_opnames(ex) @@ -946,7 +948,7 @@ def testfunc(n): x = 1 return x - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertEqual(res, 1) self.assertIsNotNone(ex) uops = get_opnames(ex) @@ -966,7 +968,7 @@ def testfunc(n): x = a == a return x - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertTrue(res) self.assertIsNotNone(ex) uops = get_opnames(ex) @@ -984,10 +986,10 @@ def testfunc(n): exec(src, ns, ns) testfunc = ns['testfunc'] ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) self.assertIsNone(ex) ns['_test_global'] = 1 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertNotIn("_GUARD_BOTH_INT", uops) @@ -998,10 +1000,10 @@ def testfunc(n): exec(src, ns, ns) testfunc = ns['testfunc'] ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) self.assertIsNone(ex) ns['_test_global'] = 3.14 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) self.assertIsNone(ex) def test_combine_stack_space_checks_sequential(self): @@ -1018,8 +1020,8 @@ def testfunc(n): a += b + c + d return a - res, ex = self._run_with_optimizer(testfunc, 32) - self.assertEqual(res, 832) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 26) self.assertIsNotNone(ex) uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] @@ -1045,8 +1047,8 @@ def testfunc(n): a += b + c return a - res, ex = self._run_with_optimizer(testfunc, 32) - self.assertEqual(res, 224) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 7) self.assertIsNotNone(ex) uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] @@ -1080,8 +1082,8 @@ def testfunc(n): a += b + c + d + e return a - res, ex = self._run_with_optimizer(testfunc, 32) - self.assertEqual(res, 800) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 25) self.assertIsNotNone(ex) uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] @@ -1116,8 +1118,8 @@ def testfunc(n): a += b + c + d + e return a - res, ex = self._run_with_optimizer(testfunc, 32) - self.assertEqual(res, 800) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 25) self.assertIsNotNone(ex) uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] @@ -1152,16 +1154,16 @@ def dummy6(x): z = dummy0(y) return dummy4(z) def testfunc(n): - a = 0; - for _ in range(32): + a = 0 + for _ in range(n): b = dummy5(1) c = dummy0(1) d = dummy6(1) a += b + c + d return a - res, ex = self._run_with_optimizer(testfunc, 32) - self.assertEqual(res, 96) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 3) self.assertIsNotNone(ex) uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] @@ -1220,8 +1222,8 @@ def testfunc(n): b += dummy15(7) return b - res, ex = self._run_with_optimizer(testfunc, 32) - self.assertEqual(res, 32 * (repetitions + 9)) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * (repetitions + 9)) self.assertIsNotNone(ex) uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] @@ -1254,8 +1256,13 @@ def testfunc(n): a += dummy15(n) return a - res, ex = self._run_with_optimizer(testfunc, 32) - self.assertEqual(res, 42 * 32) + recursion_limit = sys.getrecursionlimit() + try: + sys.setrecursionlimit(TIER2_THRESHOLD + recursion_limit) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + finally: + sys.setrecursionlimit(recursion_limit) + self.assertEqual(res, TIER2_THRESHOLD * 42) self.assertIsNotNone(ex) uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] @@ -1302,8 +1309,8 @@ def testfunc(n): for i in range(n): gen() return i - res, ex = self._run_with_optimizer(testfunc, 20) - self.assertEqual(res, 19) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD - 1) self.assertIsNotNone(ex) self.assertIn("_RETURN_GENERATOR", get_opnames(ex)) @@ -1317,8 +1324,8 @@ def testfunc(n): for i in g: s += i return s - res, ex = self._run_with_optimizer(testfunc, 20) - self.assertEqual(res, 190) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, sum(range(TIER2_THRESHOLD))) self.assertIsNotNone(ex) self.assertIn("_FOR_ITER_GEN_FRAME", get_opnames(ex)) @@ -1337,7 +1344,7 @@ def test_modified_local_is_seen_by_optimized_code(self): def test_guard_type_version_removed(self): def thing(a): x = 0 - for _ in range(100): + for _ in range(TIER2_THRESHOLD): x += a.attr x += a.attr return x @@ -1348,7 +1355,7 @@ class Foo: res, ex = self._run_with_optimizer(thing, Foo()) opnames = list(iter_opnames(ex)) self.assertIsNotNone(ex) - self.assertEqual(res, 200) + self.assertEqual(res, TIER2_THRESHOLD * 2) guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") self.assertEqual(guard_type_version_count, 1) @@ -1362,7 +1369,7 @@ def fn(): def thing(a): x = 0 - for _ in range(100): + for _ in range(TIER2_THRESHOLD): x += a.attr fn() x += a.attr @@ -1374,7 +1381,7 @@ class Foo: res, ex = self._run_with_optimizer(thing, Foo()) opnames = list(iter_opnames(ex)) self.assertIsNotNone(ex) - self.assertEqual(res, 200) + self.assertEqual(res, TIER2_THRESHOLD * 2) guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") self.assertEqual(guard_type_version_count, 1) @@ -1385,13 +1392,13 @@ def test_guard_type_version_not_removed(self): def thing(a): x = 0 - for i in range(100): + for i in range(TIER2_THRESHOLD + 100): x += a.attr - # for the first 90 iterations we set the attribute on this dummy function which shouldn't + # for the first (TIER2_THRESHOLD + 90) iterations we set the attribute on this dummy function which shouldn't # trigger the type watcher - # then after 90 it should trigger it and stop optimizing + # then after for the next 10 it should trigger it and stop optimizing # Note that the code needs to be in this weird form so it's optimized inline without any control flow - setattr((Foo, Bar)[i < 90], "attr", 2) + setattr((Foo, Bar)[i < TIER2_THRESHOLD + 90], "attr", 2) x += a.attr return x @@ -1405,7 +1412,7 @@ class Bar: opnames = list(iter_opnames(ex)) self.assertIsNotNone(ex) - self.assertEqual(res, 219) + self.assertEqual(res, (TIER2_THRESHOLD * 2) + 219) guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") self.assertEqual(guard_type_version_count, 2) @@ -1444,7 +1451,7 @@ def test_guard_type_version_executor_invalidated(self): def thing(a): x = 0 - for i in range(100): + for i in range(TIER2_THRESHOLD): x += a.attr x += a.attr return x @@ -1453,7 +1460,7 @@ class Foo: attr = 1 res, ex = self._run_with_optimizer(thing, Foo()) - self.assertEqual(res, 200) + self.assertEqual(res, TIER2_THRESHOLD * 2) self.assertIsNotNone(ex) self.assertEqual(list(iter_opnames(ex)).count("_GUARD_TYPE_VERSION"), 1) self.assertTrue(ex.is_valid()) @@ -1481,6 +1488,38 @@ def fn(a): fn(A()) + def test_func_guards_removed_or_reduced(self): + def testfunc(n): + for i in range(n): + # Only works on functions promoted to constants + global_identity(i) + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_PUSH_FRAME", uops) + # Strength reduced version + self.assertIn("_CHECK_FUNCTION_VERSION_INLINE", uops) + self.assertNotIn("_CHECK_FUNCTION_VERSION", uops) + # Removed guard + self.assertNotIn("_CHECK_FUNCTION_EXACT_ARGS", uops) + + def test_jit_error_pops(self): + """ + Tests that the correct number of pops are inserted into the + exit stub + """ + items = 17 * [None] + [[]] + with self.assertRaises(TypeError): + {item for item in items} + + +def global_identity(x): + return x if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index d492ea1d76aa3a..11054963b6ff03 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -33,6 +33,15 @@ from libclinic.cli import parse_file, Clinic +def repeat_fn(*functions): + def wrapper(test): + def wrapped(self): + for fn in functions: + with self.subTest(fn=fn): + test(self, fn) + return wrapped + return wrapper + def _make_clinic(*, filename='clinic_tests', limited_capi=False): clang = CLanguage(filename) c = Clinic(clang, filename=filename, limited_capi=limited_capi) @@ -329,7 +338,7 @@ def test_star_after_vararg(self): my_test_func pos_arg: object - *args: object + *args: tuple * kw_arg: object [clinic start generated code]*/ @@ -344,7 +353,7 @@ def test_vararg_after_star(self): pos_arg: object * - *args: object + *args: tuple kw_arg: object [clinic start generated code]*/ """ @@ -1807,7 +1816,7 @@ def test_parameters_required_after_depr_star3(self): foo.bar a: int * [from 3.14] - *args: object + *args: tuple b: int Docstring. """ @@ -1835,7 +1844,7 @@ def test_depr_star_must_come_before_vararg(self): module foo foo.bar a: int - *args: object + *args: tuple * [from 3.14] b: int Docstring. @@ -1971,7 +1980,7 @@ def test_slash_after_vararg(self): foo.bar x: int y: int - *args: object + *args: tuple z: int / """ @@ -2022,7 +2031,7 @@ def test_vararg_must_come_after_depr_slash(self): module foo foo.bar a: int - *args: object + *args: tuple / [from 3.14] b: int Docstring. @@ -2061,8 +2070,8 @@ def test_parameters_no_more_than_one_vararg(self): block = """ module foo foo.bar - *vararg1: object - *vararg2: object + *vararg1: tuple + *vararg2: tuple """ self.expect_failure(block, err, lineno=3) @@ -2147,11 +2156,11 @@ def test_indent_stack_no_tabs(self): block = """ module foo foo.bar - *vararg1: object - \t*vararg2: object + *vararg1: tuple + \t*vararg2: tuple """ err = ("Tab characters are illegal in the Clinic DSL: " - r"'\t*vararg2: object'") + r"'\t*vararg2: tuple'") self.expect_failure(block, err) def test_indent_stack_illegal_outdent(self): @@ -2496,7 +2505,7 @@ def test_vararg_cannot_take_default_value(self): err = "Vararg can't take a default value!" block = """ fn - *args: object = None + *args: tuple = None """ self.expect_failure(block, err, lineno=1) @@ -2694,7 +2703,7 @@ def test_cli_force(self): # Verify by checking the checksum. checksum = ( "/*[clinic end generated code: " - "output=0acbef4794cb933e input=9543a8d2da235301]*/\n" + "output=00512eb783a9b748 input=9543a8d2da235301]*/\n" ) with open(fn, encoding='utf-8') as f: generated = f.read() @@ -3378,30 +3387,53 @@ def test_keyword_only_parameter(self): ac_tester.keyword_only_parameter(1) self.assertEqual(ac_tester.keyword_only_parameter(a=1), (1,)) - def test_varpos(self): - # fn(*args) - fn = ac_tester.varpos - self.assertEqual(fn(), ((),)) - self.assertEqual(fn(1, 2), ((1, 2),)) - - def test_posonly_varpos(self): - # fn(a, b, /, *args) - fn = ac_tester.posonly_varpos - self.assertRaises(TypeError, fn) - self.assertRaises(TypeError, fn, 1) - self.assertRaises(TypeError, fn, 1, b=2) - self.assertEqual(fn(1, 2), (1, 2, ())) - self.assertEqual(fn(1, 2, 3, 4), (1, 2, (3, 4))) - - def test_posonly_poskw_varpos(self): - # fn(a, /, b, *args) - fn = ac_tester.posonly_poskw_varpos - self.assertRaises(TypeError, fn) - self.assertEqual(fn(1, 2), (1, 2, ())) - self.assertEqual(fn(1, b=2), (1, 2, ())) - self.assertEqual(fn(1, 2, 3, 4), (1, 2, (3, 4))) - self.assertRaises(TypeError, fn, b=4) - self.assertRaises(TypeError, fn, 1, 2, 3, b=4) + if ac_tester is not None: + @repeat_fn(ac_tester.varpos, + ac_tester.varpos_array, + ac_tester.TestClass.varpos_no_fastcall, + ac_tester.TestClass.varpos_array_no_fastcall) + def test_varpos(self, fn): + # fn(*args) + self.assertEqual(fn(), ()) + self.assertEqual(fn(1, 2), (1, 2)) + + @repeat_fn(ac_tester.posonly_varpos, + ac_tester.posonly_varpos_array, + ac_tester.TestClass.posonly_varpos_no_fastcall, + ac_tester.TestClass.posonly_varpos_array_no_fastcall) + def test_posonly_varpos(self, fn): + # fn(a, b, /, *args) + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, 1) + self.assertRaises(TypeError, fn, 1, b=2) + self.assertEqual(fn(1, 2), (1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, 2, (3, 4))) + + @repeat_fn(ac_tester.posonly_req_opt_varpos, + ac_tester.posonly_req_opt_varpos_array, + ac_tester.TestClass.posonly_req_opt_varpos_no_fastcall, + ac_tester.TestClass.posonly_req_opt_varpos_array_no_fastcall) + def test_posonly_req_opt_varpos(self, fn): + # fn(a, b=False, /, *args) + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, a=1) + self.assertEqual(fn(1), (1, False, ())) + self.assertEqual(fn(1, 2), (1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, 2, (3, 4))) + + @repeat_fn(ac_tester.posonly_poskw_varpos, + ac_tester.posonly_poskw_varpos_array, + ac_tester.TestClass.posonly_poskw_varpos_no_fastcall, + ac_tester.TestClass.posonly_poskw_varpos_array_no_fastcall) + def test_posonly_poskw_varpos(self, fn): + # fn(a, /, b, *args) + self.assertRaises(TypeError, fn) + self.assertEqual(fn(1, 2), (1, 2, ())) + self.assertEqual(fn(1, b=2), (1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, 2, (3, 4))) + self.assertRaises(TypeError, fn, b=4) + errmsg = re.escape("given by name ('b') and position (2)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, 2, 3, b=4) def test_poskw_varpos(self): # fn(a, *args) @@ -3409,7 +3441,8 @@ def test_poskw_varpos(self): self.assertRaises(TypeError, fn) self.assertRaises(TypeError, fn, 1, b=2) self.assertEqual(fn(a=1), (1, ())) - self.assertRaises(TypeError, fn, 1, a=2) + errmsg = re.escape("given by name ('a') and position (1)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, a=2) self.assertEqual(fn(1), (1, ())) self.assertEqual(fn(1, 2, 3, 4), (1, (2, 3, 4))) @@ -3417,7 +3450,8 @@ def test_poskw_varpos_kwonly_opt(self): # fn(a, *args, b=False) fn = ac_tester.poskw_varpos_kwonly_opt self.assertRaises(TypeError, fn) - self.assertRaises(TypeError, fn, 1, a=2) + errmsg = re.escape("given by name ('a') and position (1)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, a=2) self.assertEqual(fn(1, b=2), (1, (), True)) self.assertEqual(fn(1, 2, 3, 4), (1, (2, 3, 4), False)) self.assertEqual(fn(1, 2, 3, 4, b=5), (1, (2, 3, 4), True)) @@ -3428,7 +3462,8 @@ def test_poskw_varpos_kwonly_opt2(self): # fn(a, *args, b=False, c=False) fn = ac_tester.poskw_varpos_kwonly_opt2 self.assertRaises(TypeError, fn) - self.assertRaises(TypeError, fn, 1, a=2) + errmsg = re.escape("given by name ('a') and position (1)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, a=2) self.assertEqual(fn(1, b=2), (1, (), 2, False)) self.assertEqual(fn(1, b=2, c=3), (1, (), 2, 3)) self.assertEqual(fn(1, 2, 3), (1, (2, 3), False, False)) @@ -3490,9 +3525,10 @@ def test_null_or_tuple_for_varargs(self): self.assertEqual(fn(covariant=True, name='a'), ('a', (), True)) self.assertRaises(TypeError, fn, covariant=True) - self.assertRaises(TypeError, fn, 1, name='a') - self.assertRaises(TypeError, fn, 1, 2, 3, name='a', covariant=True) - self.assertRaises(TypeError, fn, 1, 2, 3, covariant=True, name='a') + errmsg = re.escape("given by name ('name') and position (1)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, name='a') + self.assertRaisesRegex(TypeError, errmsg, fn, 1, 2, 3, name='a', covariant=True) + self.assertRaisesRegex(TypeError, errmsg, fn, 1, 2, 3, covariant=True, name='a') def test_cloned_func_exception_message(self): incorrect_arg = -1 # f1() and f2() accept a single str @@ -3568,14 +3604,15 @@ def test_defclass_posonly_varpos(self): cls = ac_tester.TestClass obj = cls() fn = obj.defclass_posonly_varpos - self.assertRaises(TypeError, fn) - self.assertRaises(TypeError, fn, 1) + errmsg = 'takes at least 2 positional arguments' + self.assertRaisesRegex(TypeError, errmsg, fn) + self.assertRaisesRegex(TypeError, errmsg, fn, 1) self.assertEqual(fn(1, 2), (cls, 1, 2, ())) self.assertEqual(fn(1, 2, 3, 4), (cls, 1, 2, (3, 4))) fn = cls.defclass_posonly_varpos self.assertRaises(TypeError, fn) - self.assertRaises(TypeError, fn, obj) - self.assertRaises(TypeError, fn, obj, 1) + self.assertRaisesRegex(TypeError, errmsg, fn, obj) + self.assertRaisesRegex(TypeError, errmsg, fn, obj, 1) self.assertEqual(fn(obj, 1, 2), (cls, 1, 2, ())) self.assertEqual(fn(obj, 1, 2, 3, 4), (cls, 1, 2, (3, 4))) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 35725718152c56..634efda354407f 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -12,6 +12,7 @@ from test import support from test.support import os_helper from test.support import force_not_colorized +from test.support import threading_helper from test.support.script_helper import ( spawn_python, kill_python, assert_python_ok, assert_python_failure, interpreter_requires_environment @@ -924,7 +925,7 @@ def test_python_gil(self): self.assertEqual(proc.stderr, '') def test_python_asyncio_debug(self): - code = "import asyncio; print(asyncio.get_event_loop().get_debug())" + code = "import asyncio; print(asyncio.new_event_loop().get_debug())" rc, out, err = assert_python_ok('-c', code, PYTHONASYNCIODEBUG='1') self.assertIn(b'True', out) @@ -1068,6 +1069,57 @@ def res2int(self, res): out = res.out.strip().decode("utf-8") return tuple(int(i) for i in out.split()) + @unittest.skipUnless(support.Py_GIL_DISABLED, + "PYTHON_TLBC and -X tlbc" + " only supported in Py_GIL_DISABLED builds") + @threading_helper.requires_working_threading() + def test_disable_thread_local_bytecode(self): + code = """if 1: + import threading + def test(x, y): + return x + y + t = threading.Thread(target=test, args=(1,2)) + t.start() + t.join()""" + assert_python_ok("-W", "always", "-X", "tlbc=0", "-c", code) + assert_python_ok("-W", "always", "-c", code, PYTHON_TLBC="0") + + @unittest.skipUnless(support.Py_GIL_DISABLED, + "PYTHON_TLBC and -X tlbc" + " only supported in Py_GIL_DISABLED builds") + @threading_helper.requires_working_threading() + def test_enable_thread_local_bytecode(self): + code = """if 1: + import threading + def test(x, y): + return x + y + t = threading.Thread(target=test, args=(1,2)) + t.start() + t.join()""" + # The functionality of thread-local bytecode is tested more extensively + # in test_thread_local_bytecode + assert_python_ok("-W", "always", "-X", "tlbc=1", "-c", code) + assert_python_ok("-W", "always", "-c", code, PYTHON_TLBC="1") + + @unittest.skipUnless(support.Py_GIL_DISABLED, + "PYTHON_TLBC and -X tlbc" + " only supported in Py_GIL_DISABLED builds") + def test_invalid_thread_local_bytecode(self): + rc, out, err = assert_python_failure("-X", "tlbc") + self.assertIn(b"tlbc=n: n is missing or invalid", err) + rc, out, err = assert_python_failure("-X", "tlbc=foo") + self.assertIn(b"tlbc=n: n is missing or invalid", err) + rc, out, err = assert_python_failure("-X", "tlbc=-1") + self.assertIn(b"tlbc=n: n is missing or invalid", err) + rc, out, err = assert_python_failure("-X", "tlbc=2") + self.assertIn(b"tlbc=n: n is missing or invalid", err) + rc, out, err = assert_python_failure(PYTHON_TLBC="foo") + self.assertIn(b"PYTHON_TLBC=N: N is missing or invalid", err) + rc, out, err = assert_python_failure(PYTHON_TLBC="-1") + self.assertIn(b"PYTHON_TLBC=N: N is missing or invalid", err) + rc, out, err = assert_python_failure(PYTHON_TLBC="2") + self.assertIn(b"PYTHON_TLBC=N: N is missing or invalid", err) + @unittest.skipIf(interpreter_requires_environment(), 'Cannot run -I tests when PYTHON env vars are required.') diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 93c65a82508dcb..2a1b26e8a1ffd1 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -178,6 +178,20 @@ nlocals: 3 flags: 3 consts: ("'hello'", "'world'") + +>>> class class_with_docstring: +... '''This is a docstring for class''' +... '''This line is not docstring''' +... pass + +>>> print(class_with_docstring.__doc__) +This is a docstring for class + +>>> class class_without_docstring: +... pass + +>>> print(class_without_docstring.__doc__) +None """ import copy @@ -854,6 +868,33 @@ def f(): 3 * [(42, 42, None, None)], ) + @cpython_only + def test_docstring_under_o2(self): + code = textwrap.dedent(''' + def has_docstring(x, y): + """This is a first-line doc string""" + """This is a second-line doc string""" + a = x + y + b = x - y + return a, b + + + def no_docstring(x): + def g(y): + return x + y + return g + + + async def async_func(): + """asynf function doc string""" + pass + + + for func in [has_docstring, no_docstring(4), async_func]: + assert(func.__doc__ is None) + ''') + + rc, out, err = assert_python_ok('-OO', '-c', code) if check_impl_detail(cpython=True) and ctypes is not None: py = ctypes.pythonapi diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 290656f070503a..e51f7e0ee12b1f 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3256,7 +3256,11 @@ def test_code_page_name(self): codecs.code_page_decode, self.CP_UTF8, b'\xff', 'strict', True) def check_decode(self, cp, tests): - for raw, errors, expected in tests: + for raw, errors, expected, *rest in tests: + if rest: + altexpected, = rest + else: + altexpected = expected if expected is not None: try: decoded = codecs.code_page_decode(cp, raw, errors, True) @@ -3273,8 +3277,21 @@ def check_decode(self, cp, tests): self.assertRaises(UnicodeDecodeError, codecs.code_page_decode, cp, raw, errors, True) + if altexpected is not None: + decoded = raw.decode(f'cp{cp}', errors) + self.assertEqual(decoded, altexpected, + '%a.decode("cp%s", %r)=%a != %a' + % (raw, cp, errors, decoded, altexpected)) + else: + self.assertRaises(UnicodeDecodeError, + raw.decode, f'cp{cp}', errors) + def check_encode(self, cp, tests): - for text, errors, expected in tests: + for text, errors, expected, *rest in tests: + if rest: + altexpected, = rest + else: + altexpected = expected if expected is not None: try: encoded = codecs.code_page_encode(cp, text, errors) @@ -3285,18 +3302,26 @@ def check_encode(self, cp, tests): '%a.encode("cp%s", %r)=%a != %a' % (text, cp, errors, encoded[0], expected)) self.assertEqual(encoded[1], len(text)) + + encoded = text.encode(f'cp{cp}', errors) + self.assertEqual(encoded, altexpected, + '%a.encode("cp%s", %r)=%a != %a' + % (text, cp, errors, encoded, altexpected)) else: self.assertRaises(UnicodeEncodeError, codecs.code_page_encode, cp, text, errors) + self.assertRaises(UnicodeEncodeError, + text.encode, f'cp{cp}', errors) def test_cp932(self): self.check_encode(932, ( ('abc', 'strict', b'abc'), ('\uff44\u9a3e', 'strict', b'\x82\x84\xe9\x80'), + ('\uf8f3', 'strict', b'\xff'), # test error handlers ('\xff', 'strict', None), ('[\xff]', 'ignore', b'[]'), - ('[\xff]', 'replace', b'[y]'), + ('[\xff]', 'replace', b'[y]', b'[?]'), ('[\u20ac]', 'replace', b'[?]'), ('[\xff]', 'backslashreplace', b'[\\xff]'), ('[\xff]', 'namereplace', @@ -3310,12 +3335,12 @@ def test_cp932(self): (b'abc', 'strict', 'abc'), (b'\x82\x84\xe9\x80', 'strict', '\uff44\u9a3e'), # invalid bytes - (b'[\xff]', 'strict', None), - (b'[\xff]', 'ignore', '[]'), - (b'[\xff]', 'replace', '[\ufffd]'), - (b'[\xff]', 'backslashreplace', '[\\xff]'), - (b'[\xff]', 'surrogateescape', '[\udcff]'), - (b'[\xff]', 'surrogatepass', None), + (b'[\xff]', 'strict', None, '[\uf8f3]'), + (b'[\xff]', 'ignore', '[]', '[\uf8f3]'), + (b'[\xff]', 'replace', '[\ufffd]', '[\uf8f3]'), + (b'[\xff]', 'backslashreplace', '[\\xff]', '[\uf8f3]'), + (b'[\xff]', 'surrogateescape', '[\udcff]', '[\uf8f3]'), + (b'[\xff]', 'surrogatepass', None, '[\uf8f3]'), (b'\x81\x00abc', 'strict', None), (b'\x81\x00abc', 'ignore', '\x00abc'), (b'\x81\x00abc', 'replace', '\ufffd\x00abc'), @@ -3330,7 +3355,7 @@ def test_cp1252(self): # test error handlers ('\u0141', 'strict', None), ('\u0141', 'ignore', b''), - ('\u0141', 'replace', b'L'), + ('\u0141', 'replace', b'L', b'?'), ('\udc98', 'surrogateescape', b'\x98'), ('\udc98', 'surrogatepass', None), )) @@ -3340,6 +3365,59 @@ def test_cp1252(self): (b'\xff', 'strict', '\xff'), )) + def test_cp708(self): + self.check_encode(708, ( + ('abc2%', 'strict', b'abc2%'), + ('\u060c\u0621\u064a', 'strict', b'\xac\xc1\xea'), + ('\u2562\xe7\xa0', 'strict', b'\x86\x87\xff'), + ('\x9a\x9f', 'strict', b'\x9a\x9f'), + ('\u256b', 'strict', b'\xc0'), + # test error handlers + ('[\u0662]', 'strict', None), + ('[\u0662]', 'ignore', b'[]'), + ('[\u0662]', 'replace', b'[?]'), + ('\udca0', 'surrogateescape', b'\xa0'), + ('\udca0', 'surrogatepass', None), + )) + self.check_decode(708, ( + (b'abc2%', 'strict', 'abc2%'), + (b'\xac\xc1\xea', 'strict', '\u060c\u0621\u064a'), + (b'\x86\x87\xff', 'strict', '\u2562\xe7\xa0'), + (b'\x9a\x9f', 'strict', '\x9a\x9f'), + (b'\xc0', 'strict', '\u256b'), + # test error handlers + (b'\xa0', 'strict', None), + (b'[\xa0]', 'ignore', '[]'), + (b'[\xa0]', 'replace', '[\ufffd]'), + (b'[\xa0]', 'backslashreplace', '[\\xa0]'), + (b'[\xa0]', 'surrogateescape', '[\udca0]'), + (b'[\xa0]', 'surrogatepass', None), + )) + + def test_cp20106(self): + self.check_encode(20106, ( + ('abc', 'strict', b'abc'), + ('\xa7\xc4\xdf', 'strict', b'@[~'), + # test error handlers + ('@', 'strict', None), + ('@', 'ignore', b''), + ('@', 'replace', b'?'), + ('\udcbf', 'surrogateescape', b'\xbf'), + ('\udcbf', 'surrogatepass', None), + )) + self.check_decode(20106, ( + (b'abc', 'strict', 'abc'), + (b'@[~', 'strict', '\xa7\xc4\xdf'), + (b'\xe1\xfe', 'strict', 'a\xdf'), + # test error handlers + (b'(\xbf)', 'strict', None), + (b'(\xbf)', 'ignore', '()'), + (b'(\xbf)', 'replace', '(\ufffd)'), + (b'(\xbf)', 'backslashreplace', '(\\xbf)'), + (b'(\xbf)', 'surrogateescape', '(\udcbf)'), + (b'(\xbf)', 'surrogatepass', None), + )) + def test_cp_utf7(self): cp = 65000 self.check_encode(cp, ( @@ -3412,17 +3490,15 @@ def test_incremental(self): False) self.assertEqual(decoded, ('abc', 3)) - def test_mbcs_alias(self): - # Check that looking up our 'default' codepage will return - # mbcs when we don't have a more specific one available - code_page = 99_999 - name = f'cp{code_page}' - with mock.patch('_winapi.GetACP', return_value=code_page): - try: - codec = codecs.lookup(name) - self.assertEqual(codec.name, 'mbcs') - finally: - codecs.unregister(name) + def test_mbcs_code_page(self): + # Check that codec for the current Windows (ANSII) code page is + # always available. + try: + from _winapi import GetACP + except ImportError: + self.skipTest('requires _winapi.GetACP') + cp = GetACP() + codecs.lookup(f'cp{cp}') @support.bigmemtest(size=2**31, memuse=7, dry_run=False) def test_large_input(self, size): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 85ae71c1f77b28..f7ea923ef17672 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -2,6 +2,7 @@ import dis import io import itertools +import marshal import math import opcode import os @@ -341,6 +342,10 @@ def test_lambda_doc(self): l = lambda: "foo" self.assertIsNone(l.__doc__) + def test_lambda_consts(self): + l = lambda: "this is the only const" + self.assertEqual(l.__code__.co_consts, ("this is the only const",)) + def test_encoding(self): code = b'# -*- coding: badencoding -*-\npass\n' self.assertRaises(SyntaxError, compile, code, 'tmp', 'exec') @@ -789,10 +794,10 @@ def check_same_constant(const): # Merge constants in tuple or frozenset f1, f2 = lambda: "not a name", lambda: ("not a name",) f3 = lambda x: x in {("not a name",)} - self.assertIs(f1.__code__.co_consts[1], - f2.__code__.co_consts[1][0]) - self.assertIs(next(iter(f3.__code__.co_consts[1])), - f2.__code__.co_consts[1]) + self.assertIs(f1.__code__.co_consts[0], + f2.__code__.co_consts[0][0]) + self.assertIs(next(iter(f3.__code__.co_consts[0])), + f2.__code__.co_consts[0]) # {0} is converted to a constant frozenset({0}) by the peephole # optimizer @@ -901,6 +906,9 @@ def with_fstring(): def with_const_expression(): "also" + " not docstring" + + def multiple_const_strings(): + "not docstring " * 3 """) for opt in [0, 1, 2]: @@ -917,6 +925,7 @@ def with_const_expression(): self.assertIsNone(ns['two_strings'].__doc__) self.assertIsNone(ns['with_fstring'].__doc__) self.assertIsNone(ns['with_const_expression'].__doc__) + self.assertIsNone(ns['multiple_const_strings'].__doc__) @support.cpython_only def test_docstring_interactive_mode(self): @@ -1385,52 +1394,91 @@ def check_op_count(func, op, expected): self.assertEqual(actual, expected) def check_consts(func, typ, expected): - slice_consts = 0 + expected = set([repr(x) for x in expected]) + all_consts = set() consts = func.__code__.co_consts for instr in dis.Bytecode(func): if instr.opname == "LOAD_CONST" and isinstance(consts[instr.oparg], typ): - slice_consts += 1 - self.assertEqual(slice_consts, expected) + all_consts.add(repr(consts[instr.oparg])) + self.assertEqual(all_consts, expected) def load(): return x[a:b] + x [a:] + x[:b] + x[:] + check_op_count(load, "BINARY_SLICE", 3) + check_op_count(load, "BUILD_SLICE", 0) + check_consts(load, slice, [slice(None, None, None)]) + check_op_count(load, "BINARY_SUBSCR", 1) + def store(): x[a:b] = y x [a:] = y x[:b] = y x[:] = y + check_op_count(store, "STORE_SLICE", 3) + check_op_count(store, "BUILD_SLICE", 0) + check_op_count(store, "STORE_SUBSCR", 1) + check_consts(store, slice, [slice(None, None, None)]) + def long_slice(): return x[a:b:c] + check_op_count(long_slice, "BUILD_SLICE", 1) + check_op_count(long_slice, "BINARY_SLICE", 0) + check_consts(long_slice, slice, []) + check_op_count(long_slice, "BINARY_SUBSCR", 1) + def aug(): x[a:b] += y + check_op_count(aug, "BINARY_SLICE", 1) + check_op_count(aug, "STORE_SLICE", 1) + check_op_count(aug, "BUILD_SLICE", 0) + check_op_count(aug, "BINARY_SUBSCR", 0) + check_op_count(aug, "STORE_SUBSCR", 0) + check_consts(aug, slice, []) + def aug_const(): x[1:2] += y + check_op_count(aug_const, "BINARY_SLICE", 0) + check_op_count(aug_const, "STORE_SLICE", 0) + check_op_count(aug_const, "BINARY_SUBSCR", 1) + check_op_count(aug_const, "STORE_SUBSCR", 1) + check_consts(aug_const, slice, [slice(1, 2)]) + def compound_const_slice(): x[1:2:3, 4:5:6] = y - check_op_count(load, "BINARY_SLICE", 3) - check_op_count(load, "BUILD_SLICE", 0) - check_consts(load, slice, 1) - check_op_count(store, "STORE_SLICE", 3) - check_op_count(store, "BUILD_SLICE", 0) - check_consts(store, slice, 1) - check_op_count(long_slice, "BUILD_SLICE", 1) - check_op_count(long_slice, "BINARY_SLICE", 0) - check_op_count(aug, "BINARY_SLICE", 1) - check_op_count(aug, "STORE_SLICE", 1) - check_op_count(aug, "BUILD_SLICE", 0) - check_op_count(aug_const, "BINARY_SLICE", 0) - check_op_count(aug_const, "STORE_SLICE", 0) - check_consts(aug_const, slice, 1) check_op_count(compound_const_slice, "BINARY_SLICE", 0) check_op_count(compound_const_slice, "BUILD_SLICE", 0) - check_consts(compound_const_slice, slice, 0) - check_consts(compound_const_slice, tuple, 1) + check_op_count(compound_const_slice, "STORE_SLICE", 0) + check_op_count(compound_const_slice, "STORE_SUBSCR", 1) + check_consts(compound_const_slice, slice, []) + check_consts(compound_const_slice, tuple, [(slice(1, 2, 3), slice(4, 5, 6))]) + + def mutable_slice(): + x[[]:] = y + + check_consts(mutable_slice, slice, {}) + + def different_but_equal(): + x[:0] = y + x[:0.0] = y + x[:False] = y + x[:None] = y + + check_consts( + different_but_equal, + slice, + [ + slice(None, 0, None), + slice(None, 0.0, None), + slice(None, False, None), + slice(None, None, None) + ] + ) def test_compare_positions(self): for opname_prefix, op in [ diff --git a/Lib/test/test_context.py b/Lib/test/test_context.py index b06b9df9f5b0b8..82d1797ab3b79e 100644 --- a/Lib/test/test_context.py +++ b/Lib/test/test_context.py @@ -1,3 +1,4 @@ +import collections.abc import concurrent.futures import contextvars import functools @@ -350,6 +351,19 @@ def ctx2_fun(): ctx1.run(ctx1_fun) + def test_context_isinstance(self): + ctx = contextvars.Context() + self.assertIsInstance(ctx, collections.abc.Mapping) + self.assertTrue(issubclass(contextvars.Context, collections.abc.Mapping)) + + mapping_methods = ( + '__contains__', '__eq__', '__getitem__', '__iter__', '__len__', + '__ne__', 'get', 'items', 'keys', 'values', + ) + for name in mapping_methods: + with self.subTest(name=name): + self.assertTrue(callable(getattr(ctx, name))) + @isolated_context @threading_helper.requires_working_threading() def test_context_threads_1(self): diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index b2595eccc82f70..65720871d5c5f0 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -30,6 +30,22 @@ def test_bad_counter_during_dealloc(self): self.assertEqual(cm.unraisable.exc_type, TypeError) + def test_crash_with_not_enough_args(self): + # gh-126220 + import _lsprof + + for profile in [_lsprof.Profiler(), cProfile.Profile()]: + for method in [ + "_pystart_callback", + "_pyreturn_callback", + "_ccall_callback", + "_creturn_callback", + ]: + with self.subTest(profile=profile, method=method): + method_obj = getattr(profile, method) + with self.assertRaises(TypeError): + method_obj() # should not crash + def test_evil_external_timer(self): # gh-120289 # Disabling profiler in external timer should not crash diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index ce5c03659f1979..4af8f7f480e759 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001,2002 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # csv package unit tests import copy diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py index fa5144a3ca01bb..eb77d6d7782478 100644 --- a/Lib/test/test_ctypes/test_c_simple_type_meta.py +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -85,3 +85,68 @@ class Sub(CtBase): self.assertIsInstance(POINTER(Sub), p_meta) self.assertTrue(issubclass(POINTER(Sub), Sub)) + + def test_creating_pointer_in_dunder_init_1(self): + class ct_meta(type): + def __init__(self, name, bases, namespace): + super().__init__(name, bases, namespace) + + # Avoid recursion. + # (See test_creating_pointer_in_dunder_new_1) + if bases == (c_void_p,): + return + if issubclass(self, PtrBase): + return + if bases == (object,): + ptr_bases = (self, PtrBase) + else: + ptr_bases = (self, POINTER(bases[0])) + p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) + ctypes._pointer_type_cache[self] = p + + class p_meta(PyCSimpleType, ct_meta): + pass + + class PtrBase(c_void_p, metaclass=p_meta): + pass + + class CtBase(object, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + class Sub2(Sub): + pass + + self.assertIsInstance(POINTER(Sub2), p_meta) + self.assertTrue(issubclass(POINTER(Sub2), Sub2)) + self.assertTrue(issubclass(POINTER(Sub2), POINTER(Sub))) + self.assertTrue(issubclass(POINTER(Sub), POINTER(CtBase))) + + def test_creating_pointer_in_dunder_init_2(self): + class ct_meta(type): + def __init__(self, name, bases, namespace): + super().__init__(name, bases, namespace) + + # Avoid recursion. + # (See test_creating_pointer_in_dunder_new_2) + if isinstance(self, p_meta): + return + p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) + ctypes._pointer_type_cache[self] = p + + class p_meta(PyCSimpleType, ct_meta): + pass + + class Core(object): + pass + + class CtBase(Core, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + self.assertIsInstance(POINTER(Sub), p_meta) + self.assertTrue(issubclass(POINTER(Sub), Sub)) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py new file mode 100644 index 00000000000000..4441e30cd7a2a7 --- /dev/null +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -0,0 +1,123 @@ +import os +import sys +import unittest +import platform + +FOO_C = r""" +#include + +/* This is a 'GNU indirect function' (IFUNC) that will be called by + dlsym() to resolve the symbol "foo" to an address. Typically, such + a function would return the address of an actual function, but it + can also just return NULL. For some background on IFUNCs, see + https://willnewton.name/uncategorized/using-gnu-indirect-functions. + + Adapted from Michael Kerrisk's answer: https://stackoverflow.com/a/53590014. +*/ + +asm (".type foo STT_GNU_IFUNC"); + +void *foo(void) +{ + write($DESCRIPTOR, "OK", 2); + return NULL; +} +""" + + +@unittest.skipUnless(sys.platform.startswith('linux'), + 'Test only valid for Linux') +class TestNullDlsym(unittest.TestCase): + """GH-126554: Ensure that we catch NULL dlsym return values + + In rare cases, such as when using GNU IFUNCs, dlsym(), + the C function that ctypes' CDLL uses to get the address + of symbols, can return NULL. + + The objective way of telling if an error during symbol + lookup happened is to call glibc's dlerror() and check + for a non-NULL return value. + + However, there can be cases where dlsym() returns NULL + and dlerror() is also NULL, meaning that glibc did not + encounter any error. + + In the case of ctypes, we subjectively treat that as + an error, and throw a relevant exception. + + This test case ensures that we correctly enforce + this 'dlsym returned NULL -> throw Error' rule. + """ + + def test_null_dlsym(self): + import subprocess + import tempfile + + # To avoid ImportErrors on Windows, where _ctypes does not have + # dlopen and dlsym, + # import here, i.e., inside the test function. + # The skipUnless('linux') decorator ensures that we're on linux + # if we're executing these statements. + from ctypes import CDLL, c_int + from _ctypes import dlopen, dlsym + + retcode = subprocess.call(["gcc", "--version"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + if retcode != 0: + self.skipTest("gcc is missing") + + pipe_r, pipe_w = os.pipe() + self.addCleanup(os.close, pipe_r) + self.addCleanup(os.close, pipe_w) + + with tempfile.TemporaryDirectory() as d: + # Create a C file with a GNU Indirect Function (FOO_C) + # and compile it into a shared library. + srcname = os.path.join(d, 'foo.c') + dstname = os.path.join(d, 'libfoo.so') + with open(srcname, 'w') as f: + f.write(FOO_C.replace('$DESCRIPTOR', str(pipe_w))) + args = ['gcc', '-fPIC', '-shared', '-o', dstname, srcname] + p = subprocess.run(args, capture_output=True) + + if p.returncode != 0: + # IFUNC is not supported on all architectures. + if platform.machine() == 'x86_64': + # It should be supported here. Something else went wrong. + p.check_returncode() + else: + # IFUNC might not be supported on this machine. + self.skipTest(f"could not compile indirect function: {p}") + + # Case #1: Test 'PyCFuncPtr_FromDll' from Modules/_ctypes/_ctypes.c + L = CDLL(dstname) + with self.assertRaisesRegex(AttributeError, "function 'foo' not found"): + # Try accessing the 'foo' symbol. + # It should resolve via dlsym() to NULL, + # and since we subjectively treat NULL + # addresses as errors, we should get + # an error. + L.foo + + # Assert that the IFUNC was called + self.assertEqual(os.read(pipe_r, 2), b'OK') + + # Case #2: Test 'CDataType_in_dll_impl' from Modules/_ctypes/_ctypes.c + with self.assertRaisesRegex(ValueError, "symbol 'foo' not found"): + c_int.in_dll(L, "foo") + + # Assert that the IFUNC was called + self.assertEqual(os.read(pipe_r, 2), b'OK') + + # Case #3: Test 'py_dl_sym' from Modules/_ctypes/callproc.c + L = dlopen(dstname) + with self.assertRaisesRegex(OSError, "symbol 'foo' not found"): + dlsym(L, "foo") + + # Assert that the IFUNC was called + self.assertEqual(os.read(pipe_r, 2), b'OK') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 4030716efb51f9..c94dc2df4f0a7f 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -880,115 +880,6 @@ class C(object): gc.collect() self.assertIs(ref(), None, "Cycle was not collected") - def _not_tracked(self, t): - # Nested containers can take several collections to untrack - gc.collect() - gc.collect() - self.assertFalse(gc.is_tracked(t), t) - - def _tracked(self, t): - self.assertTrue(gc.is_tracked(t), t) - gc.collect() - gc.collect() - self.assertTrue(gc.is_tracked(t), t) - - def test_string_keys_can_track_values(self): - # Test that this doesn't leak. - for i in range(10): - d = {} - for j in range(10): - d[str(j)] = j - d["foo"] = d - - @support.cpython_only - def test_track_literals(self): - # Test GC-optimization of dict literals - x, y, z, w = 1.5, "a", (1, None), [] - - self._not_tracked({}) - self._not_tracked({x:(), y:x, z:1}) - self._not_tracked({1: "a", "b": 2}) - self._not_tracked({1: 2, (None, True, False, ()): int}) - self._not_tracked({1: object()}) - - # Dicts with mutable elements are always tracked, even if those - # elements are not tracked right now. - self._tracked({1: []}) - self._tracked({1: ([],)}) - self._tracked({1: {}}) - self._tracked({1: set()}) - - @support.cpython_only - def test_track_dynamic(self): - # Test GC-optimization of dynamically-created dicts - class MyObject(object): - pass - x, y, z, w, o = 1.5, "a", (1, object()), [], MyObject() - - d = dict() - self._not_tracked(d) - d[1] = "a" - self._not_tracked(d) - d[y] = 2 - self._not_tracked(d) - d[z] = 3 - self._not_tracked(d) - self._not_tracked(d.copy()) - d[4] = w - self._tracked(d) - self._tracked(d.copy()) - d[4] = None - self._not_tracked(d) - self._not_tracked(d.copy()) - - # dd isn't tracked right now, but it may mutate and therefore d - # which contains it must be tracked. - d = dict() - dd = dict() - d[1] = dd - self._not_tracked(dd) - self._tracked(d) - dd[1] = d - self._tracked(dd) - - d = dict.fromkeys([x, y, z]) - self._not_tracked(d) - dd = dict() - dd.update(d) - self._not_tracked(dd) - d = dict.fromkeys([x, y, z, o]) - self._tracked(d) - dd = dict() - dd.update(d) - self._tracked(dd) - - d = dict(x=x, y=y, z=z) - self._not_tracked(d) - d = dict(x=x, y=y, z=z, w=w) - self._tracked(d) - d = dict() - d.update(x=x, y=y, z=z) - self._not_tracked(d) - d.update(w=w) - self._tracked(d) - - d = dict([(x, y), (z, 1)]) - self._not_tracked(d) - d = dict([(x, y), (z, w)]) - self._tracked(d) - d = dict() - d.update([(x, y), (z, 1)]) - self._not_tracked(d) - d.update([(x, y), (z, w)]) - self._tracked(d) - - @support.cpython_only - def test_track_subtypes(self): - # Dict subtypes are always tracked - class MyDict(dict): - pass - self._tracked(MyDict()) - def make_shared_key_dict(self, n): class C: pass diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 3c6570afa50d45..f26411ace8fa73 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1262,25 +1262,7 @@ def test_super_instructions(self): @cpython_only @requires_specialization - def test_binary_specialize(self): - binary_op_quicken = """\ - 0 RESUME_CHECK 0 - - 1 LOAD_NAME 0 (a) - LOAD_NAME 1 (b) - %s - RETURN_VALUE -""" - co_int = compile('a + b', "", "eval") - self.code_quicken(lambda: exec(co_int, {}, {'a': 1, 'b': 2})) - got = self.get_disassembly(co_int, adaptive=True) - self.do_disassembly_compare(got, binary_op_quicken % "BINARY_OP_ADD_INT 0 (+)") - - co_unicode = compile('a + b', "", "eval") - self.code_quicken(lambda: exec(co_unicode, {}, {'a': 'a', 'b': 'b'})) - got = self.get_disassembly(co_unicode, adaptive=True) - self.do_disassembly_compare(got, binary_op_quicken % "BINARY_OP_ADD_UNICODE 0 (+)") - + def test_binary_subscr_specialize(self): binary_subscr_quicken = """\ 0 RESUME_CHECK 0 diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index 171412cb7cb08c..b1e165fe16b54f 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -2410,25 +2410,6 @@ def test_DocFileSuite(): >>> suite.run(unittest.TestResult()) - Support for using a package's __loader__.get_data() is also - provided. - - >>> import unittest, pkgutil, test - >>> added_loader = False - >>> if not hasattr(test, '__loader__'): - ... test.__loader__ = pkgutil.get_loader(test) - ... added_loader = True - >>> try: - ... suite = doctest.DocFileSuite('test_doctest.txt', - ... 'test_doctest2.txt', - ... 'test_doctest4.txt', - ... package='test.test_doctest') - ... suite.run(unittest.TestResult()) - ... finally: - ... if added_loader: - ... del test.__loader__ - - '/' should be used as a path separator. It will be converted to a native separator at run time: diff --git a/Lib/test/test_email/test_asian_codecs.py b/Lib/test/test_email/test_asian_codecs.py index 1e0caeeaed0810..ca44f54c69b39b 100644 --- a/Lib/test/test_email/test_asian_codecs.py +++ b/Lib/test/test_email/test_asian_codecs.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2006 Python Software Foundation +# Copyright (C) 2002 Python Software Foundation # Contact: email-sig@python.org # email package unit tests for (optional) Asian codecs diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 65ddbabcaa1997..abe9ef2e94409f 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2010 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Contact: email-sig@python.org # email package unit tests diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py index 034f7626c1fc7c..96979db27f3a21 100644 --- a/Lib/test/test_email/test_message.py +++ b/Lib/test/test_email/test_message.py @@ -1,6 +1,6 @@ -import unittest import textwrap -from email import policy, message_from_string +import unittest +from email import message_from_bytes, message_from_string, policy from email.message import EmailMessage, MIMEPart from test.test_email import TestEmailBase, parameterize @@ -958,6 +958,52 @@ def test_folding_with_utf8_encoding_8(self): b'123456789-123456789\n 123456789 Hello ' b'=?utf-8?q?W=C3=B6rld!?= 123456789 123456789\n\n') + def test_folding_with_short_nospace_1(self): + # bpo-36520 + # + # Fold a line that contains a long whitespace after + # the fold point. + + m = EmailMessage(policy.default) + m['Message-ID'] = '123456789' * 3 + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + + def test_folding_with_long_nospace_default_policy_1(self): + # Fixed: https://github.com/python/cpython/issues/124452 + # + # When the value is too long, it should be converted back + # to its original form without any modifications. + + m = EmailMessage(policy.default) + message = '123456789' * 10 + m['Message-ID'] = message + self.assertEqual(m.as_bytes(), + f'Message-ID:\n {message}\n\n'.encode()) + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + + def test_folding_with_long_nospace_compat32_policy_1(self): + m = EmailMessage(policy.compat32) + message = '123456789' * 10 + m['Message-ID'] = message + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + + def test_folding_with_long_nospace_smtp_policy_1(self): + m = EmailMessage(policy.SMTP) + message = '123456789' * 10 + m['Message-ID'] = message + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + + def test_folding_with_long_nospace_http_policy_1(self): + m = EmailMessage(policy.HTTP) + message = '123456789' * 10 + m['Message-ID'] = message + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + def test_get_body_malformed(self): """test for bpo-42892""" msg = textwrap.dedent("""\ diff --git a/Lib/test/test_email/torture_test.py b/Lib/test/test_email/torture_test.py index 9cf9362c9b77e0..d15948a38b25dd 100644 --- a/Lib/test/test_email/torture_test.py +++ b/Lib/test/test_email/torture_test.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2004 Python Software Foundation +# Copyright (C) 2002 Python Software Foundation # # A torture test of the email package. This should not be run as part of the # standard Python test suite since it requires several meg of email messages diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 4ea43edccda63e..bf861ef06ee2d3 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1,5 +1,6 @@ # Run the tests in Programs/_testembed.c (tests for the CPython embedding APIs) from test import support +from test.libregrtest.utils import get_build_info from test.support import import_helper, os_helper, threading_helper, MS_WINDOWS import unittest @@ -643,6 +644,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): CONFIG_COMPAT['run_presite'] = None if support.Py_GIL_DISABLED: CONFIG_COMPAT['enable_gil'] = -1 + CONFIG_COMPAT['tlbc_enabled'] = GET_DEFAULT_CONFIG if MS_WINDOWS: CONFIG_COMPAT.update({ 'legacy_windows_stdio': False, @@ -1780,8 +1782,10 @@ def test_initconfig_api(self): 'perf_profiling': 2, } config_dev_mode(preconfig, config) + # Temporarily enable ignore_stderr=True to ignore warnings on JIT builds + # See gh-126255 for more information self.check_all_configs("test_initconfig_api", config, preconfig, - api=API_ISOLATED) + api=API_ISOLATED, ignore_stderr=True) def test_initconfig_get_api(self): self.run_embedded_interpreter("test_initconfig_get_api") diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index 0611d1749f41c1..d60aabcdf1ae22 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -11,7 +11,7 @@ from test.support import ( cpython_only, swap_attr, gc_collect, is_emscripten, is_wasi, - infinite_recursion, + infinite_recursion, strace_helper ) from test.support.os_helper import ( TESTFN, TESTFN_ASCII, TESTFN_UNICODE, make_bad_fd, @@ -24,6 +24,9 @@ import _pyio # Python implementation of io +_strace_flags=["--trace=%file,%desc"] + + class AutoFileTests: # file tests for which a test file is automatically set up @@ -359,6 +362,139 @@ def testErrnoOnClosedReadinto(self, f): a = array('b', b'x'*10) f.readinto(a) + @strace_helper.requires_strace() + def test_syscalls_read(self): + """Check that the set of system calls produced by the I/O stack is what + is expected for various read cases. + + It's expected as bits of the I/O implementation change, this will need + to change. The goal is to catch changes that unintentionally add + additional systemcalls (ex. additional calls have been looked at in + bpo-21679 and gh-120754). + """ + self.f.write(b"Hello, World!") + self.f.close() + + + def check_readall(name, code, prelude="", cleanup="", + extra_checks=None): + with self.subTest(name=name): + syscalls = strace_helper.get_events(code, _strace_flags, + prelude=prelude, + cleanup=cleanup) + + # The first call should be an open that returns a + # file descriptor (fd). Afer that calls may vary. Once the file + # is opened, check calls refer to it by fd as the filename + # could be removed from the filesystem, renamed, etc. See: + # Time-of-check time-of-use (TOCTOU) software bug class. + # + # There are a number of related but distinct open system calls + # so not checking precise name here. + self.assertGreater( + len(syscalls), + 1, + f"Should have had at least an open call|calls={syscalls}") + fd_str = syscalls[0].returncode + + # All other calls should contain the fd in their argument set. + for ev in syscalls[1:]: + self.assertIn( + fd_str, + ev.args, + f"Looking for file descriptor in arguments|ev={ev}" + ) + + # There are a number of related syscalls used to implement + # behaviors in a libc (ex. fstat, newfstatat, statx, open, openat). + # Allow any that use the same substring. + def count_similarname(name): + return len([ev for ev in syscalls if name in ev.syscall]) + + checks = [ + # Should open and close the file exactly once + ("open", 1), + ("close", 1), + # There should no longer be an isatty call (All files being + # tested are block devices / not character devices). + ('ioctl', 0), + # Should only have one fstat (bpo-21679, gh-120754) + # note: It's important this uses a fd rather than filename, + # That is validated by the `fd` check above. + # note: fstat, newfstatat, and statx have all been observed + # here in the underlying C library implementations. + ("stat", 1) + ] + + if extra_checks: + checks += extra_checks + + for call, count in checks: + self.assertEqual( + count_similarname(call), + count, + msg=f"call={call}|count={count}|syscalls={syscalls}" + ) + + # "open, read, close" file using different common patterns. + check_readall( + "open builtin with default options", + f""" + f = open('{TESTFN}') + f.read() + f.close() + """ + ) + + check_readall( + "open in binary mode", + f""" + f = open('{TESTFN}', 'rb') + f.read() + f.close() + """ + ) + + check_readall( + "open in text mode", + f""" + f = open('{TESTFN}', 'rt') + f.read() + f.close() + """, + # GH-122111: read_text uses BufferedIO which requires looking up + # position in file. `read_bytes` disables that buffering and avoids + # these calls which is tested the `pathlib read_bytes` case. + extra_checks=[("seek", 1)] + ) + + check_readall( + "pathlib read_bytes", + "p.read_bytes()", + prelude=f"""from pathlib import Path; p = Path("{TESTFN}")""", + # GH-122111: Buffering is disabled so these calls are avoided. + extra_checks=[("seek", 0)] + ) + + check_readall( + "pathlib read_text", + "p.read_text()", + prelude=f"""from pathlib import Path; p = Path("{TESTFN}")""" + ) + + # Focus on just `read()`. + calls = strace_helper.get_syscalls( + prelude=f"f = open('{TESTFN}')", + code="f.read()", + cleanup="f.close()", + strace_flags=_strace_flags + ) + # One to read all the bytes + # One to read the EOF and get a size 0 return. + self.assertEqual(calls.count("read"), 2) + + + class CAutoFileTests(AutoFileTests, unittest.TestCase): FileIO = _io.FileIO modulename = '_io' diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index d590af090abc6e..6d60f6941c4c5d 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1005,6 +1005,29 @@ def __getitem__(self, i): d = {"one": 1, "two": 2, "three": 3} self.assertEqual(self.reduce(add, d), "".join(d.keys())) + # test correctness of keyword usage of `initial` in `reduce` + def test_initial_keyword(self): + def add(x, y): + return x + y + self.assertEqual( + self.reduce(add, ['a', 'b', 'c'], ''), + self.reduce(add, ['a', 'b', 'c'], initial=''), + ) + self.assertEqual( + self.reduce(add, [['a', 'c'], [], ['d', 'w']], []), + self.reduce(add, [['a', 'c'], [], ['d', 'w']], initial=[]), + ) + self.assertEqual( + self.reduce(lambda x, y: x*y, range(2,8), 1), + self.reduce(lambda x, y: x*y, range(2,8), initial=1), + ) + self.assertEqual( + self.reduce(lambda x, y: x*y, range(2,21), 1), + self.reduce(lambda x, y: x*y, range(2,21), initial=1), + ) + self.assertRaises(TypeError, self.reduce, add, [0, 1], initial="") + self.assertEqual(self.reduce(42, "", initial="1"), "1") # func is never called with one item + @unittest.skipUnless(c_functools, 'requires the C _functools module') class TestReduceC(TestReduce, unittest.TestCase): diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 2b3c0d3baddeaf..e02ec7c65212e0 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -31,6 +31,11 @@ def __new__(cls, *args, **kwargs): return C ContainerNoGC = None +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + ### Support code ############################################################################### @@ -1082,6 +1087,44 @@ def __del__(self): gc.collect() self.assertTrue(collected) + def test_traverse_frozen_objects(self): + # See GH-126312: Objects that were not frozen could traverse over + # a frozen object on the free-threaded build, which would cause + # a negative reference count. + x = [1, 2, 3] + gc.freeze() + y = [x] + y.append(y) + del y + gc.collect() + gc.unfreeze() + + def test_deferred_refcount_frozen(self): + # Also from GH-126312: objects that use deferred reference counting + # weren't ignored if they were frozen. Unfortunately, it's pretty + # difficult to come up with a case that triggers this. + # + # Calling gc.collect() while the garbage collector is frozen doesn't + # trigger this normally, but it *does* if it's inside unittest for whatever + # reason. We can't call unittest from inside a test, so it has to be + # in a subprocess. + source = textwrap.dedent(""" + import gc + import unittest + + + class Test(unittest.TestCase): + def test_something(self): + gc.freeze() + gc.collect() + gc.unfreeze() + + + if __name__ == "__main__": + unittest.main() + """) + assert_python_ok("-c", source) + class IncrementalGCTests(unittest.TestCase): @@ -1092,6 +1135,7 @@ def setUp(self): def tearDown(self): gc.disable() + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") @requires_gil_enabled("Free threading does not support incremental GC") # Use small increments to emulate longer running process in a shorter time @gc_threshold(200, 10) @@ -1117,32 +1161,18 @@ def make_ll(depth): return head head = make_ll(1000) - count = 1000 - - # There will be some objects we aren't counting, - # e.g. the gc stats dicts. This test checks - # that the counts don't grow, so we try to - # correct for the uncounted objects - # This is just an estimate. - CORRECTION = 20 enabled = gc.isenabled() gc.enable() olds = [] + initial_heap_size = _testinternalcapi.get_heap_size() for i in range(20_000): newhead = make_ll(20) - count += 20 newhead.surprise = head olds.append(newhead) if len(olds) == 20: - stats = gc.get_stats() - young = stats[0] - incremental = stats[1] - old = stats[2] - collected = young['collected'] + incremental['collected'] + old['collected'] - count += CORRECTION - live = count - collected - self.assertLess(live, 25000) + new_objects = _testinternalcapi.get_heap_size() - initial_heap_size + self.assertLess(new_objects, 25_000) del olds[:] if not enabled: gc.disable() @@ -1284,7 +1314,8 @@ def test_refcount_errors(self): from test.support import gc_collect, SuppressCrashReport a = [1, 2, 3] - b = [a] + b = [a, a] + a.append(b) # Avoid coredump when Py_FatalError() calls abort() SuppressCrashReport().__enter__() @@ -1294,6 +1325,8 @@ def test_refcount_errors(self): # (to avoid deallocating it): import ctypes ctypes.pythonapi.Py_DecRef(ctypes.py_object(a)) + del a + del b # The garbage collector should now have a fatal error # when it reaches the broken object @@ -1322,7 +1355,7 @@ def test_refcount_errors(self): self.assertRegex(stderr, br'object type name: list') self.assertRegex(stderr, - br'object repr : \[1, 2, 3\]') + br'object repr : \[1, 2, 3, \[\[...\], \[...\]\]\]') class GCTogglingTests(unittest.TestCase): diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 173e405b785ddc..ff9a52b7adac8a 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1429,6 +1429,39 @@ def test_instruction_size_macro(self): with self.assertRaisesRegex(SyntaxError, "All instructions containing a uop"): self.run_cases_test(input, output) + def test_escaping_call_next_to_cmacro(self): + input = """ + inst(OP, (--)) { + #ifdef Py_GIL_DISABLED + escaping_call(); + #else + another_escaping_call(); + #endif + yet_another_escaping_call(); + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + #else + _PyFrame_SetStackPointer(frame, stack_pointer); + another_escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + yet_another_escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: diff --git a/Lib/test/test_getopt.py b/Lib/test/test_getopt.py index c8b3442de4aa77..ed967ad27619ae 100644 --- a/Lib/test/test_getopt.py +++ b/Lib/test/test_getopt.py @@ -1,11 +1,12 @@ # test_getopt.py # David Goodger 2000-08-19 -from test.support.os_helper import EnvironmentVarGuard import doctest -import unittest - import getopt +import sys +import unittest +from test.support.i18n_helper import TestTranslationsBase, update_translation_snapshots +from test.support.os_helper import EnvironmentVarGuard sentinel = object() @@ -19,21 +20,34 @@ def assertError(self, *args, **kwargs): self.assertRaises(getopt.GetoptError, *args, **kwargs) def test_short_has_arg(self): - self.assertTrue(getopt.short_has_arg('a', 'a:')) - self.assertFalse(getopt.short_has_arg('a', 'a')) + self.assertIs(getopt.short_has_arg('a', 'a:'), True) + self.assertIs(getopt.short_has_arg('a', 'a'), False) + self.assertEqual(getopt.short_has_arg('a', 'a::'), '?') self.assertError(getopt.short_has_arg, 'a', 'b') def test_long_has_args(self): has_arg, option = getopt.long_has_args('abc', ['abc=']) - self.assertTrue(has_arg) + self.assertIs(has_arg, True) self.assertEqual(option, 'abc') has_arg, option = getopt.long_has_args('abc', ['abc']) - self.assertFalse(has_arg) + self.assertIs(has_arg, False) self.assertEqual(option, 'abc') + has_arg, option = getopt.long_has_args('abc', ['abc=?']) + self.assertEqual(has_arg, '?') + self.assertEqual(option, 'abc') + + has_arg, option = getopt.long_has_args('abc', ['abcd=']) + self.assertIs(has_arg, True) + self.assertEqual(option, 'abcd') + has_arg, option = getopt.long_has_args('abc', ['abcd']) - self.assertFalse(has_arg) + self.assertIs(has_arg, False) + self.assertEqual(option, 'abcd') + + has_arg, option = getopt.long_has_args('abc', ['abcd=?']) + self.assertEqual(has_arg, '?') self.assertEqual(option, 'abcd') self.assertError(getopt.long_has_args, 'abc', ['def']) @@ -49,9 +63,9 @@ def test_do_shorts(self): self.assertEqual(opts, [('-a', '1')]) self.assertEqual(args, []) - #opts, args = getopt.do_shorts([], 'a=1', 'a:', []) - #self.assertEqual(opts, [('-a', '1')]) - #self.assertEqual(args, []) + opts, args = getopt.do_shorts([], 'a=1', 'a:', []) + self.assertEqual(opts, [('-a', '=1')]) + self.assertEqual(args, []) opts, args = getopt.do_shorts([], 'a', 'a:', ['1']) self.assertEqual(opts, [('-a', '1')]) @@ -61,6 +75,14 @@ def test_do_shorts(self): self.assertEqual(opts, [('-a', '1')]) self.assertEqual(args, ['2']) + opts, args = getopt.do_shorts([], 'a', 'a::', ['1']) + self.assertEqual(opts, [('-a', '')]) + self.assertEqual(args, ['1']) + + opts, args = getopt.do_shorts([], 'a1', 'a::', []) + self.assertEqual(opts, [('-a', '1')]) + self.assertEqual(args, []) + self.assertError(getopt.do_shorts, [], 'a1', 'a', []) self.assertError(getopt.do_shorts, [], 'a', 'a:', []) @@ -77,6 +99,22 @@ def test_do_longs(self): self.assertEqual(opts, [('--abcd', '1')]) self.assertEqual(args, []) + opts, args = getopt.do_longs([], 'abc', ['abc=?'], ['1']) + self.assertEqual(opts, [('--abc', '')]) + self.assertEqual(args, ['1']) + + opts, args = getopt.do_longs([], 'abc', ['abcd=?'], ['1']) + self.assertEqual(opts, [('--abcd', '')]) + self.assertEqual(args, ['1']) + + opts, args = getopt.do_longs([], 'abc=1', ['abc=?'], []) + self.assertEqual(opts, [('--abc', '1')]) + self.assertEqual(args, []) + + opts, args = getopt.do_longs([], 'abc=1', ['abcd=?'], []) + self.assertEqual(opts, [('--abcd', '1')]) + self.assertEqual(args, []) + opts, args = getopt.do_longs([], 'abc', ['ab', 'abc', 'abcd'], []) self.assertEqual(opts, [('--abc', '')]) self.assertEqual(args, []) @@ -95,7 +133,7 @@ def test_getopt(self): # note: the empty string between '-a' and '--beta' is significant: # it simulates an empty string option argument ('-a ""') on the # command line. - cmdline = ['-a', '1', '-b', '--alpha=2', '--beta', '-a', '3', '-a', + cmdline = ['-a1', '-b', '--alpha=2', '--beta', '-a', '3', '-a', '', '--beta', 'arg1', 'arg2'] opts, args = getopt.getopt(cmdline, 'a:b', ['alpha=', 'beta']) @@ -106,33 +144,53 @@ def test_getopt(self): # accounted for in the code that calls getopt(). self.assertEqual(args, ['arg1', 'arg2']) + cmdline = ['-a1', '--alpha=2', '--alpha=', '-a', '--alpha', 'arg1', 'arg2'] + opts, args = getopt.getopt(cmdline, 'a::', ['alpha=?']) + self.assertEqual(opts, [('-a', '1'), ('--alpha', '2'), ('--alpha', ''), + ('-a', ''), ('--alpha', '')]) + self.assertEqual(args, ['arg1', 'arg2']) + self.assertError(getopt.getopt, cmdline, 'a:b', ['alpha', 'beta']) def test_gnu_getopt(self): # Test handling of GNU style scanning mode. - cmdline = ['-a', 'arg1', '-b', '1', '--alpha', '--beta=2'] + cmdline = ['-a', 'arg1', '-b', '1', '--alpha', '--beta=2', '--beta', + '3', 'arg2'] # GNU style opts, args = getopt.gnu_getopt(cmdline, 'ab:', ['alpha', 'beta=']) - self.assertEqual(args, ['arg1']) - self.assertEqual(opts, [('-a', ''), ('-b', '1'), - ('--alpha', ''), ('--beta', '2')]) + self.assertEqual(args, ['arg1', 'arg2']) + self.assertEqual(opts, [('-a', ''), ('-b', '1'), ('--alpha', ''), + ('--beta', '2'), ('--beta', '3')]) + + opts, args = getopt.gnu_getopt(cmdline, 'ab::', ['alpha', 'beta=?']) + self.assertEqual(args, ['arg1', '1', '3', 'arg2']) + self.assertEqual(opts, [('-a', ''), ('-b', ''), ('--alpha', ''), + ('--beta', '2'), ('--beta', '')]) # recognize "-" as an argument opts, args = getopt.gnu_getopt(['-a', '-', '-b', '-'], 'ab:', []) self.assertEqual(args, ['-']) self.assertEqual(opts, [('-a', ''), ('-b', '-')]) + # Return positional arguments intermixed with options. + opts, args = getopt.gnu_getopt(cmdline, '-ab:', ['alpha', 'beta=']) + self.assertEqual(args, ['arg2']) + self.assertEqual(opts, [('-a', ''), (None, ['arg1']), ('-b', '1'), ('--alpha', ''), + ('--beta', '2'), ('--beta', '3')]) + # Posix style via + opts, args = getopt.gnu_getopt(cmdline, '+ab:', ['alpha', 'beta=']) self.assertEqual(opts, [('-a', '')]) - self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2']) + self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2', + '--beta', '3', 'arg2']) # Posix style via POSIXLY_CORRECT self.env["POSIXLY_CORRECT"] = "1" opts, args = getopt.gnu_getopt(cmdline, 'ab:', ['alpha', 'beta=']) self.assertEqual(opts, [('-a', '')]) - self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2']) + self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2', + '--beta', '3', 'arg2']) def test_issue4629(self): longopts, shortopts = getopt.getopt(['--help='], '', ['help=']) @@ -173,10 +231,20 @@ def test_libref_examples(): ['a1', 'a2'] """ + +class TestTranslations(TestTranslationsBase): + def test_translations(self): + self.assertMsgidsEqual(getopt) + + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite()) return tests -if __name__ == "__main__": +if __name__ == '__main__': + # To regenerate translation snapshots + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + update_translation_snapshots(getopt) + sys.exit(0) unittest.main() diff --git a/Lib/test/test_global.py b/Lib/test/test_global.py index f5b38c25ea0728..11d0bd54e8b69b 100644 --- a/Lib/test/test_global.py +++ b/Lib/test/test_global.py @@ -1,7 +1,19 @@ -"""Verify that warnings are issued for global statements following use.""" +"""This module includes tests for syntax errors that occur when a name +declared as `global` is used in ways that violate the language +specification, such as after assignment, usage, or annotation. The tests +verify that syntax errors are correctly raised for improper `global` +statements following variable use or assignment within functions. +Additionally, it tests various name-binding scenarios for global +variables to ensure correct behavior. +See `test_scope.py` for additional related behavioral tests covering +variable scoping and usage in different contexts. +""" + +import contextlib from test.support import check_syntax_error from test.support.warnings_helper import check_warnings +from types import SimpleNamespace import unittest import warnings @@ -12,40 +24,185 @@ def setUp(self): self.enterContext(check_warnings()) warnings.filterwarnings("error", module="") - def test1(self): - prog_text_1 = """\ -def wrong1(): - a = 1 - b = 2 - global a - global b + ###################################################### + ### Syntax error cases as covered in Python/symtable.c + ###################################################### + + def test_name_param(self): + prog_text = """\ +def fn(name_param): + global name_param """ - check_syntax_error(self, prog_text_1, lineno=4, offset=5) + check_syntax_error(self, prog_text, lineno=2, offset=5) - def test2(self): - prog_text_2 = """\ -def wrong2(): - print(x) - global x + def test_name_after_assign(self): + prog_text = """\ +def fn(): + name_assign = 1 + global name_assign """ - check_syntax_error(self, prog_text_2, lineno=3, offset=5) + check_syntax_error(self, prog_text, lineno=3, offset=5) - def test3(self): - prog_text_3 = """\ -def wrong3(): - print(x) - x = 2 - global x + def test_name_after_use(self): + prog_text = """\ +def fn(): + print(name_use) + global name_use """ - check_syntax_error(self, prog_text_3, lineno=4, offset=5) + check_syntax_error(self, prog_text, lineno=3, offset=5) - def test4(self): - prog_text_4 = """\ -global x -x = 2 + def test_name_annot(self): + prog_text_3 = """\ +def fn(): + name_annot: int + global name_annot """ - # this should work - compile(prog_text_4, "", "exec") + check_syntax_error(self, prog_text_3, lineno=3, offset=5) + + ############################################################# + ### Tests for global variables across all name binding cases, + ### as described in executionmodel.rst + ############################################################# + + def test_assignment_statement(self): + global name_assignment_statement + value = object() + name_assignment_statement = value + self.assertIs(globals()["name_assignment_statement"], value) + del name_assignment_statement + + def test_unpacking_assignment(self): + global name_unpacking_assignment + value = object() + _, name_unpacking_assignment = [None, value] + self.assertIs(globals()["name_unpacking_assignment"], value) + del name_unpacking_assignment + + def test_assignment_expression(self): + global name_assignment_expression + value = object() + if name_assignment_expression := value: + pass + self.assertIs(globals()["name_assignment_expression"], value) + del name_assignment_expression + + def test_iteration_variable(self): + global name_iteration_variable + value = object() + for name_iteration_variable in [value]: + pass + self.assertIs(globals()["name_iteration_variable"], value) + del name_iteration_variable + + def test_func_def(self): + global name_func_def + + def name_func_def(): + pass + + value = name_func_def + self.assertIs(globals()["name_func_def"], value) + del name_func_def + + def test_class_def(self): + global name_class_def + + class name_class_def: + pass + + value = name_class_def + self.assertIs(globals()["name_class_def"], value) + del name_class_def + + def test_type_alias(self): + global name_type_alias + type name_type_alias = tuple[int, int] + value = name_type_alias + self.assertIs(globals()["name_type_alias"], value) + del name_type_alias + + def test_caught_exception(self): + global name_caught_exc + + try: + 1 / 0 + except ZeroDivisionError as name_caught_exc: + value = name_caught_exc + # `name_caught_exc` is cleared automatically after the except block + self.assertIs(globals()["name_caught_exc"], value) + + def test_caught_exception_group(self): + global name_caught_exc_group + try: + try: + 1 / 0 + except ZeroDivisionError as exc: + raise ExceptionGroup("eg", [exc]) + except* ZeroDivisionError as name_caught_exc_group: + value = name_caught_exc_group + # `name_caught_exc` is cleared automatically after the except block + self.assertIs(globals()["name_caught_exc_group"], value) + + def test_enter_result(self): + global name_enter_result + value = object() + with contextlib.nullcontext(value) as name_enter_result: + pass + self.assertIs(globals()["name_enter_result"], value) + del name_enter_result + + def test_import_result(self): + global name_import_result + value = contextlib + import contextlib as name_import_result + + self.assertIs(globals()["name_import_result"], value) + del name_import_result + + def test_match(self): + global name_match + value = object() + match value: + case name_match: + pass + self.assertIs(globals()["name_match"], value) + del name_match + + def test_match_as(self): + global name_match_as + value = object() + match value: + case _ as name_match_as: + pass + self.assertIs(globals()["name_match_as"], value) + del name_match_as + + def test_match_seq(self): + global name_match_seq + value = object() + match (None, value): + case (_, name_match_seq): + pass + self.assertIs(globals()["name_match_seq"], value) + del name_match_seq + + def test_match_map(self): + global name_match_map + value = object() + match {"key": value}: + case {"key": name_match_map}: + pass + self.assertIs(globals()["name_match_map"], value) + del name_match_map + + def test_match_attr(self): + global name_match_attr + value = object() + match SimpleNamespace(key=value): + case SimpleNamespace(key=name_match_attr): + pass + self.assertIs(globals()["name_match_attr"], value) + del name_match_attr def setUpModule(): diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 668042782bdc5f..0bdd1b4b82e544 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -6,12 +6,14 @@ importlib_util = util.import_importlib('importlib.util') import importlib.util +from importlib import _bootstrap_external import os import pathlib import re import string import sys from test import support +from test.support import os_helper import textwrap import types import unittest @@ -775,5 +777,35 @@ def test_complete_multi_phase_init_module(self): self.run_with_own_gil(script) +class MiscTests(unittest.TestCase): + def test_atomic_write_should_notice_incomplete_writes(self): + import _pyio + + oldwrite = os.write + seen_write = False + + truncate_at_length = 100 + + # Emulate an os.write that only writes partial data. + def write(fd, data): + nonlocal seen_write + seen_write = True + return oldwrite(fd, data[:truncate_at_length]) + + # Need to patch _io to be _pyio, so that io.FileIO is affected by the + # os.write patch. + with (support.swap_attr(_bootstrap_external, '_io', _pyio), + support.swap_attr(os, 'write', write)): + with self.assertRaises(OSError): + # Make sure we write something longer than the point where we + # truncate. + content = b'x' * (truncate_at_length * 2) + _bootstrap_external._write_atomic(os_helper.TESTFN, content) + assert seen_write + + with self.assertRaises(OSError): + os.stat(support.os_helper.TESTFN) # Check that the file did not get written. + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 2250b7e76dac01..a92627a4d60f68 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -1960,6 +1960,19 @@ def g(local_ref): builtin_vars, unbound_names) self.assertEqual(inspect.getclosurevars(C().f(_arg)), expected) + def test_attribute_same_name_as_global_var(self): + class C: + _global_ref = object() + def f(): + print(C._global_ref, _global_ref) + nonlocal_vars = {"C": C} + global_vars = {"_global_ref": _global_ref} + builtin_vars = {"print": print} + unbound_names = {"_global_ref"} + expected = inspect.ClosureVars(nonlocal_vars, global_vars, + builtin_vars, unbound_names) + self.assertEqual(inspect.getclosurevars(f), expected) + def test_nonlocal_vars(self): # More complex tests of nonlocal resolution def _nonlocal_vars(f): @@ -5708,8 +5721,8 @@ def test_faulthandler_module_has_signatures(self): self._test_module_has_signatures(faulthandler, unsupported_signature=unsupported_signature) def test_functools_module_has_signatures(self): - no_signature = {'reduce'} - self._test_module_has_signatures(functools, no_signature) + unsupported_signature = {"reduce"} + self._test_module_has_signatures(functools, unsupported_signature=unsupported_signature) def test_gc_module_has_signatures(self): import gc diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 5e3d7a052bae91..a9befbba64daa0 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -54,6 +54,9 @@ def test_in_main(self): self.assertIsInstance(interp, interpreters.Interpreter) self.assertIn(interp, interpreters.list_all()) + # GH-126221: Passing an invalid Unicode character used to cause a SystemError + self.assertRaises(UnicodeEncodeError, _interpreters.create, '\udc80') + def test_in_thread(self): lock = threading.Lock() interp = None diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 8469de998ba014..b94d688738f9e8 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -494,6 +494,8 @@ def test_count(self): self.assertEqual(take(2, zip('abc',count(-3))), [('a', -3), ('b', -2)]) self.assertRaises(TypeError, count, 2, 3, 4) self.assertRaises(TypeError, count, 'a') + self.assertEqual(take(3, count(maxsize)), + [maxsize, maxsize + 1, maxsize + 2]) self.assertEqual(take(10, count(maxsize-5)), list(range(maxsize-5, maxsize+5))) self.assertEqual(take(10, count(-maxsize-5)), @@ -540,6 +542,12 @@ def test_count_with_step(self): self.assertEqual(take(20, count(-maxsize-15, 3)), take(20, range(-maxsize-15,-maxsize+100, 3))) self.assertEqual(take(3, count(10, maxsize+5)), list(range(10, 10+3*(maxsize+5), maxsize+5))) + self.assertEqual(take(3, count(maxsize, 2)), + [maxsize, maxsize + 2, maxsize + 4]) + self.assertEqual(take(3, count(maxsize, maxsize)), + [maxsize, 2 * maxsize, 3 * maxsize]) + self.assertEqual(take(3, count(-maxsize, maxsize)), + [-maxsize, 0, maxsize]) self.assertEqual(take(3, count(2, 1.25)), [2, 3.25, 4.5]) self.assertEqual(take(3, count(2, 3.25-4j)), [2, 5.25-4j, 8.5-8j]) self.assertEqual(take(3, count(Decimal('1.1'), Decimal('.1'))), @@ -2433,10 +2441,10 @@ class subclass(cls): subclass(*args, newarg=3) for cls, args, result in testcases: - # Constructors of repeat, zip, compress accept keyword arguments. + # Constructors of repeat, zip, map, compress accept keyword arguments. # Their subclasses need overriding __new__ to support new # keyword arguments. - if cls in [repeat, zip, compress]: + if cls in [repeat, zip, map, compress]: continue with self.subTest(cls): class subclass_with_init(cls): diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 64ee1ba867d592..93b8684c725d24 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -28,6 +28,13 @@ def helper(self, sample, *extra): finally: os_helper.unlink(os_helper.TESTFN) +def omit_last_byte(data): + """return data[:-1]""" + # This file's code is used in CompatibilityTestCase, + # but slices need marshal version 5. + # Avoid the slice literal. + return data[slice(0, -1)] + class IntTestCase(unittest.TestCase, HelperMixin): def test_ints(self): # Test a range of Python ints larger than the machine word size. @@ -241,7 +248,8 @@ def test_bug_5888452(self): def test_patch_873224(self): self.assertRaises(Exception, marshal.loads, b'0') self.assertRaises(Exception, marshal.loads, b'f') - self.assertRaises(Exception, marshal.loads, marshal.dumps(2**65)[:-1]) + self.assertRaises(Exception, marshal.loads, + omit_last_byte(marshal.dumps(2**65))) def test_version_argument(self): # Python 2.4.0 crashes for any call to marshal.dumps(x, y) @@ -594,6 +602,19 @@ def testNoIntern(self): s2 = sys.intern(s) self.assertNotEqual(id(s2), id(s)) +class SliceTestCase(unittest.TestCase, HelperMixin): + def test_slice(self): + for obj in ( + slice(None), slice(1), slice(1, 2), slice(1, 2, 3), + slice({'set'}, ('tuple', {'with': 'dict'}, ), self.helper.__code__) + ): + with self.subTest(obj=str(obj)): + self.helper(obj) + + for version in range(4): + with self.assertRaises(ValueError): + marshal.dumps(obj, version) + @support.cpython_only @unittest.skipUnless(_testcapi, 'requires _testcapi') class CAPI_TestCase(unittest.TestCase, HelperMixin): @@ -654,7 +675,7 @@ def test_read_last_object_from_file(self): self.assertEqual(r, obj) with open(os_helper.TESTFN, 'wb') as f: - f.write(data[:1]) + f.write(omit_last_byte(data)) with self.assertRaises(EOFError): _testcapi.pymarshal_read_last_object_from_file(os_helper.TESTFN) os_helper.unlink(os_helper.TESTFN) @@ -671,7 +692,7 @@ def test_read_object_from_file(self): self.assertEqual(p, len(data)) with open(os_helper.TESTFN, 'wb') as f: - f.write(data[:1]) + f.write(omit_last_byte(data)) with self.assertRaises(EOFError): _testcapi.pymarshal_read_object_from_file(os_helper.TESTFN) os_helper.unlink(os_helper.TESTFN) diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 58f6a4dfae08ba..0a5b511e75537c 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -223,26 +223,46 @@ def test_guess_known_extensions(self): def test_preferred_extension(self): def check_extensions(): - self.assertEqual(mimetypes.guess_extension('application/octet-stream'), '.bin') - self.assertEqual(mimetypes.guess_extension('application/postscript'), '.ps') - self.assertEqual(mimetypes.guess_extension('application/vnd.apple.mpegurl'), '.m3u') - self.assertEqual(mimetypes.guess_extension('application/vnd.ms-excel'), '.xls') - self.assertEqual(mimetypes.guess_extension('application/vnd.ms-powerpoint'), '.ppt') - self.assertEqual(mimetypes.guess_extension('application/x-texinfo'), '.texi') - self.assertEqual(mimetypes.guess_extension('application/x-troff'), '.roff') - self.assertEqual(mimetypes.guess_extension('application/xml'), '.xsl') - self.assertEqual(mimetypes.guess_extension('audio/mpeg'), '.mp3') - self.assertEqual(mimetypes.guess_extension('image/avif'), '.avif') - self.assertEqual(mimetypes.guess_extension('image/webp'), '.webp') - self.assertEqual(mimetypes.guess_extension('image/jpeg'), '.jpg') - self.assertEqual(mimetypes.guess_extension('image/tiff'), '.tiff') - self.assertEqual(mimetypes.guess_extension('message/rfc822'), '.eml') - self.assertEqual(mimetypes.guess_extension('text/html'), '.html') - self.assertEqual(mimetypes.guess_extension('text/plain'), '.txt') - self.assertEqual(mimetypes.guess_extension('text/rtf'), '.rtf') - self.assertEqual(mimetypes.guess_extension('text/x-rst'), '.rst') - self.assertEqual(mimetypes.guess_extension('video/mpeg'), '.mpeg') - self.assertEqual(mimetypes.guess_extension('video/quicktime'), '.mov') + for mime_type, ext in ( + ("application/octet-stream", ".bin"), + ("application/postscript", ".ps"), + ("application/vnd.apple.mpegurl", ".m3u"), + ("application/vnd.ms-excel", ".xls"), + ("application/vnd.ms-fontobject", ".eot"), + ("application/vnd.ms-powerpoint", ".ppt"), + ("application/x-texinfo", ".texi"), + ("application/x-troff", ".roff"), + ("application/xml", ".xsl"), + ("audio/matroska", ".mka"), + ("audio/mpeg", ".mp3"), + ("font/otf", ".otf"), + ("font/ttf", ".ttf"), + ("font/woff", ".woff"), + ("font/woff2", ".woff2"), + ("image/avif", ".avif"), + ("image/emf", ".emf"), + ("image/fits", ".fits"), + ("image/g3fax", ".g3"), + ("image/jp2", ".jp2"), + ("image/jpm", ".jpm"), + ("image/t38", ".t38"), + ("image/webp", ".webp"), + ("image/wmf", ".wmf"), + ("image/jpeg", ".jpg"), + ("image/tiff", ".tiff"), + ("image/tiff-fx", ".tfx"), + ("message/rfc822", ".eml"), + ("text/html", ".html"), + ("text/plain", ".txt"), + ("text/rtf", ".rtf"), + ("text/x-rst", ".rst"), + ("video/matroska", ".mkv"), + ("video/matroska-3d", ".mk3d"), + ("video/mpeg", ".mpeg"), + ("video/quicktime", ".mov"), + ): + with self.subTest(mime_type=mime_type, ext=ext): + self.assertEqual(mimetypes.guess_extension(mime_type), ext) check_extensions() mimetypes.init() diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 64cbfaaaaa0690..4f59184dfcfdc7 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -806,6 +806,9 @@ def test_abspath(self): tester('ntpath.abspath("C:\\spam. . .")', "C:\\spam") tester('ntpath.abspath("C:/nul")', "\\\\.\\nul") tester('ntpath.abspath("C:\\nul")', "\\\\.\\nul") + self.assertTrue(ntpath.isabs(ntpath.abspath("C:spam"))) + self.assertEqual(ntpath.abspath("C:\x00"), ntpath.join(ntpath.abspath("C:"), "\x00")) + self.assertEqual(ntpath.abspath("\x00:spam"), "\x00:\\spam") tester('ntpath.abspath("//..")', "\\\\") tester('ntpath.abspath("//../")', "\\\\..\\") tester('ntpath.abspath("//../..")', "\\\\..\\") diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index cdcddb0d717f23..78e4bf44f7ea0c 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -4,7 +4,9 @@ import threading import types import unittest -from test.support import threading_helper, check_impl_detail, requires_specialization +from test.support import (threading_helper, check_impl_detail, + requires_specialization, requires_specialization_ft, + cpython_only) from test.support.import_helper import import_module # Skip this module on other interpreters, it is cpython specific: @@ -34,6 +36,11 @@ def assert_specialized(self, f, opname): opnames = {instruction.opname for instruction in instructions} self.assertIn(opname, opnames) + def assert_no_opcode(self, f, opname): + instructions = dis.get_instructions(f, adaptive=True) + opnames = {instruction.opname for instruction in instructions} + self.assertNotIn(opname, opnames) + class TestLoadSuperAttrCache(unittest.TestCase): def test_descriptor_not_double_executed_on_spec_fail(self): @@ -1200,5 +1207,55 @@ def f(o, n): self.assertEqual(test_obj.b, 0) +class TestSpecializer(TestBase): + + @cpython_only + @requires_specialization_ft + def test_binary_op(self): + def f(): + for _ in range(100): + a, b = 1, 2 + c = a + b + self.assertEqual(c, 3) + + f() + self.assert_specialized(f, "BINARY_OP_ADD_INT") + self.assert_no_opcode(f, "BINARY_OP") + + def g(): + for _ in range(100): + a, b = "foo", "bar" + c = a + b + self.assertEqual(c, "foobar") + + g() + self.assert_specialized(g, "BINARY_OP_ADD_UNICODE") + self.assert_no_opcode(g, "BINARY_OP") + + @cpython_only + @requires_specialization_ft + def test_contain_op(self): + def f(): + for _ in range(100): + a, b = 1, {1: 2, 2: 5} + self.assertTrue(a in b) + self.assertFalse(3 in b) + + f() + self.assert_specialized(f, "CONTAINS_OP_DICT") + self.assert_no_opcode(f, "CONTAINS_OP") + + def g(): + for _ in range(100): + a, b = 1, {1, 2} + self.assertTrue(a in b) + self.assertFalse(3 in b) + + g() + self.assert_specialized(g, "CONTAINS_OP_SET") + self.assert_no_opcode(g, "CONTAINS_OP") + + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index 28b274462388ed..8655a0537a5e56 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -15,7 +15,7 @@ from io import StringIO from test import support from test.support import os_helper - +from test.support.i18n_helper import TestTranslationsBase, update_translation_snapshots import optparse from optparse import make_option, Option, \ @@ -1656,5 +1656,14 @@ def test__all__(self): support.check__all__(self, optparse, not_exported=not_exported) +class TestTranslations(TestTranslationsBase): + def test_translations(self): + self.assertMsgidsEqual(optparse) + + if __name__ == '__main__': + # To regenerate translation snapshots + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + update_translation_snapshots(optparse) + sys.exit(0) unittest.main() diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 307f0f11ddc33f..919ed92ddb425f 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1298,8 +1298,8 @@ def test_ror_operator(self): self._test_underlying_process_env('_A_', '') self._test_underlying_process_env(overridden_key, original_value) - def test_refresh(self): - # Test os.environ.refresh() + def test_reload_environ(self): + # Test os.reload_environ() has_environb = hasattr(os, 'environb') # Test with putenv() which doesn't update os.environ @@ -1309,7 +1309,7 @@ def test_refresh(self): if has_environb: self.assertEqual(os.environb[b'test_env'], b'python_value') - os.environ.refresh() + os.reload_environ() self.assertEqual(os.environ['test_env'], 'new_value') if has_environb: self.assertEqual(os.environb[b'test_env'], b'new_value') @@ -1320,28 +1320,28 @@ def test_refresh(self): if has_environb: self.assertEqual(os.environb[b'test_env'], b'new_value') - os.environ.refresh() + os.reload_environ() self.assertNotIn('test_env', os.environ) if has_environb: self.assertNotIn(b'test_env', os.environb) if has_environb: - # test os.environb.refresh() with putenv() + # test reload_environ() on os.environb with putenv() os.environb[b'test_env'] = b'python_value2' os.putenv("test_env", "new_value2") self.assertEqual(os.environb[b'test_env'], b'python_value2') self.assertEqual(os.environ['test_env'], 'python_value2') - os.environb.refresh() + os.reload_environ() self.assertEqual(os.environb[b'test_env'], b'new_value2') self.assertEqual(os.environ['test_env'], 'new_value2') - # test os.environb.refresh() with unsetenv() + # test reload_environ() on os.environb with unsetenv() os.unsetenv('test_env') self.assertEqual(os.environb[b'test_env'], b'new_value2') self.assertEqual(os.environ['test_env'], 'new_value2') - os.environb.refresh() + os.reload_environ() self.assertNotIn(b'test_env', os.environb) self.assertNotIn('test_env', os.environ) @@ -3967,10 +3967,10 @@ def _check_xattrs_str(self, s, getxattr, setxattr, removexattr, listxattr, **kwa xattr.remove("user.test") self.assertEqual(set(listxattr(fn)), xattr) self.assertEqual(getxattr(fn, s("user.test2"), **kwargs), b"foo") - setxattr(fn, s("user.test"), b"a"*1024, **kwargs) - self.assertEqual(getxattr(fn, s("user.test"), **kwargs), b"a"*1024) + setxattr(fn, s("user.test"), b"a"*256, **kwargs) + self.assertEqual(getxattr(fn, s("user.test"), **kwargs), b"a"*256) removexattr(fn, s("user.test"), **kwargs) - many = sorted("user.test{}".format(i) for i in range(100)) + many = sorted("user.test{}".format(i) for i in range(32)) for thing in many: setxattr(fn, thing, b"x", **kwargs) self.assertEqual(set(listxattr(fn)), set(init_xattr) | set(many)) diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index c7104bfda90f6c..46966b6df2d7b0 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -861,6 +861,28 @@ def test_move_into_other_os(self): def test_move_into_empty_name_other_os(self): self.test_move_into_empty_name() + def _check_complex_symlinks(self, link0_target): + super()._check_complex_symlinks(link0_target) + P = self.cls(self.base) + # Resolve relative paths. + old_path = os.getcwd() + os.chdir(self.base) + try: + p = self.cls('link0').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = self.cls('link1').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = self.cls('link2').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = self.cls('link3').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + finally: + os.chdir(old_path) + def test_resolve_nonexist_relative_issue38671(self): p = self.cls('non', 'exist') diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 08355a71453807..b69d674e1cf1ed 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1,4 +1,5 @@ import collections +import contextlib import io import os import errno @@ -85,11 +86,6 @@ def test_unsupported_operation_pure(self): p.suffix with self.assertRaises(e): p.suffixes - with self.assertRaises(e): - p / 'bar' - with self.assertRaises(e): - 'bar' / p - self.assertRaises(e, p.joinpath, 'bar') self.assertRaises(e, p.with_name, 'bar') self.assertRaises(e, p.with_stem, 'bar') self.assertRaises(e, p.with_suffix, '.txt') @@ -152,6 +148,7 @@ def test_constructor_common(self): P = self.cls p = P('a') self.assertIsInstance(p, P) + P() P('a', 'b', 'c') P('/a', 'b', 'c') P('a/b/c') @@ -1424,6 +1421,24 @@ def close(self): 'st_mode st_ino st_dev st_nlink st_uid st_gid st_size st_atime st_mtime st_ctime') +class DummyDirEntry: + """ + Minimal os.DirEntry-like object. Returned from DummyPath.scandir(). + """ + __slots__ = ('name', '_is_symlink', '_is_dir') + + def __init__(self, name, is_symlink, is_dir): + self.name = name + self._is_symlink = is_symlink + self._is_dir = is_dir + + def is_symlink(self): + return self._is_symlink + + def is_dir(self, *, follow_symlinks=True): + return self._is_dir and (follow_symlinks or not self._is_symlink) + + class DummyPath(PathBase): """ Simple implementation of PathBase that keeps files and directories in @@ -1491,14 +1506,25 @@ def open(self, mode='r', buffering=-1, encoding=None, stream = io.TextIOWrapper(stream, encoding=encoding, errors=errors, newline=newline) return stream - def iterdir(self): - path = str(self.resolve()) - if path in self._files: - raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) - elif path in self._directories: - return iter([self / name for name in self._directories[path]]) + @contextlib.contextmanager + def scandir(self): + path = self.resolve() + path_str = str(path) + if path_str in self._files: + raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path_str) + elif path_str in self._directories: + yield iter([path.joinpath(name)._dir_entry for name in self._directories[path_str]]) else: - raise FileNotFoundError(errno.ENOENT, "File not found", path) + raise FileNotFoundError(errno.ENOENT, "File not found", path_str) + + @property + def _dir_entry(self): + path_str = str(self) + is_symlink = path_str in self._symlinks + is_directory = (path_str in self._directories + if not is_symlink + else self._symlinks[path_str][1]) + return DummyDirEntry(self.name, is_symlink, is_directory) def mkdir(self, mode=0o777, parents=False, exist_ok=False): path = str(self.parent.resolve() / self.name) @@ -1602,9 +1628,11 @@ def setUp(self): if self.can_symlink: p.joinpath('linkA').symlink_to('fileA') p.joinpath('brokenLink').symlink_to('non-existing') - p.joinpath('linkB').symlink_to('dirB') - p.joinpath('dirA', 'linkC').symlink_to(parser.join('..', 'dirB')) - p.joinpath('dirB', 'linkD').symlink_to(parser.join('..', 'dirB')) + p.joinpath('linkB').symlink_to('dirB', target_is_directory=True) + p.joinpath('dirA', 'linkC').symlink_to( + parser.join('..', 'dirB'), target_is_directory=True) + p.joinpath('dirB', 'linkD').symlink_to( + parser.join('..', 'dirB'), target_is_directory=True) p.joinpath('brokenLinkLoop').symlink_to('brokenLinkLoop') def tearDown(self): @@ -1919,7 +1947,7 @@ def ordered_walk(path): if self.can_symlink: # Add some symlinks source.joinpath('linkC').symlink_to('fileC') - source.joinpath('linkD').symlink_to('dirD') + source.joinpath('linkD').symlink_to('dirD', target_is_directory=True) # Perform the copy target = base / 'copyC' @@ -2187,6 +2215,23 @@ def test_iterdir_nodir(self): self.assertIn(cm.exception.errno, (errno.ENOTDIR, errno.ENOENT, errno.EINVAL)) + def test_scandir(self): + p = self.cls(self.base) + with p.scandir() as entries: + self.assertTrue(list(entries)) + with p.scandir() as entries: + for entry in entries: + child = p / entry.name + self.assertIsNotNone(entry) + self.assertEqual(entry.name, child.name) + self.assertEqual(entry.is_symlink(), + child.is_symlink()) + self.assertEqual(entry.is_dir(follow_symlinks=False), + child.is_dir(follow_symlinks=False)) + if entry.name != 'brokenLinkLoop': + self.assertEqual(entry.is_dir(), child.is_dir()) + + def test_glob_common(self): def _check(glob, expected): self.assertEqual(set(glob), { P(self.base, q) for q in expected }) @@ -2432,7 +2477,7 @@ def test_glob_permissions(self): if i % 2: link.symlink_to(P(self.base, "dirE", "nonexistent")) else: - link.symlink_to(P(self.base, "dirC")) + link.symlink_to(P(self.base, "dirC"), target_is_directory=True) self.assertEqual(len(set(base.glob("*"))), 100) self.assertEqual(len(set(base.glob("*/"))), 50) @@ -2448,6 +2493,23 @@ def test_glob_long_symlink(self): bad_link.symlink_to("bad" * 200) self.assertEqual(sorted(base.glob('**/*')), [bad_link]) + @needs_posix + def test_absolute_posix(self): + P = self.cls + # The default implementation uses '/' as the current directory + self.assertEqual(str(P('').absolute()), '/') + self.assertEqual(str(P('a').absolute()), '/a') + self.assertEqual(str(P('a/b').absolute()), '/a/b') + + self.assertEqual(str(P('/').absolute()), '/') + self.assertEqual(str(P('/a').absolute()), '/a') + self.assertEqual(str(P('/a/b').absolute()), '/a/b') + + # '//'-prefixed absolute path (supported by POSIX). + self.assertEqual(str(P('//').absolute()), '//') + self.assertEqual(str(P('//a').absolute()), '//a') + self.assertEqual(str(P('//a/b').absolute()), '//a/b') + @needs_symlinks def test_readlink(self): P = self.cls(self.base) @@ -2765,29 +2827,6 @@ def _check_complex_symlinks(self, link0_target): self.assertEqual(p, P) self.assertEqualNormCase(str(p), self.base) - # Resolve relative paths. - try: - self.cls('').absolute() - except UnsupportedOperation: - return - old_path = os.getcwd() - os.chdir(self.base) - try: - p = self.cls('link0').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - p = self.cls('link1').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - p = self.cls('link2').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - p = self.cls('link3').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - finally: - os.chdir(old_path) - @needs_symlinks def test_complex_symlinks_absolute(self): self._check_complex_symlinks(self.base) @@ -2920,7 +2959,7 @@ def setUpWalk(self): f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n") if self.can_symlink: - self.link_path.symlink_to(t2_path) + self.link_path.symlink_to(t2_path, target_is_directory=True) broken_link_path.symlink_to('broken') broken_link2_path.symlink_to(self.cls('tmp3', 'broken')) self.sub2_tree = (self.sub2_path, [], ["broken_link", "broken_link2", "link", "tmp3"]) @@ -3038,7 +3077,7 @@ class DummyPathWithSymlinks(DummyPath): def readlink(self): path = str(self.parent.resolve() / self.name) if path in self._symlinks: - return self.with_segments(self._symlinks[path]) + return self.with_segments(self._symlinks[path][0]) elif path in self._files or path in self._directories: raise OSError(errno.EINVAL, "Not a symlink", path) else: @@ -3050,7 +3089,7 @@ def symlink_to(self, target, target_is_directory=False): if path in self._symlinks: raise FileExistsError(errno.EEXIST, "File exists", path) self._directories[parent].add(self.name) - self._symlinks[path] = str(target) + self._symlinks[path] = str(target), target_is_directory class DummyPathWithSymlinksTest(DummyPathTest): diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index b55d441759eb69..1e74990878007a 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -210,14 +210,14 @@ def test_sys_api_with_existing_trampoline(self): sys.activate_stack_trampoline("perf") sys.activate_stack_trampoline("perf") """ - assert_python_ok("-c", code) + assert_python_ok("-c", code, PYTHON_JIT="0") def test_sys_api_with_invalid_trampoline(self): code = """if 1: import sys sys.activate_stack_trampoline("invalid") """ - rc, out, err = assert_python_failure("-c", code) + rc, out, err = assert_python_failure("-c", code, PYTHON_JIT="0") self.assertIn("invalid backend: invalid", err.decode()) def test_sys_api_get_status(self): @@ -228,7 +228,7 @@ def test_sys_api_get_status(self): sys.deactivate_stack_trampoline() assert sys.is_stack_trampoline_active() is False """ - assert_python_ok("-c", code) + assert_python_ok("-c", code, PYTHON_JIT="0") def is_unwinding_reliable_with_frame_pointers(): diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index c84e507cdf645f..4ec966d8351490 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -224,25 +224,109 @@ def persistent_load(pid): def test_pickler_super(self): class PersPickler(self.pickler): def persistent_id(subself, obj): + called.append(obj) self.assertIsNone(super().persistent_id(obj)) return obj for proto in range(pickle.HIGHEST_PROTOCOL + 1): f = io.BytesIO() pickler = PersPickler(f, proto) + called = [] pickler.dump('abc') + self.assertEqual(called, ['abc']) self.assertEqual(self.loads(f.getvalue()), 'abc') def test_unpickler_super(self): class PersUnpickler(self.unpickler): def persistent_load(subself, pid): + called.append(pid) with self.assertRaises(self.persistent_load_error): super().persistent_load(pid) return pid for proto in range(pickle.HIGHEST_PROTOCOL + 1): unpickler = PersUnpickler(io.BytesIO(self.dumps('abc', proto))) + called = [] self.assertEqual(unpickler.load(), 'abc') + self.assertEqual(called, ['abc']) + + def test_pickler_instance_attribute(self): + def persistent_id(obj): + called.append(obj) + return obj + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f = io.BytesIO() + pickler = self.pickler(f, proto) + called = [] + old_persistent_id = pickler.persistent_id + pickler.persistent_id = persistent_id + self.assertEqual(pickler.persistent_id, persistent_id) + pickler.dump('abc') + self.assertEqual(called, ['abc']) + self.assertEqual(self.loads(f.getvalue()), 'abc') + del pickler.persistent_id + self.assertEqual(pickler.persistent_id, old_persistent_id) + + def test_unpickler_instance_attribute(self): + def persistent_load(pid): + called.append(pid) + return pid + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + unpickler = self.unpickler(io.BytesIO(self.dumps('abc', proto))) + called = [] + old_persistent_load = unpickler.persistent_load + unpickler.persistent_load = persistent_load + self.assertEqual(unpickler.persistent_load, persistent_load) + self.assertEqual(unpickler.load(), 'abc') + self.assertEqual(called, ['abc']) + del unpickler.persistent_load + self.assertEqual(unpickler.persistent_load, old_persistent_load) + + def test_pickler_super_instance_attribute(self): + class PersPickler(self.pickler): + def persistent_id(subself, obj): + raise AssertionError('should never be called') + def _persistent_id(subself, obj): + called.append(obj) + self.assertIsNone(super().persistent_id(obj)) + return obj + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f = io.BytesIO() + pickler = PersPickler(f, proto) + called = [] + old_persistent_id = pickler.persistent_id + pickler.persistent_id = pickler._persistent_id + self.assertEqual(pickler.persistent_id, pickler._persistent_id) + pickler.dump('abc') + self.assertEqual(called, ['abc']) + self.assertEqual(self.loads(f.getvalue()), 'abc') + del pickler.persistent_id + self.assertEqual(pickler.persistent_id, old_persistent_id) + + def test_unpickler_super_instance_attribute(self): + class PersUnpickler(self.unpickler): + def persistent_load(subself, pid): + raise AssertionError('should never be called') + def _persistent_load(subself, pid): + called.append(pid) + with self.assertRaises(self.persistent_load_error): + super().persistent_load(pid) + return pid + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + unpickler = PersUnpickler(io.BytesIO(self.dumps('abc', proto))) + called = [] + old_persistent_load = unpickler.persistent_load + unpickler.persistent_load = unpickler._persistent_load + self.assertEqual(unpickler.persistent_load, unpickler._persistent_load) + self.assertEqual(unpickler.load(), 'abc') + self.assertEqual(called, ['abc']) + del unpickler.persistent_load + self.assertEqual(unpickler.persistent_load, old_persistent_load) + class PyPicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests, unittest.TestCase): @@ -367,7 +451,7 @@ class SizeofTests(unittest.TestCase): check_sizeof = support.check_sizeof def test_pickler(self): - basesize = support.calcobjsize('6P2n3i2n3i2P') + basesize = support.calcobjsize('7P2n3i2n3i2P') p = _pickle.Pickler(io.BytesIO()) self.assertEqual(object.__sizeof__(p), basesize) MT_size = struct.calcsize('3nP0n') @@ -384,7 +468,7 @@ def test_pickler(self): 0) # Write buffer is cleared after every dump(). def test_unpickler(self): - basesize = support.calcobjsize('2P2nP 2P2n2i5P 2P3n8P2n2i') + basesize = support.calcobjsize('2P2n2P 2P2n2i5P 2P3n8P2n2i') unpickler = _pickle.Unpickler P = struct.calcsize('P') # Size of memo table entry. n = struct.calcsize('n') # Size of mark table entry. diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index ca6927554b053c..736b83711def03 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -607,73 +607,6 @@ class ImportlibMigrationTests(unittest.TestCase): # PEP 302 emulation in this module is in the process of being # deprecated in favour of importlib proper - @unittest.skipIf(__name__ == '__main__', 'not compatible with __main__') - @ignore_warnings(category=DeprecationWarning) - def test_get_loader_handles_missing_loader_attribute(self): - global __loader__ - this_loader = __loader__ - del __loader__ - try: - self.assertIsNotNone(pkgutil.get_loader(__name__)) - finally: - __loader__ = this_loader - - @ignore_warnings(category=DeprecationWarning) - def test_get_loader_handles_missing_spec_attribute(self): - name = 'spam' - mod = type(sys)(name) - del mod.__spec__ - with CleanImport(name): - try: - sys.modules[name] = mod - loader = pkgutil.get_loader(name) - finally: - sys.modules.pop(name, None) - self.assertIsNone(loader) - - @ignore_warnings(category=DeprecationWarning) - def test_get_loader_handles_spec_attribute_none(self): - name = 'spam' - mod = type(sys)(name) - mod.__spec__ = None - with CleanImport(name): - try: - sys.modules[name] = mod - loader = pkgutil.get_loader(name) - finally: - sys.modules.pop(name, None) - self.assertIsNone(loader) - - @ignore_warnings(category=DeprecationWarning) - def test_get_loader_None_in_sys_modules(self): - name = 'totally bogus' - sys.modules[name] = None - try: - loader = pkgutil.get_loader(name) - finally: - del sys.modules[name] - self.assertIsNone(loader) - - def test_get_loader_is_deprecated(self): - with check_warnings( - (r".*\bpkgutil.get_loader\b.*", DeprecationWarning), - ): - res = pkgutil.get_loader("sys") - self.assertIsNotNone(res) - - def test_find_loader_is_deprecated(self): - with check_warnings( - (r".*\bpkgutil.find_loader\b.*", DeprecationWarning), - ): - res = pkgutil.find_loader("sys") - self.assertIsNotNone(res) - - @ignore_warnings(category=DeprecationWarning) - def test_find_loader_missing_module(self): - name = 'totally bogus' - loader = pkgutil.find_loader(name) - self.assertIsNone(loader) - def test_get_importer_avoids_emulation(self): # We use an illegal path so *none* of the path hooks should fire with check_warnings() as w: diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 40d5fb338ce563..e04ad142061ad3 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -83,6 +83,38 @@ def clear_caches(self): platform._uname_cache = None platform._os_release_cache = None + def test_invalidate_caches(self): + self.clear_caches() + + self.assertDictEqual(platform._platform_cache, {}) + self.assertDictEqual(platform._sys_version_cache, {}) + self.assertIsNone(platform._uname_cache) + self.assertIsNone(platform._os_release_cache) + + # fill the cached entries (some have side effects on others) + platform.platform() # for platform._platform_cache + platform.python_implementation() # for platform._sys_version_cache + platform.uname() # for platform._uname_cache + + # check that the cache are filled + self.assertNotEqual(platform._platform_cache, {}) + self.assertNotEqual(platform._sys_version_cache, {}) + self.assertIsNotNone(platform._uname_cache) + + try: + platform.freedesktop_os_release() + except OSError: + self.assertIsNone(platform._os_release_cache) + else: + self.assertIsNotNone(platform._os_release_cache) + + with self.subTest('clear platform caches'): + platform.invalidate_caches() + self.assertDictEqual(platform._platform_cache, {}) + self.assertDictEqual(platform._sys_version_cache, {}) + self.assertIsNone(platform._uname_cache) + self.assertIsNone(platform._os_release_cache) + def test_architecture(self): res = platform.architecture() diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index b231b05f864ab9..a0c76e5dec5ebe 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -1,4 +1,4 @@ -# Copyright (C) 2003-2013 Python Software Foundation +# Copyright (C) 2003 Python Software Foundation import copy import operator import pickle diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 35016b83a477fc..ef9d617f66feec 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -6,12 +6,14 @@ from test.support import warnings_helper from test.support.script_helper import assert_python_ok +import copy import errno import sys import signal import time import os import platform +import pickle import stat import tempfile import unittest @@ -1317,6 +1319,25 @@ def test_get_and_set_scheduler_and_param(self): param = posix.sched_param(sched_priority=-large) self.assertRaises(OverflowError, posix.sched_setparam, 0, param) + @requires_sched + def test_sched_param(self): + param = posix.sched_param(1) + for proto in range(pickle.HIGHEST_PROTOCOL+1): + newparam = pickle.loads(pickle.dumps(param, proto)) + self.assertEqual(newparam, param) + newparam = copy.copy(param) + self.assertIsNot(newparam, param) + self.assertEqual(newparam, param) + newparam = copy.deepcopy(param) + self.assertIsNot(newparam, param) + self.assertEqual(newparam, param) + newparam = copy.replace(param) + self.assertIsNot(newparam, param) + self.assertEqual(newparam, param) + newparam = copy.replace(param, sched_priority=0) + self.assertNotEqual(newparam, param) + self.assertEqual(newparam.sched_priority, 0) + @unittest.skipUnless(hasattr(posix, "sched_rr_get_interval"), "no function") def test_sched_rr_get_interval(self): try: diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index ca5cf42f8fcd71..b39255ebc79ac1 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -695,6 +695,65 @@ def test_realpath_unreadable_symlink(self): os.chmod(ABSTFN, 0o755, follow_symlinks=False) os.unlink(ABSTFN) + @skip_if_ABSTFN_contains_backslash + def test_realpath_nonterminal_file(self): + try: + with open(ABSTFN, 'w') as f: + f.write('test_posixpath wuz ere') + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN) + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "/subdir") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + finally: + os_helper.unlink(ABSTFN) + + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + def test_realpath_nonterminal_symlink_to_file(self): + try: + with open(ABSTFN + "1", 'w') as f: + f.write('test_posixpath wuz ere') + os.symlink(ABSTFN + "1", ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN + "1") + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "1") + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN + "1") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN + "1") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "1/subdir") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + finally: + os_helper.unlink(ABSTFN) + + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + def test_realpath_nonterminal_symlink_to_symlinks_to_file(self): + try: + with open(ABSTFN + "2", 'w') as f: + f.write('test_posixpath wuz ere') + os.symlink(ABSTFN + "2", ABSTFN + "1") + os.symlink(ABSTFN + "1", ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN + "2") + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "2") + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN + "2") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN + "2") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "2/subdir") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + finally: + os_helper.unlink(ABSTFN) + def test_relpath(self): (real_getcwd, os.getcwd) = (os.getcwd, lambda: r"/home/user/bar") try: diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index ff95f54026e172..0d3599be87f228 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1136,6 +1136,39 @@ def test_ignore_case_set(self): self.assertTrue(re.match(br'[19a]', b'a', re.I)) self.assertTrue(re.match(br'[19a]', b'A', re.I)) self.assertTrue(re.match(br'[19A]', b'a', re.I)) + self.assertTrue(re.match(r'[19\xc7]', '\xc7', re.I)) + self.assertTrue(re.match(r'[19\xc7]', '\xe7', re.I)) + self.assertTrue(re.match(r'[19\xe7]', '\xc7', re.I)) + self.assertTrue(re.match(r'[19\xe7]', '\xe7', re.I)) + self.assertTrue(re.match(r'[19\u0400]', '\u0400', re.I)) + self.assertTrue(re.match(r'[19\u0400]', '\u0450', re.I)) + self.assertTrue(re.match(r'[19\u0450]', '\u0400', re.I)) + self.assertTrue(re.match(r'[19\u0450]', '\u0450', re.I)) + self.assertTrue(re.match(r'[19\U00010400]', '\U00010400', re.I)) + self.assertTrue(re.match(r'[19\U00010400]', '\U00010428', re.I)) + self.assertTrue(re.match(r'[19\U00010428]', '\U00010400', re.I)) + self.assertTrue(re.match(r'[19\U00010428]', '\U00010428', re.I)) + + self.assertTrue(re.match(br'[19A]', b'A', re.I)) + self.assertTrue(re.match(br'[19a]', b'a', re.I)) + self.assertTrue(re.match(br'[19a]', b'A', re.I)) + self.assertTrue(re.match(br'[19A]', b'a', re.I)) + self.assertTrue(re.match(r'[19A]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[19a]', 'a', re.I|re.A)) + self.assertTrue(re.match(r'[19a]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[19A]', 'a', re.I|re.A)) + self.assertTrue(re.match(r'[19\xc7]', '\xc7', re.I|re.A)) + self.assertIsNone(re.match(r'[19\xc7]', '\xe7', re.I|re.A)) + self.assertIsNone(re.match(r'[19\xe7]', '\xc7', re.I|re.A)) + self.assertTrue(re.match(r'[19\xe7]', '\xe7', re.I|re.A)) + self.assertTrue(re.match(r'[19\u0400]', '\u0400', re.I|re.A)) + self.assertIsNone(re.match(r'[19\u0400]', '\u0450', re.I|re.A)) + self.assertIsNone(re.match(r'[19\u0450]', '\u0400', re.I|re.A)) + self.assertTrue(re.match(r'[19\u0450]', '\u0450', re.I|re.A)) + self.assertTrue(re.match(r'[19\U00010400]', '\U00010400', re.I|re.A)) + self.assertIsNone(re.match(r'[19\U00010400]', '\U00010428', re.I|re.A)) + self.assertIsNone(re.match(r'[19\U00010428]', '\U00010400', re.I|re.A)) + self.assertTrue(re.match(r'[19\U00010428]', '\U00010428', re.I|re.A)) # Two different characters have the same lowercase. assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K' @@ -1172,8 +1205,10 @@ def test_ignore_case_range(self): self.assertTrue(re.match(br'[9-a]', b'_', re.I)) self.assertIsNone(re.match(br'[9-A]', b'_', re.I)) self.assertTrue(re.match(r'[\xc0-\xde]', '\xd7', re.I)) + self.assertTrue(re.match(r'[\xc0-\xde]', '\xe7', re.I)) self.assertIsNone(re.match(r'[\xc0-\xde]', '\xf7', re.I)) self.assertTrue(re.match(r'[\xe0-\xfe]', '\xf7', re.I)) + self.assertTrue(re.match(r'[\xe0-\xfe]', '\xc7', re.I)) self.assertIsNone(re.match(r'[\xe0-\xfe]', '\xd7', re.I)) self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0450', re.I)) self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0400', re.I)) @@ -1184,6 +1219,26 @@ def test_ignore_case_range(self): self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010428', re.I)) self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400', re.I)) + self.assertTrue(re.match(r'[\xc0-\xde]', '\xd7', re.I|re.A)) + self.assertIsNone(re.match(r'[\xc0-\xde]', '\xe7', re.I|re.A)) + self.assertTrue(re.match(r'[\xe0-\xfe]', '\xf7', re.I|re.A)) + self.assertIsNone(re.match(r'[\xe0-\xfe]', '\xc7', re.I|re.A)) + self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0450', re.I|re.A)) + self.assertIsNone(re.match(r'[\u0430-\u045f]', '\u0400', re.I|re.A)) + self.assertIsNone(re.match(r'[\u0400-\u042f]', '\u0450', re.I|re.A)) + self.assertTrue(re.match(r'[\u0400-\u042f]', '\u0400', re.I|re.A)) + self.assertTrue(re.match(r'[\U00010428-\U0001044f]', '\U00010428', re.I|re.A)) + self.assertIsNone(re.match(r'[\U00010428-\U0001044f]', '\U00010400', re.I|re.A)) + self.assertIsNone(re.match(r'[\U00010400-\U00010427]', '\U00010428', re.I|re.A)) + self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400', re.I|re.A)) + + self.assertTrue(re.match(r'[N-\x7f]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[n-\x7f]', 'Z', re.I|re.A)) + self.assertTrue(re.match(r'[N-\uffff]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[n-\uffff]', 'Z', re.I|re.A)) + self.assertTrue(re.match(r'[N-\U00010000]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[n-\U00010000]', 'Z', re.I|re.A)) + # Two different characters have the same lowercase. assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K' self.assertTrue(re.match(r'[J-M]', '\u212a', re.I)) @@ -2585,6 +2640,12 @@ def test_bug_gh100061(self): self.assertEqual(re.match("(?>(?:ab?c){1,3})", "aca").span(), (0, 2)) self.assertEqual(re.match("(?:ab?c){1,3}+", "aca").span(), (0, 2)) + def test_bug_gh101955(self): + # Possessive quantifier with nested alternative with capture groups + self.assertEqual(re.match('((x)|y|z)*+', 'xyz').groups(), ('z', 'x')) + self.assertEqual(re.match('((x)|y|z){3}+', 'xyz').groups(), ('z', 'x')) + self.assertEqual(re.match('((x)|y|z){3,}+', 'xyz').groups(), ('z', 'x')) + @unittest.skipIf(multiprocessing is None, 'test requires multiprocessing') def test_regression_gh94675(self): pattern = re.compile(r'(?<=[({}])(((//[^\n]*)?[\n])([\000-\040])*)*' @@ -2626,6 +2687,29 @@ def test_character_set_none(self): self.assertIsNone(re.search(p, s)) self.assertIsNone(re.search('(?s:.)' + p, s)) + def check_interrupt(self, pattern, string, maxcount): + class Interrupt(Exception): + pass + p = re.compile(pattern) + for n in range(maxcount): + try: + p._fail_after(n, Interrupt) + p.match(string) + return n + except Interrupt: + pass + finally: + p._fail_after(-1, None) + + @unittest.skipUnless(hasattr(re.Pattern, '_fail_after'), 'requires debug build') + def test_memory_leaks(self): + self.check_interrupt(r'(.)*:', 'abc:', 100) + self.check_interrupt(r'([^:])*?:', 'abc:', 100) + self.check_interrupt(r'([^:])*+:', 'abc:', 100) + self.check_interrupt(r'(.){2,4}:', 'abc:', 100) + self.check_interrupt(r'([^:]){2,4}?:', 'abc:', 100) + self.check_interrupt(r'([^:]){2,4}+:', 'abc:', 100) + def get_debug_out(pat): with captured_stdout() as out: diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index f2bc52ba6e8701..dc8a4a44f37e65 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -5334,6 +5334,8 @@ def _testMakefileClose(self): self.write_file.write(self.write_msg) self.write_file.flush() + @unittest.skipUnless(hasattr(sys, 'getrefcount'), + 'test needs sys.getrefcount()') def testMakefileCloseSocketDestroy(self): refcount_before = sys.getrefcount(self.cli_conn) self.read_file.close() diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index de5110a1cc4b6d..59f37b3f9a7575 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -2299,7 +2299,6 @@ def wrap_conn(self): # See also http://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/ if e.errno != errno.EPROTOTYPE and sys.platform != "darwin": self.running = False - self.server.stop() self.close() return False else: @@ -2436,10 +2435,6 @@ def run(self): self.close() self.running = False - # normally, we'd just stop here, but for the test - # harness, we want to stop the server - self.server.stop() - def __init__(self, certificate=None, ssl_version=None, certreqs=None, cacerts=None, chatty=True, connectionchatty=False, starttls_server=False, @@ -2473,21 +2468,33 @@ def __init__(self, certificate=None, ssl_version=None, self.conn_errors = [] threading.Thread.__init__(self) self.daemon = True + self._in_context = False def __enter__(self): + if self._in_context: + raise ValueError('Re-entering ThreadedEchoServer context') + self._in_context = True self.start(threading.Event()) self.flag.wait() return self def __exit__(self, *args): + assert self._in_context + self._in_context = False self.stop() self.join() def start(self, flag=None): + if not self._in_context: + raise ValueError( + 'ThreadedEchoServer must be used as a context manager') self.flag = flag threading.Thread.start(self) def run(self): + if not self._in_context: + raise ValueError( + 'ThreadedEchoServer must be used as a context manager') self.sock.settimeout(1.0) self.sock.listen(5) self.active = True @@ -5029,7 +5036,7 @@ def non_linux_skip_if_other_okay_error(self, err): return # Expect the full test setup to always work on Linux. if (isinstance(err, ConnectionResetError) or (isinstance(err, OSError) and err.errno == errno.EINVAL) or - re.search('wrong.version.number', getattr(err, "reason", ""), re.I)): + re.search('wrong.version.number', str(getattr(err, "reason", "")), re.I)): # On Windows the TCP RST leads to a ConnectionResetError # (ECONNRESET) which Linux doesn't appear to surface to userspace. # If wrap_socket() winds up on the "if connected:" path and doing diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index f065b9c9bb1c2c..e45701dfe033a6 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -4,6 +4,7 @@ from test.support import check_sanitizer from test.support import import_helper from test.support import os_helper +from test.support import strace_helper from test.support import warnings_helper from test.support.script_helper import assert_python_ok import subprocess @@ -3415,7 +3416,7 @@ def __del__(self): @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), "vfork() not enabled by configure.") - @unittest.skipIf(sys.platform != "linux", "Linux only, requires strace.") + @strace_helper.requires_strace() @mock.patch("subprocess._USE_POSIX_SPAWN", new=False) def test_vfork_used_when_expected(self): # This is a performance regression test to ensure we default to using @@ -3423,36 +3424,25 @@ def test_vfork_used_when_expected(self): # Technically this test could pass when posix_spawn is used as well # because libc tends to implement that internally using vfork. But # that'd just be testing a libc+kernel implementation detail. - strace_binary = "/usr/bin/strace" - # The only system calls we are interested in. - strace_filter = "--trace=clone,clone2,clone3,fork,vfork,exit,exit_group" - true_binary = "/bin/true" - strace_command = [strace_binary, strace_filter] - try: - does_strace_work_process = subprocess.run( - strace_command + [true_binary], - stderr=subprocess.PIPE, - stdout=subprocess.DEVNULL, - ) - rc = does_strace_work_process.returncode - stderr = does_strace_work_process.stderr - except OSError: - rc = -1 - stderr = "" - if rc or (b"+++ exited with 0 +++" not in stderr): - self.skipTest("strace not found or not working as expected.") + # Are intersted in the system calls: + # clone,clone2,clone3,fork,vfork,exit,exit_group + # Unfortunately using `--trace` with that list to strace fails because + # not all are supported on all platforms (ex. clone2 is ia64 only...) + # So instead use `%process` which is recommended by strace, and contains + # the above. + true_binary = "/bin/true" + strace_args = ["--trace=%process"] with self.subTest(name="default_is_vfork"): - vfork_result = assert_python_ok( - "-c", - textwrap.dedent(f"""\ - import subprocess - subprocess.check_call([{true_binary!r}])"""), - __run_using_command=strace_command, + vfork_result = strace_helper.strace_python( + f"""\ + import subprocess + subprocess.check_call([{true_binary!r}])""", + strace_args ) # Match both vfork() and clone(..., flags=...|CLONE_VFORK|...) - self.assertRegex(vfork_result.err, br"(?i)vfork") + self.assertRegex(vfork_result.event_bytes, br"(?i)vfork") # Do NOT check that fork() or other clones did not happen. # If the OS denys the vfork it'll fallback to plain fork(). @@ -3465,9 +3455,8 @@ def test_vfork_used_when_expected(self): ("setgroups", "", "extra_groups=[]", True), ): with self.subTest(name=sub_name): - non_vfork_result = assert_python_ok( - "-c", - textwrap.dedent(f"""\ + non_vfork_result = strace_helper.strace_python( + f"""\ import subprocess {preamble} try: @@ -3475,11 +3464,11 @@ def test_vfork_used_when_expected(self): [{true_binary!r}], **dict({sp_kwarg})) except PermissionError: if not {expect_permission_error}: - raise"""), - __run_using_command=strace_command, + raise""", + strace_args ) # Ensure neither vfork() or clone(..., flags=...|CLONE_VFORK|...). - self.assertNotRegex(non_vfork_result.err, br"(?i)vfork") + self.assertNotRegex(non_vfork_result.event_bytes, br"(?i)vfork") @unittest.skipUnless(mswindows, "Windows specific tests") diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index c0862d7d15f39e..d839893d2c657e 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1094,7 +1094,14 @@ def test_getallocatedblocks(self): # While we could imagine a Python session where the number of # multiple buffer objects would exceed the sharing of references, # it is unlikely to happen in a normal test run. - self.assertLess(a, sys.gettotalrefcount()) + # + # In free-threaded builds each code object owns an array of + # pointers to copies of the bytecode. When the number of + # code objects is a large fraction of the total number of + # references, this can cause the total number of allocated + # blocks to exceed the total number of references. + if not support.Py_GIL_DISABLED: + self.assertLess(a, sys.gettotalrefcount()) except AttributeError: # gettotalrefcount() not available pass @@ -1613,7 +1620,10 @@ class C(object): pass def func(): return sys._getframe() x = func() - INTERPRETER_FRAME = '9PhcP' + if support.Py_GIL_DISABLED: + INTERPRETER_FRAME = '10PhcP' + else: + INTERPRETER_FRAME = '9PhcP' check(x, size('3PiccPP' + INTERPRETER_FRAME + 'P')) # function def func(): pass diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 1ade49281b4e26..9bbf8d0c6cf2da 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -5,6 +5,8 @@ import os import subprocess import shutil +import json +import textwrap from copy import copy from test.support import ( @@ -17,6 +19,7 @@ from test.support.import_helper import import_module from test.support.os_helper import (TESTFN, unlink, skip_unless_symlink, change_cwd) +from test.support.venv import VirtualEnvironment import sysconfig from sysconfig import (get_paths, get_platform, get_config_vars, @@ -101,6 +104,12 @@ def _cleanup_testfn(self): elif os.path.isdir(path): shutil.rmtree(path) + def venv(self, **venv_create_args): + return VirtualEnvironment.from_tmpdir( + prefix=f'{self.id()}-venv-', + **venv_create_args, + ) + def test_get_path_names(self): self.assertEqual(get_path_names(), sysconfig._SCHEME_KEYS) @@ -582,6 +591,105 @@ def test_osx_ext_suffix(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') self.assertTrue(suffix.endswith('-darwin.so'), suffix) + @requires_subprocess() + def test_config_vars_depend_on_site_initialization(self): + script = textwrap.dedent(""" + import sysconfig + + config_vars = sysconfig.get_config_vars() + + import json + print(json.dumps(config_vars, indent=2)) + """) + + with self.venv() as venv: + site_config_vars = json.loads(venv.run('-c', script).stdout) + no_site_config_vars = json.loads(venv.run('-S', '-c', script).stdout) + + self.assertNotEqual(site_config_vars, no_site_config_vars) + # With the site initialization, the virtual environment should be enabled. + self.assertEqual(site_config_vars['base'], venv.prefix) + self.assertEqual(site_config_vars['platbase'], venv.prefix) + #self.assertEqual(site_config_vars['prefix'], venv.prefix) # # FIXME: prefix gets overwriten by _init_posix + # Without the site initialization, the virtual environment should be disabled. + self.assertEqual(no_site_config_vars['base'], site_config_vars['installed_base']) + self.assertEqual(no_site_config_vars['platbase'], site_config_vars['installed_platbase']) + + @requires_subprocess() + def test_config_vars_recalculation_after_site_initialization(self): + script = textwrap.dedent(""" + import sysconfig + + before = sysconfig.get_config_vars() + + import site + site.main() + + after = sysconfig.get_config_vars() + + import json + print(json.dumps({'before': before, 'after': after}, indent=2)) + """) + + with self.venv() as venv: + config_vars = json.loads(venv.run('-S', '-c', script).stdout) + + self.assertNotEqual(config_vars['before'], config_vars['after']) + self.assertEqual(config_vars['after']['base'], venv.prefix) + #self.assertEqual(config_vars['after']['prefix'], venv.prefix) # FIXME: prefix gets overwriten by _init_posix + #self.assertEqual(config_vars['after']['exec_prefix'], venv.prefix) # FIXME: exec_prefix gets overwriten by _init_posix + + @requires_subprocess() + def test_paths_depend_on_site_initialization(self): + script = textwrap.dedent(""" + import sysconfig + + paths = sysconfig.get_paths() + + import json + print(json.dumps(paths, indent=2)) + """) + + with self.venv() as venv: + site_paths = json.loads(venv.run('-c', script).stdout) + no_site_paths = json.loads(venv.run('-S', '-c', script).stdout) + + self.assertNotEqual(site_paths, no_site_paths) + + @requires_subprocess() + def test_makefile_overwrites_config_vars(self): + script = textwrap.dedent(""" + import sys, sysconfig + + data = { + 'prefix': sys.prefix, + 'exec_prefix': sys.exec_prefix, + 'base_prefix': sys.base_prefix, + 'base_exec_prefix': sys.base_exec_prefix, + 'config_vars': sysconfig.get_config_vars(), + } + + import json + print(json.dumps(data, indent=2)) + """) + + # We need to run the test inside a virtual environment so that + # sys.prefix/sys.exec_prefix have a different value from the + # prefix/exec_prefix Makefile variables. + with self.venv() as venv: + data = json.loads(venv.run('-c', script).stdout) + + # We expect sysconfig.get_config_vars to correctly reflect sys.prefix/sys.exec_prefix + self.assertEqual(data['prefix'], data['config_vars']['prefix']) + self.assertEqual(data['exec_prefix'], data['config_vars']['exec_prefix']) + # As a sanity check, just make sure sys.prefix/sys.exec_prefix really + # are different from the Makefile values. + # sys.base_prefix/sys.base_exec_prefix should reflect the value of the + # prefix/exec_prefix Makefile variables, so we use them in the comparison. + self.assertNotEqual(data['prefix'], data['base_prefix']) + self.assertNotEqual(data['exec_prefix'], data['base_exec_prefix']) + + class MakefileTests(unittest.TestCase): @unittest.skipIf(sys.platform.startswith('win'), diff --git a/Lib/test/test_thread_local_bytecode.py b/Lib/test/test_thread_local_bytecode.py new file mode 100644 index 00000000000000..7a8809c5ae7697 --- /dev/null +++ b/Lib/test/test_thread_local_bytecode.py @@ -0,0 +1,198 @@ +"""Tests for thread-local bytecode.""" +import dis +import textwrap +import unittest + +from test import support +from test.support import cpython_only, import_helper, requires_specialization_ft +from test.support.script_helper import assert_python_ok +from test.support.threading_helper import requires_working_threading + +# Skip this test if the _testinternalcapi module isn't available +_testinternalcapi = import_helper.import_module("_testinternalcapi") + + +@cpython_only +@requires_working_threading() +@unittest.skipUnless(support.Py_GIL_DISABLED, "only in free-threaded builds") +class TLBCTests(unittest.TestCase): + @requires_specialization_ft + def test_new_threads_start_with_unspecialized_code(self): + code = textwrap.dedent(""" + import dis + import queue + import threading + + from _testinternalcapi import get_tlbc + + def all_opnames(bc): + return {i.opname for i in dis._get_instructions_bytes(bc)} + + def f(a, b, q=None): + if q is not None: + q.put(get_tlbc(f)) + return a + b + + for _ in range(100): + # specialize + f(1, 2) + + q = queue.Queue() + t = threading.Thread(target=f, args=('a', 'b', q)) + t.start() + t.join() + + assert "BINARY_OP_ADD_INT" in all_opnames(get_tlbc(f)) + assert "BINARY_OP_ADD_INT" not in all_opnames(q.get()) + """) + assert_python_ok("-X", "tlbc=1", "-c", code) + + @requires_specialization_ft + def test_threads_specialize_independently(self): + code = textwrap.dedent(""" + import dis + import queue + import threading + + from _testinternalcapi import get_tlbc + + def all_opnames(bc): + return {i.opname for i in dis._get_instructions_bytes(bc)} + + def f(a, b): + return a + b + + def g(a, b, q=None): + for _ in range(100): + f(a, b) + if q is not None: + q.put(get_tlbc(f)) + + # specialize in main thread + g(1, 2) + + # specialize in other thread + q = queue.Queue() + t = threading.Thread(target=g, args=('a', 'b', q)) + t.start() + t.join() + + assert "BINARY_OP_ADD_INT" in all_opnames(get_tlbc(f)) + t_opnames = all_opnames(q.get()) + assert "BINARY_OP_ADD_INT" not in t_opnames + assert "BINARY_OP_ADD_UNICODE" in t_opnames + """) + assert_python_ok("-X", "tlbc=1", "-c", code) + + def test_reuse_tlbc_across_threads_different_lifetimes(self): + code = textwrap.dedent(""" + import queue + import threading + + from _testinternalcapi import get_tlbc_id + + def f(a, b, q=None): + if q is not None: + q.put(get_tlbc_id(f)) + return a + b + + q = queue.Queue() + tlbc_ids = [] + for _ in range(3): + t = threading.Thread(target=f, args=('a', 'b', q)) + t.start() + t.join() + tlbc_ids.append(q.get()) + + assert tlbc_ids[0] == tlbc_ids[1] + assert tlbc_ids[1] == tlbc_ids[2] + """) + assert_python_ok("-X", "tlbc=1", "-c", code) + + def test_no_copies_if_tlbc_disabled(self): + code = textwrap.dedent(""" + import queue + import threading + + from _testinternalcapi import get_tlbc_id + + def f(a, b, q=None): + if q is not None: + q.put(get_tlbc_id(f)) + return a + b + + q = queue.Queue() + threads = [] + for _ in range(3): + t = threading.Thread(target=f, args=('a', 'b', q)) + t.start() + threads.append(t) + + tlbc_ids = [] + for t in threads: + t.join() + tlbc_ids.append(q.get()) + + main_tlbc_id = get_tlbc_id(f) + assert main_tlbc_id is not None + assert tlbc_ids[0] == main_tlbc_id + assert tlbc_ids[1] == main_tlbc_id + assert tlbc_ids[2] == main_tlbc_id + """) + assert_python_ok("-X", "tlbc=0", "-c", code) + + def test_no_specialization_if_tlbc_disabled(self): + code = textwrap.dedent(""" + import dis + import queue + import threading + + from _testinternalcapi import get_tlbc + + def all_opnames(f): + bc = get_tlbc(f) + return {i.opname for i in dis._get_instructions_bytes(bc)} + + def f(a, b): + return a + b + + for _ in range(100): + f(1, 2) + + assert "BINARY_OP_ADD_INT" not in all_opnames(f) + """) + assert_python_ok("-X", "tlbc=0", "-c", code) + + def test_generator_throw(self): + code = textwrap.dedent(""" + import queue + import threading + + from _testinternalcapi import get_tlbc_id + + def g(): + try: + yield + except: + yield get_tlbc_id(g) + + def f(q): + gen = g() + next(gen) + q.put(gen.throw(ValueError)) + + q = queue.Queue() + t = threading.Thread(target=f, args=(q,)) + t.start() + t.join() + + gen = g() + next(gen) + main_id = gen.throw(ValueError) + assert main_id != q.get() + """) + assert_python_ok("-X", "tlbc=1", "-c", code) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index b0b9ed60040443..579ce2af9fa0bf 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -66,9 +66,10 @@ def test_tk_busy(self): f.tk_busy_forget() self.assertFalse(f.tk_busy_status()) self.assertFalse(f.tk_busy_current()) - with self.assertRaisesRegex(TclError, "can't find busy window"): + errmsg = r"can(no|')t find busy window.*" + with self.assertRaisesRegex(TclError, errmsg): f.tk_busy_configure() - with self.assertRaisesRegex(TclError, "can't find busy window"): + with self.assertRaisesRegex(TclError, errmsg): f.tk_busy_forget() @requires_tk(8, 6, 6) @@ -87,7 +88,8 @@ def test_tk_busy_with_cursor(self): self.assertEqual(f.tk_busy_configure('cursor')[4], 'heart') f.tk_busy_forget() - with self.assertRaisesRegex(TclError, "can't find busy window"): + errmsg = r"can(no|')t find busy window.*" + with self.assertRaisesRegex(TclError, errmsg): f.tk_busy_cget('cursor') def test_tk_setPalette(self): diff --git a/Lib/test/test_tkinter/test_widgets.py b/Lib/test/test_tkinter/test_widgets.py index 9ea764ca2a39d8..f6e77973061956 100644 --- a/Lib/test/test_tkinter/test_widgets.py +++ b/Lib/test/test_tkinter/test_widgets.py @@ -7,9 +7,13 @@ from test.test_tkinter.support import (requires_tk, tk_version, get_tk_patchlevel, widget_eq, AbstractDefaultRootTest) + from test.test_tkinter.widget_tests import ( - add_standard_options, - AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests) + add_configure_tests, + AbstractWidgetTest, + StandardOptionsTests, + IntegerSizeTests, + PixelSizeTests) requires('gui') @@ -20,9 +24,17 @@ def float_round(x): return float(round(x)) - class AbstractToplevelTest(AbstractWidgetTest, PixelSizeTests): - _conv_pad_pixels = False + if tk_version < (9, 0): + _no_round = {'padx', 'pady'} + else: + _no_round = {'borderwidth', 'height', 'highlightthickness', 'padx', + 'pady', 'width'} + if tk_version < (9, 0): + _clipped = {'highlightthickness'} + else: + _clipped = {'borderwidth', 'height', 'highlightthickness', 'padx', + 'pady', 'width'} def test_configure_class(self): widget = self.create() @@ -58,7 +70,7 @@ def test_configure_visual(self): self.assertEqual(widget2['visual'], 'default') -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class ToplevelTest(AbstractToplevelTest, unittest.TestCase): OPTIONS = ( 'background', 'backgroundimage', 'borderwidth', @@ -101,7 +113,7 @@ def test_configure_use(self): self.assertEqual(widget2['use'], wid) -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class FrameTest(AbstractToplevelTest, unittest.TestCase): OPTIONS = ( 'background', 'backgroundimage', 'borderwidth', @@ -109,12 +121,17 @@ class FrameTest(AbstractToplevelTest, unittest.TestCase): 'highlightbackground', 'highlightcolor', 'highlightthickness', 'padx', 'pady', 'relief', 'takefocus', 'tile', 'visual', 'width', ) + if tk_version < (9, 0): + _no_round = {'padx', 'pady'} + else: + _no_round = {'borderwidth', 'height', 'highlightthickness', 'padx', + 'pady', 'width'} def create(self, **kwargs): return tkinter.Frame(self.root, **kwargs) -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class LabelFrameTest(AbstractToplevelTest, unittest.TestCase): OPTIONS = ( 'background', 'borderwidth', @@ -124,6 +141,11 @@ class LabelFrameTest(AbstractToplevelTest, unittest.TestCase): 'labelanchor', 'labelwidget', 'padx', 'pady', 'relief', 'takefocus', 'text', 'visual', 'width', ) + if tk_version < (9, 0): + _no_round = {'padx', 'pady'} + else: + _no_round = {'borderwidth', 'height', 'highlightthickness', 'padx', + 'pady', 'width'} def create(self, **kwargs): return tkinter.LabelFrame(self.root, **kwargs) @@ -141,15 +163,16 @@ def test_configure_labelwidget(self): self.checkParam(widget, 'labelwidget', label, expected='.foo') label.destroy() - +# Label, Button, Checkbutton, Radiobutton, MenuButton class AbstractLabelTest(AbstractWidgetTest, IntegerSizeTests): - _conv_pixels = False - _clip_highlightthickness = tk_version >= (8, 7) - _clip_pad = tk_version >= (8, 7) - _clip_borderwidth = tk_version >= (8, 7) - - -@add_standard_options(StandardOptionsTests) + _rounds_pixels = False + if tk_version < (9, 0): + _clipped = {} + else: + _clipped = {'borderwidth', 'insertborderwidth', 'highlightthickness', + 'padx', 'pady'} + +@add_configure_tests(StandardOptionsTests) class LabelTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'activeforeground', 'anchor', @@ -165,7 +188,7 @@ def create(self, **kwargs): return tkinter.Label(self.root, **kwargs) -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class ButtonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'activeforeground', 'anchor', @@ -186,7 +209,7 @@ def test_configure_default(self): self.checkEnumParam(widget, 'default', 'active', 'disabled', 'normal') -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class CheckbuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'activeforeground', 'anchor', @@ -240,8 +263,7 @@ def test_same_name(self): b2.deselect() self.assertEqual(v.get(), 0) - -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class RadiobuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'activeforeground', 'anchor', @@ -264,7 +286,7 @@ def test_configure_value(self): self.checkParams(widget, 'value', 1, 2.3, '', 'any string') -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class MenubuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'activeforeground', 'anchor', @@ -277,10 +299,11 @@ class MenubuttonTest(AbstractLabelTest, unittest.TestCase): 'takefocus', 'text', 'textvariable', 'underline', 'width', 'wraplength', ) - _conv_pixels = round - _clip_highlightthickness = True - _clip_pad = True - _clip_borderwidth = False + _rounds_pixels = (tk_version < (9, 0)) + if tk_version < (9, 0): + _clipped = {'highlightthickness', 'padx', 'pady'} + else: + _clipped ={ 'insertborderwidth', 'highlightthickness', 'padx', 'pady'} def create(self, **kwargs): return tkinter.Menubutton(self.root, **kwargs) @@ -298,7 +321,10 @@ def test_configure_image(self): widget = self.create() image = tkinter.PhotoImage(master=self.root, name='image1') self.checkParam(widget, 'image', image, conv=str) - errmsg = 'image "spam" doesn\'t exist' + if tk_version < (9, 0): + errmsg = 'image "spam" doesn\'t exist' + else: + errmsg = 'image "spam" does not exist' with self.assertRaises(tkinter.TclError) as cm: widget['image'] = 'spam' if errmsg is not None: @@ -328,9 +354,15 @@ def test_bad_kwarg(self): with self.assertRaisesRegex(TclError, r"^unknown option -image$"): tkinter.OptionMenu(self.root, None, 'b', image='') - -@add_standard_options(IntegerSizeTests, StandardOptionsTests) +@add_configure_tests(IntegerSizeTests, StandardOptionsTests) class EntryTest(AbstractWidgetTest, unittest.TestCase): + _rounds_pixels = (tk_version < (9, 0)) + if tk_version < (9, 0): + _clipped = {'highlightthickness'} + else: + _clipped = {'highlightthickness', 'borderwidth', 'insertborderwidth', + 'selectborderwidth'} + OPTIONS = ( 'background', 'borderwidth', 'cursor', 'disabledbackground', 'disabledforeground', @@ -355,16 +387,23 @@ def test_configure_disabledbackground(self): def test_configure_insertborderwidth(self): widget = self.create(insertwidth=100) self.checkPixelsParam(widget, 'insertborderwidth', - 0, 1.3, 2.6, 6, -2, '10p') + 0, 1.3, 2.6, 6, '10p') + self.checkParam(widget, 'insertborderwidth', -2) # insertborderwidth is bounded above by a half of insertwidth. - self.checkParam(widget, 'insertborderwidth', 60, expected=100//2) + expected = 100 // 2 if tk_version < (9, 0) else 60 + self.checkParam(widget, 'insertborderwidth', 60, expected=expected) def test_configure_insertwidth(self): widget = self.create() self.checkPixelsParam(widget, 'insertwidth', 1.3, 3.6, '10p') - self.checkParam(widget, 'insertwidth', 0.1, expected=2) - self.checkParam(widget, 'insertwidth', -2, expected=2) - self.checkParam(widget, 'insertwidth', 0.9, expected=1) + if tk_version < (9, 0): + self.checkParam(widget, 'insertwidth', 0.1, expected=2) + self.checkParam(widget, 'insertwidth', -2, expected=2) + self.checkParam(widget, 'insertwidth', 0.9, expected=1) + else: + self.checkParam(widget, 'insertwidth', 0.1) + self.checkParam(widget, 'insertwidth', -2, expected=0) + self.checkParam(widget, 'insertwidth', 0.9) def test_configure_invalidcommand(self): widget = self.create() @@ -422,7 +461,7 @@ def test_selection_methods(self): widget.selection_adjust(0) -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class SpinboxTest(EntryTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'background', 'borderwidth', @@ -559,7 +598,7 @@ def test_selection_element(self): self.assertEqual(widget.selection_element(), "buttondown") -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class TextTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'autoseparators', 'background', 'blockcursor', 'borderwidth', @@ -574,6 +613,9 @@ class TextTest(AbstractWidgetTest, unittest.TestCase): 'tabs', 'tabstyle', 'takefocus', 'undo', 'width', 'wrap', 'xscrollcommand', 'yscrollcommand', ) + _rounds_pixels = (tk_version < (9, 0)) + _no_round = {'selectborderwidth'} + _clipped = {'highlightthickness'} def create(self, **kwargs): return tkinter.Text(self.root, **kwargs) @@ -602,8 +644,10 @@ def test_configure_endline(self): def test_configure_height(self): widget = self.create() self.checkPixelsParam(widget, 'height', 100, 101.2, 102.6, '3c') - self.checkParam(widget, 'height', -100, expected=1) - self.checkParam(widget, 'height', 0, expected=1) + self.checkParam(widget, 'height', -100, + expected=1 if tk_version < (9, 0) else -100) + self.checkParam(widget, 'height', 0, + expected=1 if tk_version < (9, 0) else 0 ) def test_configure_maxundo(self): widget = self.create() @@ -696,7 +740,7 @@ def test_bbox(self): self.assertRaises(TypeError, widget.bbox, '1.1', 'end') -@add_standard_options(PixelSizeTests, StandardOptionsTests) +@add_configure_tests(PixelSizeTests, StandardOptionsTests) class CanvasTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'background', 'borderwidth', @@ -710,8 +754,15 @@ class CanvasTest(AbstractWidgetTest, unittest.TestCase): 'xscrollcommand', 'xscrollincrement', 'yscrollcommand', 'yscrollincrement', 'width', ) - - _conv_pixels = round + _rounds_pixels = True + if tk_version < (9, 0): + _noround = {} + _clipped = {'highlightthickness'} + else: + _no_round = {'borderwidth', 'height', 'highlightthickness', 'width', + 'xscrollincrement', 'yscrollincrement'} + _clipped = {'borderwidth', 'height', 'highlightthickness', 'width', + 'xscrollincrement', 'yscrollincrement'} _stringify = True def create(self, **kwargs): @@ -953,7 +1004,7 @@ def test_moveto(self): self.assertEqual(y2_2 - y1_2, y2_3 - y1_3) -@add_standard_options(IntegerSizeTests, StandardOptionsTests) +@add_configure_tests(IntegerSizeTests, StandardOptionsTests) class ListboxTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'activestyle', 'background', 'borderwidth', 'cursor', @@ -965,6 +1016,11 @@ class ListboxTest(AbstractWidgetTest, unittest.TestCase): 'selectmode', 'setgrid', 'state', 'takefocus', 'width', 'xscrollcommand', 'yscrollcommand', ) + _rounds_pixels = (tk_version < (9, 0)) + if tk_version < (9, 0): + _clipped = {'highlightthickness'} + else: + _clipped = { 'borderwidth', 'highlightthickness', 'selectborderwidth'} def create(self, **kwargs): return tkinter.Listbox(self.root, **kwargs) @@ -1091,7 +1147,7 @@ def test_get(self): self.assertRaises(TclError, lb.get, 2.4) -@add_standard_options(PixelSizeTests, StandardOptionsTests) +@add_configure_tests(PixelSizeTests, StandardOptionsTests) class ScaleTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'background', 'bigincrement', 'borderwidth', @@ -1102,6 +1158,8 @@ class ScaleTest(AbstractWidgetTest, unittest.TestCase): 'resolution', 'showvalue', 'sliderlength', 'sliderrelief', 'state', 'takefocus', 'tickinterval', 'to', 'troughcolor', 'variable', 'width', ) + _rounds_pixels = (tk_version < (9, 0)) + _clipped = {'highlightthickness'} default_orient = 'vertical' def create(self, **kwargs): @@ -1159,7 +1217,7 @@ def test_configure_to(self): conv=float_round) -@add_standard_options(PixelSizeTests, StandardOptionsTests) +@add_configure_tests(PixelSizeTests, StandardOptionsTests) class ScrollbarTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'activerelief', @@ -1170,7 +1228,14 @@ class ScrollbarTest(AbstractWidgetTest, unittest.TestCase): 'repeatdelay', 'repeatinterval', 'takefocus', 'troughcolor', 'width', ) - _conv_pixels = round + _rounds_pixels = True + if tk_version >= (9, 0): + _no_round = {'borderwidth', 'elementborderwidth', 'highlightthickness', + 'width'} + if tk_version < (9, 0): + _clipped = {'highlightthickness'} + else: + _clipped = {'borderwidth', 'highlightthickness', 'width'} _stringify = True default_orient = 'vertical' @@ -1208,7 +1273,7 @@ def test_set(self): self.assertRaises(TypeError, sb.set, 0.6, 0.7, 0.8) -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class PanedWindowTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'background', 'borderwidth', 'cursor', @@ -1219,6 +1284,15 @@ class PanedWindowTest(AbstractWidgetTest, unittest.TestCase): 'sashcursor', 'sashpad', 'sashrelief', 'sashwidth', 'showhandle', 'width', ) + _rounds_pixels = True + if tk_version < (9, 0): + _no_round = {'handlesize', 'height', 'proxyborderwidth', 'sashwidth', + 'selectborderwidth', 'width'} + else: + _no_round = {'borderwidth', 'handlepad', 'handlesize', 'height', + 'proxyborderwidth', 'sashpad', 'sashwidth', + 'selectborderwidth', 'width'} + _clipped = {} default_orient = 'horizontal' def create(self, **kwargs): @@ -1347,13 +1421,13 @@ def test_paneconfigure_minsize(self): def test_paneconfigure_padx(self): p, b, c = self.create2() - self.check_paneconfigure(p, b, 'padx', 1.3, 1) + self.check_paneconfigure(p, b, 'padx', 1.3, 1 if tk_version < (9, 0) else 1.3) self.check_paneconfigure_bad(p, b, 'padx', EXPECTED_SCREEN_DISTANCE_ERRMSG.format('badValue')) def test_paneconfigure_pady(self): p, b, c = self.create2() - self.check_paneconfigure(p, b, 'pady', 1.3, 1) + self.check_paneconfigure(p, b, 'pady', 1.3, 1 if tk_version < (9, 0) else 1.3) self.check_paneconfigure_bad(p, b, 'pady', EXPECTED_SCREEN_DISTANCE_ERRMSG.format('badValue')) @@ -1379,17 +1453,17 @@ def test_paneconfigure_width(self): EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG.format('badValue')) -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class MenuTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'activeborderwidth', 'activeforeground', - 'activerelief', - 'background', 'borderwidth', 'cursor', + 'activerelief', 'background', 'borderwidth', 'cursor', 'disabledforeground', 'font', 'foreground', 'postcommand', 'relief', 'selectcolor', 'takefocus', 'tearoff', 'tearoffcommand', 'title', 'type', ) - _conv_pixels = False + _rounds_pixels = False + _clipped = {} def create(self, **kwargs): return tkinter.Menu(self.root, **kwargs) @@ -1458,7 +1532,7 @@ def test_entryconfigure_variable(self): self.assertEqual(str(m1.entrycget(1, 'variable')), str(v2)) -@add_standard_options(PixelSizeTests, StandardOptionsTests) +@add_configure_tests(PixelSizeTests, StandardOptionsTests) class MessageTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'anchor', 'aspect', 'background', 'borderwidth', @@ -1467,11 +1541,12 @@ class MessageTest(AbstractWidgetTest, unittest.TestCase): 'justify', 'padx', 'pady', 'relief', 'takefocus', 'text', 'textvariable', 'width', ) - _conv_pad_pixels = False - if tk_version >= (8, 7): - _conv_pixels = False - _clip_pad = tk_version >= (8, 7) - _clip_borderwidth = tk_version >= (8, 7) + _rounds_pixels = (tk_version < (9, 0)) + _no_round = {'padx', 'pady'} + if tk_version < (9, 0): + _clipped = {'highlightthickness'} + else: + _clipped = {'borderwidth', 'highlightthickness', 'padx', 'pady'} def create(self, **kwargs): return tkinter.Message(self.root, **kwargs) @@ -1482,16 +1557,14 @@ def test_configure_aspect(self): def test_configure_padx(self): widget = self.create() - self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, '12m', - conv=self._conv_pad_pixels) - expected = self._default_pixels if self._clip_pad else -2 + self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, '12m') + expected = -2 if tk_version < (9, 0) else self._default_pixels self.checkParam(widget, 'padx', -2, expected=expected) def test_configure_pady(self): widget = self.create() - self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, '12m', - conv=self._conv_pad_pixels) - expected = self._default_pixels if self._clip_pad else -2 + self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, '12m') + expected = -2 if tk_version < (9, 0) else self._default_pixels self.checkParam(widget, 'pady', -2, expected=expected) def test_configure_width(self): diff --git a/Lib/test/test_tkinter/widget_tests.py b/Lib/test/test_tkinter/widget_tests.py index 8ab2f74245095d..ac7fb5977e04fc 100644 --- a/Lib/test/test_tkinter/widget_tests.py +++ b/Lib/test/test_tkinter/widget_tests.py @@ -6,17 +6,16 @@ pixels_conv, tcl_obj_eq) import test.support - _sentinel = object() +# Options which accept all values allowed by Tk_GetPixels +# borderwidth = bd + class AbstractWidgetTest(AbstractTkTest): - _default_pixels = '' if tk_version >= (9, 0) else -1 if tk_version >= (8, 7) else '' - _conv_pixels = round - _conv_pad_pixels = None - _stringify = False - _clip_highlightthickness = True - _clip_pad = False - _clip_borderwidth = False + _default_pixels = '' # Value for unset pixel options. + _rounds_pixels = True # True if some pixel options are rounded. + _no_round = {} # Pixel options which are not rounded nonetheless + _stringify = False # Whether to convert tuples to strings _allow_empty_justify = False @property @@ -44,6 +43,9 @@ def checkParam(self, widget, name, value, *, expected=_sentinel, widget[name] = value if expected is _sentinel: expected = value + if name in self._clipped: + if not isinstance(expected, str): + expected = max(expected, 0) if conv: expected = conv(expected) if self._stringify or not self.wantobjects: @@ -140,14 +142,17 @@ def checkEnumParam(self, widget, name, *values, errmsg = 'bad' + errmsg2 self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) - def checkPixelsParam(self, widget, name, *values, - conv=None, **kwargs): - if conv is None: - conv = self._conv_pixels + def checkPixelsParam(self, widget, name, *values, conv=None, **kwargs): + if not self._rounds_pixels or name in self._no_round: + conv = False + elif conv != str: + conv = round for value in values: expected = _sentinel conv1 = conv if isinstance(value, str): + if not getattr(self, '_converts_pixels', True): + conv1 = str if conv1 and conv1 is not str: expected = pixels_conv(value) * self.scaling conv1 = round @@ -172,8 +177,12 @@ def checkReliefParam(self, widget, name, *, allow_empty=False): def checkImageParam(self, widget, name): image = tkinter.PhotoImage(master=self.root, name='image1') self.checkParam(widget, name, image, conv=str) + if tk_version < (9, 0): + errmsg = 'image "spam" doesn\'t exist' + else: + errmsg = 'image "spam" does not exist' self.checkInvalidParam(widget, name, 'spam', - errmsg='image "spam" doesn\'t exist') + errmsg=errmsg) widget[name] = '' def checkVariableParam(self, widget, name, var): @@ -215,31 +224,80 @@ def test_keys(self): print('%s.OPTIONS doesn\'t contain "%s"' % (self.__class__.__name__, k)) +class PixelOptionsTests: + """Standard options that accept all formats acceptable to Tk_GetPixels. -class StandardOptionsTests: - STANDARD_OPTIONS = ( - 'activebackground', 'activeborderwidth', 'activeforeground', 'anchor', - 'background', 'bitmap', 'borderwidth', 'compound', 'cursor', - 'disabledforeground', 'exportselection', 'font', 'foreground', - 'highlightbackground', 'highlightcolor', 'highlightthickness', - 'image', 'insertbackground', 'insertborderwidth', - 'insertofftime', 'insertontime', 'insertwidth', - 'jump', 'justify', 'orient', 'padx', 'pady', 'relief', - 'repeatdelay', 'repeatinterval', - 'selectbackground', 'selectborderwidth', 'selectforeground', - 'setgrid', 'takefocus', 'text', 'textvariable', 'troughcolor', - 'underline', 'wraplength', 'xscrollcommand', 'yscrollcommand', - ) - - def test_configure_activebackground(self): - widget = self.create() - self.checkColorParam(widget, 'activebackground') + In addition to numbers, these options can be set with distances + specified as a string consisting of a number followed by a single + character giving the unit of distance. The allowed units are: + millimeters ('m'), centimeters ('c'), inches ('i') or points ('p'). + In Tk 9 a cget call for one of these options returns a Tcl_Obj of + type "pixels", whose string representation is the distance string + passed to configure. + """ + PIXEL_OPTIONS = ('activeborderwidth', 'borderwidth', 'highlightthickness', + 'insertborderwidth', 'insertwidth', 'padx', 'pady', 'selectborderwidth') def test_configure_activeborderwidth(self): widget = self.create() self.checkPixelsParam(widget, 'activeborderwidth', 0, 1.3, 2.9, 6, -2, '10p') + def test_configure_borderwidth(self): + widget = self.create() + self.checkPixelsParam(widget, 'borderwidth', + 0, 1.3, 2.6, 6, '10p') + self.checkParam(widget, 'borderwidth', -2) + if 'bd' in self.OPTIONS: + self.checkPixelsParam(widget, 'bd', 0, 1.3, 2.6, 6, '10p') + self.checkParam(widget, 'bd', -2, expected=expected) + + def test_configure_highlightthickness(self): + widget = self.create() + self.checkPixelsParam(widget, 'highlightthickness', + 0, 1.3, 2.6, 6, '10p') + self.checkParam(widget, 'highlightthickness', -2) + + def test_configure_insertborderwidth(self): + widget = self.create() + self.checkPixelsParam(widget, 'insertborderwidth', + 0, 1.3, 2.6, 6, '10p') + self.checkParam(widget, 'insertborderwidth', -2) + + def test_configure_insertwidth(self): + widget = self.create() + self.checkPixelsParam(widget, 'insertwidth', 1.3, 2.6, -2, '10p') + + def test_configure_padx(self): + widget = self.create() + self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, '12m') + self.checkParam(widget, 'padx', -2) + + def test_configure_pady(self): + widget = self.create() + self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, '12m') + self.checkParam(widget, 'pady', -2) + + def test_configure_selectborderwidth(self): + widget = self.create() + self.checkPixelsParam(widget, 'selectborderwidth', 1.3, 2.6, -2, '10p') + +class StandardOptionsTests(PixelOptionsTests): + + STANDARD_OPTIONS = ( 'activebackground', 'activeforeground', + 'anchor', 'background', 'bitmap', 'compound', 'cursor', + 'disabledforeground', 'exportselection', 'font', 'foreground', + 'highlightbackground', 'highlightcolor', 'image', + 'insertbackground', 'insertofftime', 'insertontime', 'jump', + 'justify', 'orient', 'relief', 'repeatdelay', 'repeatinterval', + 'selectbackground', 'selectforeground', 'setgrid', 'takefocus', + 'text', 'textvariable', 'troughcolor', 'underline', 'wraplength', + 'xscrollcommand', 'yscrollcommand', ) + PixelOptionsTests.PIXEL_OPTIONS + + def test_configure_activebackground(self): + widget = self.create() + self.checkColorParam(widget, 'activebackground') + def test_configure_activeforeground(self): widget = self.create() self.checkColorParam(widget, 'activeforeground') @@ -277,18 +335,6 @@ def test_configure_bitmap(self): self.checkInvalidParam(widget, 'bitmap', 'spam', errmsg='bitmap "spam" not defined') - def test_configure_borderwidth(self): - widget = self.create() - self.checkPixelsParam(widget, 'borderwidth', - 0, 1.3, 2.6, 6, '10p') - expected = 0 if self._clip_borderwidth else -2 - self.checkParam(widget, 'borderwidth', -2, expected=expected, - conv=self._conv_pixels) - if 'bd' in self.OPTIONS: - self.checkPixelsParam(widget, 'bd', 0, 1.3, 2.6, 6, '10p') - self.checkParam(widget, 'bd', -2, expected=expected, - conv=self._conv_pixels) - def test_configure_compound(self): widget = self.create() self.checkEnumParam(widget, 'compound', @@ -312,8 +358,8 @@ def test_configure_font(self): '-Adobe-Helvetica-Medium-R-Normal--*-120-*-*-*-*-*-*') is_ttk = widget.__class__.__module__ == 'tkinter.ttk' if not is_ttk: - self.checkInvalidParam(widget, 'font', '', - errmsg='font "" doesn\'t exist') + errmsg = 'font "" does ?n[o\']t exist' + self.checkInvalidParam(widget, 'font', '', errmsg=errmsg) def test_configure_foreground(self): widget = self.create() @@ -329,14 +375,6 @@ def test_configure_highlightcolor(self): widget = self.create() self.checkColorParam(widget, 'highlightcolor') - def test_configure_highlightthickness(self): - widget = self.create() - self.checkPixelsParam(widget, 'highlightthickness', - 0, 1.3, 2.6, 6, '10p') - expected = 0 if self._clip_highlightthickness else -2 - self.checkParam(widget, 'highlightthickness', -2, expected=expected, - conv=self._conv_pixels) - def test_configure_image(self): widget = self.create() self.checkImageParam(widget, 'image') @@ -345,11 +383,6 @@ def test_configure_insertbackground(self): widget = self.create() self.checkColorParam(widget, 'insertbackground') - def test_configure_insertborderwidth(self): - widget = self.create() - self.checkPixelsParam(widget, 'insertborderwidth', - 0, 1.3, 2.6, 6, -2, '10p') - def test_configure_insertofftime(self): widget = self.create() self.checkIntegerParam(widget, 'insertofftime', 100) @@ -358,10 +391,6 @@ def test_configure_insertontime(self): widget = self.create() self.checkIntegerParam(widget, 'insertontime', 100) - def test_configure_insertwidth(self): - widget = self.create() - self.checkPixelsParam(widget, 'insertwidth', 1.3, 2.6, -2, '10p') - def test_configure_jump(self): widget = self.create() self.checkBooleanParam(widget, 'jump') @@ -379,22 +408,6 @@ def test_configure_orient(self): self.assertEqual(str(widget['orient']), self.default_orient) self.checkEnumParam(widget, 'orient', 'horizontal', 'vertical') - def test_configure_padx(self): - widget = self.create() - self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, '12m', - conv=self._conv_pad_pixels) - expected = 0 if self._clip_pad else -2 - self.checkParam(widget, 'padx', -2, expected=expected, - conv=self._conv_pad_pixels) - - def test_configure_pady(self): - widget = self.create() - self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, '12m', - conv=self._conv_pad_pixels) - expected = 0 if self._clip_pad else -2 - self.checkParam(widget, 'pady', -2, expected=expected, - conv=self._conv_pad_pixels) - @requires_tk(8, 7) def test_configure_placeholder(self): widget = self.create() @@ -421,10 +434,6 @@ def test_configure_selectbackground(self): widget = self.create() self.checkColorParam(widget, 'selectbackground') - def test_configure_selectborderwidth(self): - widget = self.create() - self.checkPixelsParam(widget, 'selectborderwidth', 1.3, 2.6, -2, '10p') - def test_configure_selectforeground(self): widget = self.create() self.checkColorParam(widget, 'selectforeground') @@ -534,6 +543,7 @@ def test_configure_variable(self): class IntegerSizeTests: + """ Tests widgets which only accept integral width and height.""" def test_configure_height(self): widget = self.create() self.checkIntegerParam(widget, 'height', 100, -100, 0) @@ -544,6 +554,7 @@ def test_configure_width(self): class PixelSizeTests: + """ Tests widgets which accept screen distances for width and height.""" def test_configure_height(self): widget = self.create() self.checkPixelsParam(widget, 'height', 100, 101.2, 102.6, -100, 0, '3c') @@ -553,7 +564,7 @@ def test_configure_width(self): self.checkPixelsParam(widget, 'width', 402, 403.4, 404.6, -402, 0, '5i') -def add_standard_options(*source_classes): +def add_configure_tests(*source_classes): # This decorator adds test_configure_xxx methods from source classes for # every xxx option in the OPTIONS class attribute if they are not defined # explicitly. diff --git a/Lib/test/test_tomllib/test_error.py b/Lib/test/test_tomllib/test_error.py index d2ef59a29ca350..3a8587492859ca 100644 --- a/Lib/test/test_tomllib/test_error.py +++ b/Lib/test/test_tomllib/test_error.py @@ -49,7 +49,9 @@ def test_type_error(self): self.assertEqual(str(exc_info.exception), "Expected str object, not 'bool'") def test_module_name(self): - self.assertEqual(tomllib.TOMLDecodeError().__module__, tomllib.__name__) + self.assertEqual( + tomllib.TOMLDecodeError("", "", 0).__module__, tomllib.__name__ + ) def test_invalid_parse_float(self): def dict_returner(s: str) -> dict: @@ -64,3 +66,33 @@ def list_returner(s: str) -> list: self.assertEqual( str(exc_info.exception), "parse_float must not return dicts or lists" ) + + def test_deprecated_tomldecodeerror(self): + for args in [ + (), + ("err msg",), + (None,), + (None, "doc"), + ("err msg", None), + (None, "doc", None), + ("err msg", "doc", None), + ("one", "two", "three", "four"), + ("one", "two", 3, "four", "five"), + ]: + with self.assertWarns(DeprecationWarning): + e = tomllib.TOMLDecodeError(*args) # type: ignore[arg-type] + self.assertEqual(e.args, args) + + def test_tomldecodeerror(self): + msg = "error parsing" + doc = "v=1\n[table]\nv='val'" + pos = 13 + formatted_msg = "error parsing (at line 3, column 2)" + e = tomllib.TOMLDecodeError(msg, doc, pos) + self.assertEqual(e.args, (formatted_msg,)) + self.assertEqual(str(e), formatted_msg) + self.assertEqual(e.msg, msg) + self.assertEqual(e.doc, doc) + self.assertEqual(e.pos, pos) + self.assertEqual(e.lineno, 3) + self.assertEqual(e.colno, 2) diff --git a/Lib/test/test_tools/i18n_data/docstrings.pot b/Lib/test/test_tools/i18n_data/docstrings.pot new file mode 100644 index 00000000000000..5af1d41422ff62 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/docstrings.pot @@ -0,0 +1,40 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: docstrings.py:7 +#, docstring +msgid "" +msgstr "" + +#: docstrings.py:18 +#, docstring +msgid "" +"multiline\n" +" docstring\n" +" " +msgstr "" + +#: docstrings.py:25 +#, docstring +msgid "docstring1" +msgstr "" + +#: docstrings.py:30 +#, docstring +msgid "Hello, {}!" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/docstrings.py b/Lib/test/test_tools/i18n_data/docstrings.py new file mode 100644 index 00000000000000..85d7f159d37775 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/docstrings.py @@ -0,0 +1,41 @@ +# Test docstring extraction +from gettext import gettext as _ + + +# Empty docstring +def test(x): + """""" + + +# Leading empty line +def test2(x): + + """docstring""" # XXX This should be extracted but isn't. + + +# XXX Multiline docstrings should be cleaned with `inspect.cleandoc`. +def test3(x): + """multiline + docstring + """ + + +# Multiple docstrings - only the first should be extracted +def test4(x): + """docstring1""" + """docstring2""" + + +def test5(x): + """Hello, {}!""".format("world!") # XXX This should not be extracted. + + +# Nested docstrings +def test6(x): + def inner(y): + """nested docstring""" # XXX This should be extracted but isn't. + + +class Outer: + class Inner: + "nested class docstring" # XXX This should be extracted but isn't. diff --git a/Lib/test/test_tools/i18n_data/fileloc.pot b/Lib/test/test_tools/i18n_data/fileloc.pot new file mode 100644 index 00000000000000..dbd28687a73556 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/fileloc.pot @@ -0,0 +1,35 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: fileloc.py:5 fileloc.py:6 +msgid "foo" +msgstr "" + +#: fileloc.py:9 +msgid "bar" +msgstr "" + +#: fileloc.py:14 fileloc.py:18 +#, docstring +msgid "docstring" +msgstr "" + +#: fileloc.py:22 fileloc.py:26 +#, docstring +msgid "baz" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/fileloc.py b/Lib/test/test_tools/i18n_data/fileloc.py new file mode 100644 index 00000000000000..c5d4d0595fea52 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/fileloc.py @@ -0,0 +1,26 @@ +# Test file locations +from gettext import gettext as _ + +# Duplicate strings +_('foo') +_('foo') + +# Duplicate strings on the same line should only add one location to the output +_('bar'), _('bar') + + +# Duplicate docstrings +class A: + """docstring""" + + +def f(): + """docstring""" + + +# Duplicate message and docstring +_('baz') + + +def g(): + """baz""" diff --git a/Lib/test/test_tools/i18n_data/messages.pot b/Lib/test/test_tools/i18n_data/messages.pot new file mode 100644 index 00000000000000..ddfbd18349ef4f --- /dev/null +++ b/Lib/test/test_tools/i18n_data/messages.pot @@ -0,0 +1,67 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: messages.py:5 +msgid "" +msgstr "" + +#: messages.py:8 messages.py:9 +msgid "parentheses" +msgstr "" + +#: messages.py:12 +msgid "Hello, world!" +msgstr "" + +#: messages.py:15 +msgid "" +"Hello,\n" +" multiline!\n" +msgstr "" + +#: messages.py:29 +msgid "Hello, {}!" +msgstr "" + +#: messages.py:33 +msgid "1" +msgstr "" + +#: messages.py:33 +msgid "2" +msgstr "" + +#: messages.py:34 messages.py:35 +msgid "A" +msgstr "" + +#: messages.py:34 messages.py:35 +msgid "B" +msgstr "" + +#: messages.py:36 +msgid "set" +msgstr "" + +#: messages.py:42 +msgid "nested string" +msgstr "" + +#: messages.py:47 +msgid "baz" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/messages.py b/Lib/test/test_tools/i18n_data/messages.py new file mode 100644 index 00000000000000..f220294b8d5c67 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/messages.py @@ -0,0 +1,64 @@ +# Test message extraction +from gettext import gettext as _ + +# Empty string +_("") + +# Extra parentheses +(_("parentheses")) +((_("parentheses"))) + +# Multiline strings +_("Hello, " + "world!") + +_("""Hello, + multiline! +""") + +# Invalid arguments +_() +_(None) +_(1) +_(False) +_(x="kwargs are not allowed") +_("foo", "bar") +_("something", x="something else") + +# .format() +_("Hello, {}!").format("world") # valid +_("Hello, {}!".format("world")) # invalid + +# Nested structures +_("1"), _("2") +arr = [_("A"), _("B")] +obj = {'a': _("A"), 'b': _("B")} +{{{_('set')}}} + + +# Nested functions and classes +def test(): + _("nested string") # XXX This should be extracted but isn't. + [_("nested string")] + + +class Foo: + def bar(self): + return _("baz") + + +def bar(x=_('default value')): # XXX This should be extracted but isn't. + pass + + +def baz(x=[_('default value')]): # XXX This should be extracted but isn't. + pass + + +# Shadowing _() +def _(x): + pass + + +def _(x="don't extract me"): + pass diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index c083a04475e726..6f71f0976819f1 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -1,9 +1,11 @@ """Tests to cover the Tools/i18n package""" import os +import re import sys import unittest from textwrap import dedent +from pathlib import Path from test.support.script_helper import assert_python_ok from test.test_tools import skip_if_missing, toolsdir @@ -12,20 +14,47 @@ skip_if_missing() +DATA_DIR = Path(__file__).resolve().parent / 'i18n_data' + + +def normalize_POT_file(pot): + """Normalize the POT creation timestamp, charset and + file locations to make the POT file easier to compare. + + """ + # Normalize the creation date. + date_pattern = re.compile(r'"POT-Creation-Date: .+?\\n"') + header = r'"POT-Creation-Date: 2000-01-01 00:00+0000\\n"' + pot = re.sub(date_pattern, header, pot) + + # Normalize charset to UTF-8 (currently there's no way to specify the output charset). + charset_pattern = re.compile(r'"Content-Type: text/plain; charset=.+?\\n"') + charset = r'"Content-Type: text/plain; charset=UTF-8\\n"' + pot = re.sub(charset_pattern, charset, pot) + + # Normalize file location path separators in case this test is + # running on Windows (which uses '\'). + fileloc_pattern = re.compile(r'#:.+') + + def replace(match): + return match[0].replace(os.sep, "/") + pot = re.sub(fileloc_pattern, replace, pot) + return pot + class Test_pygettext(unittest.TestCase): """Tests for the pygettext.py tool""" - script = os.path.join(toolsdir,'i18n', 'pygettext.py') + script = Path(toolsdir, 'i18n', 'pygettext.py') def get_header(self, data): """ utility: return the header of a .po file as a dictionary """ headers = {} for line in data.split('\n'): - if not line or line.startswith(('#', 'msgid','msgstr')): + if not line or line.startswith(('#', 'msgid', 'msgstr')): continue line = line.strip('"') - key, val = line.split(':',1) + key, val = line.split(':', 1) headers[key] = val.strip() return headers @@ -53,23 +82,34 @@ def get_msgids(self, data): return msgids - def extract_docstrings_from_str(self, module_content): - """ utility: return all msgids extracted from module_content """ - filename = 'test_docstrings.py' - with temp_cwd(None) as cwd: + def assert_POT_equal(self, expected, actual): + """Check if two POT files are equal""" + self.maxDiff = None + self.assertEqual(normalize_POT_file(expected), normalize_POT_file(actual)) + + def extract_from_str(self, module_content, *, args=(), strict=True): + """Return all msgids extracted from module_content.""" + filename = 'test.py' + with temp_cwd(None): with open(filename, 'w', encoding='utf-8') as fp: fp.write(module_content) - assert_python_ok(self.script, '-D', filename) + res = assert_python_ok('-Xutf8', self.script, *args, filename) + if strict: + self.assertEqual(res.err, b'') with open('messages.pot', encoding='utf-8') as fp: data = fp.read() return self.get_msgids(data) + def extract_docstrings_from_str(self, module_content): + """Return all docstrings extracted from module_content.""" + return self.extract_from_str(module_content, args=('--docstrings',), strict=False) + def test_header(self): """Make sure the required fields are in the header, according to: http://www.gnu.org/software/gettext/manual/gettext.html#Header-Entry """ with temp_cwd(None) as cwd: - assert_python_ok(self.script) + assert_python_ok('-Xutf8', self.script) with open('messages.pot', encoding='utf-8') as fp: data = fp.read() header = self.get_header(data) @@ -96,7 +136,7 @@ def test_POT_Creation_Date(self): """ Match the date format from xgettext for POT-Creation-Date """ from datetime import datetime with temp_cwd(None) as cwd: - assert_python_ok(self.script) + assert_python_ok('-Xutf8', self.script) with open('messages.pot', encoding='utf-8') as fp: data = fp.read() header = self.get_header(data) @@ -310,6 +350,37 @@ def test_calls_in_fstring_with_partially_wrong_expression(self): self.assertNotIn('foo', msgids) self.assertIn('bar', msgids) + def test_function_and_class_names(self): + """Test that function and class names are not mistakenly extracted.""" + msgids = self.extract_from_str(dedent('''\ + def _(x): + pass + + def _(x="foo"): + pass + + async def _(x): + pass + + class _(object): + pass + ''')) + self.assertEqual(msgids, ['']) + + def test_pygettext_output(self): + """Test that the pygettext output exactly matches snapshots.""" + for input_file in DATA_DIR.glob('*.py'): + output_file = input_file.with_suffix('.pot') + with self.subTest(input_file=f'i18n_data/{input_file}'): + contents = input_file.read_text(encoding='utf-8') + with temp_cwd(None): + Path(input_file.name).write_text(contents) + assert_python_ok('-Xutf8', self.script, '--docstrings', input_file.name) + output = Path('messages.pot').read_text(encoding='utf-8') + + expected = output_file.read_text(encoding='utf-8') + self.assert_POT_equal(expected, output) + def test_files_list(self): """Make sure the directories are inspected for source files bpo-31920 @@ -318,21 +389,41 @@ def test_files_list(self): text2 = 'Text to translate2' text3 = 'Text to ignore' with temp_cwd(None), temp_dir(None) as sdir: - os.mkdir(os.path.join(sdir, 'pypkg')) - with open(os.path.join(sdir, 'pypkg', 'pymod.py'), 'w', - encoding='utf-8') as sfile: - sfile.write(f'_({text1!r})') - os.mkdir(os.path.join(sdir, 'pkg.py')) - with open(os.path.join(sdir, 'pkg.py', 'pymod2.py'), 'w', - encoding='utf-8') as sfile: - sfile.write(f'_({text2!r})') - os.mkdir(os.path.join(sdir, 'CVS')) - with open(os.path.join(sdir, 'CVS', 'pymod3.py'), 'w', - encoding='utf-8') as sfile: - sfile.write(f'_({text3!r})') - assert_python_ok(self.script, sdir) - with open('messages.pot', encoding='utf-8') as fp: - data = fp.read() + pymod = Path(sdir, 'pypkg', 'pymod.py') + pymod.parent.mkdir() + pymod.write_text(f'_({text1!r})', encoding='utf-8') + + pymod2 = Path(sdir, 'pkg.py', 'pymod2.py') + pymod2.parent.mkdir() + pymod2.write_text(f'_({text2!r})', encoding='utf-8') + + pymod3 = Path(sdir, 'CVS', 'pymod3.py') + pymod3.parent.mkdir() + pymod3.write_text(f'_({text3!r})', encoding='utf-8') + + assert_python_ok('-Xutf8', self.script, sdir) + data = Path('messages.pot').read_text(encoding='utf-8') self.assertIn(f'msgid "{text1}"', data) self.assertIn(f'msgid "{text2}"', data) self.assertNotIn(text3, data) + + +def update_POT_snapshots(): + for input_file in DATA_DIR.glob('*.py'): + output_file = input_file.with_suffix('.pot') + contents = input_file.read_bytes() + with temp_cwd(None): + Path(input_file.name).write_bytes(contents) + assert_python_ok('-Xutf8', Test_pygettext.script, '--docstrings', input_file.name) + output = Path('messages.pot').read_text(encoding='utf-8') + + output = normalize_POT_file(output) + output_file.write_text(output, encoding='utf-8') + + +if __name__ == '__main__': + # To regenerate POT files + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + update_POT_snapshots() + sys.exit(0) + unittest.main() diff --git a/Lib/test/test_ttk/test_style.py b/Lib/test/test_ttk/test_style.py index eeaf5de2e303f6..19918772514ad4 100644 --- a/Lib/test/test_ttk/test_style.py +++ b/Lib/test/test_ttk/test_style.py @@ -205,7 +205,8 @@ def test_element_create_from_errors(self): style = self.style with self.assertRaises(IndexError): style.element_create('plain.newelem', 'from') - with self.assertRaisesRegex(TclError, 'theme "spam" doesn\'t exist'): + with self.assertRaisesRegex(TclError, + 'theme "spam" (does not|doesn\'t) exist'): style.element_create('plain.newelem', 'from', 'spam') def test_element_create_image(self): diff --git a/Lib/test/test_ttk/test_widgets.py b/Lib/test/test_ttk/test_widgets.py index 10bec33be617a1..d5620becfa7187 100644 --- a/Lib/test/test_ttk/test_widgets.py +++ b/Lib/test/test_ttk/test_widgets.py @@ -8,7 +8,7 @@ from test.test_tkinter.support import ( AbstractTkTest, requires_tk, tk_version, get_tk_patchlevel, simulate_mouse_click, AbstractDefaultRootTest) -from test.test_tkinter.widget_tests import (add_standard_options, +from test.test_tkinter.widget_tests import (add_configure_tests, AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests) requires('gui') @@ -125,10 +125,11 @@ def test_cb(arg1, **kw): class AbstractToplevelTest(AbstractWidgetTest, PixelSizeTests): - _conv_pixels = False + _rounds_pixels = False + _clipped = {} -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class FrameTest(AbstractToplevelTest, unittest.TestCase): OPTIONS = ( 'borderwidth', 'class', 'cursor', 'height', @@ -140,7 +141,7 @@ def create(self, **kwargs): return ttk.Frame(self.root, **kwargs) -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class LabelFrameTest(AbstractToplevelTest, unittest.TestCase): OPTIONS = ( 'borderwidth', 'class', 'cursor', 'height', @@ -168,6 +169,8 @@ def test_configure_labelwidget(self): class AbstractLabelTest(AbstractWidgetTest): _allow_empty_justify = True + _rounds_pixels = False + _clipped = {} def checkImageParam(self, widget, name): image = tkinter.PhotoImage(master=self.root, name='image1') @@ -179,8 +182,11 @@ def checkImageParam(self, widget, name): expected=('image1', 'active', 'image2')) self.checkParam(widget, name, 'image1 active image2', expected=('image1', 'active', 'image2')) - self.checkInvalidParam(widget, name, 'spam', - errmsg='image "spam" doesn\'t exist') + if tk_version < (9, 0): + errmsg = 'image "spam" doesn\'t exist' + else: + errmsg = 'image "spam" does not exist' + self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) def test_configure_compound(self): values = ('none', 'text', 'image', 'center', 'top', 'bottom', 'left', 'right') @@ -196,7 +202,7 @@ def test_configure_width(self): self.checkParams(widget, 'width', 402, -402, 0) -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class LabelTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'anchor', 'background', 'borderwidth', @@ -214,7 +220,7 @@ def create(self, **kwargs): test_configure_justify = StandardOptionsTests.test_configure_justify -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class ButtonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'compound', 'cursor', 'default', @@ -239,7 +245,7 @@ def test_invoke(self): self.assertTrue(success) -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class CheckbuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'compound', 'cursor', @@ -326,7 +332,7 @@ def test_unique_variables2(self): self.assertEqual(len(set(variables)), len(buttons), variables) -@add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) +@add_configure_tests(IntegerSizeTests, StandardTtkOptionsTests) class EntryTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'background', 'class', 'cursor', @@ -336,6 +342,8 @@ class EntryTest(AbstractWidgetTest, unittest.TestCase): 'show', 'state', 'style', 'takefocus', 'textvariable', 'validate', 'validatecommand', 'width', 'xscrollcommand', ) + _rounds_pixels = False + _clipped = {} # bpo-27313: macOS Tk/Tcl may or may not report 'Entry.field'. IDENTIFY_AS = {'Entry.field', 'textarea'} @@ -371,8 +379,12 @@ def test_bbox(self): self.assertRaises(tkinter.TclError, self.entry.bbox, None) def test_identify(self): + if (tk_version >= (9, 0) and sys.platform == 'darwin' + and isinstance(self.entry, ttk.Combobox)): + self.skipTest('Test does not work on macOS Tk 9.') + # https://core.tcl-lang.org/tk/tktview/8b49e9cfa6 self.entry.pack() - self.entry.update() + self.root.update() self.assertIn(self.entry.identify(5, 5), self.IDENTIFY_AS) self.assertEqual(self.entry.identify(-1, -1), "") @@ -450,7 +462,7 @@ def validate(content): self.assertEqual(self.entry.state(), ()) -@add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) +@add_configure_tests(IntegerSizeTests, StandardTtkOptionsTests) class ComboboxTest(EntryTest, unittest.TestCase): OPTIONS = ( 'background', 'class', 'cursor', 'exportselection', @@ -479,11 +491,14 @@ def _show_drop_down_listbox(self): x, y = width - 5, 5 if sys.platform != 'darwin': # there's no down arrow on macOS self.assertRegex(self.combo.identify(x, y), r'.*downarrow\Z') - self.combo.event_generate('', x=x, y=y) + self.combo.event_generate('', x=x, y=y) self.combo.event_generate('', x=x, y=y) - self.combo.update_idletasks() def test_virtual_event(self): + if (tk_version >= (9, 0) and sys.platform == 'darwin' + and isinstance(self.entry, ttk.Combobox)): + self.skipTest('Test does not work on macOS Tk 9.') + # https://core.tcl-lang.org/tk/tktview/8b49e9cfa6 success = [] self.combo['values'] = [1] @@ -501,6 +516,10 @@ def test_virtual_event(self): self.assertTrue(success) def test_configure_postcommand(self): + if (tk_version >= (9, 0) and sys.platform == 'darwin' + and isinstance(self.entry, ttk.Combobox)): + self.skipTest('Test does not work on macOS Tk 9.') + # https://core.tcl-lang.org/tk/tktview/8b49e9cfa6 success = [] self.combo['postcommand'] = lambda: success.append(True) @@ -576,12 +595,14 @@ def check_get_current(getval, currval): combo2.destroy() -@add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) +@add_configure_tests(IntegerSizeTests, StandardTtkOptionsTests) class PanedWindowTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'cursor', 'height', 'orient', 'style', 'takefocus', 'width', ) + _rounds_pixels = False + _clipped = {} def setUp(self): super().setUp() @@ -712,7 +733,7 @@ def test_sashpos(self): self.assertIsInstance(self.paned.sashpos(0), int) -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class RadiobuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'compound', 'cursor', @@ -791,13 +812,14 @@ def test_configure_menu(self): menu.destroy() -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class ScaleTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'cursor', 'from', 'length', 'orient', 'state', 'style', 'takefocus', 'to', 'value', 'variable', ) - _conv_pixels = False + _rounds_pixels = False + _clipped = {} default_orient = 'horizontal' def setUp(self): @@ -899,7 +921,7 @@ def test_set(self): self.assertRaises(tkinter.TclError, self.scale.set, None) -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class ProgressbarTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'anchor', 'class', 'cursor', 'font', 'foreground', 'justify', @@ -907,7 +929,8 @@ class ProgressbarTest(AbstractWidgetTest, unittest.TestCase): 'mode', 'maximum', 'phase', 'text', 'wraplength', 'style', 'takefocus', 'value', 'variable', ) - _conv_pixels = False + _rounds_pixels = False + _clipped = {} _allow_empty_justify = True default_orient = 'horizontal' @@ -952,24 +975,27 @@ def test_configure_value(self): @unittest.skipIf(sys.platform == 'darwin', 'ttk.Scrollbar is special on MacOSX') -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class ScrollbarTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'cursor', 'orient', 'style', 'takefocus', ) + _rounds_pixels = False + _clipped = {} default_orient = 'vertical' def create(self, **kwargs): return ttk.Scrollbar(self.root, **kwargs) -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class NotebookTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'cursor', 'height', 'padding', 'style', 'takefocus', 'width', ) - if tk_version >= (8, 7): - _conv_pixels = False + _rounds_pixels = (tk_version < (9,0)) + _converts_pixels = False + _clipped = {} def setUp(self): super().setUp() @@ -987,14 +1013,14 @@ def test_configure_height(self): if get_tk_patchlevel(self.root) < (8, 6, 15): self.checkIntegerParam(widget, 'height', 402, -402, 0) else: - self.checkPixelsParam(widget, 'height', '10c', 402, -402, 0, conv=False) + self.checkPixelsParam(widget, 'height', '10c', 402, -402, 0) def test_configure_width(self): widget = self.create() if get_tk_patchlevel(self.root) < (8, 6, 15): self.checkIntegerParam(widget, 'width', 402, -402, 0) else: - self.checkPixelsParam(widget, 'width', '10c', 402, -402, 0, conv=False) + self.checkPixelsParam(widget, 'width', '10c', 402, -402, 0) def test_tab_identifiers(self): self.nb.forget(0) @@ -1160,7 +1186,12 @@ def test_traversal(self): self.nb.select(0) - focus_identify_as = 'focus' if sys.platform != 'darwin' else '' + if sys.platform == 'darwin': + focus_identify_as = '' + elif sys.platform == 'win32': + focus_identify_as = 'focus' + else: + focus_identify_as = 'focus' if tk_version < (9,0) else 'padding' self.assertEqual(self.nb.identify(5, 5), focus_identify_as) simulate_mouse_click(self.nb, 5, 5) self.nb.focus_force() @@ -1193,7 +1224,7 @@ def test_traversal(self): self.assertEqual(self.nb.select(), str(self.child2)) -@add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) +@add_configure_tests(IntegerSizeTests, StandardTtkOptionsTests) class SpinboxTest(EntryTest, unittest.TestCase): OPTIONS = ( 'background', 'class', 'command', 'cursor', 'exportselection', @@ -1370,7 +1401,7 @@ def test_configure_values(self): spin2.destroy() -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class TreeviewTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'columns', 'cursor', 'displaycolumns', @@ -1378,6 +1409,8 @@ class TreeviewTest(AbstractWidgetTest, unittest.TestCase): 'style', 'takefocus', 'titlecolumns', 'titleitems', 'xscrollcommand', 'yscrollcommand', ) + _rounds_pixels = False + _clipped = {} def setUp(self): super().setUp() @@ -1413,8 +1446,10 @@ def test_configure_displaycolumns(self): def test_configure_height(self): widget = self.create() - self.checkPixelsParam(widget, 'height', 100, -100, 0, '3c', conv=False) - self.checkPixelsParam(widget, 'height', 101.2, 102.6, conv=False) + self.checkPixelsParam(widget, 'height', 100, -100, 0, '3c', + conv=False) + self.checkPixelsParam(widget, 'height', 101.2, 102.6, '3c', + conv=False) def test_configure_selectmode(self): widget = self.create() @@ -1936,24 +1971,28 @@ def test_tag_has(self): self.assertEqual(self.tv.tag_has('tag3'), ()) -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class SeparatorTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'cursor', 'orient', 'style', 'takefocus', # 'state'? ) + _rounds_pixels = False + _clipped = {} default_orient = 'horizontal' def create(self, **kwargs): return ttk.Separator(self.root, **kwargs) -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class SizegripTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'cursor', 'style', 'takefocus', # 'state'? ) + _rounds_pixels = False + _clipped = {} def create(self, **kwargs): return ttk.Sizegrip(self.root, **kwargs) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 2f1f9e86a0bce4..aa42beca5f9256 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -8,6 +8,7 @@ import inspect import itertools import operator +import os import pickle import re import sys @@ -4252,9 +4253,15 @@ def test_builtin_protocol_allowlist(self): class CustomProtocol(TestCase, Protocol): pass + class CustomPathLikeProtocol(os.PathLike, Protocol): + pass + class CustomContextManager(typing.ContextManager, Protocol): pass + class CustomAsyncIterator(typing.AsyncIterator, Protocol): + pass + def test_non_runtime_protocol_isinstance_check(self): class P(Protocol): x: int diff --git a/Lib/test/test_unittest/testmock/testpatch.py b/Lib/test/test_unittest/testmock/testpatch.py index f26e74ce0bc1ba..037c021e6eafcf 100644 --- a/Lib/test/test_unittest/testmock/testpatch.py +++ b/Lib/test/test_unittest/testmock/testpatch.py @@ -745,6 +745,54 @@ def test_stop_idempotent(self): self.assertIsNone(patcher.stop()) + def test_exit_idempotent(self): + patcher = patch(foo_name, 'bar', 3) + with patcher: + patcher.stop() + + + def test_second_start_failure(self): + patcher = patch(foo_name, 'bar', 3) + patcher.start() + try: + self.assertRaises(RuntimeError, patcher.start) + finally: + patcher.stop() + + + def test_second_enter_failure(self): + patcher = patch(foo_name, 'bar', 3) + with patcher: + self.assertRaises(RuntimeError, patcher.start) + + + def test_second_start_after_stop(self): + patcher = patch(foo_name, 'bar', 3) + patcher.start() + patcher.stop() + patcher.start() + patcher.stop() + + + def test_property_setters(self): + mock_object = Mock() + mock_bar = mock_object.bar + patcher = patch.object(mock_object, 'bar', 'x') + with patcher: + self.assertEqual(patcher.is_local, False) + self.assertIs(patcher.target, mock_object) + self.assertEqual(patcher.temp_original, mock_bar) + patcher.is_local = True + patcher.target = mock_bar + patcher.temp_original = mock_object + self.assertEqual(patcher.is_local, True) + self.assertIs(patcher.target, mock_bar) + self.assertEqual(patcher.temp_original, mock_object) + # if changes are left intact, they may lead to disruption as shown below (it might be what someone needs though) + self.assertEqual(mock_bar.bar, mock_object) + self.assertEqual(mock_object.bar, 'x') + + def test_patchobject_start_stop(self): original = something patcher = patch.object(PTModule, 'something', 'foo') @@ -1098,7 +1146,7 @@ def test_new_callable_patch(self): self.assertIsNot(m1, m2) for mock in m1, m2: - self.assertNotCallable(m1) + self.assertNotCallable(mock) def test_new_callable_patch_object(self): @@ -1111,7 +1159,7 @@ def test_new_callable_patch_object(self): self.assertIsNot(m1, m2) for mock in m1, m2: - self.assertNotCallable(m1) + self.assertNotCallable(mock) def test_new_callable_keyword_arguments(self): diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 3ee17f96b817e1..2c53ce3f99e675 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -1526,8 +1526,10 @@ def test_pathname2url_win(self): self.assertEqual(fn('\\\\?\\C:\\dir'), '///C:/dir') self.assertEqual(fn('\\\\?\\unc\\server\\share\\dir'), '//server/share/dir') self.assertEqual(fn("C:"), '///C:') - self.assertEqual(fn("C:\\"), '///C:') + self.assertEqual(fn("C:\\"), '///C:/') self.assertEqual(fn('C:\\a\\b.c'), '///C:/a/b.c') + self.assertEqual(fn('C:\\a\\b.c\\'), '///C:/a/b.c/') + self.assertEqual(fn('C:\\a\\\\b.c'), '///C:/a//b.c') self.assertEqual(fn('C:\\a\\b%#c'), '///C:/a/b%25%23c') self.assertEqual(fn('C:\\a\\b\xe9'), '///C:/a/b%C3%A9') self.assertEqual(fn('C:\\foo\\bar\\spam.foo'), "///C:/foo/bar/spam.foo") @@ -1540,9 +1542,14 @@ def test_pathname2url_win(self): self.assertEqual(fn('\\\\some\\share\\'), '//some/share/') self.assertEqual(fn('\\\\some\\share\\a\\b.c'), '//some/share/a/b.c') self.assertEqual(fn('\\\\some\\share\\a\\b%#c\xe9'), '//some/share/a/b%25%23c%C3%A9') + # Alternate path separator + self.assertEqual(fn('C:/a/b.c'), '///C:/a/b.c') + self.assertEqual(fn('//some/share/a/b.c'), '//some/share/a/b.c') + self.assertEqual(fn('//?/C:/dir'), '///C:/dir') + self.assertEqual(fn('//?/unc/server/share/dir'), '//server/share/dir') # Round-tripping urls = ['///C:', - '///folder/test/', + '/folder/test/', '///C:/foo/bar/spam.foo'] for url in urls: self.assertEqual(fn(urllib.request.url2pathname(url)), url) @@ -1563,13 +1570,15 @@ def test_url2pathname_win(self): self.assertEqual(fn("///C|"), 'C:') self.assertEqual(fn("///C:"), 'C:') self.assertEqual(fn('///C:/'), 'C:\\') - self.assertEqual(fn('/C|//'), 'C:\\') + self.assertEqual(fn('/C|//'), 'C:\\\\') self.assertEqual(fn('///C|/path'), 'C:\\path') # No DOS drive - self.assertEqual(fn("///C/test/"), '\\\\\\C\\test\\') + self.assertEqual(fn("///C/test/"), '\\C\\test\\') self.assertEqual(fn("////C/test/"), '\\\\C\\test\\') # DOS drive paths self.assertEqual(fn('C:/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('C:/path/to/file/'), 'C:\\path\\to\\file\\') + self.assertEqual(fn('C:/path/to//file'), 'C:\\path\\to\\\\file') self.assertEqual(fn('C|/path/to/file'), 'C:\\path\\to\\file') self.assertEqual(fn('/C|/path/to/file'), 'C:\\path\\to\\file') self.assertEqual(fn('///C|/path/to/file'), 'C:\\path\\to\\file') @@ -1583,9 +1592,12 @@ def test_url2pathname_win(self): # Localhost paths self.assertEqual(fn('//localhost/C:/path/to/file'), 'C:\\path\\to\\file') self.assertEqual(fn('//localhost/C|/path/to/file'), 'C:\\path\\to\\file') + # Percent-encoded forward slashes are preserved for backwards compatibility + self.assertEqual(fn('C:/foo%2fbar'), 'C:\\foo/bar') + self.assertEqual(fn('//server/share/foo%2fbar'), '\\\\server\\share\\foo/bar') # Round-tripping paths = ['C:', - r'\\\C\test\\', + r'\C\test\\', r'C:\foo\bar\spam.foo'] for path in paths: self.assertEqual(fn(urllib.request.pathname2url(path)), path) @@ -1596,8 +1608,8 @@ def test_url2pathname_posix(self): fn = urllib.request.url2pathname self.assertEqual(fn('/foo/bar'), '/foo/bar') self.assertEqual(fn('//foo/bar'), '//foo/bar') - self.assertEqual(fn('///foo/bar'), '///foo/bar') - self.assertEqual(fn('////foo/bar'), '////foo/bar') + self.assertEqual(fn('///foo/bar'), '/foo/bar') + self.assertEqual(fn('////foo/bar'), '//foo/bar') self.assertEqual(fn('//localhost/foo/bar'), '//localhost/foo/bar') class Utility_Tests(unittest.TestCase): diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index d49e4388696ab4..4516bdea6adb19 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -623,6 +623,78 @@ def test_urljoins(self): self.checkJoin(RFC1808_BASE, 'https:;', 'https:;') self.checkJoin(RFC1808_BASE, 'https:;x', 'https:;x') + def test_urljoins_relative_base(self): + # According to RFC 3986, Section 5.1, a base URI must conform to + # the absolute-URI syntax rule (Section 4.3). But urljoin() lacks + # a context to establish missed components of the relative base URI. + # It still has to return a sensible result for backwards compatibility. + # The following tests are figments of the imagination and artifacts + # of the current implementation that are not based on any standard. + self.checkJoin('', '', '') + self.checkJoin('', '//', '//', relroundtrip=False) + self.checkJoin('', '//v', '//v') + self.checkJoin('', '//v/w', '//v/w') + self.checkJoin('', '/w', '/w') + self.checkJoin('', '///w', '///w', relroundtrip=False) + self.checkJoin('', 'w', 'w') + + self.checkJoin('//', '', '//') + self.checkJoin('//', '//', '//') + self.checkJoin('//', '//v', '//v') + self.checkJoin('//', '//v/w', '//v/w') + self.checkJoin('//', '/w', '///w') + self.checkJoin('//', '///w', '///w') + self.checkJoin('//', 'w', '///w') + + self.checkJoin('//a', '', '//a') + self.checkJoin('//a', '//', '//a') + self.checkJoin('//a', '//v', '//v') + self.checkJoin('//a', '//v/w', '//v/w') + self.checkJoin('//a', '/w', '//a/w') + self.checkJoin('//a', '///w', '//a/w') + self.checkJoin('//a', 'w', '//a/w') + + for scheme in '', 'http:': + self.checkJoin('http:', scheme + '', 'http:') + self.checkJoin('http:', scheme + '//', 'http:') + self.checkJoin('http:', scheme + '//v', 'http://v') + self.checkJoin('http:', scheme + '//v/w', 'http://v/w') + self.checkJoin('http:', scheme + '/w', 'http:/w') + self.checkJoin('http:', scheme + '///w', 'http:/w') + self.checkJoin('http:', scheme + 'w', 'http:/w') + + self.checkJoin('http://', scheme + '', 'http://') + self.checkJoin('http://', scheme + '//', 'http://') + self.checkJoin('http://', scheme + '//v', 'http://v') + self.checkJoin('http://', scheme + '//v/w', 'http://v/w') + self.checkJoin('http://', scheme + '/w', 'http:///w') + self.checkJoin('http://', scheme + '///w', 'http:///w') + self.checkJoin('http://', scheme + 'w', 'http:///w') + + self.checkJoin('http://a', scheme + '', 'http://a') + self.checkJoin('http://a', scheme + '//', 'http://a') + self.checkJoin('http://a', scheme + '//v', 'http://v') + self.checkJoin('http://a', scheme + '//v/w', 'http://v/w') + self.checkJoin('http://a', scheme + '/w', 'http://a/w') + self.checkJoin('http://a', scheme + '///w', 'http://a/w') + self.checkJoin('http://a', scheme + 'w', 'http://a/w') + + self.checkJoin('/b/c', '', '/b/c') + self.checkJoin('/b/c', '//', '/b/c') + self.checkJoin('/b/c', '//v', '//v') + self.checkJoin('/b/c', '//v/w', '//v/w') + self.checkJoin('/b/c', '/w', '/w') + self.checkJoin('/b/c', '///w', '/w') + self.checkJoin('/b/c', 'w', '/b/w') + + self.checkJoin('///b/c', '', '///b/c') + self.checkJoin('///b/c', '//', '///b/c') + self.checkJoin('///b/c', '//v', '//v') + self.checkJoin('///b/c', '//v/w', '//v/w') + self.checkJoin('///b/c', '/w', '///w') + self.checkJoin('///b/c', '///w', '///w') + self.checkJoin('///b/c', 'w', '///b/w') + def test_RFC2732(self): str_cases = [ ('http://Test.python.org:5432/foo/', 'test.python.org', 5432), @@ -1242,9 +1314,17 @@ def test_parse_qsl_bytes(self): def test_parse_qsl_false_value(self): kwargs = dict(keep_blank_values=True, strict_parsing=True) - for x in '', b'', None, 0, 0.0, [], {}, memoryview(b''): + for x in '', b'', None, memoryview(b''): self.assertEqual(urllib.parse.parse_qsl(x, **kwargs), []) self.assertRaises(ValueError, urllib.parse.parse_qsl, x, separator=1) + for x in 0, 0.0, [], {}: + with self.assertWarns(DeprecationWarning) as cm: + self.assertEqual(urllib.parse.parse_qsl(x, **kwargs), []) + self.assertEqual(cm.filename, __file__) + with self.assertWarns(DeprecationWarning) as cm: + self.assertEqual(urllib.parse.parse_qs(x, **kwargs), {}) + self.assertEqual(cm.filename, __file__) + self.assertRaises(ValueError, urllib.parse.parse_qsl, x, separator=1) def test_parse_qsl_errors(self): self.assertRaises(TypeError, urllib.parse.parse_qsl, list(b'a=b')) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index e177464c00f7a6..7bd26a8ca34b62 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -8,8 +8,10 @@ import io import os import pickle +import random import sys import weakref +from itertools import product from unittest import mock py_uuid = import_helper.import_fresh_module('uuid', blocked=['_uuid']) @@ -267,7 +269,7 @@ def test_exceptions(self): # Version number out of range. badvalue(lambda: self.uuid.UUID('00'*16, version=0)) - badvalue(lambda: self.uuid.UUID('00'*16, version=6)) + badvalue(lambda: self.uuid.UUID('00'*16, version=42)) # Integer value out of range. badvalue(lambda: self.uuid.UUID(int=-1)) @@ -681,6 +683,37 @@ def test_uuid5(self): equal(u, self.uuid.UUID(v)) equal(str(u), v) + def test_uuid8(self): + equal = self.assertEqual + u = self.uuid.uuid8() + + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 8) + + for (_, hi, mid, lo) in product( + range(10), # repeat 10 times + [None, 0, random.getrandbits(48)], + [None, 0, random.getrandbits(12)], + [None, 0, random.getrandbits(62)], + ): + u = self.uuid.uuid8(hi, mid, lo) + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 8) + if hi is not None: + equal((u.int >> 80) & 0xffffffffffff, hi) + if mid is not None: + equal((u.int >> 64) & 0xfff, mid) + if lo is not None: + equal(u.int & 0x3fffffffffffffff, lo) + + def test_uuid8_uniqueness(self): + # Test that UUIDv8-generated values are unique + # (up to a negligible probability of failure). + u1 = self.uuid.uuid8() + u2 = self.uuid.uuid8() + self.assertNotEqual(u1.int, u2.int) + self.assertEqual(u1.version, u2.version) + @support.requires_fork() def testIssue8621(self): # On at least some versions of OSX self.uuid.uuid4 generates diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 8b59630717e790..4e3c877896f295 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -533,6 +533,18 @@ def test_skip_file_prefixes(self): warning_tests.package("prefix02", stacklevel=3) self.assertIn("unittest", w[-1].filename) + def test_skip_file_prefixes_file_path(self): + # see: gh-126209 + with warnings_state(self.module): + skipped = warning_tests.__file__ + with original_warnings.catch_warnings( + record=True, module=self.module, + ) as w: + warning_tests.outer("msg", skip_file_prefixes=(skipped,)) + + self.assertEqual(len(w), 1) + self.assertNotEqual(w[-1].filename, skipped) + def test_skip_file_prefixes_type_errors(self): with warnings_state(self.module): warn = warning_tests.warnings.warn diff --git a/Lib/test/test_warnings/data/stacklevel.py b/Lib/test/test_warnings/data/stacklevel.py index c6dd24733b3b74..fe36242d3d20c2 100644 --- a/Lib/test/test_warnings/data/stacklevel.py +++ b/Lib/test/test_warnings/data/stacklevel.py @@ -4,11 +4,13 @@ import warnings from test.test_warnings.data import package_helper -def outer(message, stacklevel=1): - inner(message, stacklevel) -def inner(message, stacklevel=1): - warnings.warn(message, stacklevel=stacklevel) +def outer(message, stacklevel=1, skip_file_prefixes=()): + inner(message, stacklevel, skip_file_prefixes) + +def inner(message, stacklevel=1, skip_file_prefixes=()): + warnings.warn(message, stacklevel=stacklevel, + skip_file_prefixes=skip_file_prefixes) def package(message, *, stacklevel): package_helper.inner_api(message, stacklevel=stacklevel, diff --git a/Lib/test/test_yield_from.py b/Lib/test/test_yield_from.py index 1a60357a1bcd62..b90e15e20027dc 100644 --- a/Lib/test/test_yield_from.py +++ b/Lib/test/test_yield_from.py @@ -1576,6 +1576,19 @@ def outer(): self.assertIsNone(caught.exception.__context__) self.assert_stop_iteration(g) + def test_throws_in_iter(self): + # See GH-126366: NULL pointer dereference if __iter__ + # threw an exception. + class Silly: + def __iter__(self): + raise RuntimeError("nobody expects the spanish inquisition") + + def my_generator(): + yield from Silly() + + with self.assertRaisesRegex(RuntimeError, "nobody expects the spanish inquisition"): + next(iter(my_generator())) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 8b4bb8750f8f5c..4d97fe56f3a094 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -505,20 +505,16 @@ def test_flushes(self): for sync in sync_opt: for level in range(10): - try: + with self.subTest(sync=sync, level=level): obj = zlib.compressobj( level ) a = obj.compress( data[:3000] ) b = obj.flush( sync ) c = obj.compress( data[3000:] ) d = obj.flush() - except: - print("Error for flush mode={}, level={}" - .format(sync, level)) - raise - self.assertEqual(zlib.decompress(b''.join([a,b,c,d])), - data, ("Decompress failed: flush " - "mode=%i, level=%i") % (sync, level)) - del obj + self.assertEqual(zlib.decompress(b''.join([a,b,c,d])), + data, ("Decompress failed: flush " + "mode=%i, level=%i") % (sync, level)) + del obj @unittest.skipUnless(hasattr(zlib, 'Z_SYNC_FLUSH'), 'requires zlib.Z_SYNC_FLUSH') diff --git a/Lib/test/translationdata/getopt/msgids.txt b/Lib/test/translationdata/getopt/msgids.txt new file mode 100644 index 00000000000000..1ffab1f31abad5 --- /dev/null +++ b/Lib/test/translationdata/getopt/msgids.txt @@ -0,0 +1,6 @@ +option -%s not recognized +option -%s requires argument +option --%s must not have an argument +option --%s not a unique prefix +option --%s not recognized +option --%s requires argument \ No newline at end of file diff --git a/Lib/test/translationdata/optparse/msgids.txt b/Lib/test/translationdata/optparse/msgids.txt new file mode 100644 index 00000000000000..ac5317c736af8c --- /dev/null +++ b/Lib/test/translationdata/optparse/msgids.txt @@ -0,0 +1,14 @@ +%prog [options] +%s option does not take a value +Options +Usage +Usage: %s\n +ambiguous option: %s (%s?) +complex +floating-point +integer +no such option: %s +option %s: invalid %s value: %r +option %s: invalid choice: %r (choose from %s) +show program's version number and exit +show this help message and exit \ No newline at end of file diff --git a/Lib/textwrap.py b/Lib/textwrap.py index 7ca393d1c371aa..1bf07aa46cad99 100644 --- a/Lib/textwrap.py +++ b/Lib/textwrap.py @@ -2,7 +2,7 @@ """ # Copyright (C) 1999-2001 Gregory P. Ward. -# Copyright (C) 2002, 2003 Python Software Foundation. +# Copyright (C) 2002 Python Software Foundation. # Written by Greg Ward import re diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index 073b3ae20797c3..8ddb7f97e3b233 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -321,6 +321,8 @@ def _tclobj_to_py(val): elif hasattr(val, 'typename'): # some other (single) Tcl object val = _convert_stringval(val) + if isinstance(val, tuple) and len(val) == 0: + return '' return val def tclobjs_to_py(adict): diff --git a/Lib/tomllib/_parser.py b/Lib/tomllib/_parser.py index 5671326646ca5a..4d208bcfb4a9a6 100644 --- a/Lib/tomllib/_parser.py +++ b/Lib/tomllib/_parser.py @@ -8,6 +8,7 @@ import string from types import MappingProxyType from typing import Any, BinaryIO, NamedTuple +import warnings from ._re import ( RE_DATETIME, @@ -50,8 +51,68 @@ ) +class DEPRECATED_DEFAULT: + """Sentinel to be used as default arg during deprecation + period of TOMLDecodeError's free-form arguments.""" + + class TOMLDecodeError(ValueError): - """An error raised if a document is not valid TOML.""" + """An error raised if a document is not valid TOML. + + Adds the following attributes to ValueError: + msg: The unformatted error message + doc: The TOML document being parsed + pos: The index of doc where parsing failed + lineno: The line corresponding to pos + colno: The column corresponding to pos + """ + + def __init__( + self, + msg: str = DEPRECATED_DEFAULT, # type: ignore[assignment] + doc: str = DEPRECATED_DEFAULT, # type: ignore[assignment] + pos: Pos = DEPRECATED_DEFAULT, # type: ignore[assignment] + *args: Any, + ): + if ( + args + or not isinstance(msg, str) + or not isinstance(doc, str) + or not isinstance(pos, int) + ): + warnings.warn( + "Free-form arguments for TOMLDecodeError are deprecated. " + "Please set 'msg' (str), 'doc' (str) and 'pos' (int) arguments only.", + DeprecationWarning, + stacklevel=2, + ) + if pos is not DEPRECATED_DEFAULT: # type: ignore[comparison-overlap] + args = pos, *args + if doc is not DEPRECATED_DEFAULT: # type: ignore[comparison-overlap] + args = doc, *args + if msg is not DEPRECATED_DEFAULT: # type: ignore[comparison-overlap] + args = msg, *args + ValueError.__init__(self, *args) + return + + lineno = doc.count("\n", 0, pos) + 1 + if lineno == 1: + colno = pos + 1 + else: + colno = pos - doc.rindex("\n", 0, pos) + + if pos >= len(doc): + coord_repr = "end of document" + else: + coord_repr = f"line {lineno}, column {colno}" + errmsg = f"{msg} (at {coord_repr})" + ValueError.__init__(self, errmsg) + + self.msg = msg + self.doc = doc + self.pos = pos + self.lineno = lineno + self.colno = colno def load(fp: BinaryIO, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: @@ -118,7 +179,7 @@ def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # n pos, header = create_dict_rule(src, pos, out) pos = skip_chars(src, pos, TOML_WS) elif char != "#": - raise suffixed_err(src, pos, "Invalid statement") + raise TOMLDecodeError("Invalid statement", src, pos) # 3. Skip comment pos = skip_comment(src, pos) @@ -129,8 +190,8 @@ def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # n except IndexError: break if char != "\n": - raise suffixed_err( - src, pos, "Expected newline or end of document after a statement" + raise TOMLDecodeError( + "Expected newline or end of document after a statement", src, pos ) pos += 1 @@ -256,12 +317,12 @@ def skip_until( except ValueError: new_pos = len(src) if error_on_eof: - raise suffixed_err(src, new_pos, f"Expected {expect!r}") from None + raise TOMLDecodeError(f"Expected {expect!r}", src, new_pos) from None if not error_on.isdisjoint(src[pos:new_pos]): while src[pos] not in error_on: pos += 1 - raise suffixed_err(src, pos, f"Found invalid character {src[pos]!r}") + raise TOMLDecodeError(f"Found invalid character {src[pos]!r}", src, pos) return new_pos @@ -292,15 +353,17 @@ def create_dict_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: pos, key = parse_key(src, pos) if out.flags.is_(key, Flags.EXPLICIT_NEST) or out.flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot declare {key} twice") + raise TOMLDecodeError(f"Cannot declare {key} twice", src, pos) out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False) try: out.data.get_or_create_nest(key) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if not src.startswith("]", pos): - raise suffixed_err(src, pos, "Expected ']' at the end of a table declaration") + raise TOMLDecodeError( + "Expected ']' at the end of a table declaration", src, pos + ) return pos + 1, key @@ -310,7 +373,7 @@ def create_list_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: pos, key = parse_key(src, pos) if out.flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") + raise TOMLDecodeError(f"Cannot mutate immutable namespace {key}", src, pos) # Free the namespace now that it points to another empty list item... out.flags.unset_all(key) # ...but this key precisely is still prohibited from table declaration @@ -318,10 +381,12 @@ def create_list_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: try: out.data.append_nest_to_list(key) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if not src.startswith("]]", pos): - raise suffixed_err(src, pos, "Expected ']]' at the end of an array declaration") + raise TOMLDecodeError( + "Expected ']]' at the end of an array declaration", src, pos + ) return pos + 2, key @@ -336,22 +401,22 @@ def key_value_rule( for cont_key in relative_path_cont_keys: # Check that dotted key syntax does not redefine an existing table if out.flags.is_(cont_key, Flags.EXPLICIT_NEST): - raise suffixed_err(src, pos, f"Cannot redefine namespace {cont_key}") + raise TOMLDecodeError(f"Cannot redefine namespace {cont_key}", src, pos) # Containers in the relative path can't be opened with the table syntax or # dotted key/value syntax in following table sections. out.flags.add_pending(cont_key, Flags.EXPLICIT_NEST) if out.flags.is_(abs_key_parent, Flags.FROZEN): - raise suffixed_err( - src, pos, f"Cannot mutate immutable namespace {abs_key_parent}" + raise TOMLDecodeError( + f"Cannot mutate immutable namespace {abs_key_parent}", src, pos ) try: nest = out.data.get_or_create_nest(abs_key_parent) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if key_stem in nest: - raise suffixed_err(src, pos, "Cannot overwrite a value") + raise TOMLDecodeError("Cannot overwrite a value", src, pos) # Mark inline table and array namespaces recursively immutable if isinstance(value, (dict, list)): out.flags.set(header + key, Flags.FROZEN, recursive=True) @@ -368,7 +433,7 @@ def parse_key_value_pair( except IndexError: char = None if char != "=": - raise suffixed_err(src, pos, "Expected '=' after a key in a key/value pair") + raise TOMLDecodeError("Expected '=' after a key in a key/value pair", src, pos) pos += 1 pos = skip_chars(src, pos, TOML_WS) pos, value = parse_value(src, pos, parse_float) @@ -406,7 +471,7 @@ def parse_key_part(src: str, pos: Pos) -> tuple[Pos, str]: return parse_literal_str(src, pos) if char == '"': return parse_one_line_basic_str(src, pos) - raise suffixed_err(src, pos, "Invalid initial character for a key part") + raise TOMLDecodeError("Invalid initial character for a key part", src, pos) def parse_one_line_basic_str(src: str, pos: Pos) -> tuple[Pos, str]: @@ -430,7 +495,7 @@ def parse_array(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, list] if c == "]": return pos + 1, array if c != ",": - raise suffixed_err(src, pos, "Unclosed array") + raise TOMLDecodeError("Unclosed array", src, pos) pos += 1 pos = skip_comments_and_array_ws(src, pos) @@ -450,20 +515,20 @@ def parse_inline_table(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos pos, key, value = parse_key_value_pair(src, pos, parse_float) key_parent, key_stem = key[:-1], key[-1] if flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") + raise TOMLDecodeError(f"Cannot mutate immutable namespace {key}", src, pos) try: nest = nested_dict.get_or_create_nest(key_parent, access_lists=False) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if key_stem in nest: - raise suffixed_err(src, pos, f"Duplicate inline table key {key_stem!r}") + raise TOMLDecodeError(f"Duplicate inline table key {key_stem!r}", src, pos) nest[key_stem] = value pos = skip_chars(src, pos, TOML_WS) c = src[pos : pos + 1] if c == "}": return pos + 1, nested_dict.dict if c != ",": - raise suffixed_err(src, pos, "Unclosed inline table") + raise TOMLDecodeError("Unclosed inline table", src, pos) if isinstance(value, (dict, list)): flags.set(key, Flags.FROZEN, recursive=True) pos += 1 @@ -485,7 +550,7 @@ def parse_basic_str_escape( except IndexError: return pos, "" if char != "\n": - raise suffixed_err(src, pos, "Unescaped '\\' in a string") + raise TOMLDecodeError("Unescaped '\\' in a string", src, pos) pos += 1 pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE) return pos, "" @@ -496,7 +561,7 @@ def parse_basic_str_escape( try: return pos, BASIC_STR_ESCAPE_REPLACEMENTS[escape_id] except KeyError: - raise suffixed_err(src, pos, "Unescaped '\\' in a string") from None + raise TOMLDecodeError("Unescaped '\\' in a string", src, pos) from None def parse_basic_str_escape_multiline(src: str, pos: Pos) -> tuple[Pos, str]: @@ -506,11 +571,13 @@ def parse_basic_str_escape_multiline(src: str, pos: Pos) -> tuple[Pos, str]: def parse_hex_char(src: str, pos: Pos, hex_len: int) -> tuple[Pos, str]: hex_str = src[pos : pos + hex_len] if len(hex_str) != hex_len or not HEXDIGIT_CHARS.issuperset(hex_str): - raise suffixed_err(src, pos, "Invalid hex value") + raise TOMLDecodeError("Invalid hex value", src, pos) pos += hex_len hex_int = int(hex_str, 16) if not is_unicode_scalar_value(hex_int): - raise suffixed_err(src, pos, "Escaped character is not a Unicode scalar value") + raise TOMLDecodeError( + "Escaped character is not a Unicode scalar value", src, pos + ) return pos, chr(hex_int) @@ -567,7 +634,7 @@ def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]: try: char = src[pos] except IndexError: - raise suffixed_err(src, pos, "Unterminated string") from None + raise TOMLDecodeError("Unterminated string", src, pos) from None if char == '"': if not multiline: return pos + 1, result + src[start_pos:pos] @@ -582,7 +649,7 @@ def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]: start_pos = pos continue if char in error_on: - raise suffixed_err(src, pos, f"Illegal character {char!r}") + raise TOMLDecodeError(f"Illegal character {char!r}", src, pos) pos += 1 @@ -630,7 +697,7 @@ def parse_value( # noqa: C901 try: datetime_obj = match_to_datetime(datetime_match) except ValueError as e: - raise suffixed_err(src, pos, "Invalid date or datetime") from e + raise TOMLDecodeError("Invalid date or datetime", src, pos) from e return datetime_match.end(), datetime_obj localtime_match = RE_LOCALTIME.match(src, pos) if localtime_match: @@ -651,24 +718,7 @@ def parse_value( # noqa: C901 if first_four in {"-inf", "+inf", "-nan", "+nan"}: return pos + 4, parse_float(first_four) - raise suffixed_err(src, pos, "Invalid value") - - -def suffixed_err(src: str, pos: Pos, msg: str) -> TOMLDecodeError: - """Return a `TOMLDecodeError` where error message is suffixed with - coordinates in source.""" - - def coord_repr(src: str, pos: Pos) -> str: - if pos >= len(src): - return "end of document" - line = src.count("\n", 0, pos) + 1 - if line == 1: - column = pos + 1 - else: - column = pos - src.rindex("\n", 0, pos) - return f"line {line}, column {column}" - - return TOMLDecodeError(f"{msg} (at {coord_repr(src, pos)})") + raise TOMLDecodeError("Invalid value", src, pos) def is_unicode_scalar_value(codepoint: int) -> bool: diff --git a/Lib/typing.py b/Lib/typing.py index c924c767042552..938e52922aee03 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1940,9 +1940,11 @@ def _allow_reckless_class_checks(depth=2): _PROTO_ALLOWLIST = { 'collections.abc': [ 'Callable', 'Awaitable', 'Iterable', 'Iterator', 'AsyncIterable', - 'Hashable', 'Sized', 'Container', 'Collection', 'Reversible', 'Buffer', + 'AsyncIterator', 'Hashable', 'Sized', 'Container', 'Collection', + 'Reversible', 'Buffer', ], 'contextlib': ['AbstractContextManager', 'AbstractAsyncContextManager'], + 'os': ['PathLike'], } diff --git a/Lib/unittest/__init__.py b/Lib/unittest/__init__.py index 324e5d038aef03..78ff6bb4fdcce5 100644 --- a/Lib/unittest/__init__.py +++ b/Lib/unittest/__init__.py @@ -27,7 +27,7 @@ def testMultiply(self): http://docs.python.org/library/unittest.html Copyright (c) 1999-2003 Steve Purcell -Copyright (c) 2003-2010 Python Software Foundation +Copyright (c) 2003 Python Software Foundation This module is free software, and you may redistribute it and/or modify it under the same terms as Python itself, so long as this copyright message and disclaimer are retained in their original form. diff --git a/Lib/unittest/async_case.py b/Lib/unittest/async_case.py index bd06eb3207697a..6000af1cef0a78 100644 --- a/Lib/unittest/async_case.py +++ b/Lib/unittest/async_case.py @@ -5,6 +5,7 @@ from .case import TestCase +__unittest = True class IsolatedAsyncioTestCase(TestCase): # Names intentionally have a long prefix diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 21ca061a77c26f..55cb4b1f6aff90 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -1360,6 +1360,7 @@ def __init__( self.autospec = autospec self.kwargs = kwargs self.additional_patchers = [] + self.is_started = False def copy(self): @@ -1472,6 +1473,9 @@ def get_original(self): def __enter__(self): """Perform the patch.""" + if self.is_started: + raise RuntimeError("Patch is already started") + new, spec, spec_set = self.new, self.spec, self.spec_set autospec, kwargs = self.autospec, self.kwargs new_callable = self.new_callable @@ -1603,6 +1607,7 @@ def __enter__(self): self.temp_original = original self.is_local = local self._exit_stack = contextlib.ExitStack() + self.is_started = True try: setattr(self.target, self.attribute, new_attr) if self.attribute_name is not None: @@ -1622,6 +1627,9 @@ def __enter__(self): def __exit__(self, *exc_info): """Undo the patch.""" + if not self.is_started: + return + if self.is_local and self.temp_original is not DEFAULT: setattr(self.target, self.attribute, self.temp_original) else: @@ -1638,6 +1646,7 @@ def __exit__(self, *exc_info): del self.target exit_stack = self._exit_stack del self._exit_stack + self.is_started = False return exit_stack.__exit__(*exc_info) diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 5b00ab25c6b4ca..8d7631d5693ece 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -577,9 +577,9 @@ def urljoin(base, url, allow_fragments=True): if scheme is None: scheme = bscheme - if scheme != bscheme or scheme not in uses_relative: + if scheme != bscheme or (scheme and scheme not in uses_relative): return _coerce_result(url) - if scheme in uses_netloc: + if not scheme or scheme in uses_netloc: if netloc: return _coerce_result(_urlunsplit(scheme, netloc, path, query, fragment)) @@ -753,7 +753,8 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, parsed_result = {} pairs = parse_qsl(qs, keep_blank_values, strict_parsing, encoding=encoding, errors=errors, - max_num_fields=max_num_fields, separator=separator) + max_num_fields=max_num_fields, separator=separator, + _stacklevel=2) for name, value in pairs: if name in parsed_result: parsed_result[name].append(value) @@ -763,7 +764,7 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, - encoding='utf-8', errors='replace', max_num_fields=None, separator='&'): + encoding='utf-8', errors='replace', max_num_fields=None, separator='&', *, _stacklevel=1): """Parse a query given as a string argument. Arguments: @@ -791,7 +792,6 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, Returns a list, as G-d intended. """ - if not separator or not isinstance(separator, (str, bytes)): raise ValueError("Separator must be of type string or bytes.") if isinstance(qs, str): @@ -800,12 +800,21 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, eq = '=' def _unquote(s): return unquote_plus(s, encoding=encoding, errors=errors) + elif qs is None: + return [] else: - if not qs: - return [] - # Use memoryview() to reject integers and iterables, - # acceptable by the bytes constructor. - qs = bytes(memoryview(qs)) + try: + # Use memoryview() to reject integers and iterables, + # acceptable by the bytes constructor. + qs = bytes(memoryview(qs)) + except TypeError: + if not qs: + warnings.warn(f"Accepting {type(qs).__name__} objects with " + f"false value in urllib.parse.parse_qsl() is " + f"deprecated as of 3.14", + DeprecationWarning, stacklevel=_stacklevel + 1) + return [] + raise if isinstance(separator, str): separator = bytes(separator, 'ascii') eq = b'=' diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index bc35d8a80e5d03..18a837dd57ed59 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1656,6 +1656,10 @@ def data_open(self, req): def url2pathname(pathname): """OS-specific conversion from a relative URL of the 'file' scheme to a file system path; not recommended for general use.""" + if pathname[:3] == '///': + # URL has an empty authority section, so the path begins on the + # third character. + pathname = pathname[2:] return unquote(pathname) def pathname2url(pathname): diff --git a/Lib/uuid.py b/Lib/uuid.py index 4d4f06cfc9ebbe..9c6ad9643cf6d5 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -1,8 +1,8 @@ -r"""UUID objects (universally unique identifiers) according to RFC 4122. +r"""UUID objects (universally unique identifiers) according to RFC 4122/9562. This module provides immutable UUID objects (class UUID) and the functions -uuid1(), uuid3(), uuid4(), uuid5() for generating version 1, 3, 4, and 5 -UUIDs as specified in RFC 4122. +uuid1(), uuid3(), uuid4(), uuid5(), and uuid8() for generating version 1, 3, +4, 5, and 8 UUIDs as specified in RFC 4122/9562. If all you want is a unique ID, you should probably call uuid1() or uuid4(). Note that uuid1() may compromise privacy since it creates a UUID containing @@ -124,12 +124,12 @@ class UUID: int the UUID as a 128-bit integer - urn the UUID as a URN as specified in RFC 4122 + urn the UUID as a URN as specified in RFC 4122/9562 variant the UUID variant (one of the constants RESERVED_NCS, RFC_4122, RESERVED_MICROSOFT, or RESERVED_FUTURE) - version the UUID version number (1 through 5, meaningful only + version the UUID version number (1 through 8, meaningful only when the variant is RFC_4122) is_safe An enum indicating whether the UUID has been generated in @@ -214,9 +214,9 @@ def __init__(self, hex=None, bytes=None, bytes_le=None, fields=None, if not 0 <= int < 1<<128: raise ValueError('int is out of range (need a 128-bit value)') if version is not None: - if not 1 <= version <= 5: + if not 1 <= version <= 8: raise ValueError('illegal version number') - # Set the variant to RFC 4122. + # Set the variant to RFC 4122/9562. int &= ~(0xc000 << 48) int |= 0x8000 << 48 # Set the version number. @@ -355,7 +355,7 @@ def variant(self): @property def version(self): - # The version bits are only meaningful for RFC 4122 UUIDs. + # The version bits are only meaningful for RFC 4122/9562 UUIDs. if self.variant == RFC_4122: return int((self.int >> 76) & 0xf) @@ -719,6 +719,28 @@ def uuid5(namespace, name): hash = sha1(namespace.bytes + name).digest() return UUID(bytes=hash[:16], version=5) +def uuid8(a=None, b=None, c=None): + """Generate a UUID from three custom blocks. + + * 'a' is the first 48-bit chunk of the UUID (octets 0-5); + * 'b' is the mid 12-bit chunk (octets 6-7); + * 'c' is the last 62-bit chunk (octets 8-15). + + When a value is not specified, a pseudo-random value is generated. + """ + if a is None: + import random + a = random.getrandbits(48) + if b is None: + import random + b = random.getrandbits(12) + if c is None: + import random + c = random.getrandbits(62) + int_uuid_8 = (a & 0xffff_ffff_ffff) << 80 + int_uuid_8 |= (b & 0xfff) << 64 + int_uuid_8 |= c & 0x3fff_ffff_ffff_ffff + return UUID(int=int_uuid_8, version=8) def main(): """Run the uuid command line interface.""" @@ -726,7 +748,8 @@ def main(): "uuid1": uuid1, "uuid3": uuid3, "uuid4": uuid4, - "uuid5": uuid5 + "uuid5": uuid5, + "uuid8": uuid8, } uuid_namespace_funcs = ("uuid3", "uuid5") namespaces = { diff --git a/Lib/wsgiref/headers.py b/Lib/wsgiref/headers.py index 05d2ba4c664e5e..c78879f80c7df2 100644 --- a/Lib/wsgiref/headers.py +++ b/Lib/wsgiref/headers.py @@ -1,7 +1,7 @@ """Manage HTTP Response Headers Much of this module is red-handedly pilfered from email.message in the stdlib, -so portions are Copyright (C) 2001,2002 Python Software Foundation, and were +so portions are Copyright (C) 2001 Python Software Foundation, and were written by Barry Warsaw. """ diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index e2aaf8bab4913d..08c83cfb760250 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -309,7 +309,7 @@ def _EndRecData(fpin): fpin.seek(-sizeEndCentDir, 2) except OSError: return None - data = fpin.read() + data = fpin.read(sizeEndCentDir) if (len(data) == sizeEndCentDir and data[0:4] == stringEndArchive and data[-2:] == b"\000\000"): @@ -329,9 +329,9 @@ def _EndRecData(fpin): # record signature. The comment is the last item in the ZIP file and may be # up to 64K long. It is assumed that the "end of central directory" magic # number does not appear in the comment. - maxCommentStart = max(filesize - (1 << 16) - sizeEndCentDir, 0) + maxCommentStart = max(filesize - ZIP_MAX_COMMENT - sizeEndCentDir, 0) fpin.seek(maxCommentStart, 0) - data = fpin.read() + data = fpin.read(ZIP_MAX_COMMENT + sizeEndCentDir) start = data.rfind(stringEndArchive) if start >= 0: # found the magic number; attempt to unpack and interpret diff --git a/Lib/zipfile/_path/__init__.py b/Lib/zipfile/_path/__init__.py index c0e53e273cfaac..5ae16ec970dda4 100644 --- a/Lib/zipfile/_path/__init__.py +++ b/Lib/zipfile/_path/__init__.py @@ -339,7 +339,7 @@ def open(self, mode='r', *args, pwd=None, **kwargs): if self.is_dir(): raise IsADirectoryError(self) zip_mode = mode[0] - if not self.exists() and zip_mode == 'r': + if zip_mode == 'r' and not self.exists(): raise FileNotFoundError(self) stream = self.root.open(self.at, zip_mode, pwd=pwd) if 'b' in mode: diff --git a/Mac/BuildScript/resources/License.rtf b/Mac/BuildScript/resources/License.rtf index 1255d1ce48ed6c..b5cb8ec41c86e2 100644 --- a/Mac/BuildScript/resources/License.rtf +++ b/Mac/BuildScript/resources/License.rtf @@ -64,7 +64,7 @@ Some software incorporated into Python is under different licenses. The licenses \f1\b0 \ 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated documentation.\ \ -2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright \'a9 2001-2020 Python Software Foundation; All Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.\ +2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright \'a9 2001 Python Software Foundation; All Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.\ \ 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in any such work a brief summary of the changes made to Python.\ \ diff --git a/Mac/PythonLauncher/Info.plist.in b/Mac/PythonLauncher/Info.plist.in index 233694788ac2b7..ce8f27cd7d4de7 100644 --- a/Mac/PythonLauncher/Info.plist.in +++ b/Mac/PythonLauncher/Info.plist.in @@ -40,9 +40,9 @@ CFBundleExecutable Python Launcher NSHumanReadableCopyright - Copyright © 2001-2024 Python Software Foundation + Copyright © 2001 Python Software Foundation CFBundleGetInfoString - %VERSION%, © 2001-2024 Python Software Foundation + %VERSION%, © 2001 Python Software Foundation CFBundleIconFile PythonLauncher.icns CFBundleIdentifier diff --git a/Makefile.pre.in b/Makefile.pre.in index aa7fa4e29d84c2..8d94ba361fd934 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -460,6 +460,7 @@ PYTHON_OBJS= \ Python/hashtable.o \ Python/import.o \ Python/importdl.o \ + Python/index_pool.o \ Python/initconfig.o \ Python/interpconfig.o \ Python/instrumentation.o \ @@ -904,7 +905,7 @@ coverage-report: regen-token regen-frozen # Run "Argument Clinic" over all source files .PHONY: clinic clinic: check-clean-src - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --exclude Lib/test/clinic.test.c --srcdir $(srcdir) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --force --make --exclude Lib/test/clinic.test.c --srcdir $(srcdir) .PHONY: clinic-tests clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c @@ -1202,6 +1203,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_context.h \ $(srcdir)/Include/internal/pycore_critical_section.h \ $(srcdir)/Include/internal/pycore_crossinterp.h \ + $(srcdir)/Include/internal/pycore_crossinterp_data_registry.h \ $(srcdir)/Include/internal/pycore_debug_offsets.h \ $(srcdir)/Include/internal/pycore_descrobject.h \ $(srcdir)/Include/internal/pycore_dict.h \ @@ -1228,6 +1230,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_hashtable.h \ $(srcdir)/Include/internal/pycore_import.h \ $(srcdir)/Include/internal/pycore_importdl.h \ + $(srcdir)/Include/internal/pycore_index_pool.h \ $(srcdir)/Include/internal/pycore_initconfig.h \ $(srcdir)/Include/internal/pycore_instruments.h \ $(srcdir)/Include/internal/pycore_instruction_sequence.h \ @@ -2545,6 +2548,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_tomllib/data/valid/dates-and-times \ test/test_tomllib/data/valid/multiline-basic-str \ test/test_tools \ + test/test_tools/i18n_data \ test/test_ttk \ test/test_unittest \ test/test_unittest/namespace_test_pkg \ @@ -2563,6 +2567,8 @@ TESTSUBDIRS= idlelib/idle_test \ test/tracedmodules \ test/translationdata \ test/translationdata/argparse \ + test/translationdata/getopt \ + test/translationdata/optparse \ test/typinganndata \ test/wheeldata \ test/xmltestdata \ diff --git a/Misc/ACKS b/Misc/ACKS index a1769d9601a2ea..08cd293eac3835 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -485,6 +485,7 @@ Luke Dunstan Virgil Dupras Bruno Dupuis Andy Dustman +Sayandip Dutta Gary Duzan Eugene Dvurechenski Karmen Dykstra @@ -732,6 +733,7 @@ Larry Hastings Tim Hatch Zac Hatfield-Dodds Shane Hathaway +Akinori Hattori Michael Haubenwallner Janko Hauser Flavian Hautbois @@ -820,6 +822,7 @@ Tomáš Hrnčiar Miro Hrončok Chiu-Hsiang Hsu Chih-Hao Huang +Xuanteng Huang Christian Hudon Benoît Hudson Lawrence Hudson @@ -952,6 +955,7 @@ Sanyam Khurana Tyler Kieft Mads Kiilerich Jason Killen +Beomsoo Bombs Kim Derek D. Kim Gihwan Kim Jan Kim @@ -1164,6 +1168,7 @@ Grzegorz Makarewicz David Malcolm Greg Malcolm William Mallard +Cody Maloney Ken Manheimer Vladimir Marangozov Colin Marc @@ -1272,6 +1277,7 @@ Emily Morehouse Derek Morr James A Morrison Martin Morrison +Stephen Morton Derek McTavish Mounce Alessandro Moura Pablo Mouzo diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst index 7126e08a20c7fd..3a3870ac9fe621 100644 --- a/Misc/NEWS.d/3.12.0b1.rst +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -1750,7 +1750,7 @@ Remove the long-deprecated ``imp`` module. .. nonce: N46coo .. section: Library -Deprecate :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` in +Deprecate :func:`!pkgutil.find_loader` and :func:`!pkgutil.get_loader` in favor of :func:`importlib.util.find_spec`. .. diff --git a/Misc/NEWS.d/next/Build/2024-10-30-17-47-15.gh-issue-126187.0jFCZB.rst b/Misc/NEWS.d/next/Build/2024-10-30-17-47-15.gh-issue-126187.0jFCZB.rst new file mode 100644 index 00000000000000..c295a91c2225a3 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-10-30-17-47-15.gh-issue-126187.0jFCZB.rst @@ -0,0 +1 @@ +Introduced ``Tools/wasm/emscripten.py`` to simplify doing Emscripten builds. diff --git a/Misc/NEWS.d/next/Build/2024-10-31-15-37-05.gh-issue-126206.oC6z2i.rst b/Misc/NEWS.d/next/Build/2024-10-31-15-37-05.gh-issue-126206.oC6z2i.rst new file mode 100644 index 00000000000000..24b172e1747403 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-10-31-15-37-05.gh-issue-126206.oC6z2i.rst @@ -0,0 +1,2 @@ +``make clinic`` now runs Argument Clinic using the ``--force`` option, +thus forcefully regenerating generated code. diff --git a/Misc/NEWS.d/next/Build/2024-11-04-09-42-04.gh-issue-89640.QBv05o.rst b/Misc/NEWS.d/next/Build/2024-11-04-09-42-04.gh-issue-89640.QBv05o.rst new file mode 100644 index 00000000000000..4fa44a1d6493b4 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-11-04-09-42-04.gh-issue-89640.QBv05o.rst @@ -0,0 +1 @@ +Hard-code float word ordering as little endian on WASM. diff --git a/Misc/NEWS.d/next/Build/2024-11-06-11-12-04.gh-issue-126458.7vzHtx.rst b/Misc/NEWS.d/next/Build/2024-11-06-11-12-04.gh-issue-126458.7vzHtx.rst new file mode 100644 index 00000000000000..cc06dd8a30e30a --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-11-06-11-12-04.gh-issue-126458.7vzHtx.rst @@ -0,0 +1 @@ +Disable SIMD support for HACL under WASI. diff --git a/Misc/NEWS.d/next/Build/2024-11-07-11-09-31.gh-issue-123877.CVdd0b.rst b/Misc/NEWS.d/next/Build/2024-11-07-11-09-31.gh-issue-123877.CVdd0b.rst new file mode 100644 index 00000000000000..080d2f2ab12af9 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-11-07-11-09-31.gh-issue-123877.CVdd0b.rst @@ -0,0 +1,3 @@ +Use ``wasm32-wasip1`` as the target triple for WASI instead of +``wasm32-wasi``. The latter will eventually be reclaimed for WASI 1.0 while +CPython currently only supports WASI preview1. diff --git a/Misc/NEWS.d/next/Build/2024-11-13-15-47-09.gh-issue-126691.ni4K-b.rst b/Misc/NEWS.d/next/Build/2024-11-13-15-47-09.gh-issue-126691.ni4K-b.rst new file mode 100644 index 00000000000000..9a2196dab8d711 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-11-13-15-47-09.gh-issue-126691.ni4K-b.rst @@ -0,0 +1,3 @@ +Removed the ``--with-emscripten-target`` configure flag. We unified the +``node`` and ``browser`` options and the same build can now be used, independent +of target runtime. diff --git a/Misc/NEWS.d/next/C_API/2024-06-04-13-38-44.gh-issue-120026.uhEvJ9.rst b/Misc/NEWS.d/next/C_API/2024-06-04-13-38-44.gh-issue-120026.uhEvJ9.rst new file mode 100644 index 00000000000000..d43a138724ad33 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-06-04-13-38-44.gh-issue-120026.uhEvJ9.rst @@ -0,0 +1 @@ +The :c:macro:`!Py_HUGE_VAL` macro is :term:`soft deprecated`. diff --git a/Misc/NEWS.d/next/C_API/2024-09-03-13-33-33.gh-issue-123619.HhgUUI.rst b/Misc/NEWS.d/next/C_API/2024-09-03-13-33-33.gh-issue-123619.HhgUUI.rst new file mode 100644 index 00000000000000..ac821b5326026e --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-09-03-13-33-33.gh-issue-123619.HhgUUI.rst @@ -0,0 +1,2 @@ +Added the :c:func:`PyUnstable_Object_EnableDeferredRefcount` function for +enabling :pep:`703` deferred reference counting. diff --git a/Misc/NEWS.d/next/C_API/2024-10-28-15-56-03.gh-issue-126061.Py51_1.rst b/Misc/NEWS.d/next/C_API/2024-10-28-15-56-03.gh-issue-126061.Py51_1.rst new file mode 100644 index 00000000000000..0a4ad4ea2874cf --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-10-28-15-56-03.gh-issue-126061.Py51_1.rst @@ -0,0 +1,3 @@ +Add :c:func:`PyLong_IsPositive`, :c:func:`PyLong_IsNegative` +and :c:func:`PyLong_IsZero` for checking if a :c:type:`PyLongObject` +is positive, negative, or zero, respectively. diff --git a/Misc/NEWS.d/next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst b/Misc/NEWS.d/next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst new file mode 100644 index 00000000000000..6af89c7d4709ec --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst @@ -0,0 +1,2 @@ +Fix error handling in :class:`ctypes.CDLL` objects +which could result in a crash in rare situations. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-06-13-19-12-49.gh-issue-119793.FDVCDk.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-06-13-19-12-49.gh-issue-119793.FDVCDk.rst new file mode 100644 index 00000000000000..976d6712e4b6af --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-06-13-19-12-49.gh-issue-119793.FDVCDk.rst @@ -0,0 +1,3 @@ +The :func:`map` built-in now has an optional keyword-only *strict* flag +like :func:`zip` to check that all the iterables are of equal length. +Patch by Wannes Boeykens. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-30-18-16-10.gh-issue-126195.6ezBpr.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-30-18-16-10.gh-issue-126195.6ezBpr.rst new file mode 100644 index 00000000000000..01424d8a545d78 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-30-18-16-10.gh-issue-126195.6ezBpr.rst @@ -0,0 +1 @@ +Improve JIT performance by 1.4% on macOS Apple Silicon by using platform-specific memory protection APIs. Patch by Diego Russo. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-31-21-49-00.gh-issue-126072.o9k8Ns.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-31-21-49-00.gh-issue-126072.o9k8Ns.rst new file mode 100644 index 00000000000000..2464ac78cf429b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-31-21-49-00.gh-issue-126072.o9k8Ns.rst @@ -0,0 +1,2 @@ +Following :gh:`126101`, for :ref:`codeobjects` like lambda, annotation and type alias, +we no longer add ``None`` to its :attr:`~codeobject.co_consts`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-01-09-58-06.gh-issue-103951.6qduwj.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-01-09-58-06.gh-issue-103951.6qduwj.rst new file mode 100644 index 00000000000000..39b54e0b72556e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-01-09-58-06.gh-issue-103951.6qduwj.rst @@ -0,0 +1,2 @@ +Relax optimization requirements to allow fast attribute access to module +subclasses. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-02-14-43-46.gh-issue-126312.LMHzLT.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-02-14-43-46.gh-issue-126312.LMHzLT.rst new file mode 100644 index 00000000000000..19c8f0a348731c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-02-14-43-46.gh-issue-126312.LMHzLT.rst @@ -0,0 +1,2 @@ +Fix crash during garbage collection on an object frozen by :func:`gc.freeze` on the +free-threaded build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-02-18-01-31.gh-issue-126209.2ZIhrS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-02-18-01-31.gh-issue-126209.2ZIhrS.rst new file mode 100644 index 00000000000000..727f7f8180ab22 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-02-18-01-31.gh-issue-126209.2ZIhrS.rst @@ -0,0 +1,3 @@ +Fix an issue with ``skip_file_prefixes`` parameter which resulted in an inconsistent +behaviour between the C and Python implementations of :func:`warnings.warn`. +Patch by Daehee Kim. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-03-15-15-36.gh-issue-126366.8BBdGU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-03-15-15-36.gh-issue-126366.8BBdGU.rst new file mode 100644 index 00000000000000..a47233602e4eff --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-03-15-15-36.gh-issue-126366.8BBdGU.rst @@ -0,0 +1,2 @@ +Fix crash when using ``yield from`` on an object that raises an exception in +its ``__iter__``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-06-15-22-34.gh-issue-126491.n9VyZc.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-06-15-22-34.gh-issue-126491.n9VyZc.rst new file mode 100644 index 00000000000000..f69977852c994a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-06-15-22-34.gh-issue-126491.n9VyZc.rst @@ -0,0 +1,4 @@ +Add a marking phase to the GC. All objects that can be transitively +reached from builtin modules or the stacks are marked as reachable +before cycle detection. This reduces the amount of work done by the +GC by approximately half. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-06-16-34-11.gh-issue-126222.9NBfTn.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-06-16-34-11.gh-issue-126222.9NBfTn.rst new file mode 100644 index 00000000000000..ebf6673782f02c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-06-16-34-11.gh-issue-126222.9NBfTn.rst @@ -0,0 +1,3 @@ +Do not include count of "peek" items in ``_PyUop_num_popped``. This ensures +that the correct number of items are popped from the stack when a micro-op +exits with an error. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-09-16-10-22.gh-issue-126066.9zs4m4.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-09-16-10-22.gh-issue-126066.9zs4m4.rst new file mode 100644 index 00000000000000..9c0072304ded63 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-09-16-10-22.gh-issue-126066.9zs4m4.rst @@ -0,0 +1,3 @@ +Fix :mod:`importlib` to not write an incomplete .pyc files when a ulimit or some +other operating system mechanism is preventing the write to go through +fully. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-11-17-02-48.gh-issue-126688.QiOXUi.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-11-17-02-48.gh-issue-126688.QiOXUi.rst new file mode 100644 index 00000000000000..30aa5722f0ea02 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-11-17-02-48.gh-issue-126688.QiOXUi.rst @@ -0,0 +1,2 @@ +Fix a crash when calling :func:`os.fork` on some operating systems, +including SerenityOS. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-12-19-24-00.gh-issue-126341.5SdAe1.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-12-19-24-00.gh-issue-126341.5SdAe1.rst new file mode 100644 index 00000000000000..c2436d2ebf4d09 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-12-19-24-00.gh-issue-126341.5SdAe1.rst @@ -0,0 +1 @@ +Now :exc:`ValueError` is raised instead of :exc:`SystemError` when trying to iterate over a released :class:`memoryview` object. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-13-17-18-13.gh-issue-126795._JBX9e.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-13-17-18-13.gh-issue-126795._JBX9e.rst new file mode 100644 index 00000000000000..16799534ef37cf --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-13-17-18-13.gh-issue-126795._JBX9e.rst @@ -0,0 +1,2 @@ +Increase the threshold for JIT code warmup. Depending on platform and workload, +this can result in performance gains of 1-9% and memory savings of 3-5%. diff --git a/Misc/NEWS.d/next/Documentation/2024-11-09-19-43-10.gh-issue-126622.YacfDc.rst b/Misc/NEWS.d/next/Documentation/2024-11-09-19-43-10.gh-issue-126622.YacfDc.rst new file mode 100644 index 00000000000000..a2181b5712873b --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-11-09-19-43-10.gh-issue-126622.YacfDc.rst @@ -0,0 +1,3 @@ +Added stub pages for removed modules explaining their removal, where to find +replacements, and linking to the last Python version that supported them. +Contributed by Ned Batchelder. diff --git a/Misc/NEWS.d/next/Library/2020-05-19-01-12-47.gh-issue-84852.FEjHJW.rst b/Misc/NEWS.d/next/Library/2020-05-19-01-12-47.gh-issue-84852.FEjHJW.rst new file mode 100644 index 00000000000000..2581697591af62 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-05-19-01-12-47.gh-issue-84852.FEjHJW.rst @@ -0,0 +1,2 @@ +Add MIME types for MS Embedded OpenType, OpenType Layout, TrueType, +WOFF 1.0 and 2.0 fonts. Patch by Sahil Prajapati and Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2021-12-19-10-47-24.bpo-46128.Qv3EK1.rst b/Misc/NEWS.d/next/Library/2021-12-19-10-47-24.bpo-46128.Qv3EK1.rst new file mode 100644 index 00000000000000..7d11d20d94e8a3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-12-19-10-47-24.bpo-46128.Qv3EK1.rst @@ -0,0 +1,2 @@ +Strip :class:`unittest.IsolatedAsyncioTestCase` stack frames from reported +stacktraces. diff --git a/Misc/NEWS.d/next/Library/2022-10-15-10-18-20.gh-issue-71936.MzJjc_.rst b/Misc/NEWS.d/next/Library/2022-10-15-10-18-20.gh-issue-71936.MzJjc_.rst new file mode 100644 index 00000000000000..a0959cc086fa9e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-15-10-18-20.gh-issue-71936.MzJjc_.rst @@ -0,0 +1 @@ +Fix a race condition in :class:`multiprocessing.pool.Pool`. diff --git a/Misc/NEWS.d/next/Library/2023-10-26-16-36-22.gh-issue-101955.Ixu3IF.rst b/Misc/NEWS.d/next/Library/2023-10-26-16-36-22.gh-issue-101955.Ixu3IF.rst new file mode 100644 index 00000000000000..89431010f784f8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-26-16-36-22.gh-issue-101955.Ixu3IF.rst @@ -0,0 +1,2 @@ +Fix SystemError when match regular expression pattern containing some +combination of possessive quantifier, alternative and capture group. diff --git a/Misc/NEWS.d/next/Library/2024-03-16-13-38-27.gh-issue-116897.UDQTjp.rst b/Misc/NEWS.d/next/Library/2024-03-16-13-38-27.gh-issue-116897.UDQTjp.rst new file mode 100644 index 00000000000000..6c8e4b16f20de8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-16-13-38-27.gh-issue-116897.UDQTjp.rst @@ -0,0 +1,4 @@ +Accepting objects with false values (like ``0`` and ``[]``) except empty +strings, byte-like objects and ``None`` in :mod:`urllib.parse` functions +:func:`~urllib.parse.parse_qsl` and :func:`~urllib.parse.parse_qs` is now +deprecated. diff --git a/Misc/NEWS.d/next/Library/2024-05-28-14-35-23.gh-issue-97850.dCtjel.rst b/Misc/NEWS.d/next/Library/2024-05-28-14-35-23.gh-issue-97850.dCtjel.rst new file mode 100644 index 00000000000000..bb94f7d8ad124d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-28-14-35-23.gh-issue-97850.dCtjel.rst @@ -0,0 +1 @@ +Remove deprecated :func:`!pkgutil.get_loader` and :func:`!pkgutil.find_loader`. diff --git a/Misc/NEWS.d/next/Library/2024-06-02-11-48-19.gh-issue-119826.N1obGa.rst b/Misc/NEWS.d/next/Library/2024-06-02-11-48-19.gh-issue-119826.N1obGa.rst new file mode 100644 index 00000000000000..6901e7475dd082 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-02-11-48-19.gh-issue-119826.N1obGa.rst @@ -0,0 +1 @@ +Always return an absolute path for :func:`os.path.abspath` on Windows. diff --git a/Misc/NEWS.d/next/Library/2024-06-05-19-09-36.gh-issue-118289.moL9_d.rst b/Misc/NEWS.d/next/Library/2024-06-05-19-09-36.gh-issue-118289.moL9_d.rst new file mode 100644 index 00000000000000..522572e160ba7b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-05-19-09-36.gh-issue-118289.moL9_d.rst @@ -0,0 +1,2 @@ +:func:`!posixpath.realpath` now raises :exc:`NotADirectoryError` when *strict* +mode is enabled and a non-directory path with a trailing slash is supplied. diff --git a/Misc/NEWS.d/next/Library/2024-06-06-04-06-05.gh-issue-70764.6511hw.rst b/Misc/NEWS.d/next/Library/2024-06-06-04-06-05.gh-issue-70764.6511hw.rst new file mode 100644 index 00000000000000..4cfb66a6ccc6ee --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-06-04-06-05.gh-issue-70764.6511hw.rst @@ -0,0 +1 @@ +Fixed an issue where :func:`inspect.getclosurevars` would incorrectly classify an attribute name as a global variable when the name exists both as an attribute name and a global variable. diff --git a/Misc/NEWS.d/next/Library/2024-07-23-02-24-50.gh-issue-120754.nHb5mG.rst b/Misc/NEWS.d/next/Library/2024-07-23-02-24-50.gh-issue-120754.nHb5mG.rst new file mode 100644 index 00000000000000..6c33e7b7ec7716 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-23-02-24-50.gh-issue-120754.nHb5mG.rst @@ -0,0 +1 @@ +Update unbounded ``read`` calls in :mod:`zipfile` to specify an explicit ``size`` putting a limit on how much data they may read. This also updates handling around ZIP max comment size to match the standard instead of reading comments that are one byte too long. diff --git a/Misc/NEWS.d/next/Library/2024-08-01-11-15-55.gh-issue-122549.ztV4Kz.rst b/Misc/NEWS.d/next/Library/2024-08-01-11-15-55.gh-issue-122549.ztV4Kz.rst new file mode 100644 index 00000000000000..6b2cbc0a6c9b7a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-01-11-15-55.gh-issue-122549.ztV4Kz.rst @@ -0,0 +1 @@ +Add :func:`platform.invalidate_caches` to invalidate cached results. diff --git a/Misc/NEWS.d/next/Library/2024-08-22-12-12-35.gh-issue-89083.b6zFh0.rst b/Misc/NEWS.d/next/Library/2024-08-22-12-12-35.gh-issue-89083.b6zFh0.rst new file mode 100644 index 00000000000000..d37d585d51b490 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-22-12-12-35.gh-issue-89083.b6zFh0.rst @@ -0,0 +1,2 @@ +Add :func:`uuid.uuid8` for generating UUIDv8 objects as specified in +:rfc:`9562`. Patch by Bénédikt Tran diff --git a/Misc/NEWS.d/next/Library/2024-09-17-10-38-26.gh-issue-124111.Hd53VN.rst b/Misc/NEWS.d/next/Library/2024-09-17-10-38-26.gh-issue-124111.Hd53VN.rst new file mode 100644 index 00000000000000..aba082a7ac1ad4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-17-10-38-26.gh-issue-124111.Hd53VN.rst @@ -0,0 +1,4 @@ +The tkinter module can now be built to use either the new version 9.0.0 of +Tcl/Tk or the latest release 8.6.15 of Tcl/Tk 8. Tcl/Tk 9 includes many +improvements, both to the Tcl language and to the appearance and utility of +the graphical user interface provided by Tk. diff --git a/Misc/NEWS.d/next/Library/2024-10-19-11-06-06.gh-issue-125631.BlhVvR.rst b/Misc/NEWS.d/next/Library/2024-10-19-11-06-06.gh-issue-125631.BlhVvR.rst new file mode 100644 index 00000000000000..e870abbf87803a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-19-11-06-06.gh-issue-125631.BlhVvR.rst @@ -0,0 +1,4 @@ +Restore ability to set :attr:`~pickle.Pickler.persistent_id` and +:attr:`~pickle.Unpickler.persistent_load` attributes of instances of the +:class:`!Pickler` and :class:`!Unpickler` classes in the :mod:`pickle` +module. diff --git a/Misc/NEWS.d/next/Library/2024-10-23-20-44-30.gh-issue-117941.Y9jdlW.rst b/Misc/NEWS.d/next/Library/2024-10-23-20-44-30.gh-issue-117941.Y9jdlW.rst new file mode 100644 index 00000000000000..9c2553f0f0e8cd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-23-20-44-30.gh-issue-117941.Y9jdlW.rst @@ -0,0 +1,2 @@ +:class:`!argparse.BooleanOptionalAction` now rejects option names starting +with ``--no-``. diff --git a/Misc/NEWS.d/next/Library/2024-10-24-10-49-47.gh-issue-124452.eqTRgx.rst b/Misc/NEWS.d/next/Library/2024-10-24-10-49-47.gh-issue-124452.eqTRgx.rst new file mode 100644 index 00000000000000..b0d63794022db4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-24-10-49-47.gh-issue-124452.eqTRgx.rst @@ -0,0 +1,4 @@ +Fix an issue in :meth:`email.policy.EmailPolicy.header_source_parse` and +:meth:`email.policy.Compat32.header_source_parse` that introduced spurious +leading whitespaces into header values when the header includes a newline +character after the header name delimiter (``:``) and before the value. diff --git a/Misc/NEWS.d/next/Library/2024-10-24-13-40-20.gh-issue-126916.MAgz6D.rst b/Misc/NEWS.d/next/Library/2024-10-24-13-40-20.gh-issue-126916.MAgz6D.rst new file mode 100644 index 00000000000000..cbe2fc166ba6af --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-24-13-40-20.gh-issue-126916.MAgz6D.rst @@ -0,0 +1,2 @@ +Allow the *initial* parameter of :func:`functools.reduce` to be passed as a keyword argument. +Patch by Sayandip Dutta. diff --git a/Misc/NEWS.d/next/Library/2024-10-25-20-52-15.gh-issue-125926.pp8rtZ.rst b/Misc/NEWS.d/next/Library/2024-10-25-20-52-15.gh-issue-125926.pp8rtZ.rst new file mode 100644 index 00000000000000..7f98bcdc38e566 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-25-20-52-15.gh-issue-125926.pp8rtZ.rst @@ -0,0 +1,4 @@ +Fix :func:`urllib.parse.urljoin` for base URI with undefined authority. +Although :rfc:`3986` only specify reference resolution for absolute base +URI, :func:`!urljoin` should continue to return sensible result for relative +base URI. diff --git a/Misc/NEWS.d/next/Library/2024-10-28-01-24-52.gh-issue-125413.Jat5kq.rst b/Misc/NEWS.d/next/Library/2024-10-28-01-24-52.gh-issue-125413.Jat5kq.rst new file mode 100644 index 00000000000000..ddf1f9725d9695 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-28-01-24-52.gh-issue-125413.Jat5kq.rst @@ -0,0 +1,3 @@ +Add :meth:`pathlib.Path.scandir` method to efficiently fetch directory +children and their file attributes. This is a trivial wrapper of +:func:`os.scandir`. diff --git a/Misc/NEWS.d/next/Library/2024-10-29-10-38-28.gh-issue-126080.qKRBuo.rst b/Misc/NEWS.d/next/Library/2024-10-29-10-38-28.gh-issue-126080.qKRBuo.rst new file mode 100644 index 00000000000000..e54ac17b217c92 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-29-10-38-28.gh-issue-126080.qKRBuo.rst @@ -0,0 +1,3 @@ +Fix a use-after-free crash on :class:`asyncio.Task` objects for which the +underlying event loop implements an evil :meth:`~object.__getattribute__`. +Reported by Nico-Posada. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-10-30-00-12-22.gh-issue-126156.BOSqv0.rst b/Misc/NEWS.d/next/Library/2024-10-30-00-12-22.gh-issue-126156.BOSqv0.rst new file mode 100644 index 00000000000000..4fe18275ab9384 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-30-00-12-22.gh-issue-126156.BOSqv0.rst @@ -0,0 +1 @@ +Improved performances of creating :py:class:`~http.cookies.Morsel` objects by a factor of 3.8x. diff --git a/Misc/NEWS.d/next/Library/2024-10-30-23-42-44.gh-issue-126223.k2qooc.rst b/Misc/NEWS.d/next/Library/2024-10-30-23-42-44.gh-issue-126223.k2qooc.rst new file mode 100644 index 00000000000000..fee391c030b941 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-30-23-42-44.gh-issue-126223.k2qooc.rst @@ -0,0 +1,2 @@ +Raise a :exc:`UnicodeEncodeError` instead of a :exc:`SystemError` upon +calling :func:`!_interpreters.create` with an invalid Unicode character. diff --git a/Misc/NEWS.d/next/Library/2024-10-30-23-59-36.gh-issue-126212._9uYjT.rst b/Misc/NEWS.d/next/Library/2024-10-30-23-59-36.gh-issue-126212._9uYjT.rst new file mode 100644 index 00000000000000..047fe0f68048b5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-30-23-59-36.gh-issue-126212._9uYjT.rst @@ -0,0 +1,3 @@ +Fix issue where :func:`urllib.request.pathname2url` and +:func:`~urllib.request.url2pathname` removed slashes from Windows DOS drive +paths and URLs. diff --git a/Misc/NEWS.d/next/Library/2024-10-31-14-06-28.gh-issue-126220.uJAJCU.rst b/Misc/NEWS.d/next/Library/2024-10-31-14-06-28.gh-issue-126220.uJAJCU.rst new file mode 100644 index 00000000000000..555f2f3bafbf33 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-31-14-06-28.gh-issue-126220.uJAJCU.rst @@ -0,0 +1,2 @@ +Fix crash in :class:`!cProfile.Profile` and :class:`!_lsprof.Profiler` when their +callbacks were directly called with 0 arguments. diff --git a/Misc/NEWS.d/next/Library/2024-11-01-10-35-49.gh-issue-120057.YWy81Q.rst b/Misc/NEWS.d/next/Library/2024-11-01-10-35-49.gh-issue-120057.YWy81Q.rst new file mode 100644 index 00000000000000..ded60a3f57bca3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-01-10-35-49.gh-issue-120057.YWy81Q.rst @@ -0,0 +1,2 @@ +Replace the ``os.environ.refresh()`` method with a new +:func:`os.reload_environ` function. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-11-01-14-31-41.gh-issue-126138.yTniOG.rst b/Misc/NEWS.d/next/Library/2024-11-01-14-31-41.gh-issue-126138.yTniOG.rst new file mode 100644 index 00000000000000..459eebc82bd42a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-01-14-31-41.gh-issue-126138.yTniOG.rst @@ -0,0 +1,3 @@ +Fix a use-after-free crash on :class:`asyncio.Task` objects +whose underlying coroutine yields an object that implements +an evil :meth:`~object.__getattribute__`. Patch by Nico Posada. diff --git a/Misc/NEWS.d/next/Library/2024-11-02-19-20-44.gh-issue-126303.yVvyWB.rst b/Misc/NEWS.d/next/Library/2024-11-02-19-20-44.gh-issue-126303.yVvyWB.rst new file mode 100644 index 00000000000000..0072c97338c251 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-02-19-20-44.gh-issue-126303.yVvyWB.rst @@ -0,0 +1 @@ +Fix pickling and copying of :class:`os.sched_param` objects. diff --git a/Misc/NEWS.d/next/Library/2024-11-03-09-42-42.gh-issue-126313.EFP6Dl.rst b/Misc/NEWS.d/next/Library/2024-11-03-09-42-42.gh-issue-126313.EFP6Dl.rst new file mode 100644 index 00000000000000..dad348d8898f13 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-03-09-42-42.gh-issue-126313.EFP6Dl.rst @@ -0,0 +1,2 @@ +Fix an issue in :func:`curses.napms` when :func:`curses.initscr` has not yet +been called. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-11-03-10-48-07.gh-issue-126353.ChDzot.rst b/Misc/NEWS.d/next/Library/2024-11-03-10-48-07.gh-issue-126353.ChDzot.rst new file mode 100644 index 00000000000000..16d508b7ec6a20 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-03-10-48-07.gh-issue-126353.ChDzot.rst @@ -0,0 +1,2 @@ +:func:`asyncio.get_event_loop` now does not implicitly creates an event loop. +It now raises a :exc:`RuntimeError` if there is no set event loop. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2024-11-03-14-43-51.gh-issue-126363.Xus7vU.rst b/Misc/NEWS.d/next/Library/2024-11-03-14-43-51.gh-issue-126363.Xus7vU.rst new file mode 100644 index 00000000000000..20fea9b9ef99a0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-03-14-43-51.gh-issue-126363.Xus7vU.rst @@ -0,0 +1,2 @@ +Speed up pattern parsing in :meth:`pathlib.Path.glob` by skipping creation +of a :class:`pathlib.Path` object for the pattern. diff --git a/Misc/NEWS.d/next/Library/2024-11-03-23-25-07.gh-issue-126374.Xu_THP.rst b/Misc/NEWS.d/next/Library/2024-11-03-23-25-07.gh-issue-126374.Xu_THP.rst new file mode 100644 index 00000000000000..ad7ecfb6af9ec8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-03-23-25-07.gh-issue-126374.Xu_THP.rst @@ -0,0 +1 @@ +Add support for options with optional arguments in the :mod:`getopt` module. diff --git a/Misc/NEWS.d/next/Library/2024-11-04-13-16-18.gh-issue-126390.Cxvqa5.rst b/Misc/NEWS.d/next/Library/2024-11-04-13-16-18.gh-issue-126390.Cxvqa5.rst new file mode 100644 index 00000000000000..3b32bb512f6556 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-04-13-16-18.gh-issue-126390.Cxvqa5.rst @@ -0,0 +1,2 @@ +Add support for returning intermixed options and non-option arguments in +order in :func:`getopt.gnu_getopt`. diff --git a/Misc/NEWS.d/next/Library/2024-11-04-16-40-02.gh-issue-126417.OWPqn0.rst b/Misc/NEWS.d/next/Library/2024-11-04-16-40-02.gh-issue-126417.OWPqn0.rst new file mode 100644 index 00000000000000..c4a366343382f3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-04-16-40-02.gh-issue-126417.OWPqn0.rst @@ -0,0 +1,3 @@ +Register the :class:`!multiprocessing.managers.DictProxy` and :class:`!multiprocessing.managers.ListProxy` types in +:mod:`multiprocessing.managers` to :class:`collections.abc.MutableMapping` and +:class:`collections.abc.MutableSequence`, respectively. diff --git a/Misc/NEWS.d/next/Library/2024-11-04-22-53-09.gh-issue-89416.YVQaas.rst b/Misc/NEWS.d/next/Library/2024-11-04-22-53-09.gh-issue-89416.YVQaas.rst new file mode 100644 index 00000000000000..f1a2fcbaff2564 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-04-22-53-09.gh-issue-89416.YVQaas.rst @@ -0,0 +1,2 @@ +Add :rfc:`9559` MIME types for Matroska audiovisual container formats. Patch +by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2024-11-05-09-54-49.gh-issue-126175.spnjJr.rst b/Misc/NEWS.d/next/Library/2024-11-05-09-54-49.gh-issue-126175.spnjJr.rst new file mode 100644 index 00000000000000..de7ce88c8d0f28 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-05-09-54-49.gh-issue-126175.spnjJr.rst @@ -0,0 +1,2 @@ +Add ``msg``, ``doc``, ``pos``, ``lineno`` and ``colno`` attributes to :exc:`tomllib.TOMLDecodeError`. +Deprecate instantiating with free-form arguments. diff --git a/Misc/NEWS.d/next/Library/2024-11-05-11-28-45.gh-issue-126451.XJMtqz.rst b/Misc/NEWS.d/next/Library/2024-11-05-11-28-45.gh-issue-126451.XJMtqz.rst new file mode 100644 index 00000000000000..563cb2515eca60 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-05-11-28-45.gh-issue-126451.XJMtqz.rst @@ -0,0 +1,2 @@ +Register the :class:`contextvars.Context` type to +:class:`collections.abc.Mapping`. diff --git a/Misc/NEWS.d/next/Library/2024-11-06-13-41-38.gh-issue-126489.toaf-0.rst b/Misc/NEWS.d/next/Library/2024-11-06-13-41-38.gh-issue-126489.toaf-0.rst new file mode 100644 index 00000000000000..8a6573cdea7b42 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-06-13-41-38.gh-issue-126489.toaf-0.rst @@ -0,0 +1,3 @@ +The Python implementation of :mod:`pickle` no longer calls +:meth:`pickle.Pickler.persistent_id` for the result of +:meth:`!persistent_id`. diff --git a/Misc/NEWS.d/next/Library/2024-11-06-18-30-50.gh-issue-126476.F1wh3c.rst b/Misc/NEWS.d/next/Library/2024-11-06-18-30-50.gh-issue-126476.F1wh3c.rst new file mode 100644 index 00000000000000..f558c29e8b087f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-06-18-30-50.gh-issue-126476.F1wh3c.rst @@ -0,0 +1,2 @@ +Raise :class:`calendar.IllegalMonthError` (now a subclass of :class:`IndexError`) for :func:`calendar.month` +when the input month is not correct. diff --git a/Misc/NEWS.d/next/Library/2024-11-06-23-40-28.gh-issue-125679.Qq9xF5.rst b/Misc/NEWS.d/next/Library/2024-11-06-23-40-28.gh-issue-125679.Qq9xF5.rst new file mode 100644 index 00000000000000..ac6851e2689692 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-06-23-40-28.gh-issue-125679.Qq9xF5.rst @@ -0,0 +1,2 @@ +The :class:`multiprocessing.Lock` and :class:`multiprocessing.RLock` +``repr`` values no longer say "unknown" on macOS. diff --git a/Misc/NEWS.d/next/Library/2024-11-07-01-40-11.gh-issue-117378.o9O5uM.rst b/Misc/NEWS.d/next/Library/2024-11-07-01-40-11.gh-issue-117378.o9O5uM.rst new file mode 100644 index 00000000000000..d7d4477ec17814 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-07-01-40-11.gh-issue-117378.o9O5uM.rst @@ -0,0 +1,17 @@ +Fixed the :mod:`multiprocessing` ``"forkserver"`` start method forkserver +process to correctly inherit the parent's :data:`sys.path` during the importing +of :func:`multiprocessing.set_forkserver_preload` modules in the same manner as +:data:`sys.path` is configured in workers before executing work items. + +This bug caused some forkserver module preloading to silently fail to preload. +This manifested as a performance degration in child processes when the +``sys.path`` was required due to additional repeated work in every worker. + +It could also have a side effect of ``""`` remaining in :data:`sys.path` during +forkserver preload imports instead of the absolute path from :func:`os.getcwd` +at multiprocessing import time used in the worker ``sys.path``. + +The ``sys.path`` differences between phases in the child process could +potentially have caused preload to import incorrect things from the wrong +location. We are unaware of that actually having happened in practice. + diff --git a/Misc/NEWS.d/next/Library/2024-11-07-22-41-47.gh-issue-126505.iztYE1.rst b/Misc/NEWS.d/next/Library/2024-11-07-22-41-47.gh-issue-126505.iztYE1.rst new file mode 100644 index 00000000000000..0a0f893a2688a0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-07-22-41-47.gh-issue-126505.iztYE1.rst @@ -0,0 +1,4 @@ +Fix bugs in compiling case-insensitive :mod:`regular expressions ` with +character classes containing non-BMP characters: upper-case non-BMP +character did was ignored and the ASCII flag was ignored when +matching a character range whose upper bound is beyond the BMP region. diff --git a/Misc/NEWS.d/next/Library/2024-11-08-11-06-14.gh-issue-126565.dFFO22.rst b/Misc/NEWS.d/next/Library/2024-11-08-11-06-14.gh-issue-126565.dFFO22.rst new file mode 100644 index 00000000000000..22858570bbe03c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-08-11-06-14.gh-issue-126565.dFFO22.rst @@ -0,0 +1 @@ +Improve performances of :meth:`zipfile.Path.open` for non-reading modes. diff --git a/Misc/NEWS.d/next/Library/2024-11-08-17-05-10.gh-issue-120423.7rdLVV.rst b/Misc/NEWS.d/next/Library/2024-11-08-17-05-10.gh-issue-120423.7rdLVV.rst new file mode 100644 index 00000000000000..b475257ceb6610 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-08-17-05-10.gh-issue-120423.7rdLVV.rst @@ -0,0 +1,2 @@ +Fix issue where :func:`urllib.request.pathname2url` mishandled Windows paths +with embedded forward slashes. diff --git a/Misc/NEWS.d/next/Library/2024-11-09-10-31-10.gh-issue-126595.A-7MyC.rst b/Misc/NEWS.d/next/Library/2024-11-09-10-31-10.gh-issue-126595.A-7MyC.rst new file mode 100644 index 00000000000000..84a5dc0b23922f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-09-10-31-10.gh-issue-126595.A-7MyC.rst @@ -0,0 +1,2 @@ +Fix a crash when instantiating :class:`itertools.count` with an initial +count of :data:`sys.maxsize` on debug builds. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-11-10-18-14-51.gh-issue-104745.zAa5Ke.rst b/Misc/NEWS.d/next/Library/2024-11-10-18-14-51.gh-issue-104745.zAa5Ke.rst new file mode 100644 index 00000000000000..c83a10769820cf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-10-18-14-51.gh-issue-104745.zAa5Ke.rst @@ -0,0 +1,3 @@ +Limit starting a patcher (from :func:`unittest.mock.patch` or +:func:`unittest.mock.patch.object`) more than +once without stopping it diff --git a/Misc/NEWS.d/next/Library/2024-11-11-13-00-21.gh-issue-126654.4gfP2y.rst b/Misc/NEWS.d/next/Library/2024-11-11-13-00-21.gh-issue-126654.4gfP2y.rst new file mode 100644 index 00000000000000..750158e6d4d3ae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-11-13-00-21.gh-issue-126654.4gfP2y.rst @@ -0,0 +1,2 @@ +Fix crash when non-dict was passed to several functions in ``_interpreters`` +module. diff --git a/Misc/NEWS.d/next/Library/2024-11-11-13-24-22.gh-issue-126699.ONGbMd.rst b/Misc/NEWS.d/next/Library/2024-11-11-13-24-22.gh-issue-126699.ONGbMd.rst new file mode 100644 index 00000000000000..9741294487d716 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-11-13-24-22.gh-issue-126699.ONGbMd.rst @@ -0,0 +1 @@ +Allow :class:`collections.abc.AsyncIterator` to be a base for Protocols. diff --git a/Misc/NEWS.d/next/Library/2024-11-11-14-52-21.gh-issue-126705.0W7jFW.rst b/Misc/NEWS.d/next/Library/2024-11-11-14-52-21.gh-issue-126705.0W7jFW.rst new file mode 100644 index 00000000000000..f49c9c765d778f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-11-14-52-21.gh-issue-126705.0W7jFW.rst @@ -0,0 +1 @@ +Allow :class:`os.PathLike` to be a base for Protocols. diff --git a/Misc/NEWS.d/next/Library/2024-11-12-21-43-12.gh-issue-126766.oi2KJ7.rst b/Misc/NEWS.d/next/Library/2024-11-12-21-43-12.gh-issue-126766.oi2KJ7.rst new file mode 100644 index 00000000000000..e3936305164883 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-12-21-43-12.gh-issue-126766.oi2KJ7.rst @@ -0,0 +1,2 @@ +Fix issue where :func:`urllib.request.url2pathname` failed to discard two +leading slashes introducing an empty authority section. diff --git a/Misc/NEWS.d/next/Library/2024-11-13-20-03-18.gh-issue-126188.RJLKk-.rst b/Misc/NEWS.d/next/Library/2024-11-13-20-03-18.gh-issue-126188.RJLKk-.rst new file mode 100644 index 00000000000000..bb13662e6ae62c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-13-20-03-18.gh-issue-126188.RJLKk-.rst @@ -0,0 +1 @@ +Update bundled pip to 24.3.1 diff --git a/Misc/NEWS.d/next/Library/2024-11-13-22-25-57.gh-issue-126789.lKzlc7.rst b/Misc/NEWS.d/next/Library/2024-11-13-22-25-57.gh-issue-126789.lKzlc7.rst new file mode 100644 index 00000000000000..09d4d2e5ab9037 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-13-22-25-57.gh-issue-126789.lKzlc7.rst @@ -0,0 +1,4 @@ +Fixed the values of :py:func:`sysconfig.get_config_vars`, +:py:func:`sysconfig.get_paths`, and their siblings when the :py:mod:`site` +initialization happens after :py:mod:`sysconfig` has built a cache for +:py:func:`sysconfig.get_config_vars`. diff --git a/Misc/NEWS.d/next/Library/2024-11-14-13-16-20.gh-issue-125063.kJ-WnH.rst b/Misc/NEWS.d/next/Library/2024-11-14-13-16-20.gh-issue-125063.kJ-WnH.rst new file mode 100644 index 00000000000000..5ddf41206db07e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-14-13-16-20.gh-issue-125063.kJ-WnH.rst @@ -0,0 +1,2 @@ +:mod:`marshal` now supports :class:`slice` objects. The marshal format +version was increased to 5. diff --git a/Misc/NEWS.d/next/Library/2024-11-14-22-25-49.gh-issue-67877.G9hw0w.rst b/Misc/NEWS.d/next/Library/2024-11-14-22-25-49.gh-issue-67877.G9hw0w.rst new file mode 100644 index 00000000000000..021b4ae2e100bc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-14-22-25-49.gh-issue-67877.G9hw0w.rst @@ -0,0 +1,2 @@ +Fix memory leaks when :mod:`regular expression ` matching terminates +abruptly, either because of a signal or because memory allocation fails. diff --git a/Misc/NEWS.d/next/Library/2024-11-17-01-14-59.gh-issue-126920.s8-f_L.rst b/Misc/NEWS.d/next/Library/2024-11-17-01-14-59.gh-issue-126920.s8-f_L.rst new file mode 100644 index 00000000000000..6966aec380fae9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-17-01-14-59.gh-issue-126920.s8-f_L.rst @@ -0,0 +1,5 @@ +Fix the ``prefix`` and ``exec_prefix`` keys from +:py:func:`sysconfig.get_config_vars` incorrectly having the same value as +:py:const:`sys.base_prefix` and :py:const:`sys.base_exec_prefix`, +respectively, inside virtual environments. They now accurately reflect +:py:const:`sys.prefix` and :py:const:`sys.exec_prefix`. diff --git a/Misc/NEWS.d/next/Library/2024-11-18-15-33-25.gh-issue-85957.8gT3B-.rst b/Misc/NEWS.d/next/Library/2024-11-18-15-33-25.gh-issue-85957.8gT3B-.rst new file mode 100644 index 00000000000000..02d3fe3077e4d1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-18-15-33-25.gh-issue-85957.8gT3B-.rst @@ -0,0 +1,2 @@ +Add missing MIME types for images with RFCs: emf, fits, g3fax, jp2, jpm, +jpx, t38, tiff-fx and wmf. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Security/2024-11-13-11-09-12.gh-issue-126623.TO7NnR.rst b/Misc/NEWS.d/next/Security/2024-11-13-11-09-12.gh-issue-126623.TO7NnR.rst new file mode 100644 index 00000000000000..f09a158af2a475 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-11-13-11-09-12.gh-issue-126623.TO7NnR.rst @@ -0,0 +1 @@ +Upgrade libexpat to 2.6.4 diff --git a/Misc/NEWS.d/next/Tests/2024-11-17-16-56-48.gh-issue-126909.60VTxW.rst b/Misc/NEWS.d/next/Tests/2024-11-17-16-56-48.gh-issue-126909.60VTxW.rst new file mode 100644 index 00000000000000..68bd9ac70cd1f4 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-11-17-16-56-48.gh-issue-126909.60VTxW.rst @@ -0,0 +1,2 @@ +Fix test_os extended attribute tests to work on filesystems with 1 KiB xattr size +limit. diff --git a/Misc/NEWS.d/next/Tools-Demos/2024-10-30-13-59-07.gh-issue-126167.j5cCWE.rst b/Misc/NEWS.d/next/Tools-Demos/2024-10-30-13-59-07.gh-issue-126167.j5cCWE.rst new file mode 100644 index 00000000000000..338160e69fd522 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2024-10-30-13-59-07.gh-issue-126167.j5cCWE.rst @@ -0,0 +1,2 @@ +The iOS testbed was modified so that it can be used by third-party projects +for testing purposes. diff --git a/Misc/NEWS.d/next/Tools-Demos/2024-11-13-22-23-36.gh-issue-126807.vpaWuN.rst b/Misc/NEWS.d/next/Tools-Demos/2024-11-13-22-23-36.gh-issue-126807.vpaWuN.rst new file mode 100644 index 00000000000000..310286ce8319ea --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2024-11-13-22-23-36.gh-issue-126807.vpaWuN.rst @@ -0,0 +1,2 @@ +Fix extraction warnings in :program:`pygettext.py` caused by mistaking +function definitions for function calls. diff --git a/Misc/NEWS.d/next/Windows/2024-09-07-15-16-24.gh-issue-123803.J9VNQU.rst b/Misc/NEWS.d/next/Windows/2024-09-07-15-16-24.gh-issue-123803.J9VNQU.rst new file mode 100644 index 00000000000000..3ad4d12eb0dc3e --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-09-07-15-16-24.gh-issue-123803.J9VNQU.rst @@ -0,0 +1 @@ +All Windows code pages are now supported as "cpXXX" codecs on Windows. diff --git a/Misc/NEWS.d/next/Windows/2024-10-29-20-09-52.gh-issue-126074.83ZzZs.rst b/Misc/NEWS.d/next/Windows/2024-10-29-20-09-52.gh-issue-126074.83ZzZs.rst new file mode 100644 index 00000000000000..d4d06b090b5922 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-10-29-20-09-52.gh-issue-126074.83ZzZs.rst @@ -0,0 +1 @@ +Removed unnecessary DLLs from Windows embeddable package diff --git a/Misc/NEWS.d/next/Windows/2024-11-07-20-42-31.gh-issue-126497.EARpd-.rst b/Misc/NEWS.d/next/Windows/2024-11-07-20-42-31.gh-issue-126497.EARpd-.rst new file mode 100644 index 00000000000000..c902b9d6da8c65 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-11-07-20-42-31.gh-issue-126497.EARpd-.rst @@ -0,0 +1,2 @@ +Fixes venv failure due to missing redirector executables in experimental +free-threaded installs. diff --git a/Misc/NEWS.d/next/Windows/2024-11-12-22-31-13.gh-issue-118973._lfxW6.rst b/Misc/NEWS.d/next/Windows/2024-11-12-22-31-13.gh-issue-118973._lfxW6.rst new file mode 100644 index 00000000000000..c173c3ee081cc2 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-11-12-22-31-13.gh-issue-118973._lfxW6.rst @@ -0,0 +1,3 @@ +Ensures the experimental free-threaded install includes the ``_tkinter`` +module. The optional Tcl/Tk component must also be installed in order for +the module to work. diff --git a/Misc/NEWS.d/next/Windows/2024-11-16-22-08-41.gh-issue-126911.HchCZZ.rst b/Misc/NEWS.d/next/Windows/2024-11-16-22-08-41.gh-issue-126911.HchCZZ.rst new file mode 100644 index 00000000000000..32481cde930fcd --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-11-16-22-08-41.gh-issue-126911.HchCZZ.rst @@ -0,0 +1 @@ +Update credits command output. diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index cc73e93009b43f..739e005646ba97 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -48,11 +48,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "6aaee1b194bea30f0a60d1cce71eada8b14d3526" + "checksumValue": "373cc00d87782a736970644d863ff2ebbd0e4886" }, { "algorithm": "SHA256", - "checksumValue": "7bd4e53a8015534b5bbb58afe1a131b3989d3d4fca29bca685c44d34bcaa2555" + "checksumValue": "0f750bc336e510d14ac9a3e63fc2399f60f3f04f0061c426e86751ed5fba90e4" } ], "fileName": "Modules/expat/expat.h" @@ -62,11 +62,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "b70ce53fdc25ae482681ae2f6623c3c8edc9c1b7" + "checksumValue": "9e615c6e5c3ba00670f674a6b071bb855b0b563d" }, { "algorithm": "SHA256", - "checksumValue": "86afb425ec9999eb4f1ec9ab2fb41c58c4aa5cb9bf934b8c94264670fc5a961d" + "checksumValue": "3d90a4b65c40a3f848c36100f4d73b933a015c7b7cd85c28e4331a6b845c1ad0" } ], "fileName": "Modules/expat/expat_external.h" @@ -128,18 +128,18 @@ "fileName": "Modules/expat/nametab.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-expat-pyexpatns.h", + "SPDXID": "SPDXRef-FILE-Modules-expat-refresh.sh", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "f50c899172acd93fc539007bfb43315b83d407e4" + "checksumValue": "a9b0a33b8359cfe94b23972a1605daf8dcc605d9" }, { "algorithm": "SHA256", - "checksumValue": "d571b8258cfaa067a20adef553e5fcedd6671ca4a8841483496de031bd904567" + "checksumValue": "19eb541460bc2ca8b87118acd3c048f6af77affbf8719ac29aa7b6c8d70f83fd" } ], - "fileName": "Modules/expat/pyexpatns.h" + "fileName": "Modules/expat/refresh.sh" }, { "SPDXID": "SPDXRef-FILE-Modules-expat-siphash.h", @@ -188,11 +188,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "b2ec0ad170ccc21e63fbcfc8d7404cdd756eedd3" + "checksumValue": "3199fbd38b6fb158f73d5c8de6b6e6e3812ef803" }, { "algorithm": "SHA256", - "checksumValue": "92159d4e17393e56ee85f47d9fb31348695a58589899aa01e7536cdc88f60b85" + "checksumValue": "c1518244dd5ea397e345d00e12cc45d42f43453ed208218559c981c97a0583e2" } ], "fileName": "Modules/expat/xmlparse.c" @@ -300,11 +300,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "a34e821b68ef5334eccf4f729b28bb7bb65b965e" + "checksumValue": "1cd3cda98e0e6882a13a59268b88640c542350fd" }, { "algorithm": "SHA256", - "checksumValue": "4582db9143c0810b98838a5357c577e0b32ae77f3018486159df4e0dfd3fce3c" + "checksumValue": "41a420bc9355e451720e60e9536e66f04dc6e416ca9217c4ab18d827887a2e08" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2b.c" @@ -328,11 +328,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0ffe60c6d5eed5dd222515e820d461d319d16b1f" + "checksumValue": "0ceef306590ec12251db03a31fc08ecba697486d" }, { "algorithm": "SHA256", - "checksumValue": "4804cb3ce68bfdcf98853d6f1d77b4a844a3c2796f776b39770ba327e400d402" + "checksumValue": "1575a23b21319e55e670f74194fc2dfd1777eb5a3816cad43750e03da6e44db9" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c" @@ -370,11 +370,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "cf035ffeff875bc74345a47373ce25dc408ea9dc" + "checksumValue": "9616a9f8d795d64487bf86a96719f943729621e2" }, { "algorithm": "SHA256", - "checksumValue": "579059b002c45fab0fed6381e85c3f5eaf1d959400ca64b103542ac6c35bade3" + "checksumValue": "5ecde5ddc8ec073cffe64d60e868535d995f33fb0f87f9b50e68bd2a694b7434" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2s.c" @@ -398,11 +398,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "9bb53022d158a9c349edb52a8def8aac7d098a4e" + "checksumValue": "5b950ce0a5c8f0c2c56b4ac96e1943b504255d45" }, { "algorithm": "SHA256", - "checksumValue": "2abde0c6b5da0402e91b4bedfe786c24b908fbdc04e08e74651c7624729254d9" + "checksumValue": "5a5f5d8e376dc30d89fd6c6c435157fe9ffa5308030e7abb1256afaee0765536" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c" @@ -496,11 +496,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "f2aa3ed6acce621c162bc3a0592780ce5aa3bc4d" + "checksumValue": "b0aa8810339adb09623ffa429246b4324fac4565" }, { "algorithm": "SHA256", - "checksumValue": "30638efb75c8b185bb09c3df6977e3f3c5d21a1e696218cf7ade6bc4d5201b31" + "checksumValue": "2288f8f860efe80eed4f1e14ef570079b7459aeb41f87e94e691d7cf5e0e7adb" } ], "fileName": "Modules/_hacl/Hacl_Hash_SHA2.c" @@ -524,11 +524,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "fc2c3ef83a71bef42eb3f73b78e4ef6642a4634e" + "checksumValue": "ef374b9d0951ebb38006af944dd4b38a6cf3abb2" }, { "algorithm": "SHA256", - "checksumValue": "e4f3ed9d1e8f661482cbd2d04b197e15cc3b698c5ef2ddedf0eb65df320dbbc4" + "checksumValue": "164df19f229143006c5f9a3c0bd771415f152bfbc7efb61c337fa0f903003eb3" } ], "fileName": "Modules/_hacl/Hacl_Hash_SHA3.c" @@ -566,11 +566,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "47ce34375d43a27312e1fffb96b8965610b05855" + "checksumValue": "2e08072c0c57dac02b67f3f71d77068c537ac02e" }, { "algorithm": "SHA256", - "checksumValue": "8affd767d7644150064d8bccd05d7bf4c4ae41fd4bb5bf5b8e943eabf09f3d74" + "checksumValue": "e69fd3e84f77873ecb414f5300761b686321d01f5710ccf2517765236b08fc25" } ], "fileName": "Modules/_hacl/Lib_Memzero0.c" @@ -748,11 +748,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0018e084339058dd454b4e49d10d236b4f896bf8" + "checksumValue": "2e9ae174142fc491f20567ab8b5c08cef9b07cfe" }, { "algorithm": "SHA256", - "checksumValue": "10e959a92b3288a6165a404c8fae2bbcd7fb00a9abbae2b7809fa55d6fe9068d" + "checksumValue": "07100964adcf4b5f8bd4773e25f475b34cd180b90df8b1c0052e55c008b7cc49" } ], "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA2.h" @@ -1640,14 +1640,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "935ae51d0ff0bf1403f0ecc1ff02b8f685d09053618558c07fbe4bd2abbc5dd1" + "checksumValue": "40de5297b032d2676fc0039049b4e8dab1f2730eebb5ecff6a40c04fa0356339" } ], - "downloadLocation": "https://github.com/hacl-star/hacl-star/archive/315a9e491d2bc347b9dae99e0ea506995ea84d9d.zip", + "downloadLocation": "https://github.com/hacl-star/hacl-star/archive/f218923ef2417d963d7efc7951593ae6aef613f7.zip", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:hacl-star:hacl-star:315a9e491d2bc347b9dae99e0ea506995ea84d9d:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:hacl-star:hacl-star:f218923ef2417d963d7efc7951593ae6aef613f7:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1655,7 +1655,7 @@ "name": "hacl-star", "originator": "Organization: HACL* Developers", "primaryPackagePurpose": "SOURCE", - "versionInfo": "315a9e491d2bc347b9dae99e0ea506995ea84d9d" + "versionInfo": "f218923ef2417d963d7efc7951593ae6aef613f7" }, { "SPDXID": "SPDXRef-PACKAGE-macholib", @@ -1749,7 +1749,7 @@ "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-pyexpatns.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-refresh.sh", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index c2500fbd692d4d..f883125a2c70b2 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -8,7 +8,7 @@ #include "pycore_freelist.h" // _Py_FREELIST_POP() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_object.h" // _Py_SetImmortalUntracked +#include "pycore_object.h" // _Py_SetImmortalUntracked() #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -85,8 +85,6 @@ typedef struct { # define ASYNCIO_STATE_UNLOCK(state) ((void)state) #endif -typedef struct futureiterobject futureiterobject; - /* State of the _asyncio module */ typedef struct { #ifdef Py_GIL_DISABLED @@ -138,47 +136,18 @@ typedef struct { /* Counter for autogenerated Task names */ uint64_t task_name_counter; - /* Linked-list of all tasks which are instances of asyncio.Task or subclasses + /* Circular linked-list of all tasks which are instances of asyncio.Task or subclasses of it. Third party tasks implementations which don't inherit from asyncio.Task are tracked separately using the 'non_asyncio_tasks' WeakSet. - `tail` is used as a sentinel to mark the end of the linked-list. It avoids one + `first` is used as a sentinel to mark the end of the linked-list. It avoids one branch in checking for empty list when adding a new task, the list is - initialized with `head` pointing to `tail` to mark an empty list. - - Invariants: - * When the list is empty: - - asyncio_tasks.head == &asyncio_tasks.tail - - asyncio_tasks.head->prev == NULL - - asyncio_tasks.head->next == NULL - - * After adding the first task 'task1': - - asyncio_tasks.head == task1 - - task1->next == &asyncio_tasks.tail - - task1->prev == NULL - - asyncio_tasks.tail.prev == task1 - - * After adding a second task 'task2': - - asyncio_tasks.head == task2 - - task2->next == task1 - - task2->prev == NULL - - task1->prev == task2 - - asyncio_tasks.tail.prev == task1 - - * After removing task 'task1': - - asyncio_tasks.head == task2 - - task2->next == &asyncio_tasks.tail - - task2->prev == NULL - - asyncio_tasks.tail.prev == task2 - - * After removing task 'task2', the list is empty: - - asyncio_tasks.head == &asyncio_tasks.tail - - asyncio_tasks.head->prev == NULL - - asyncio_tasks.tail.prev == NULL - - asyncio_tasks.tail.next == NULL + initialized with `head`, `head->next` and `head->prev` pointing to `first` + to mark an empty list. + */ struct { - TaskObj tail; + TaskObj first; TaskObj *head; } asyncio_tasks; @@ -1048,8 +1017,10 @@ _asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls, if (len == 1) { PyObject *cb_tup = PyList_GET_ITEM(self->fut_callbacks, 0); + Py_INCREF(cb_tup); int cmp = PyObject_RichCompareBool( PyTuple_GET_ITEM(cb_tup, 0), fn, Py_EQ); + Py_DECREF(cb_tup); if (cmp == -1) { return NULL; } @@ -1925,7 +1896,7 @@ register_task(asyncio_state *state, TaskObj *task) { ASYNCIO_STATE_LOCK(state); assert(Task_Check(state, task)); - assert(task != &state->asyncio_tasks.tail); + assert(task != &state->asyncio_tasks.first); if (task->next != NULL) { // already registered goto exit; @@ -1934,8 +1905,10 @@ register_task(asyncio_state *state, TaskObj *task) assert(state->asyncio_tasks.head != NULL); task->next = state->asyncio_tasks.head; + task->prev = state->asyncio_tasks.head->prev; + state->asyncio_tasks.head->prev->next = task; state->asyncio_tasks.head->prev = task; - state->asyncio_tasks.head = task; + exit: ASYNCIO_STATE_UNLOCK(state); } @@ -1951,7 +1924,7 @@ unregister_task(asyncio_state *state, TaskObj *task) { ASYNCIO_STATE_LOCK(state); assert(Task_Check(state, task)); - assert(task != &state->asyncio_tasks.tail); + assert(task != &state->asyncio_tasks.first); if (task->next == NULL) { // not registered assert(task->prev == NULL); @@ -1959,12 +1932,7 @@ unregister_task(asyncio_state *state, TaskObj *task) goto exit; } task->next->prev = task->prev; - if (task->prev == NULL) { - assert(state->asyncio_tasks.head == task); - state->asyncio_tasks.head = task->next; - } else { - task->prev->next = task->next; - } + task->prev->next = task->next; task->next = NULL; task->prev = NULL; assert(state->asyncio_tasks.head != task); @@ -2738,7 +2706,11 @@ task_call_step_soon(asyncio_state *state, TaskObj *task, PyObject *arg) return -1; } - int ret = call_soon(state, task->task_loop, cb, NULL, task->task_context); + // Beware: An evil call_soon could alter task_context. + // See: https://github.com/python/cpython/issues/126080. + PyObject *task_context = Py_NewRef(task->task_context); + int ret = call_soon(state, task->task_loop, cb, NULL, task_context); + Py_DECREF(task_context); Py_DECREF(cb); return ret; } @@ -2963,8 +2935,17 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu if (task->task_must_cancel) { PyObject *r; int is_true; + + // Beware: An evil `__getattribute__` could + // prematurely delete task->task_cancel_msg before the + // task is cancelled, thereby causing a UAF crash. + // + // See https://github.com/python/cpython/issues/126138 + PyObject *task_cancel_msg = Py_NewRef(task->task_cancel_msg); r = PyObject_CallMethodOneArg(result, &_Py_ID(cancel), - task->task_cancel_msg); + task_cancel_msg); + Py_DECREF(task_cancel_msg); + if (r == NULL) { return NULL; } @@ -3056,8 +3037,17 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu if (task->task_must_cancel) { PyObject *r; int is_true; + + // Beware: An evil `__getattribute__` could + // prematurely delete task->task_cancel_msg before the + // task is cancelled, thereby causing a UAF crash. + // + // See https://github.com/python/cpython/issues/126138 + PyObject *task_cancel_msg = Py_NewRef(task->task_cancel_msg); r = PyObject_CallMethodOneArg(result, &_Py_ID(cancel), - task->task_cancel_msg); + task_cancel_msg); + Py_DECREF(task_cancel_msg); + if (r == NULL) { return NULL; } @@ -3635,12 +3625,10 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) Py_DECREF(eager_iter); int err = 0; ASYNCIO_STATE_LOCK(state); - TaskObj *head = state->asyncio_tasks.head; + TaskObj *first = &state->asyncio_tasks.first; + TaskObj *head = state->asyncio_tasks.head->next; Py_INCREF(head); - assert(head != NULL); - assert(head->prev == NULL); - TaskObj *tail = &state->asyncio_tasks.tail; - while (head != tail) + while (head != first) { if (add_one_task(state, tasks, (PyObject *)head, loop) < 0) { Py_DECREF(tasks); @@ -3853,9 +3841,12 @@ static int module_exec(PyObject *mod) { asyncio_state *state = get_asyncio_state(mod); - Py_SET_TYPE(&state->asyncio_tasks.tail, state->TaskType); - _Py_SetImmortalUntracked((PyObject *)&state->asyncio_tasks.tail); - state->asyncio_tasks.head = &state->asyncio_tasks.tail; + + Py_SET_TYPE(&state->asyncio_tasks.first, state->TaskType); + _Py_SetImmortalUntracked((PyObject *)&state->asyncio_tasks.first); + state->asyncio_tasks.head = &state->asyncio_tasks.first; + state->asyncio_tasks.head->next = &state->asyncio_tasks.first; + state->asyncio_tasks.head->prev = &state->asyncio_tasks.first; #define CREATE_TYPE(m, tp, spec, base) \ do { \ diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index eae69e484e1660..34529bce496d88 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -956,32 +956,48 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, return NULL; } +#undef USE_DLERROR #ifdef MS_WIN32 Py_BEGIN_ALLOW_THREADS address = (void *)GetProcAddress(handle, name); Py_END_ALLOW_THREADS - if (!address) { - PyErr_Format(PyExc_ValueError, - "symbol '%s' not found", - name); - return NULL; - } #else + #ifdef __CYGWIN__ + // dlerror() isn't very helpful on cygwin + #else + #define USE_DLERROR + /* dlerror() always returns the latest error. + * + * Clear the previous value before calling dlsym(), + * to ensure we can tell if our call resulted in an error. + */ + (void)dlerror(); + #endif address = (void *)dlsym(handle, name); - if (!address) { -#ifdef __CYGWIN__ -/* dlerror() isn't very helpful on cygwin */ - PyErr_Format(PyExc_ValueError, - "symbol '%s' not found", - name); -#else - PyErr_SetString(PyExc_ValueError, dlerror()); #endif - return NULL; + + if (address) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + return PyCData_AtAddress(st, type, address); } -#endif - ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); - return PyCData_AtAddress(st, type, address); + + #ifdef USE_DLERROR + const char *dlerr = dlerror(); + if (dlerr) { + PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape"); + if (message) { + PyErr_SetObject(PyExc_ValueError, message); + Py_DECREF(message); + return NULL; + } + // Ignore errors from PyUnicode_DecodeLocale, + // fall back to the generic error below. + PyErr_Clear(); + } + #endif +#undef USE_DLERROR + PyErr_Format(PyExc_ValueError, "symbol '%s' not found", name); + return NULL; } /*[clinic input] @@ -3759,6 +3775,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } +#undef USE_DLERROR #ifdef MS_WIN32 address = FindAddress(handle, name, (PyObject *)type); if (!address) { @@ -3774,20 +3791,41 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } #else + #ifdef __CYGWIN__ + //dlerror() isn't very helpful on cygwin */ + #else + #define USE_DLERROR + /* dlerror() always returns the latest error. + * + * Clear the previous value before calling dlsym(), + * to ensure we can tell if our call resulted in an error. + */ + (void)dlerror(); + #endif address = (PPROC)dlsym(handle, name); + if (!address) { -#ifdef __CYGWIN__ -/* dlerror() isn't very helpful on cygwin */ - PyErr_Format(PyExc_AttributeError, - "function '%s' not found", - name); -#else - PyErr_SetString(PyExc_AttributeError, dlerror()); -#endif + #ifdef USE_DLERROR + const char *dlerr = dlerror(); + if (dlerr) { + PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape"); + if (message) { + PyErr_SetObject(PyExc_AttributeError, message); + Py_DECREF(ftuple); + Py_DECREF(message); + return NULL; + } + // Ignore errors from PyUnicode_DecodeLocale, + // fall back to the generic error below. + PyErr_Clear(); + } + #endif + PyErr_Format(PyExc_AttributeError, "function '%s' not found", name); Py_DECREF(ftuple); return NULL; } #endif +#undef USE_DLERROR ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); if (!_validate_paramflags(st, type, paramflags)) { Py_DECREF(ftuple); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 5ac9cf16681645..218c3a9c81e05f 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1623,13 +1623,39 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) if (PySys_Audit("ctypes.dlsym/handle", "O", args) < 0) { return NULL; } +#undef USE_DLERROR + #ifdef __CYGWIN__ + // dlerror() isn't very helpful on cygwin + #else + #define USE_DLERROR + /* dlerror() always returns the latest error. + * + * Clear the previous value before calling dlsym(), + * to ensure we can tell if our call resulted in an error. + */ + (void)dlerror(); + #endif ptr = dlsym((void*)handle, name); - if (!ptr) { - PyErr_SetString(PyExc_OSError, - dlerror()); - return NULL; + if (ptr) { + return PyLong_FromVoidPtr(ptr); + } + #ifdef USE_DLERROR + const char *dlerr = dlerror(); + if (dlerr) { + PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape"); + if (message) { + PyErr_SetObject(PyExc_OSError, message); + Py_DECREF(message); + return NULL; + } + // Ignore errors from PyUnicode_DecodeLocale, + // fall back to the generic error below. + PyErr_Clear(); } - return PyLong_FromVoidPtr(ptr); + #endif + #undef USE_DLERROR + PyErr_Format(PyExc_OSError, "symbol '%s' not found", name); + return NULL; } #endif diff --git a/Modules/_ctypes/clinic/_ctypes.c.h b/Modules/_ctypes/clinic/_ctypes.c.h index e1d5a17cbe7d68..1332ba04cdfecd 100644 --- a/Modules/_ctypes/clinic/_ctypes.c.h +++ b/Modules/_ctypes/clinic/_ctypes.c.h @@ -65,7 +65,8 @@ CDataType_from_address(PyObject *type, PyTypeObject *cls, PyObject *const *args, PyObject *argsbuf[1]; PyObject *value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -112,7 +113,8 @@ CDataType_from_buffer(PyObject *type, PyTypeObject *cls, PyObject *const *args, PyObject *obj; Py_ssize_t offset = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -175,7 +177,8 @@ CDataType_from_buffer_copy(PyObject *type, PyTypeObject *cls, PyObject *const *a Py_buffer buffer = {NULL, NULL}; Py_ssize_t offset = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -245,7 +248,8 @@ CDataType_in_dll(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ss PyObject *dll; const char *name; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -301,7 +305,8 @@ CDataType_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, P PyObject *argsbuf[1]; PyObject *value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -344,7 +349,8 @@ PyCPointerType_set_type(PyTypeObject *self, PyTypeObject *cls, PyObject *const * PyObject *argsbuf[1]; PyObject *type; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -388,7 +394,8 @@ PyCPointerType_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *ar PyObject *argsbuf[1]; PyObject *value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -430,7 +437,8 @@ c_wchar_p_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, P PyObject *argsbuf[1]; PyObject *value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -472,7 +480,8 @@ c_char_p_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py PyObject *argsbuf[1]; PyObject *value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -514,7 +523,8 @@ c_void_p_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py PyObject *argsbuf[1]; PyObject *value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -558,7 +568,8 @@ PyCSimpleType_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *arg PyObject *argsbuf[1]; PyObject *value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -610,4 +621,4 @@ Simple_from_outparm(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py } return Simple_from_outparm_impl(self, cls); } -/*[clinic end generated code: output=a90886be2a294ee6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=52724c091e3a8b8d input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h index df5da783e050ed..bbf108a1a07b67 100644 --- a/Modules/_ctypes/clinic/cfield.c.h +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -54,7 +54,8 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t index; PyObject *bit_size_obj = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 6, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 5, /*maxpos*/ 6, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -110,4 +111,4 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=27c010bae9be7213 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6b450bdd861571e7 input=a9049054013a1b77]*/ diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 27d5df08de933e..040ffa81153ebe 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -3815,8 +3815,11 @@ static int _curses_napms_impl(PyObject *module, int ms) /*[clinic end generated code: output=5f292a6a724491bd input=c6d6e01f2f1df9f7]*/ { - PyCursesStatefulInitialised(module); - + if (!_PyCursesStatefulCheckFunction(module, + curses_initscr_called, + "initscr")) { + return -1; + } return napms(ms); } diff --git a/Modules/_decimal/docstrings.h b/Modules/_decimal/docstrings.h index b34bff83d3f4e9..5abd7b9d807e19 100644 --- a/Modules/_decimal/docstrings.h +++ b/Modules/_decimal/docstrings.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001-2012 Python Software Foundation. All Rights Reserved. + * Copyright (c) 2001 Python Software Foundation. All Rights Reserved. * Modified and extended by Stefan Krah. */ diff --git a/Modules/_decimal/tests/bench.py b/Modules/_decimal/tests/bench.py index 640290f2ec7962..6605e9a92e2dde 100644 --- a/Modules/_decimal/tests/bench.py +++ b/Modules/_decimal/tests/bench.py @@ -1,5 +1,5 @@ # -# Copyright (C) 2001-2012 Python Software Foundation. All Rights Reserved. +# Copyright (C) 2001 Python Software Foundation. All Rights Reserved. # Modified and extended by Stefan Krah. # diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index da4e088e54621e..24b38063dde9e5 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -18,7 +18,7 @@ class _functools._lru_cache_wrapper "PyObject *" "&lru_cache_type_spec" /* _functools module written and maintained by Hye-Shik Chang with adaptations by Raymond Hettinger - Copyright (c) 2004, 2005, 2006 Python Software Foundation. + Copyright (c) 2004 Python Software Foundation. All rights reserved. */ @@ -932,15 +932,31 @@ _functools_cmp_to_key_impl(PyObject *module, PyObject *mycmp) /* reduce (used to be a builtin) ********************************************/ -// Not converted to argument clinic, because of `args` in-place modification. -// AC will affect performance. +/*[clinic input] +_functools.reduce + + function as func: object + iterable as seq: object + / + initial as result: object = NULL + +Apply a function of two arguments cumulatively to the items of an iterable, from left to right. + +This effectively reduces the iterable to a single value. If initial is present, +it is placed before the items of the iterable in the calculation, and serves as +a default when the iterable is empty. + +For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) +calculates ((((1 + 2) + 3) + 4) + 5). +[clinic start generated code]*/ + static PyObject * -functools_reduce(PyObject *self, PyObject *args) +_functools_reduce_impl(PyObject *module, PyObject *func, PyObject *seq, + PyObject *result) +/*[clinic end generated code: output=30d898fe1267c79d input=1511e9a8c38581ac]*/ { - PyObject *seq, *func, *result = NULL, *it; + PyObject *args, *it; - if (!PyArg_UnpackTuple(args, "reduce", 2, 3, &func, &seq, &result)) - return NULL; if (result != NULL) Py_INCREF(result); @@ -1006,18 +1022,6 @@ functools_reduce(PyObject *self, PyObject *args) return NULL; } -PyDoc_STRVAR(functools_reduce_doc, -"reduce(function, iterable[, initial], /) -> value\n\ -\n\ -Apply a function of two arguments cumulatively to the items of an iterable, from left to right.\n\ -\n\ -This effectively reduces the iterable to a single value. If initial is present,\n\ -it is placed before the items of the iterable in the calculation, and serves as\n\ -a default when the iterable is empty.\n\ -\n\ -For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])\n\ -calculates ((((1 + 2) + 3) + 4) + 5)."); - /* lru_cache object **********************************************************/ /* There are four principal algorithmic differences from the pure python version: @@ -1722,7 +1726,7 @@ PyDoc_STRVAR(_functools_doc, "Tools that operate on functions."); static PyMethodDef _functools_methods[] = { - {"reduce", functools_reduce, METH_VARARGS, functools_reduce_doc}, + _FUNCTOOLS_REDUCE_METHODDEF _FUNCTOOLS_CMP_TO_KEY_METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_hacl/Hacl_Hash_Blake2b.c b/Modules/_hacl/Hacl_Hash_Blake2b.c index cd3b9777e09f6c..1bab75e6aaf2ab 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2b.c +++ b/Modules/_hacl/Hacl_Hash_Blake2b.c @@ -1029,7 +1029,8 @@ Hacl_Hash_Blake2b_reset_with_key_and_params( uint8_t *k ) { - index_of_state(s); + Hacl_Hash_Blake2b_index i1 = index_of_state(s); + KRML_MAYBE_UNUSED_VAR(i1); reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); } diff --git a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c index 92b2e8f539041b..19234ab9d7f9b2 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c +++ b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c @@ -855,7 +855,8 @@ Hacl_Hash_Blake2b_Simd256_reset_with_key_and_params( uint8_t *k ) { - index_of_state(s); + Hacl_Hash_Blake2b_index i1 = index_of_state(s); + KRML_MAYBE_UNUSED_VAR(i1); reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); } diff --git a/Modules/_hacl/Hacl_Hash_Blake2s.c b/Modules/_hacl/Hacl_Hash_Blake2s.c index e5e0ecd0bfde7e..ceb7385072e048 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2s.c +++ b/Modules/_hacl/Hacl_Hash_Blake2s.c @@ -1011,7 +1011,8 @@ Hacl_Hash_Blake2s_reset_with_key_and_params( uint8_t *k ) { - index_of_state(s); + Hacl_Hash_Blake2b_index i1 = index_of_state(s); + KRML_MAYBE_UNUSED_VAR(i1); reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); } diff --git a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c index f675a7f14f192f..3b68783bfad9b4 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c +++ b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c @@ -842,7 +842,8 @@ Hacl_Hash_Blake2s_Simd128_reset_with_key_and_params( uint8_t *k ) { - index_of_state(s); + Hacl_Hash_Blake2b_index i1 = index_of_state(s); + KRML_MAYBE_UNUSED_VAR(i1); reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); } diff --git a/Modules/_hacl/Hacl_Hash_SHA2.c b/Modules/_hacl/Hacl_Hash_SHA2.c index 4b6af5fc78c680..cc930bbc89e8ad 100644 --- a/Modules/_hacl/Hacl_Hash_SHA2.c +++ b/Modules/_hacl/Hacl_Hash_SHA2.c @@ -211,7 +211,7 @@ void Hacl_Hash_SHA2_sha224_init(uint32_t *hash) os[i] = x;); } -static inline void sha224_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st) +void Hacl_Hash_SHA2_sha224_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st) { Hacl_Hash_SHA2_sha256_update_nblocks(len, b, st); } @@ -825,7 +825,7 @@ void Hacl_Hash_SHA2_digest_224(Hacl_Streaming_MD_state_32 *state, uint8_t *outpu } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - sha224_update_nblocks(0U, buf_multi, tmp_block_state); + Hacl_Hash_SHA2_sha224_update_nblocks(0U, buf_multi, tmp_block_state); uint64_t prev_len_last = total_len - (uint64_t)r; Hacl_Hash_SHA2_sha224_update_last(prev_len_last + (uint64_t)r, r, buf_last, tmp_block_state); Hacl_Hash_SHA2_sha224_finish(tmp_block_state, output); @@ -847,7 +847,7 @@ void Hacl_Hash_SHA2_hash_224(uint8_t *output, uint8_t *input, uint32_t input_len Hacl_Hash_SHA2_sha224_init(st); uint32_t rem = input_len % 64U; uint64_t len_ = (uint64_t)input_len; - sha224_update_nblocks(input_len, ib, st); + Hacl_Hash_SHA2_sha224_update_nblocks(input_len, ib, st); uint32_t rem1 = input_len % 64U; uint8_t *b0 = ib; uint8_t *lb = b0 + input_len - rem1; diff --git a/Modules/_hacl/Hacl_Hash_SHA3.c b/Modules/_hacl/Hacl_Hash_SHA3.c index 9cf5abb330b180..b964e1d9c0aa69 100644 --- a/Modules/_hacl/Hacl_Hash_SHA3.c +++ b/Modules/_hacl/Hacl_Hash_SHA3.c @@ -251,7 +251,8 @@ Hacl_Hash_SHA3_update_multi_sha3( uint8_t *bl0 = b_; uint8_t *uu____0 = b0 + i * block_len(a); memcpy(bl0, uu____0, block_len(a) * sizeof (uint8_t)); - block_len(a); + uint32_t unused = block_len(a); + KRML_MAYBE_UNUSED_VAR(unused); absorb_inner_32(b_, s); } } diff --git a/Modules/_hacl/Lib_Memzero0.c b/Modules/_hacl/Lib_Memzero0.c index 5b1a2f7797db76..5c269d231de82f 100644 --- a/Modules/_hacl/Lib_Memzero0.c +++ b/Modules/_hacl/Lib_Memzero0.c @@ -36,7 +36,7 @@ void Lib_Memzero0_memzero0(void *dst, uint64_t len) { size_t len_ = (size_t) len; #ifdef _WIN32 - SecureZeroMemory(dst, len); + SecureZeroMemory(dst, len_); #elif defined(__APPLE__) && defined(__MACH__) memset_s(dst, len_, 0, len_); #elif (defined(__linux__) && !defined(LINUX_NO_EXPLICIT_BZERO)) || defined(__FreeBSD__) diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA2.h b/Modules/_hacl/internal/Hacl_Hash_SHA2.h index 0127f4373fb1a1..cb60f9e9bd4df6 100644 --- a/Modules/_hacl/internal/Hacl_Hash_SHA2.h +++ b/Modules/_hacl/internal/Hacl_Hash_SHA2.h @@ -123,6 +123,8 @@ void Hacl_Hash_SHA2_sha256_finish(uint32_t *st, uint8_t *h); void Hacl_Hash_SHA2_sha224_init(uint32_t *hash); +void Hacl_Hash_SHA2_sha224_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st); + void Hacl_Hash_SHA2_sha224_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *st); diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index 6234fea9f17bc7..4147ab302fe146 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=315a9e491d2bc347b9dae99e0ea506995ea84d9d +expected_hacl_star_rev=f218923ef2417d963d7efc7951593ae6aef613f7 hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index 8e6b21db76e01c..75d69ade1d3c9b 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -6,7 +6,7 @@ #endif #include "Python.h" -#include "pycore_crossinterp.h" // struct _xid +#include "pycore_crossinterp.h" // _PyXIData_t #include "pycore_interp.h" // _PyInterpreterState_LookUpID() #include "pycore_pystate.h" // _PyInterpreterState_GetIDObject() @@ -59,11 +59,11 @@ _globals (static struct globals): first (struct _channelitem *): next (struct _channelitem *): ... - data (_PyCrossInterpreterData *): + data (_PyXIData_t *): data (void *) obj (PyObject *) interpid (int64_t) - new_object (xid_newobjectfunc) + new_object (xid_newobjfunc) free (xid_freefunc) last (struct _channelitem *): ... @@ -80,10 +80,10 @@ The above state includes the following allocations by the module: * 1 struct _channelqueue * for each item in each channel: * 1 struct _channelitem - * 1 _PyCrossInterpreterData + * 1 _PyXIData_t The only objects in that global state are the references held by each -channel's queue, which are safely managed via the _PyCrossInterpreterData_*() +channel's queue, which are safely managed via the _PyXIData_*() API.. The module does not create any objects that are shared globally. */ @@ -102,7 +102,7 @@ API.. The module does not create any objects that are shared globally. #define XID_FREE 2 static int -_release_xid_data(_PyCrossInterpreterData *data, int flags) +_release_xid_data(_PyXIData_t *data, int flags) { int ignoreexc = flags & XID_IGNORE_EXC; PyObject *exc; @@ -111,10 +111,10 @@ _release_xid_data(_PyCrossInterpreterData *data, int flags) } int res; if (flags & XID_FREE) { - res = _PyCrossInterpreterData_ReleaseAndRawFree(data); + res = _PyXIData_ReleaseAndRawFree(data); } else { - res = _PyCrossInterpreterData_Release(data); + res = _PyXIData_Release(data); } if (res < 0) { /* The owning interpreter is already destroyed. */ @@ -519,7 +519,7 @@ typedef struct _channelitem { This is necessary because item->data might be NULL, meaning the interpreter has been destroyed. */ int64_t interpid; - _PyCrossInterpreterData *data; + _PyXIData_t *data; _waiting_t *waiting; int unboundop; struct _channelitem *next; @@ -533,7 +533,7 @@ _channelitem_ID(_channelitem *item) static void _channelitem_init(_channelitem *item, - int64_t interpid, _PyCrossInterpreterData *data, + int64_t interpid, _PyXIData_t *data, _waiting_t *waiting, int unboundop) { if (interpid < 0) { @@ -541,8 +541,8 @@ _channelitem_init(_channelitem *item, } else { assert(data == NULL - || _PyCrossInterpreterData_INTERPID(data) < 0 - || interpid == _PyCrossInterpreterData_INTERPID(data)); + || _PyXIData_INTERPID(data) < 0 + || interpid == _PyXIData_INTERPID(data)); } *item = (_channelitem){ .interpid = interpid, @@ -580,7 +580,7 @@ _channelitem_clear(_channelitem *item) } static _channelitem * -_channelitem_new(int64_t interpid, _PyCrossInterpreterData *data, +_channelitem_new(int64_t interpid, _PyXIData_t *data, _waiting_t *waiting, int unboundop) { _channelitem *item = GLOBAL_MALLOC(_channelitem); @@ -611,7 +611,7 @@ _channelitem_free_all(_channelitem *item) static void _channelitem_popped(_channelitem *item, - _PyCrossInterpreterData **p_data, _waiting_t **p_waiting, + _PyXIData_t **p_data, _waiting_t **p_waiting, int *p_unboundop) { assert(item->waiting == NULL || item->waiting->status == WAITING_ACQUIRED); @@ -634,7 +634,7 @@ _channelitem_clear_interpreter(_channelitem *item) assert(item->unboundop != UNBOUND_REMOVE); return 0; } - assert(_PyCrossInterpreterData_INTERPID(item->data) == item->interpid); + assert(_PyXIData_INTERPID(item->data) == item->interpid); switch (item->unboundop) { case UNBOUND_REMOVE: @@ -691,7 +691,7 @@ _channelqueue_free(_channelqueue *queue) static int _channelqueue_put(_channelqueue *queue, - int64_t interpid, _PyCrossInterpreterData *data, + int64_t interpid, _PyXIData_t *data, _waiting_t *waiting, int unboundop) { _channelitem *item = _channelitem_new(interpid, data, waiting, unboundop); @@ -717,7 +717,7 @@ _channelqueue_put(_channelqueue *queue, static int _channelqueue_get(_channelqueue *queue, - _PyCrossInterpreterData **p_data, _waiting_t **p_waiting, + _PyXIData_t **p_data, _waiting_t **p_waiting, int *p_unboundop) { _channelitem *item = queue->first; @@ -769,7 +769,7 @@ _channelqueue_find(_channelqueue *queue, _channelitem_id_t itemid, static void _channelqueue_remove(_channelqueue *queue, _channelitem_id_t itemid, - _PyCrossInterpreterData **p_data, _waiting_t **p_waiting) + _PyXIData_t **p_data, _waiting_t **p_waiting) { _channelitem *prev = NULL; _channelitem *item = NULL; @@ -1128,8 +1128,7 @@ _channel_free(_channel_state *chan) static int _channel_add(_channel_state *chan, int64_t interpid, - _PyCrossInterpreterData *data, _waiting_t *waiting, - int unboundop) + _PyXIData_t *data, _waiting_t *waiting, int unboundop) { int res = -1; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1156,8 +1155,7 @@ _channel_add(_channel_state *chan, int64_t interpid, static int _channel_next(_channel_state *chan, int64_t interpid, - _PyCrossInterpreterData **p_data, _waiting_t **p_waiting, - int *p_unboundop) + _PyXIData_t **p_data, _waiting_t **p_waiting, int *p_unboundop) { int err = 0; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1193,7 +1191,7 @@ _channel_next(_channel_state *chan, int64_t interpid, static void _channel_remove(_channel_state *chan, _channelitem_id_t itemid) { - _PyCrossInterpreterData *data = NULL; + _PyXIData_t *data = NULL; _waiting_t *waiting = NULL; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1760,6 +1758,11 @@ channel_send(_channels *channels, int64_t cid, PyObject *obj, } int64_t interpid = PyInterpreterState_GetID(interp); + _PyXIData_lookup_context_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return -1; + } + // Look up the channel. PyThread_type_lock mutex = NULL; _channel_state *chan = NULL; @@ -1776,12 +1779,12 @@ channel_send(_channels *channels, int64_t cid, PyObject *obj, } // Convert the object to cross-interpreter data. - _PyCrossInterpreterData *data = GLOBAL_MALLOC(_PyCrossInterpreterData); + _PyXIData_t *data = GLOBAL_MALLOC(_PyXIData_t); if (data == NULL) { PyThread_release_lock(mutex); return -1; } - if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { + if (_PyObject_GetXIData(&ctx, obj, data) != 0) { PyThread_release_lock(mutex); GLOBAL_FREE(data); return -1; @@ -1904,7 +1907,7 @@ channel_recv(_channels *channels, int64_t cid, PyObject **res, int *p_unboundop) // Past this point we are responsible for releasing the mutex. // Pop off the next item from the channel. - _PyCrossInterpreterData *data = NULL; + _PyXIData_t *data = NULL; _waiting_t *waiting = NULL; err = _channel_next(chan, interpid, &data, &waiting, p_unboundop); PyThread_release_lock(mutex); @@ -1919,7 +1922,7 @@ channel_recv(_channels *channels, int64_t cid, PyObject **res, int *p_unboundop) } // Convert the data back to an object. - PyObject *obj = _PyCrossInterpreterData_NewObject(data); + PyObject *obj = _PyXIData_NewObject(data); if (obj == NULL) { assert(PyErr_Occurred()); // It was allocated in channel_send(), so we free it. @@ -2047,7 +2050,7 @@ struct channel_info { int recv; } cur; } status; - Py_ssize_t count; + int64_t count; }; static int @@ -2061,7 +2064,7 @@ _channel_get_info(_channels *channels, int64_t cid, struct channel_info *info) if (interp == NULL) { return -1; } - Py_ssize_t interpid = PyInterpreterState_GetID(interp); + int64_t interpid = PyInterpreterState_GetID(interp); // Hold the global lock until we're done. PyThread_acquire_lock(channels->mutex, WAIT_LOCK); @@ -2545,10 +2548,9 @@ struct _channelid_xid { }; static PyObject * -_channelid_from_xid(_PyCrossInterpreterData *data) +_channelid_from_xid(_PyXIData_t *data) { - struct _channelid_xid *xid = \ - (struct _channelid_xid *)_PyCrossInterpreterData_DATA(data); + struct _channelid_xid *xid = (struct _channelid_xid *)_PyXIData_DATA(data); // It might not be imported yet, so we can't use _get_current_module(). PyObject *mod = PyImport_ImportModule(MODULE_NAME_STR); @@ -2594,18 +2596,16 @@ _channelid_from_xid(_PyCrossInterpreterData *data) } static int -_channelid_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_channelid_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { - if (_PyCrossInterpreterData_InitWithSize( + if (_PyXIData_InitWithSize( data, tstate->interp, sizeof(struct _channelid_xid), obj, _channelid_from_xid ) < 0) { return -1; } - struct _channelid_xid *xid = \ - (struct _channelid_xid *)_PyCrossInterpreterData_DATA(data); + struct _channelid_xid *xid = (struct _channelid_xid *)_PyXIData_DATA(data); xid->cid = ((channelid *)obj)->cid; xid->end = ((channelid *)obj)->end; xid->resolve = ((channelid *)obj)->resolve; @@ -2745,7 +2745,7 @@ _get_current_channelend_type(int end) } static PyObject * -_channelend_from_xid(_PyCrossInterpreterData *data) +_channelend_from_xid(_PyXIData_t *data) { channelid *cidobj = (channelid *)_channelid_from_xid(data); if (cidobj == NULL) { @@ -2762,8 +2762,7 @@ _channelend_from_xid(_PyCrossInterpreterData *data) } static int -_channelend_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_channelend_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { PyObject *cidobj = PyObject_GetAttrString(obj, "_id"); if (cidobj == NULL) { @@ -2774,7 +2773,7 @@ _channelend_shared(PyThreadState *tstate, PyObject *obj, if (res < 0) { return -1; } - _PyCrossInterpreterData_SET_NEW_OBJECT(data, _channelend_from_xid); + _PyXIData_SET_NEW_OBJECT(data, _channelend_from_xid); return 0; } diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 297a1763a98ce6..808938a9e8cd16 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -6,7 +6,7 @@ #endif #include "Python.h" -#include "pycore_crossinterp.h" // struct _xid +#include "pycore_crossinterp.h" // _PyXIData_t #define REGISTERS_HEAP_TYPES #define HAS_UNBOUND_ITEMS @@ -30,7 +30,7 @@ #define XID_FREE 2 static int -_release_xid_data(_PyCrossInterpreterData *data, int flags) +_release_xid_data(_PyXIData_t *data, int flags) { int ignoreexc = flags & XID_IGNORE_EXC; PyObject *exc; @@ -39,10 +39,10 @@ _release_xid_data(_PyCrossInterpreterData *data, int flags) } int res; if (flags & XID_FREE) { - res = _PyCrossInterpreterData_ReleaseAndRawFree(data); + res = _PyXIData_ReleaseAndRawFree(data); } else { - res = _PyCrossInterpreterData_Release(data); + res = _PyXIData_Release(data); } if (res < 0) { /* The owning interpreter is already destroyed. */ @@ -400,7 +400,7 @@ typedef struct _queueitem { This is necessary because item->data might be NULL, meaning the interpreter has been destroyed. */ int64_t interpid; - _PyCrossInterpreterData *data; + _PyXIData_t *data; int fmt; int unboundop; struct _queueitem *next; @@ -408,16 +408,15 @@ typedef struct _queueitem { static void _queueitem_init(_queueitem *item, - int64_t interpid, _PyCrossInterpreterData *data, - int fmt, int unboundop) + int64_t interpid, _PyXIData_t *data, int fmt, int unboundop) { if (interpid < 0) { interpid = _get_interpid(data); } else { assert(data == NULL - || _PyCrossInterpreterData_INTERPID(data) < 0 - || interpid == _PyCrossInterpreterData_INTERPID(data)); + || _PyXIData_INTERPID(data) < 0 + || interpid == _PyXIData_INTERPID(data)); } assert(check_unbound(unboundop)); *item = (_queueitem){ @@ -447,8 +446,7 @@ _queueitem_clear(_queueitem *item) } static _queueitem * -_queueitem_new(int64_t interpid, _PyCrossInterpreterData *data, - int fmt, int unboundop) +_queueitem_new(int64_t interpid, _PyXIData_t *data, int fmt, int unboundop) { _queueitem *item = GLOBAL_MALLOC(_queueitem); if (item == NULL) { @@ -478,7 +476,7 @@ _queueitem_free_all(_queueitem *item) static void _queueitem_popped(_queueitem *item, - _PyCrossInterpreterData **p_data, int *p_fmt, int *p_unboundop) + _PyXIData_t **p_data, int *p_fmt, int *p_unboundop) { *p_data = item->data; *p_fmt = item->fmt; @@ -498,7 +496,7 @@ _queueitem_clear_interpreter(_queueitem *item) assert(item->unboundop != UNBOUND_REMOVE); return 0; } - assert(_PyCrossInterpreterData_INTERPID(item->data) == item->interpid); + assert(_PyXIData_INTERPID(item->data) == item->interpid); switch (item->unboundop) { case UNBOUND_REMOVE: @@ -633,7 +631,7 @@ _queue_unlock(_queue *queue) } static int -_queue_add(_queue *queue, int64_t interpid, _PyCrossInterpreterData *data, +_queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data, int fmt, int unboundop) { int err = _queue_lock(queue); @@ -671,7 +669,7 @@ _queue_add(_queue *queue, int64_t interpid, _PyCrossInterpreterData *data, static int _queue_next(_queue *queue, - _PyCrossInterpreterData **p_data, int *p_fmt, int *p_unboundop) + _PyXIData_t **p_data, int *p_fmt, int *p_unboundop) { int err = _queue_lock(queue); if (err < 0) { @@ -1129,6 +1127,12 @@ queue_destroy(_queues *queues, int64_t qid) static int queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop) { + PyInterpreterState *interp = PyInterpreterState_Get(); + _PyXIData_lookup_context_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return -1; + } + // Look up the queue. _queue *queue = NULL; int err = _queues_lookup(queues, qid, &queue); @@ -1138,18 +1142,17 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop) assert(queue != NULL); // Convert the object to cross-interpreter data. - _PyCrossInterpreterData *data = GLOBAL_MALLOC(_PyCrossInterpreterData); + _PyXIData_t *data = GLOBAL_MALLOC(_PyXIData_t); if (data == NULL) { _queue_unmark_waiter(queue, queues->mutex); return -1; } - if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { + if (_PyObject_GetXIData(&ctx, obj, data) != 0) { _queue_unmark_waiter(queue, queues->mutex); GLOBAL_FREE(data); return -1; } - assert(_PyCrossInterpreterData_INTERPID(data) == \ - PyInterpreterState_GetID(PyInterpreterState_Get())); + assert(_PyXIData_INTERPID(data) == PyInterpreterState_GetID(interp)); // Add the data to the queue. int64_t interpid = -1; // _queueitem_init() will set it. @@ -1184,7 +1187,7 @@ queue_get(_queues *queues, int64_t qid, assert(queue != NULL); // Pop off the next item from the queue. - _PyCrossInterpreterData *data = NULL; + _PyXIData_t *data = NULL; err = _queue_next(queue, &data, p_fmt, p_unboundop); _queue_unmark_waiter(queue, queues->mutex); if (err != 0) { @@ -1196,7 +1199,7 @@ queue_get(_queues *queues, int64_t qid, } // Convert the data back to an object. - PyObject *obj = _PyCrossInterpreterData_NewObject(data); + PyObject *obj = _PyXIData_NewObject(data); if (obj == NULL) { assert(PyErr_Occurred()); // It was allocated in queue_put(), so we free it. @@ -1258,8 +1261,7 @@ queue_get_count(_queues *queues, int64_t qid, Py_ssize_t *p_count) /* external Queue objects ***************************************************/ -static int _queueobj_shared(PyThreadState *, - PyObject *, _PyCrossInterpreterData *); +static int _queueobj_shared(PyThreadState *, PyObject *, _PyXIData_t *); static int set_external_queue_type(module_state *state, PyTypeObject *queue_type) @@ -1339,9 +1341,9 @@ _queueid_xid_free(void *data) } static PyObject * -_queueobj_from_xid(_PyCrossInterpreterData *data) +_queueobj_from_xid(_PyXIData_t *data) { - int64_t qid = *(int64_t *)_PyCrossInterpreterData_DATA(data); + int64_t qid = *(int64_t *)_PyXIData_DATA(data); PyObject *qidobj = PyLong_FromLongLong(qid); if (qidobj == NULL) { return NULL; @@ -1367,8 +1369,7 @@ _queueobj_from_xid(_PyCrossInterpreterData *data) } static int -_queueobj_shared(PyThreadState *tstate, PyObject *queueobj, - _PyCrossInterpreterData *data) +_queueobj_shared(PyThreadState *tstate, PyObject *queueobj, _PyXIData_t *data) { PyObject *qidobj = PyObject_GetAttrString(queueobj, "_id"); if (qidobj == NULL) { @@ -1388,9 +1389,8 @@ _queueobj_shared(PyThreadState *tstate, PyObject *queueobj, if (raw == NULL) { return -1; } - _PyCrossInterpreterData_Init(data, tstate->interp, raw, NULL, - _queueobj_from_xid); - _PyCrossInterpreterData_SET_FREE(data, _queueid_xid_free); + _PyXIData_Init(data, tstate->interp, raw, NULL, _queueobj_from_xid); + _PyXIData_SET_FREE(data, _queueid_xid_free); return 0; } diff --git a/Modules/_interpreters_common.h b/Modules/_interpreters_common.h index 0d2e0c9efd3837..a6c639feea5d14 100644 --- a/Modules/_interpreters_common.h +++ b/Modules/_interpreters_common.h @@ -6,27 +6,36 @@ static int -ensure_xid_class(PyTypeObject *cls, crossinterpdatafunc getdata) +ensure_xid_class(PyTypeObject *cls, xidatafunc getdata) { - //assert(cls->tp_flags & Py_TPFLAGS_HEAPTYPE); - return _PyCrossInterpreterData_RegisterClass(cls, getdata); + PyInterpreterState *interp = PyInterpreterState_Get(); + _PyXIData_lookup_context_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return -1; + } + return _PyXIData_RegisterClass(&ctx, cls, getdata); } #ifdef REGISTERS_HEAP_TYPES static int clear_xid_class(PyTypeObject *cls) { - return _PyCrossInterpreterData_UnregisterClass(cls); + PyInterpreterState *interp = PyInterpreterState_Get(); + _PyXIData_lookup_context_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return -1; + } + return _PyXIData_UnregisterClass(&ctx, cls); } #endif static inline int64_t -_get_interpid(_PyCrossInterpreterData *data) +_get_interpid(_PyXIData_t *data) { int64_t interpid; if (data != NULL) { - interpid = _PyCrossInterpreterData_INTERPID(data); + interpid = _PyXIData_INTERPID(data); assert(!PyErr_Occurred()); } else { diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 6f3392fe6ea28d..a36823c4bb982b 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -7,7 +7,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() -#include "pycore_crossinterp.h" // struct _xid +#include "pycore_crossinterp.h" // _PyXIData_t #include "pycore_interp.h" // _PyInterpreterState_IDIncref() #include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() #include "pycore_modsupport.h" // _PyArg_BadArgument() @@ -84,18 +84,18 @@ typedef struct { } XIBufferViewObject; static PyObject * -xibufferview_from_xid(PyTypeObject *cls, _PyCrossInterpreterData *data) +xibufferview_from_xid(PyTypeObject *cls, _PyXIData_t *data) { - assert(_PyCrossInterpreterData_DATA(data) != NULL); - assert(_PyCrossInterpreterData_OBJ(data) == NULL); - assert(_PyCrossInterpreterData_INTERPID(data) >= 0); + assert(_PyXIData_DATA(data) != NULL); + assert(_PyXIData_OBJ(data) == NULL); + assert(_PyXIData_INTERPID(data) >= 0); XIBufferViewObject *self = PyObject_Malloc(sizeof(XIBufferViewObject)); if (self == NULL) { return NULL; } PyObject_Init((PyObject *)self, cls); - self->view = (Py_buffer *)_PyCrossInterpreterData_DATA(data); - self->interpid = _PyCrossInterpreterData_INTERPID(data); + self->view = (Py_buffer *)_PyXIData_DATA(data); + self->interpid = _PyXIData_INTERPID(data); return (PyObject *)self; } @@ -154,7 +154,7 @@ static PyType_Spec XIBufferViewType_spec = { static PyTypeObject * _get_current_xibufferview_type(void); static PyObject * -_memoryview_from_xid(_PyCrossInterpreterData *data) +_memoryview_from_xid(_PyXIData_t *data) { PyTypeObject *cls = _get_current_xibufferview_type(); if (cls == NULL) { @@ -168,8 +168,7 @@ _memoryview_from_xid(_PyCrossInterpreterData *data) } static int -_memoryview_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_memoryview_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { Py_buffer *view = PyMem_RawMalloc(sizeof(Py_buffer)); if (view == NULL) { @@ -179,8 +178,7 @@ _memoryview_shared(PyThreadState *tstate, PyObject *obj, PyMem_RawFree(view); return -1; } - _PyCrossInterpreterData_Init(data, tstate->interp, view, NULL, - _memoryview_from_xid); + _PyXIData_Init(data, tstate->interp, view, NULL, _memoryview_from_xid); return 0; } @@ -402,7 +400,11 @@ config_from_object(PyObject *configobj, PyInterpreterConfig *config) } } else if (PyUnicode_Check(configobj)) { - if (init_named_config(config, PyUnicode_AsUTF8(configobj)) < 0) { + const char *utf8name = PyUnicode_AsUTF8(configobj); + if (utf8name == NULL) { + return -1; + } + if (init_named_config(config, utf8name) < 0) { return -1; } } @@ -934,6 +936,11 @@ static int _interp_exec(PyObject *self, PyInterpreterState *interp, PyObject *code_arg, PyObject *shared_arg, PyObject **p_excinfo) { + if (shared_arg != NULL && !PyDict_CheckExact(shared_arg)) { + PyErr_SetString(PyExc_TypeError, "expected 'shared' to be a dict"); + return -1; + } + // Extract code. Py_ssize_t codestrlen = -1; PyObject *bytes_obj = NULL; @@ -1179,7 +1186,13 @@ object_is_shareable(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - if (_PyObject_CheckCrossInterpreterData(obj) == 0) { + PyInterpreterState *interp = PyInterpreterState_Get(); + _PyXIData_lookup_context_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return NULL; + } + + if (_PyObject_CheckXIData(&ctx, obj) == 0) { Py_RETURN_TRUE; } PyErr_Clear(); @@ -1478,6 +1491,11 @@ module_exec(PyObject *mod) PyInterpreterState *interp = PyInterpreterState_Get(); module_state *state = get_module_state(mod); + _PyXIData_lookup_context_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return -1; + } + #define ADD_WHENCE(NAME) \ if (PyModule_AddIntConstant(mod, "WHENCE_" #NAME, \ _PyInterpreterState_WHENCE_##NAME) < 0) \ @@ -1499,9 +1517,7 @@ module_exec(PyObject *mod) if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterNotFoundError) < 0) { goto error; } - PyObject *PyExc_NotShareableError = \ - _PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError; - if (PyModule_AddType(mod, (PyTypeObject *)PyExc_NotShareableError) < 0) { + if (PyModule_AddType(mod, (PyTypeObject *)ctx.PyExc_NotShareableError) < 0) { goto error; } diff --git a/Modules/_io/clinic/_iomodule.c.h b/Modules/_io/clinic/_iomodule.c.h index 112408a95df036..82932a23331ab6 100644 --- a/Modules/_io/clinic/_iomodule.c.h +++ b/Modules/_io/clinic/_iomodule.c.h @@ -175,7 +175,8 @@ _io_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw int closefd = 1; PyObject *opener = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 8, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 8, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -390,7 +391,8 @@ _io_open_code(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec PyObject *argsbuf[1]; PyObject *path; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -404,4 +406,4 @@ _io_open_code(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=5d60f4e778a600a4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ec1df2ff5265ab16 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index 708bef638887e2..b703d7e6855398 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -151,7 +151,8 @@ _io__BufferedIOBase_read(PyObject *self, PyTypeObject *cls, PyObject *const *arg PyObject *argsbuf[1]; int size = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -205,7 +206,8 @@ _io__BufferedIOBase_read1(PyObject *self, PyTypeObject *cls, PyObject *const *ar PyObject *argsbuf[1]; int size = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -262,7 +264,8 @@ _io__BufferedIOBase_write(PyObject *self, PyTypeObject *cls, PyObject *const *ar PyObject *argsbuf[1]; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -910,7 +913,8 @@ _io__Buffered_truncate(buffered *self, PyTypeObject *cls, PyObject *const *args, PyObject *argsbuf[1]; PyObject *pos = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -973,7 +977,8 @@ _io_BufferedReader___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *raw; Py_ssize_t buffer_size = DEFAULT_BUFFER_SIZE; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -1050,7 +1055,8 @@ _io_BufferedWriter___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *raw; Py_ssize_t buffer_size = DEFAULT_BUFFER_SIZE; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -1219,7 +1225,8 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *raw; Py_ssize_t buffer_size = DEFAULT_BUFFER_SIZE; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -1245,4 +1252,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=8eead000083dc5fa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=36abca5bd2f63ea1 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index 620e9e3b84ea19..98d88698c5e9b7 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -520,7 +520,8 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; PyObject *initvalue = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -534,4 +535,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=ef116925b8b9e535 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=985ff54e89f6036e input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 5b5487d63eba90..0b8b4a49ac24b6 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -94,7 +94,8 @@ _io_FileIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) int closefd = 1; PyObject *opener = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 4, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -241,7 +242,8 @@ _io_FileIO_readinto(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_s PyObject *argsbuf[1]; Py_buffer buffer = {NULL, NULL}; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -317,7 +319,8 @@ _io_FileIO_read(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize PyObject *argsbuf[1]; Py_ssize_t size = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -370,7 +373,8 @@ _io_FileIO_write(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssiz PyObject *argsbuf[1]; Py_buffer b = {NULL, NULL}; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -490,7 +494,8 @@ _io_FileIO_truncate(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_s PyObject *argsbuf[1]; PyObject *posobj = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -528,4 +533,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=e3d9446b4087020e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1c262ae135da4dcb input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index a35cac7dc0b8d7..402448545dfc51 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -56,7 +56,8 @@ _io__IOBase_seek(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ss int offset; int whence = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -132,7 +133,8 @@ _io__IOBase_truncate(PyObject *self, PyTypeObject *cls, PyObject *const *args, P PyObject *argsbuf[1]; PyObject *size = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -441,4 +443,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=dab5e9323d191e32 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9359e74d95534bef input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/stringio.c.h b/Modules/_io/clinic/stringio.c.h index 6bdb2181985f7d..ac8b4b7b85b76d 100644 --- a/Modules/_io/clinic/stringio.c.h +++ b/Modules/_io/clinic/stringio.c.h @@ -336,7 +336,8 @@ _io_StringIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *value = NULL; PyObject *newline_obj = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -555,4 +556,4 @@ _io_StringIO_newlines_get(stringio *self, void *Py_UNUSED(context)) return return_value; } -/*[clinic end generated code: output=9ffea20cd32d4cd8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8c8d4f8fa32986bb input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 669e2aa637ebbf..c9301c5a23fa86 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -70,7 +70,8 @@ _io__TextIOBase_read(PyObject *self, PyTypeObject *cls, PyObject *const *args, P PyObject *argsbuf[1]; int size = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -124,7 +125,8 @@ _io__TextIOBase_readline(PyObject *self, PyTypeObject *cls, PyObject *const *arg PyObject *argsbuf[1]; int size = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -178,7 +180,8 @@ _io__TextIOBase_write(PyObject *self, PyTypeObject *cls, PyObject *const *args, PyObject *argsbuf[1]; const char *s; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -339,7 +342,8 @@ _io_IncrementalNewlineDecoder___init__(PyObject *self, PyObject *args, PyObject int translate; PyObject *errors = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 3, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -405,7 +409,8 @@ _io_IncrementalNewlineDecoder_decode(nldecoder_object *self, PyObject *const *ar PyObject *input; int final = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -546,7 +551,8 @@ _io_TextIOWrapper___init__(PyObject *self, PyObject *args, PyObject *kwargs) int line_buffering = 0; int write_through = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 6, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 6, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -681,7 +687,8 @@ _io_TextIOWrapper_reconfigure(textio *self, PyObject *const *args, Py_ssize_t na PyObject *line_buffering_obj = Py_None; PyObject *write_through_obj = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1292,4 +1299,4 @@ _io_TextIOWrapper__CHUNK_SIZE_set(textio *self, PyObject *value, void *Py_UNUSED return return_value; } -/*[clinic end generated code: output=04cb7c67791a9ec1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=459c0e50acd772b1 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index 4696ecc5c843e6..df281dfeb13fef 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -93,7 +93,8 @@ _io__WindowsConsoleIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) int closefd = 1; PyObject *opener = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 4, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -239,7 +240,8 @@ _io__WindowsConsoleIO_readinto(winconsoleio *self, PyTypeObject *cls, PyObject * PyObject *argsbuf[1]; Py_buffer buffer = {NULL, NULL}; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -323,7 +325,8 @@ _io__WindowsConsoleIO_read(winconsoleio *self, PyTypeObject *cls, PyObject *cons PyObject *argsbuf[1]; Py_ssize_t size = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -380,7 +383,8 @@ _io__WindowsConsoleIO_write(winconsoleio *self, PyTypeObject *cls, PyObject *con PyObject *argsbuf[1]; Py_buffer b = {NULL, NULL}; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -457,4 +461,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=2c2bc86713b21dd6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=78e0f6abf4de2d6d input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index f374592eb95967..cf0f1d671b507a 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -131,6 +131,8 @@ internal_close(fileio *self) _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS } + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; if (err < 0) { errno = save_errno; PyErr_SetFromErrno(PyExc_OSError); @@ -268,8 +270,9 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, if (self->fd >= 0) { if (self->closefd) { /* Have to close the existing file first. */ - if (internal_close(self) < 0) + if (internal_close(self) < 0) { return -1; + } } else self->fd = -1; @@ -523,10 +526,8 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, internal_close(self); _PyErr_ChainExceptions1(exc); } - if (self->stat_atopen != NULL) { - PyMem_Free(self->stat_atopen); - self->stat_atopen = NULL; - } + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; done: #ifdef MS_WINDOWS diff --git a/Modules/_json.c b/Modules/_json.c index ce0093ab431d05..a99abbe72bf7a0 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -86,11 +86,11 @@ encoder_dealloc(PyObject *self); static int encoder_clear(PyEncoderObject *self); static int -encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *seq, PyObject *newline_indent); +encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *seq, Py_ssize_t indent_level, PyObject *indent_cache); static int -encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *obj, PyObject *newline_indent); +encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *obj, Py_ssize_t indent_level, PyObject *indent_cache); static int -encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *dct, PyObject *newline_indent); +encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *dct, Py_ssize_t indent_level, PyObject *indent_cache); static PyObject * _encoded_const(PyObject *obj); static void @@ -1252,17 +1252,92 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *)s; } + +/* indent_cache is a list that contains intermixed values at even and odd + * positions: + * + * 2*k : '\n' + indent * (k + initial_indent_level) + * strings written after opening and before closing brackets + * 2*k-1 : item_separator + '\n' + indent * (k + initial_indent_level) + * strings written between items + * + * Its size is always an odd number. + */ static PyObject * -_create_newline_indent(PyObject *indent, Py_ssize_t indent_level) +create_indent_cache(PyEncoderObject *s, Py_ssize_t indent_level) { PyObject *newline_indent = PyUnicode_FromOrdinal('\n'); if (newline_indent != NULL && indent_level) { PyUnicode_AppendAndDel(&newline_indent, - PySequence_Repeat(indent, indent_level)); + PySequence_Repeat(s->indent, indent_level)); + } + if (newline_indent == NULL) { + return NULL; + } + PyObject *indent_cache = PyList_New(1); + if (indent_cache == NULL) { + Py_DECREF(newline_indent); + return NULL; } - return newline_indent; + PyList_SET_ITEM(indent_cache, 0, newline_indent); + return indent_cache; +} + +/* Extend indent_cache by adding values for the next level. + * It should have values for the indent_level-1 level before the call. + */ +static int +update_indent_cache(PyEncoderObject *s, + Py_ssize_t indent_level, PyObject *indent_cache) +{ + assert(indent_level * 2 == PyList_GET_SIZE(indent_cache) + 1); + assert(indent_level > 0); + PyObject *newline_indent = PyList_GET_ITEM(indent_cache, (indent_level - 1)*2); + newline_indent = PyUnicode_Concat(newline_indent, s->indent); + if (newline_indent == NULL) { + return -1; + } + PyObject *separator_indent = PyUnicode_Concat(s->item_separator, newline_indent); + if (separator_indent == NULL) { + Py_DECREF(newline_indent); + return -1; + } + + if (PyList_Append(indent_cache, separator_indent) < 0 || + PyList_Append(indent_cache, newline_indent) < 0) + { + Py_DECREF(separator_indent); + Py_DECREF(newline_indent); + return -1; + } + Py_DECREF(separator_indent); + Py_DECREF(newline_indent); + return 0; } +static PyObject * +get_item_separator(PyEncoderObject *s, + Py_ssize_t indent_level, PyObject *indent_cache) +{ + assert(indent_level > 0); + if (indent_level * 2 > PyList_GET_SIZE(indent_cache)) { + if (update_indent_cache(s, indent_level, indent_cache) < 0) { + return NULL; + } + } + assert(indent_level * 2 < PyList_GET_SIZE(indent_cache)); + return PyList_GET_ITEM(indent_cache, indent_level * 2 - 1); +} + +static int +write_newline_indent(PyUnicodeWriter *writer, + Py_ssize_t indent_level, PyObject *indent_cache) +{ + PyObject *newline_indent = PyList_GET_ITEM(indent_cache, indent_level * 2); + return PyUnicodeWriter_WriteStr(writer, newline_indent); +} + + static PyObject * encoder_call(PyEncoderObject *self, PyObject *args, PyObject *kwds) { @@ -1280,20 +1355,20 @@ encoder_call(PyEncoderObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyObject *newline_indent = NULL; + PyObject *indent_cache = NULL; if (self->indent != Py_None) { - newline_indent = _create_newline_indent(self->indent, indent_level); - if (newline_indent == NULL) { + indent_cache = create_indent_cache(self, indent_level); + if (indent_cache == NULL) { PyUnicodeWriter_Discard(writer); return NULL; } } - if (encoder_listencode_obj(self, writer, obj, newline_indent)) { + if (encoder_listencode_obj(self, writer, obj, indent_level, indent_cache)) { PyUnicodeWriter_Discard(writer); - Py_XDECREF(newline_indent); + Py_XDECREF(indent_cache); return NULL; } - Py_XDECREF(newline_indent); + Py_XDECREF(indent_cache); PyObject *str = PyUnicodeWriter_Finish(writer); if (str == NULL) { @@ -1381,7 +1456,8 @@ _steal_accumulate(PyUnicodeWriter *writer, PyObject *stolen) static int encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, - PyObject *obj, PyObject *newline_indent) + PyObject *obj, + Py_ssize_t indent_level, PyObject *indent_cache) { /* Encode Python object obj to a JSON term */ PyObject *newobj; @@ -1421,14 +1497,14 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, else if (PyList_Check(obj) || PyTuple_Check(obj)) { if (_Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_list(s, writer, obj, newline_indent); + rv = encoder_listencode_list(s, writer, obj, indent_level, indent_cache); _Py_LeaveRecursiveCall(); return rv; } else if (PyDict_Check(obj)) { if (_Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_dict(s, writer, obj, newline_indent); + rv = encoder_listencode_dict(s, writer, obj, indent_level, indent_cache); _Py_LeaveRecursiveCall(); return rv; } @@ -1462,7 +1538,7 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, Py_XDECREF(ident); return -1; } - rv = encoder_listencode_obj(s, writer, newobj, newline_indent); + rv = encoder_listencode_obj(s, writer, newobj, indent_level, indent_cache); _Py_LeaveRecursiveCall(); Py_DECREF(newobj); @@ -1485,7 +1561,7 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, static int encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *first, PyObject *dct, PyObject *key, PyObject *value, - PyObject *newline_indent, + Py_ssize_t indent_level, PyObject *indent_cache, PyObject *item_separator) { PyObject *keystr = NULL; @@ -1541,7 +1617,7 @@ encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *firs if (PyUnicodeWriter_WriteStr(writer, s->key_separator) < 0) { return -1; } - if (encoder_listencode_obj(s, writer, value, newline_indent) < 0) { + if (encoder_listencode_obj(s, writer, value, indent_level, indent_cache) < 0) { _PyErr_FormatNote("when serializing %T item %R", dct, key); return -1; } @@ -1550,15 +1626,14 @@ encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *firs static int encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, - PyObject *dct, PyObject *newline_indent) + PyObject *dct, + Py_ssize_t indent_level, PyObject *indent_cache) { /* Encode Python dict dct a JSON term */ PyObject *ident = NULL; PyObject *items = NULL; PyObject *key, *value; bool first = true; - PyObject *new_newline_indent = NULL; - PyObject *separator_indent = NULL; if (PyDict_GET_SIZE(dct) == 0) { /* Fast path */ @@ -1585,19 +1660,13 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, goto bail; } - PyObject *current_item_separator = s->item_separator; // borrowed reference + PyObject *separator = s->item_separator; // borrowed reference if (s->indent != Py_None) { - new_newline_indent = PyUnicode_Concat(newline_indent, s->indent); - if (new_newline_indent == NULL) { - goto bail; - } - separator_indent = PyUnicode_Concat(current_item_separator, new_newline_indent); - if (separator_indent == NULL) { - goto bail; - } - // update item separator with a borrowed reference - current_item_separator = separator_indent; - if (PyUnicodeWriter_WriteStr(writer, new_newline_indent) < 0) { + indent_level++; + separator = get_item_separator(s, indent_level, indent_cache); + if (separator == NULL || + write_newline_indent(writer, indent_level, indent_cache) < 0) + { goto bail; } } @@ -1618,8 +1687,8 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, key = PyTuple_GET_ITEM(item, 0); value = PyTuple_GET_ITEM(item, 1); if (encoder_encode_key_value(s, writer, &first, dct, key, value, - new_newline_indent, - current_item_separator) < 0) + indent_level, indent_cache, + separator) < 0) goto bail; } Py_CLEAR(items); @@ -1628,8 +1697,8 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, Py_ssize_t pos = 0; while (PyDict_Next(dct, &pos, &key, &value)) { if (encoder_encode_key_value(s, writer, &first, dct, key, value, - new_newline_indent, - current_item_separator) < 0) + indent_level, indent_cache, + separator) < 0) goto bail; } } @@ -1640,10 +1709,8 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, Py_CLEAR(ident); } if (s->indent != Py_None) { - Py_CLEAR(new_newline_indent); - Py_CLEAR(separator_indent); - - if (PyUnicodeWriter_WriteStr(writer, newline_indent) < 0) { + indent_level--; + if (write_newline_indent(writer, indent_level, indent_cache) < 0) { goto bail; } } @@ -1656,20 +1723,17 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, bail: Py_XDECREF(items); Py_XDECREF(ident); - Py_XDECREF(separator_indent); - Py_XDECREF(new_newline_indent); return -1; } static int encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, - PyObject *seq, PyObject *newline_indent) + PyObject *seq, + Py_ssize_t indent_level, PyObject *indent_cache) { PyObject *ident = NULL; PyObject *s_fast = NULL; Py_ssize_t i; - PyObject *new_newline_indent = NULL; - PyObject *separator_indent = NULL; ident = NULL; s_fast = PySequence_Fast(seq, "_iterencode_list needs a sequence"); @@ -1702,20 +1766,13 @@ encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *separator = s->item_separator; // borrowed reference if (s->indent != Py_None) { - new_newline_indent = PyUnicode_Concat(newline_indent, s->indent); - if (new_newline_indent == NULL) { - goto bail; - } - - if (PyUnicodeWriter_WriteStr(writer, new_newline_indent) < 0) { - goto bail; - } - - separator_indent = PyUnicode_Concat(separator, new_newline_indent); - if (separator_indent == NULL) { + indent_level++; + separator = get_item_separator(s, indent_level, indent_cache); + if (separator == NULL || + write_newline_indent(writer, indent_level, indent_cache) < 0) + { goto bail; } - separator = separator_indent; // assign separator with borrowed reference } for (i = 0; i < PySequence_Fast_GET_SIZE(s_fast); i++) { PyObject *obj = PySequence_Fast_GET_ITEM(s_fast, i); @@ -1723,7 +1780,7 @@ encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, if (PyUnicodeWriter_WriteStr(writer, separator) < 0) goto bail; } - if (encoder_listencode_obj(s, writer, obj, new_newline_indent)) { + if (encoder_listencode_obj(s, writer, obj, indent_level, indent_cache)) { _PyErr_FormatNote("when serializing %T item %zd", seq, i); goto bail; } @@ -1735,9 +1792,8 @@ encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, } if (s->indent != Py_None) { - Py_CLEAR(new_newline_indent); - Py_CLEAR(separator_indent); - if (PyUnicodeWriter_WriteStr(writer, newline_indent) < 0) { + indent_level--; + if (write_newline_indent(writer, indent_level, indent_cache) < 0) { goto bail; } } @@ -1751,8 +1807,6 @@ encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, bail: Py_XDECREF(ident); Py_DECREF(s_fast); - Py_XDECREF(separator_indent); - Py_XDECREF(new_newline_indent); return -1; } diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 8b6906234bdc25..51ad9fc7da8492 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -606,17 +606,42 @@ setBuiltins(ProfilerObject *pObj, int nvalue) return 0; } -PyObject* pystart_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +/*[clinic input] +_lsprof.Profiler._pystart_callback + + code: object + instruction_offset: object + / + +[clinic start generated code]*/ + +static PyObject * +_lsprof_Profiler__pystart_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset) +/*[clinic end generated code: output=5fec8b7ad5ed25e8 input=b166e6953c579cda]*/ { - PyObject* code = args[0]; - ptrace_enter_call((PyObject*)self, (void *)code, (PyObject *)code); + ptrace_enter_call((PyObject*)self, (void *)code, code); Py_RETURN_NONE; } -PyObject* pyreturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +/*[clinic input] +_lsprof.Profiler._pyreturn_callback + + code: object + instruction_offset: object + retval: object + / + +[clinic start generated code]*/ + +static PyObject * +_lsprof_Profiler__pyreturn_callback_impl(ProfilerObject *self, + PyObject *code, + PyObject *instruction_offset, + PyObject *retval) +/*[clinic end generated code: output=9e2f6fc1b882c51e input=667ffaeb2fa6fd1f]*/ { - PyObject* code = args[0]; ptrace_leave_call((PyObject*)self, (void *)code); Py_RETURN_NONE; @@ -649,12 +674,24 @@ PyObject* get_cfunc_from_callable(PyObject* callable, PyObject* self_arg, PyObje return NULL; } -PyObject* ccall_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +/*[clinic input] +_lsprof.Profiler._ccall_callback + + code: object + instruction_offset: object + callable: object + self_arg: object + / + +[clinic start generated code]*/ + +static PyObject * +_lsprof_Profiler__ccall_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset, + PyObject *callable, PyObject *self_arg) +/*[clinic end generated code: output=152db83cabd18cad input=0e66687cfb95c001]*/ { if (self->flags & POF_BUILTINS) { - PyObject* callable = args[2]; - PyObject* self_arg = args[3]; - PyObject* cfunc = get_cfunc_from_callable(callable, self_arg, self->missing); if (cfunc) { @@ -667,12 +704,25 @@ PyObject* ccall_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t Py_RETURN_NONE; } -PyObject* creturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +/*[clinic input] +_lsprof.Profiler._creturn_callback + + code: object + instruction_offset: object + callable: object + self_arg: object + / + +[clinic start generated code]*/ + +static PyObject * +_lsprof_Profiler__creturn_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset, + PyObject *callable, + PyObject *self_arg) +/*[clinic end generated code: output=1e886dde8fed8fb0 input=b18afe023746923a]*/ { if (self->flags & POF_BUILTINS) { - PyObject* callable = args[2]; - PyObject* self_arg = args[3]; - PyObject* cfunc = get_cfunc_from_callable(callable, self_arg, self->missing); if (cfunc) { @@ -700,27 +750,27 @@ static const struct { {0, NULL} }; -PyDoc_STRVAR(enable_doc, "\ -enable(subcalls=True, builtins=True)\n\ -\n\ -Start collecting profiling information.\n\ -If 'subcalls' is True, also records for each function\n\ -statistics separated according to its current caller.\n\ -If 'builtins' is True, records the time spent in\n\ -built-in functions separately from their caller.\n\ -"); - -static PyObject* -profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds) + +/*[clinic input] +_lsprof.Profiler.enable + + subcalls: bool = True + If True, also records for each function + statistics separated according to its current caller. + + builtins: bool = True + If True, records the time spent in + built-in functions separately from their caller. + +Start collecting profiling information. +[clinic start generated code]*/ + +static PyObject * +_lsprof_Profiler_enable_impl(ProfilerObject *self, int subcalls, + int builtins) +/*[clinic end generated code: output=1e747f9dc1edd571 input=9ab81405107ab7f1]*/ { - int subcalls = -1; - int builtins = -1; - static char *kwlist[] = {"subcalls", "builtins", 0}; int all_events = 0; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|pp:enable", - kwlist, &subcalls, &builtins)) - return NULL; if (setSubcalls(self, subcalls) < 0 || setBuiltins(self, builtins) < 0) { return NULL; } @@ -730,34 +780,47 @@ profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds) return NULL; } - if (PyObject_CallMethod(monitoring, "use_tool_id", "is", self->tool_id, "cProfile") == NULL) { + PyObject *check = PyObject_CallMethod(monitoring, + "use_tool_id", "is", + self->tool_id, "cProfile"); + if (check == NULL) { PyErr_Format(PyExc_ValueError, "Another profiling tool is already active"); - Py_DECREF(monitoring); - return NULL; + goto error; } + Py_DECREF(check); for (int i = 0; callback_table[i].callback_method; i++) { + int event = (1 << callback_table[i].event); PyObject* callback = PyObject_GetAttrString((PyObject*)self, callback_table[i].callback_method); if (!callback) { - Py_DECREF(monitoring); - return NULL; + goto error; } - Py_XDECREF(PyObject_CallMethod(monitoring, "register_callback", "iiO", self->tool_id, - (1 << callback_table[i].event), - callback)); + PyObject *register_result = PyObject_CallMethod(monitoring, "register_callback", + "iiO", self->tool_id, + event, callback); Py_DECREF(callback); - all_events |= (1 << callback_table[i].event); + if (register_result == NULL) { + goto error; + } + Py_DECREF(register_result); + all_events |= event; } - if (!PyObject_CallMethod(monitoring, "set_events", "ii", self->tool_id, all_events)) { - Py_DECREF(monitoring); - return NULL; + PyObject *event_result = PyObject_CallMethod(monitoring, "set_events", "ii", + self->tool_id, all_events); + if (event_result == NULL) { + goto error; } + Py_DECREF(event_result); Py_DECREF(monitoring); self->flags |= POF_ENABLED; Py_RETURN_NONE; + +error: + Py_DECREF(monitoring); + return NULL; } static void @@ -776,14 +839,16 @@ flush_unmatched(ProfilerObject *pObj) } -PyDoc_STRVAR(disable_doc, "\ -disable()\n\ -\n\ -Stop collecting profiling information.\n\ -"); -static PyObject* -profiler_disable(ProfilerObject *self, PyObject* noarg) +/*[clinic input] +_lsprof.Profiler.disable + +Stop collecting profiling information. +[clinic start generated code]*/ + +static PyObject * +_lsprof_Profiler_disable_impl(ProfilerObject *self) +/*[clinic end generated code: output=838cffef7f651870 input=05700b3fc68d1f50]*/ { if (self->flags & POF_EXT_TIMER) { PyErr_SetString(PyExc_RuntimeError, @@ -834,21 +899,22 @@ profiler_disable(ProfilerObject *self, PyObject* noarg) Py_RETURN_NONE; } -PyDoc_STRVAR(clear_doc, "\ -clear()\n\ -\n\ -Clear all profiling information collected so far.\n\ -"); +/*[clinic input] +_lsprof.Profiler.clear + +Clear all profiling information collected so far. +[clinic start generated code]*/ -static PyObject* -profiler_clear(ProfilerObject *pObj, PyObject* noarg) +static PyObject * +_lsprof_Profiler_clear_impl(ProfilerObject *self) +/*[clinic end generated code: output=dd1c668fb84b1335 input=fbe1f88c28be4f98]*/ { - if (pObj->flags & POF_EXT_TIMER) { + if (self->flags & POF_EXT_TIMER) { PyErr_SetString(PyExc_RuntimeError, "cannot clear profiler in external timer"); return NULL; } - clearEntries(pObj); + clearEntries(self); Py_RETURN_NONE; } @@ -879,33 +945,40 @@ profiler_dealloc(ProfilerObject *op) Py_DECREF(tp); } +/*[clinic input] +_lsprof.Profiler.__init__ as profiler_init + + timer: object(c_default='NULL') = None + timeunit: double = 0.0 + subcalls: bool = True + builtins: bool = True + +Build a profiler object using the specified timer function. + +The default timer is a fast built-in one based on real time. +For custom timer functions returning integers, 'timeunit' can +be a float specifying a scale (that is, how long each integer unit +is, in seconds). +[clinic start generated code]*/ + static int -profiler_init(ProfilerObject *pObj, PyObject *args, PyObject *kw) +profiler_init_impl(ProfilerObject *self, PyObject *timer, double timeunit, + int subcalls, int builtins) +/*[clinic end generated code: output=ac523803ec9f9df2 input=8285ca746f96a414]*/ { - PyObject *timer = NULL; - double timeunit = 0.0; - int subcalls = 1; - int builtins = 1; - static char *kwlist[] = {"timer", "timeunit", - "subcalls", "builtins", 0}; - - if (!PyArg_ParseTupleAndKeywords(args, kw, "|Odpp:Profiler", kwlist, - &timer, &timeunit, - &subcalls, &builtins)) - return -1; - - if (setSubcalls(pObj, subcalls) < 0 || setBuiltins(pObj, builtins) < 0) + if (setSubcalls(self, subcalls) < 0 || setBuiltins(self, builtins) < 0) { return -1; - pObj->externalTimerUnit = timeunit; - Py_XSETREF(pObj->externalTimer, Py_XNewRef(timer)); - pObj->tool_id = PY_MONITORING_PROFILER_ID; + } + self->externalTimerUnit = timeunit; + Py_XSETREF(self->externalTimer, Py_XNewRef(timer)); + self->tool_id = PY_MONITORING_PROFILER_ID; PyObject* monitoring = _PyImport_GetModuleAttrString("sys", "monitoring"); if (!monitoring) { return -1; } - pObj->missing = PyObject_GetAttrString(monitoring, "MISSING"); - if (!pObj->missing) { + self->missing = PyObject_GetAttrString(monitoring, "MISSING"); + if (!self->missing) { Py_DECREF(monitoring); return -1; } @@ -915,35 +988,18 @@ profiler_init(ProfilerObject *pObj, PyObject *args, PyObject *kw) static PyMethodDef profiler_methods[] = { _LSPROF_PROFILER_GETSTATS_METHODDEF - {"enable", _PyCFunction_CAST(profiler_enable), - METH_VARARGS | METH_KEYWORDS, enable_doc}, - {"disable", (PyCFunction)profiler_disable, - METH_NOARGS, disable_doc}, - {"clear", (PyCFunction)profiler_clear, - METH_NOARGS, clear_doc}, - {"_pystart_callback", _PyCFunction_CAST(pystart_callback), - METH_FASTCALL, NULL}, - {"_pyreturn_callback", _PyCFunction_CAST(pyreturn_callback), - METH_FASTCALL, NULL}, - {"_ccall_callback", _PyCFunction_CAST(ccall_callback), - METH_FASTCALL, NULL}, - {"_creturn_callback", _PyCFunction_CAST(creturn_callback), - METH_FASTCALL, NULL}, + _LSPROF_PROFILER_ENABLE_METHODDEF + _LSPROF_PROFILER_DISABLE_METHODDEF + _LSPROF_PROFILER_CLEAR_METHODDEF + _LSPROF_PROFILER__PYSTART_CALLBACK_METHODDEF + _LSPROF_PROFILER__PYRETURN_CALLBACK_METHODDEF + _LSPROF_PROFILER__CCALL_CALLBACK_METHODDEF + _LSPROF_PROFILER__CRETURN_CALLBACK_METHODDEF {NULL, NULL} }; -PyDoc_STRVAR(profiler_doc, "\ -Profiler(timer=None, timeunit=None, subcalls=True, builtins=True)\n\ -\n\ - Builds a profiler object using the specified timer function.\n\ - The default timer is a fast built-in one based on real time.\n\ - For custom timer functions returning integers, timeunit can\n\ - be a float specifying a scale (i.e. how long each integer unit\n\ - is, in seconds).\n\ -"); - static PyType_Slot _lsprof_profiler_type_spec_slots[] = { - {Py_tp_doc, (void *)profiler_doc}, + {Py_tp_doc, (void *)profiler_init__doc__}, {Py_tp_methods, profiler_methods}, {Py_tp_dealloc, profiler_dealloc}, {Py_tp_init, profiler_init}, diff --git a/Modules/_multiprocessing/clinic/semaphore.c.h b/Modules/_multiprocessing/clinic/semaphore.c.h index 512e5a016192fb..2702c3369c76ed 100644 --- a/Modules/_multiprocessing/clinic/semaphore.c.h +++ b/Modules/_multiprocessing/clinic/semaphore.c.h @@ -58,7 +58,8 @@ _multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ int blocking = 1; PyObject *timeout_obj = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -163,7 +164,8 @@ _multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ int blocking = 1; PyObject *timeout_obj = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -263,7 +265,8 @@ _multiprocessing_SemLock(PyTypeObject *type, PyObject *args, PyObject *kwargs) const char *name; int unlink; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 5, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 5, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -573,4 +576,4 @@ _multiprocessing_SemLock___exit__(SemLockObject *self, PyObject *const *args, Py #ifndef _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #define _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #endif /* !defined(_MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF) */ -/*[clinic end generated code: output=dea36482d23a355f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9023d3e48a24afd2 input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/semaphore.c b/Modules/_multiprocessing/semaphore.c index 4de4ee6c78fbd1..9eef7c25636899 100644 --- a/Modules/_multiprocessing/semaphore.c +++ b/Modules/_multiprocessing/semaphore.c @@ -15,6 +15,7 @@ #ifdef HAVE_MP_SEMAPHORE +// These match the values in Lib/multiprocessing/synchronize.py enum { RECURSIVE_MUTEX, SEMAPHORE }; typedef struct { diff --git a/Modules/_opcode.c b/Modules/_opcode.c index dc93063aee7e54..7ccf7af6bf908f 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -422,6 +422,9 @@ _opcode_exec(PyObject *m) { if (PyModule_AddIntMacro(m, ENABLE_SPECIALIZATION) < 0) { return -1; } + if (PyModule_AddIntMacro(m, ENABLE_SPECIALIZATION_FT) < 0) { + return -1; + } return 0; } diff --git a/Modules/_pickle.c b/Modules/_pickle.c index b2bd9545c1b130..5837cd41a40cd4 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -613,6 +613,7 @@ typedef struct PicklerObject { objects to support self-referential objects pickling. */ PyObject *persistent_id; /* persistent_id() method, can be NULL */ + PyObject *persistent_id_attr; /* instance attribute, can be NULL */ PyObject *dispatch_table; /* private dispatch_table, can be NULL */ PyObject *reducer_override; /* hook for invoking user-defined callbacks instead of save_global when pickling @@ -655,6 +656,7 @@ typedef struct UnpicklerObject { size_t memo_len; /* Number of objects in the memo */ PyObject *persistent_load; /* persistent_load() method, can be NULL. */ + PyObject *persistent_load_attr; /* instance attribute, can be NULL. */ Py_buffer buffer; char *input_buffer; @@ -1108,6 +1110,7 @@ _Pickler_New(PickleState *st) self->memo = memo; self->persistent_id = NULL; + self->persistent_id_attr = NULL; self->dispatch_table = NULL; self->reducer_override = NULL; self->write = NULL; @@ -1288,6 +1291,10 @@ _Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n) else { read_size = _Unpickler_SetStringInput(self, data); Py_DECREF(data); + if (read_size < 0) { + return -1; + } + self->prefetched_idx = 0; if (n <= read_size) return n; @@ -1602,6 +1609,7 @@ _Unpickler_New(PyObject *module) self->memo_size = MEMO_SIZE; self->memo_len = 0; self->persistent_load = NULL; + self->persistent_load_attr = NULL; memset(&self->buffer, 0, sizeof(Py_buffer)); self->input_buffer = NULL; self->input_line = NULL; @@ -5088,6 +5096,33 @@ Pickler_set_memo(PicklerObject *self, PyObject *obj, void *Py_UNUSED(ignored)) return -1; } +static PyObject * +Pickler_getattr(PyObject *self, PyObject *name) +{ + if (PyUnicode_Check(name) + && PyUnicode_EqualToUTF8(name, "persistent_id") + && ((PicklerObject *)self)->persistent_id_attr) + { + return Py_NewRef(((PicklerObject *)self)->persistent_id_attr); + } + + return PyObject_GenericGetAttr(self, name); +} + +static int +Pickler_setattr(PyObject *self, PyObject *name, PyObject *value) +{ + if (PyUnicode_Check(name) + && PyUnicode_EqualToUTF8(name, "persistent_id")) + { + Py_XINCREF(value); + Py_XSETREF(((PicklerObject *)self)->persistent_id_attr, value); + return 0; + } + + return PyObject_GenericSetAttr(self, name, value); +} + static PyMemberDef Pickler_members[] = { {"bin", Py_T_INT, offsetof(PicklerObject, bin)}, {"fast", Py_T_INT, offsetof(PicklerObject, fast)}, @@ -5103,6 +5138,8 @@ static PyGetSetDef Pickler_getsets[] = { static PyType_Slot pickler_type_slots[] = { {Py_tp_dealloc, Pickler_dealloc}, + {Py_tp_getattro, Pickler_getattr}, + {Py_tp_setattro, Pickler_setattr}, {Py_tp_methods, Pickler_methods}, {Py_tp_members, Pickler_members}, {Py_tp_getset, Pickler_getsets}, @@ -7562,6 +7599,33 @@ Unpickler_set_memo(UnpicklerObject *self, PyObject *obj, void *Py_UNUSED(ignored return -1; } +static PyObject * +Unpickler_getattr(PyObject *self, PyObject *name) +{ + if (PyUnicode_Check(name) + && PyUnicode_EqualToUTF8(name, "persistent_load") + && ((UnpicklerObject *)self)->persistent_load_attr) + { + return Py_NewRef(((UnpicklerObject *)self)->persistent_load_attr); + } + + return PyObject_GenericGetAttr(self, name); +} + +static int +Unpickler_setattr(PyObject *self, PyObject *name, PyObject *value) +{ + if (PyUnicode_Check(name) + && PyUnicode_EqualToUTF8(name, "persistent_load")) + { + Py_XINCREF(value); + Py_XSETREF(((UnpicklerObject *)self)->persistent_load_attr, value); + return 0; + } + + return PyObject_GenericSetAttr(self, name, value); +} + static PyGetSetDef Unpickler_getsets[] = { {"memo", (getter)Unpickler_get_memo, (setter)Unpickler_set_memo}, {NULL} @@ -7570,6 +7634,8 @@ static PyGetSetDef Unpickler_getsets[] = { static PyType_Slot unpickler_type_slots[] = { {Py_tp_dealloc, Unpickler_dealloc}, {Py_tp_doc, (char *)_pickle_Unpickler___init____doc__}, + {Py_tp_getattro, Unpickler_getattr}, + {Py_tp_setattro, Unpickler_setattr}, {Py_tp_traverse, Unpickler_traverse}, {Py_tp_clear, Unpickler_clear}, {Py_tp_methods, Unpickler_methods}, diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index 6cb610a12b59c6..42eb6eb2f12554 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -81,7 +81,8 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } } - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 8, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 8, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -213,7 +214,8 @@ pysqlite_connection_cursor(pysqlite_Connection *self, PyObject *const *args, Py_ Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *factory = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -289,7 +291,8 @@ blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyO int readonly = 0; const char *name = "main"; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -481,7 +484,8 @@ pysqlite_connection_create_function(pysqlite_Connection *self, PyTypeObject *cls PyObject *func; int deterministic = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -574,7 +578,8 @@ create_window_function(pysqlite_Connection *self, PyTypeObject *cls, PyObject *c int num_params; PyObject *aggregate_class; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -670,7 +675,8 @@ pysqlite_connection_create_aggregate(pysqlite_Connection *self, PyTypeObject *cl int n_arg; PyObject *aggregate_class; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -770,7 +776,8 @@ pysqlite_connection_set_authorizer(pysqlite_Connection *self, PyTypeObject *cls, PyObject *argsbuf[1]; PyObject *callable; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -864,7 +871,8 @@ pysqlite_connection_set_progress_handler(pysqlite_Connection *self, PyTypeObject PyObject *callable; int n; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -951,7 +959,8 @@ pysqlite_connection_set_trace_callback(pysqlite_Connection *self, PyTypeObject * PyObject *argsbuf[1]; PyObject *callable; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1055,7 +1064,8 @@ pysqlite_connection_load_extension(pysqlite_Connection *self, PyObject *const *a const char *extension_name; const char *entrypoint = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1252,7 +1262,8 @@ pysqlite_connection_iterdump(pysqlite_Connection *self, PyObject *const *args, P Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *filter = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1320,7 +1331,8 @@ pysqlite_connection_backup(pysqlite_Connection *self, PyObject *const *args, Py_ const char *name = "main"; double sleep = 0.25; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1418,7 +1430,8 @@ pysqlite_connection_create_collation(pysqlite_Connection *self, PyTypeObject *cl const char *name; PyObject *callable; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1497,7 +1510,8 @@ serialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, Py Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; const char *name = "main"; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1587,7 +1601,8 @@ deserialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, Py_buffer data = {NULL, NULL}; const char *name = "main"; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1866,4 +1881,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=5d4d7e4abe17b164 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a8fd19301c7390cc input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/cursor.c.h b/Modules/_sqlite/clinic/cursor.c.h index a13e0d0745b58d..ca7823cf5aef5b 100644 --- a/Modules/_sqlite/clinic/cursor.c.h +++ b/Modules/_sqlite/clinic/cursor.c.h @@ -216,7 +216,8 @@ pysqlite_cursor_fetchmany(pysqlite_Cursor *self, PyObject *const *args, Py_ssize Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int maxrows = self->arraysize; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -313,4 +314,4 @@ pysqlite_cursor_close(pysqlite_Cursor *self, PyObject *Py_UNUSED(ignored)) { return pysqlite_cursor_close_impl(self); } -/*[clinic end generated code: output=a8ce095c3c80cf65 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f0804afc5f8646c1 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/module.c.h b/Modules/_sqlite/clinic/module.c.h index 529dc4e281e0eb..5569c45451647f 100644 --- a/Modules/_sqlite/clinic/module.c.h +++ b/Modules/_sqlite/clinic/module.c.h @@ -52,7 +52,8 @@ pysqlite_complete_statement(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *argsbuf[1]; const char *statement; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -208,4 +209,4 @@ pysqlite_adapt(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=457ab0fdbb9e1880 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=db9bf7ecad197343 input=a9049054013a1b77]*/ diff --git a/Modules/_sre/clinic/sre.c.h b/Modules/_sre/clinic/sre.c.h index 48336c7a2fca26..87e4785a428468 100644 --- a/Modules/_sre/clinic/sre.c.h +++ b/Modules/_sre/clinic/sre.c.h @@ -213,7 +213,8 @@ _sre_SRE_Pattern_match(PatternObject *self, PyTypeObject *cls, PyObject *const * Py_ssize_t pos = 0; Py_ssize_t endpos = PY_SSIZE_T_MAX; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -306,7 +307,8 @@ _sre_SRE_Pattern_fullmatch(PatternObject *self, PyTypeObject *cls, PyObject *con Py_ssize_t pos = 0; Py_ssize_t endpos = PY_SSIZE_T_MAX; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -401,7 +403,8 @@ _sre_SRE_Pattern_search(PatternObject *self, PyTypeObject *cls, PyObject *const Py_ssize_t pos = 0; Py_ssize_t endpos = PY_SSIZE_T_MAX; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -493,7 +496,8 @@ _sre_SRE_Pattern_findall(PatternObject *self, PyObject *const *args, Py_ssize_t Py_ssize_t pos = 0; Py_ssize_t endpos = PY_SSIZE_T_MAX; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -588,7 +592,8 @@ _sre_SRE_Pattern_finditer(PatternObject *self, PyTypeObject *cls, PyObject *cons Py_ssize_t pos = 0; Py_ssize_t endpos = PY_SSIZE_T_MAX; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -680,7 +685,8 @@ _sre_SRE_Pattern_scanner(PatternObject *self, PyTypeObject *cls, PyObject *const Py_ssize_t pos = 0; Py_ssize_t endpos = PY_SSIZE_T_MAX; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -771,7 +777,8 @@ _sre_SRE_Pattern_split(PatternObject *self, PyObject *const *args, Py_ssize_t na PyObject *string; Py_ssize_t maxsplit = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -846,7 +853,8 @@ _sre_SRE_Pattern_sub(PatternObject *self, PyTypeObject *cls, PyObject *const *ar PyObject *string; Py_ssize_t count = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -923,7 +931,8 @@ _sre_SRE_Pattern_subn(PatternObject *self, PyTypeObject *cls, PyObject *const *a PyObject *string; Py_ssize_t count = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -976,6 +985,44 @@ PyDoc_STRVAR(_sre_SRE_Pattern___deepcopy____doc__, #define _SRE_SRE_PATTERN___DEEPCOPY___METHODDEF \ {"__deepcopy__", (PyCFunction)_sre_SRE_Pattern___deepcopy__, METH_O, _sre_SRE_Pattern___deepcopy____doc__}, +#if defined(Py_DEBUG) + +PyDoc_STRVAR(_sre_SRE_Pattern__fail_after__doc__, +"_fail_after($self, count, exception, /)\n" +"--\n" +"\n" +"For debugging."); + +#define _SRE_SRE_PATTERN__FAIL_AFTER_METHODDEF \ + {"_fail_after", _PyCFunction_CAST(_sre_SRE_Pattern__fail_after), METH_FASTCALL, _sre_SRE_Pattern__fail_after__doc__}, + +static PyObject * +_sre_SRE_Pattern__fail_after_impl(PatternObject *self, int count, + PyObject *exception); + +static PyObject * +_sre_SRE_Pattern__fail_after(PatternObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int count; + PyObject *exception; + + if (!_PyArg_CheckPositional("_fail_after", nargs, 2, 2)) { + goto exit; + } + count = PyLong_AsInt(args[0]); + if (count == -1 && PyErr_Occurred()) { + goto exit; + } + exception = args[1]; + return_value = _sre_SRE_Pattern__fail_after_impl(self, count, exception); + +exit: + return return_value; +} + +#endif /* defined(Py_DEBUG) */ + PyDoc_STRVAR(_sre_compile__doc__, "compile($module, /, pattern, flags, code, groups, groupindex,\n" " indexgroup)\n" @@ -1027,7 +1074,8 @@ _sre_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *groupindex; PyObject *indexgroup; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 6, 6, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 6, /*maxpos*/ 6, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1152,7 +1200,8 @@ _sre_SRE_Match_expand(MatchObject *self, PyObject *const *args, Py_ssize_t nargs PyObject *argsbuf[1]; PyObject *template; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1211,7 +1260,8 @@ _sre_SRE_Match_groups(MatchObject *self, PyObject *const *args, Py_ssize_t nargs Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *default_value = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1274,7 +1324,8 @@ _sre_SRE_Match_groupdict(MatchObject *self, PyObject *const *args, Py_ssize_t na Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *default_value = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1461,4 +1512,8 @@ _sre_SRE_Scanner_search(ScannerObject *self, PyTypeObject *cls, PyObject *const } return _sre_SRE_Scanner_search_impl(self, cls); } -/*[clinic end generated code: output=c3e711f0b2f43d66 input=a9049054013a1b77]*/ + +#ifndef _SRE_SRE_PATTERN__FAIL_AFTER_METHODDEF + #define _SRE_SRE_PATTERN__FAIL_AFTER_METHODDEF +#endif /* !defined(_SRE_SRE_PATTERN__FAIL_AFTER_METHODDEF) */ +/*[clinic end generated code: output=f8cb77f2261f0b2e input=a9049054013a1b77]*/ diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 2c86f8869d8e58..36f542ddb4df2b 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -267,6 +267,85 @@ data_stack_grow(SRE_STATE* state, Py_ssize_t size) return 0; } +/* memory pool functions for SRE_REPEAT, this can avoid memory + leak when SRE(match) function terminates abruptly. + state->repeat_pool_used is a doubly-linked list, so that we + can remove a SRE_REPEAT node from it. + state->repeat_pool_unused is a singly-linked list, we put/get + node at the head. */ +static SRE_REPEAT * +repeat_pool_malloc(SRE_STATE *state) +{ + SRE_REPEAT *repeat; + + if (state->repeat_pool_unused) { + /* remove from unused pool (singly-linked list) */ + repeat = state->repeat_pool_unused; + state->repeat_pool_unused = repeat->pool_next; + } + else { + repeat = PyMem_Malloc(sizeof(SRE_REPEAT)); + if (!repeat) { + return NULL; + } + } + + /* add to used pool (doubly-linked list) */ + SRE_REPEAT *temp = state->repeat_pool_used; + if (temp) { + temp->pool_prev = repeat; + } + repeat->pool_prev = NULL; + repeat->pool_next = temp; + state->repeat_pool_used = repeat; + + return repeat; +} + +static void +repeat_pool_free(SRE_STATE *state, SRE_REPEAT *repeat) +{ + SRE_REPEAT *prev = repeat->pool_prev; + SRE_REPEAT *next = repeat->pool_next; + + /* remove from used pool (doubly-linked list) */ + if (prev) { + prev->pool_next = next; + } + else { + state->repeat_pool_used = next; + } + if (next) { + next->pool_prev = prev; + } + + /* add to unused pool (singly-linked list) */ + repeat->pool_next = state->repeat_pool_unused; + state->repeat_pool_unused = repeat; +} + +static void +repeat_pool_clear(SRE_STATE *state) +{ + /* clear used pool */ + SRE_REPEAT *next = state->repeat_pool_used; + state->repeat_pool_used = NULL; + while (next) { + SRE_REPEAT *temp = next; + next = temp->pool_next; + PyMem_Free(temp); + } + + /* clear unused pool */ + next = state->repeat_pool_unused; + state->repeat_pool_unused = NULL; + while (next) { + SRE_REPEAT *temp = next; + next = temp->pool_next; + PyMem_Free(temp); + } +} + /* generate 8-bit version */ #define SRE_CHAR Py_UCS1 @@ -511,6 +590,11 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string, state->pos = start; state->endpos = end; +#ifdef Py_DEBUG + state->fail_after_count = pattern->fail_after_count; + state->fail_after_exc = pattern->fail_after_exc; // borrowed ref +#endif + return string; err: /* We add an explicit cast here because MSVC has a bug when @@ -533,6 +617,8 @@ state_fini(SRE_STATE* state) /* See above PyMem_Free() for why we explicitly cast here. */ PyMem_Free((void*) state->mark); state->mark = NULL; + /* SRE_REPEAT pool */ + repeat_pool_clear(state); } /* calculate offset from start of string */ @@ -619,6 +705,9 @@ pattern_traverse(PatternObject *self, visitproc visit, void *arg) Py_VISIT(self->groupindex); Py_VISIT(self->indexgroup); Py_VISIT(self->pattern); +#ifdef Py_DEBUG + Py_VISIT(self->fail_after_exc); +#endif return 0; } @@ -628,6 +717,9 @@ pattern_clear(PatternObject *self) Py_CLEAR(self->groupindex); Py_CLEAR(self->indexgroup); Py_CLEAR(self->pattern); +#ifdef Py_DEBUG + Py_CLEAR(self->fail_after_exc); +#endif return 0; } @@ -690,7 +782,7 @@ _sre_SRE_Pattern_match_impl(PatternObject *self, PyTypeObject *cls, Py_ssize_t status; PyObject *match; - if (!state_init(&state, (PatternObject *)self, string, pos, endpos)) + if (!state_init(&state, self, string, pos, endpos)) return NULL; INIT_TRACE(&state); @@ -1381,6 +1473,29 @@ _sre_SRE_Pattern___deepcopy__(PatternObject *self, PyObject *memo) return Py_NewRef(self); } +#ifdef Py_DEBUG +/*[clinic input] +_sre.SRE_Pattern._fail_after + + count: int + exception: object + / + +For debugging. +[clinic start generated code]*/ + +static PyObject * +_sre_SRE_Pattern__fail_after_impl(PatternObject *self, int count, + PyObject *exception) +/*[clinic end generated code: output=9a6bf12135ac50c2 input=ef80a45c66c5499d]*/ +{ + self->fail_after_count = count; + Py_INCREF(exception); + Py_XSETREF(self->fail_after_exc, exception); + Py_RETURN_NONE; +} +#endif /* Py_DEBUG */ + static PyObject * pattern_repr(PatternObject *obj) { @@ -1506,6 +1621,10 @@ _sre_compile_impl(PyObject *module, PyObject *pattern, int flags, self->pattern = NULL; self->groupindex = NULL; self->indexgroup = NULL; +#ifdef Py_DEBUG + self->fail_after_count = -1; + self->fail_after_exc = NULL; +#endif self->codesize = n; @@ -2604,7 +2723,8 @@ pattern_new_match(_sremodulestate* module_state, if (!match) return NULL; - match->pattern = (PatternObject*)Py_NewRef(pattern); + Py_INCREF(pattern); + match->pattern = pattern; match->string = Py_NewRef(state->string); @@ -2740,7 +2860,7 @@ _sre_SRE_Scanner_match_impl(ScannerObject *self, PyTypeObject *cls) return NULL; } - match = pattern_new_match(module_state, (PatternObject*) self->pattern, + match = pattern_new_match(module_state, self->pattern, state, status); if (status == 0) @@ -2790,7 +2910,7 @@ _sre_SRE_Scanner_search_impl(ScannerObject *self, PyTypeObject *cls) return NULL; } - match = pattern_new_match(module_state, (PatternObject*) self->pattern, + match = pattern_new_match(module_state, self->pattern, state, status); if (status == 0) @@ -2826,7 +2946,8 @@ pattern_scanner(_sremodulestate *module_state, return NULL; } - scanner->pattern = Py_NewRef(self); + Py_INCREF(self); + scanner->pattern = self; PyObject_GC_Track(scanner); return (PyObject*) scanner; @@ -3020,6 +3141,7 @@ static PyMethodDef pattern_methods[] = { _SRE_SRE_PATTERN_SCANNER_METHODDEF _SRE_SRE_PATTERN___COPY___METHODDEF _SRE_SRE_PATTERN___DEEPCOPY___METHODDEF + _SRE_SRE_PATTERN__FAIL_AFTER_METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} diff --git a/Modules/_sre/sre.h b/Modules/_sre/sre.h index 83d89d57b11199..42681c2addf3c2 100644 --- a/Modules/_sre/sre.h +++ b/Modules/_sre/sre.h @@ -34,6 +34,11 @@ typedef struct { int flags; /* flags used when compiling pattern source */ PyObject *weakreflist; /* List of weak references */ int isbytes; /* pattern type (1 - bytes, 0 - string, -1 - None) */ +#ifdef Py_DEBUG + /* for simulation of user interruption */ + int fail_after_count; + PyObject *fail_after_exc; +#endif /* pattern code */ Py_ssize_t codesize; SRE_CODE code[1]; @@ -68,6 +73,9 @@ typedef struct SRE_REPEAT_T { const SRE_CODE* pattern; /* points to REPEAT operator arguments */ const void* last_ptr; /* helper to check for infinite loops */ struct SRE_REPEAT_T *prev; /* points to previous repeat context */ + /* for SRE_REPEAT pool */ + struct SRE_REPEAT_T *pool_prev; + struct SRE_REPEAT_T *pool_next; } SRE_REPEAT; typedef struct { @@ -95,12 +103,19 @@ typedef struct { size_t data_stack_base; /* current repeat context */ SRE_REPEAT *repeat; + /* SRE_REPEAT pool */ + SRE_REPEAT *repeat_pool_used; + SRE_REPEAT *repeat_pool_unused; unsigned int sigcount; +#ifdef Py_DEBUG + int fail_after_count; + PyObject *fail_after_exc; +#endif } SRE_STATE; typedef struct { PyObject_HEAD - PyObject* pattern; + PatternObject* pattern; SRE_STATE state; int executing; } ScannerObject; diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h index 97fbb0a75e54b6..af4bfc56083bcb 100644 --- a/Modules/_sre/sre_lib.h +++ b/Modules/_sre/sre_lib.h @@ -560,13 +560,28 @@ typedef struct { Py_ssize_t last_ctx_pos; } SRE(match_context); -#define MAYBE_CHECK_SIGNALS \ +#define _MAYBE_CHECK_SIGNALS \ do { \ if ((0 == (++sigcount & 0xfff)) && PyErr_CheckSignals()) { \ RETURN_ERROR(SRE_ERROR_INTERRUPTED); \ } \ } while (0) +#ifdef Py_DEBUG +# define MAYBE_CHECK_SIGNALS \ + do { \ + _MAYBE_CHECK_SIGNALS; \ + if (state->fail_after_count >= 0) { \ + if (state->fail_after_count-- == 0) { \ + PyErr_SetNone(state->fail_after_exc); \ + RETURN_ERROR(SRE_ERROR_INTERRUPTED); \ + } \ + } \ + } while (0) +#else +# define MAYBE_CHECK_SIGNALS _MAYBE_CHECK_SIGNALS +#endif /* Py_DEBUG */ + #ifdef HAVE_COMPUTED_GOTOS #ifndef USE_COMPUTED_GOTOS #define USE_COMPUTED_GOTOS 1 @@ -1120,12 +1135,9 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) pattern[1], pattern[2])); /* install new repeat context */ - /* TODO(https://github.com/python/cpython/issues/67877): Fix this - * potential memory leak. */ - ctx->u.rep = (SRE_REPEAT*) PyMem_Malloc(sizeof(*ctx->u.rep)); + ctx->u.rep = repeat_pool_malloc(state); if (!ctx->u.rep) { - PyErr_NoMemory(); - RETURN_FAILURE; + RETURN_ERROR(SRE_ERROR_MEMORY); } ctx->u.rep->count = -1; ctx->u.rep->pattern = pattern; @@ -1136,7 +1148,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) state->ptr = ptr; DO_JUMP(JUMP_REPEAT, jump_repeat, pattern+pattern[0]); state->repeat = ctx->u.rep->prev; - PyMem_Free(ctx->u.rep); + repeat_pool_free(state, ctx->u.rep); if (ret) { RETURN_ON_ERROR(ret); @@ -1294,6 +1306,17 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) pointer */ state->ptr = ptr; + /* Set state->repeat to non-NULL */ + ctx->u.rep = repeat_pool_malloc(state); + if (!ctx->u.rep) { + RETURN_ERROR(SRE_ERROR_MEMORY); + } + ctx->u.rep->count = -1; + ctx->u.rep->pattern = NULL; + ctx->u.rep->prev = state->repeat; + ctx->u.rep->last_ptr = NULL; + state->repeat = ctx->u.rep; + /* Initialize Count to 0 */ ctx->count = 0; @@ -1308,6 +1331,9 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) } else { state->ptr = ptr; + /* Restore state->repeat */ + state->repeat = ctx->u.rep->prev; + repeat_pool_free(state, ctx->u.rep); RETURN_FAILURE; } } @@ -1380,6 +1406,10 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) } } + /* Restore state->repeat */ + state->repeat = ctx->u.rep->prev; + repeat_pool_free(state, ctx->u.rep); + /* Evaluate Tail */ /* Jump to end of pattern indicated by skip, and then skip the SUCCESS op code that follows it. */ diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 54bac28e5beccf..b6b5ebf094c938 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -2979,7 +2979,7 @@ static PyType_Spec PySSLSocket_spec = { .name = "_ssl._SSLSocket", .basicsize = sizeof(PySSLSocket), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | - Py_TPFLAGS_HAVE_GC), + Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_DISALLOW_INSTANTIATION), .slots = PySSLSocket_slots, }; @@ -4923,7 +4923,9 @@ static unsigned int psk_client_callback(SSL *s, goto error; } - if (identity_len_ + 1 > max_identity_len || psk_len_ > max_psk_len) { + if ((size_t)identity_len_ + 1 > max_identity_len + || (size_t)psk_len_ > max_psk_len) + { Py_DECREF(result); goto error; } @@ -5036,7 +5038,7 @@ static unsigned int psk_server_callback(SSL *s, goto error; } - if (psk_len_ > max_psk_len) { + if ((size_t)psk_len_ > max_psk_len) { Py_DECREF(result); goto error; } diff --git a/Modules/_ssl/clinic/cert.c.h b/Modules/_ssl/clinic/cert.c.h index 0748d32291e3c5..19559442cd9b88 100644 --- a/Modules/_ssl/clinic/cert.c.h +++ b/Modules/_ssl/clinic/cert.c.h @@ -52,7 +52,8 @@ _ssl_Certificate_public_bytes(PySSLCertificate *self, PyObject *const *args, Py_ Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int format = PY_SSL_ENCODING_PEM; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -86,4 +87,4 @@ _ssl_Certificate_get_info(PySSLCertificate *self, PyObject *Py_UNUSED(ignored)) { return _ssl_Certificate_get_info_impl(self); } -/*[clinic end generated code: output=5b668226f933a677 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e5fa354db5fc56b4 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/exceptions.c.h b/Modules/_testcapi/clinic/exceptions.c.h index a797444c1a72b9..7c18bac0f686e2 100644 --- a/Modules/_testcapi/clinic/exceptions.c.h +++ b/Modules/_testcapi/clinic/exceptions.c.h @@ -104,7 +104,8 @@ _testcapi_make_exception_with_doc(PyObject *module, PyObject *const *args, Py_ss PyObject *base = NULL; PyObject *dict = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -456,4 +457,4 @@ _testcapi_unstable_exc_prep_reraise_star(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=0b11ef105030a48e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d917e9ec082e69ee input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 2b5e85d5707522..ebea09080ef11c 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -105,6 +105,30 @@ pylong_getsign(PyObject *module, PyObject *arg) } +static PyObject * +pylong_ispositive(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + RETURN_INT(PyLong_IsPositive(arg)); +} + + +static PyObject * +pylong_isnegative(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + RETURN_INT(PyLong_IsNegative(arg)); +} + + +static PyObject * +pylong_iszero(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + RETURN_INT(PyLong_IsZero(arg)); +} + + static PyObject * pylong_aspid(PyObject *module, PyObject *arg) { @@ -124,6 +148,9 @@ static PyMethodDef test_methods[] = { {"pylong_fromnativebytes", pylong_fromnativebytes, METH_VARARGS}, {"pylong_getsign", pylong_getsign, METH_O}, {"pylong_aspid", pylong_aspid, METH_O}, + {"pylong_ispositive", pylong_ispositive, METH_O}, + {"pylong_isnegative", pylong_isnegative, METH_O}, + {"pylong_iszero", pylong_iszero, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index 1c76e766a790f0..3af5429ef00985 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -124,13 +124,20 @@ pyobject_clear_weakrefs_no_callbacks(PyObject *self, PyObject *obj) Py_RETURN_NONE; } +static PyObject * +pyobject_enable_deferred_refcount(PyObject *self, PyObject *obj) +{ + int result = PyUnstable_Object_EnableDeferredRefcount(obj); + return PyLong_FromLong(result); +} + static PyMethodDef test_methods[] = { {"call_pyobject_print", call_pyobject_print, METH_VARARGS}, {"pyobject_print_null", pyobject_print_null, METH_VARARGS}, {"pyobject_print_noref_object", pyobject_print_noref_object, METH_VARARGS}, {"pyobject_print_os_error", pyobject_print_os_error, METH_VARARGS}, {"pyobject_clear_weakrefs_no_callbacks", pyobject_clear_weakrefs_no_callbacks, METH_O}, - + {"pyobject_enable_deferred_refcount", pyobject_enable_deferred_refcount, METH_O}, {NULL}, }; diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index e3c8ba9b0b5074..a5c4a4c40b3949 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -59,19 +59,20 @@ pack_arguments_newref(int argc, ...) } static PyObject * -pack_varargs_to_tuple(Py_ssize_t varargssize, PyObject *const *args) +pack_arguments_2pos_varpos(PyObject *a, PyObject *b, + PyObject * const *args, Py_ssize_t args_length) +/*[clinic end generated code: output=267032f41bd039cc input=86ee3064b7853e86]*/ { - assert(!PyErr_Occurred()); - PyObject *tuple = PyTuple_New(varargssize); - if (!tuple) { + PyObject *tuple = _PyTuple_FromArray(args, args_length); + if (tuple == NULL) { return NULL; } - for (Py_ssize_t i = 0; i < varargssize; i++) { - PyTuple_SET_ITEM(tuple, i, Py_NewRef(args[i])); - } - return tuple; + PyObject *result = pack_arguments_newref(3, a, b, tuple); + Py_DECREF(tuple); + return result; } + /* Pack arguments to a tuple. * `wrapper` is function which converts primitive type to PyObject. * `arg_type` is type that arguments should be converted to before wrapped. */ @@ -979,21 +980,15 @@ keyword_only_parameter_impl(PyObject *module, PyObject *a) /*[clinic input] varpos - *args: object + *args: tuple [clinic start generated code]*/ static PyObject * -varpos_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args) -/*[clinic end generated code: output=b65096f423fb5dcc input=f87cd674145d394c]*/ +varpos_impl(PyObject *module, PyObject *args) +/*[clinic end generated code: output=7b0b9545872bdca4 input=ae7ccecd997aaff4]*/ { - PyObject *vararg_tuple = pack_varargs_to_tuple(nargs, args); - if (!vararg_tuple) { - return NULL; - } - PyObject *result = pack_arguments_newref(1, vararg_tuple); - Py_DECREF(vararg_tuple); - return result; + return Py_NewRef(args); } @@ -1003,22 +998,35 @@ posonly_varpos a: object b: object / - *args: object + *args: tuple [clinic start generated code]*/ static PyObject * posonly_varpos_impl(PyObject *module, PyObject *a, PyObject *b, - Py_ssize_t nargs, PyObject *const *args) -/*[clinic end generated code: output=d10d43d86d117ab3 input=c9fd7895cfbaabba]*/ + PyObject *args) +/*[clinic end generated code: output=5dae5eb2a0d623cd input=6dd74417b62cbe67]*/ { - PyObject *vararg_tuple = pack_varargs_to_tuple(nargs, args); - if (!vararg_tuple) { - return NULL; - } - PyObject *result = pack_arguments_newref(3, a, b, vararg_tuple); - Py_DECREF(vararg_tuple); - return result; + return pack_arguments_newref(3, a, b, args); +} + + +/*[clinic input] +posonly_req_opt_varpos + + a: object + b: object = False + / + *args: tuple + +[clinic start generated code]*/ + +static PyObject * +posonly_req_opt_varpos_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *args) +/*[clinic end generated code: output=67f82f90838e166a input=e08ed48926a5b760]*/ +{ + return pack_arguments_newref(3, a, b, args); } @@ -1028,14 +1036,14 @@ posonly_poskw_varpos a: object / b: object - *args: object + *args: tuple [clinic start generated code]*/ static PyObject * posonly_poskw_varpos_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *args) -/*[clinic end generated code: output=bffdb7649941c939 input=b3d7a734e0625f68]*/ +/*[clinic end generated code: output=bffdb7649941c939 input=e5a2c4cab6745ca1]*/ { return pack_arguments_newref(3, a, b, args); } @@ -1045,13 +1053,13 @@ posonly_poskw_varpos_impl(PyObject *module, PyObject *a, PyObject *b, poskw_varpos a: object - *args: object + *args: tuple [clinic start generated code]*/ static PyObject * poskw_varpos_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=2413ddfb5ef22794 input=a1dff12d00422484]*/ +/*[clinic end generated code: output=2413ddfb5ef22794 input=e78114436a07aefe]*/ { return pack_arguments_newref(2, a, args); } @@ -1061,7 +1069,7 @@ poskw_varpos_impl(PyObject *module, PyObject *a, PyObject *args) poskw_varpos_kwonly_opt a: object - *args: object + *args: tuple b: bool = False [clinic start generated code]*/ @@ -1069,7 +1077,7 @@ poskw_varpos_kwonly_opt static PyObject * poskw_varpos_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *args, int b) -/*[clinic end generated code: output=f36d35ba6133463b input=1721d43dc5f6d856]*/ +/*[clinic end generated code: output=f36d35ba6133463b input=ebecfb189f4a3380]*/ { PyObject *obj_b = b ? Py_True : Py_False; return pack_arguments_newref(3, a, args, obj_b); @@ -1080,7 +1088,7 @@ poskw_varpos_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *args, poskw_varpos_kwonly_opt2 a: object - *args: object + *args: tuple b: object = False c: object = False @@ -1089,7 +1097,7 @@ poskw_varpos_kwonly_opt2 static PyObject * poskw_varpos_kwonly_opt2_impl(PyObject *module, PyObject *a, PyObject *args, PyObject *b, PyObject *c) -/*[clinic end generated code: output=846cef62c6c40463 input=bb4b8d1577a8a408]*/ +/*[clinic end generated code: output=846cef62c6c40463 input=1aff29829431f711]*/ { return pack_arguments_newref(4, a, args, b, c); } @@ -1098,14 +1106,14 @@ poskw_varpos_kwonly_opt2_impl(PyObject *module, PyObject *a, PyObject *args, /*[clinic input] varpos_kwonly_opt - *args: object + *args: tuple b: object = False [clinic start generated code]*/ static PyObject * varpos_kwonly_opt_impl(PyObject *module, PyObject *args, PyObject *b) -/*[clinic end generated code: output=3b7bf98b091f5731 input=380fb00deec847e8]*/ +/*[clinic end generated code: output=3b7bf98b091f5731 input=1bec50dc49fca2eb]*/ { return pack_arguments_newref(2, args, b); } @@ -1114,7 +1122,7 @@ varpos_kwonly_opt_impl(PyObject *module, PyObject *args, PyObject *b) /*[clinic input] varpos_kwonly_req_opt - *args: object + *args: tuple a: object b: object = False c: object = False @@ -1124,19 +1132,94 @@ varpos_kwonly_req_opt static PyObject * varpos_kwonly_req_opt_impl(PyObject *module, PyObject *args, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=165274e1fd037ae9 input=530794afd0690c22]*/ +/*[clinic end generated code: output=165274e1fd037ae9 input=355271f6119bb6c8]*/ { return pack_arguments_newref(4, args, a, b, c); } +/*[clinic input] +varpos_array + + *args: array + +[clinic start generated code]*/ + +static PyObject * +varpos_array_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=a25f42f39c9b13ad input=97b8bdcf87e019c7]*/ +{ + return _PyTuple_FromArray(args, args_length); +} + + +/*[clinic input] +posonly_varpos_array + + a: object + b: object + / + *args: array + +[clinic start generated code]*/ + +static PyObject * +posonly_varpos_array_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject * const *args, Py_ssize_t args_length) +/*[clinic end generated code: output=267032f41bd039cc input=86ee3064b7853e86]*/ +{ + return pack_arguments_2pos_varpos(a, b, args, args_length); +} + + +/*[clinic input] +posonly_req_opt_varpos_array + + a: object + b: object = False + / + *args: array + +[clinic start generated code]*/ + +static PyObject * +posonly_req_opt_varpos_array_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=f2f93c77ead93699 input=b01d7728164fd93e]*/ +{ + return pack_arguments_2pos_varpos(a, b, args, args_length); +} + + +/*[clinic input] +posonly_poskw_varpos_array + + a: object + / + b: object + *args: array + +[clinic start generated code]*/ + +static PyObject * +posonly_poskw_varpos_array_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=155811a8b2d65a12 input=5fb08cdc6afb9d7c]*/ +{ + return pack_arguments_2pos_varpos(a, b, args, args_length); +} + + /*[clinic input] gh_32092_oob pos1: object pos2: object - *varargs: object + *varargs: tuple kw1: object = None kw2: object = None @@ -1147,7 +1230,7 @@ Proof-of-concept of GH-32092 OOB bug. static PyObject * gh_32092_oob_impl(PyObject *module, PyObject *pos1, PyObject *pos2, PyObject *varargs, PyObject *kw1, PyObject *kw2) -/*[clinic end generated code: output=ee259c130054653f input=46d15c881608f8ff]*/ +/*[clinic end generated code: output=ee259c130054653f input=63aeeca881979b91]*/ { Py_RETURN_NONE; } @@ -1157,7 +1240,7 @@ gh_32092_oob_impl(PyObject *module, PyObject *pos1, PyObject *pos2, gh_32092_kw_pass pos: object - *args: object + *args: tuple kw: object = None Proof-of-concept of GH-32092 keyword args passing bug. @@ -1167,7 +1250,7 @@ Proof-of-concept of GH-32092 keyword args passing bug. static PyObject * gh_32092_kw_pass_impl(PyObject *module, PyObject *pos, PyObject *args, PyObject *kw) -/*[clinic end generated code: output=4a2bbe4f7c8604e9 input=5c0bd5b9079a0cce]*/ +/*[clinic end generated code: output=4a2bbe4f7c8604e9 input=258987971f3ee97a]*/ { Py_RETURN_NONE; } @@ -1176,16 +1259,15 @@ gh_32092_kw_pass_impl(PyObject *module, PyObject *pos, PyObject *args, /*[clinic input] gh_99233_refcount - *args: object + *args: tuple Proof-of-concept of GH-99233 refcount error bug. [clinic start generated code]*/ static PyObject * -gh_99233_refcount_impl(PyObject *module, Py_ssize_t nargs, - PyObject *const *args) -/*[clinic end generated code: output=b570007e61e5c670 input=eecfdc2092d90dc3]*/ +gh_99233_refcount_impl(PyObject *module, PyObject *args) +/*[clinic end generated code: output=585855abfbca9a7f input=f5204359fd852613]*/ { Py_RETURN_NONE; } @@ -1213,7 +1295,7 @@ gh_99240_double_free_impl(PyObject *module, char *a, char *b) null_or_tuple_for_varargs name: object - *constraints: object + *constraints: tuple covariant: bool = False See https://github.com/python/cpython/issues/110864 @@ -1222,7 +1304,7 @@ See https://github.com/python/cpython/issues/110864 static PyObject * null_or_tuple_for_varargs_impl(PyObject *module, PyObject *name, PyObject *constraints, int covariant) -/*[clinic end generated code: output=a785b35421358983 input=c9bce186637956b3]*/ +/*[clinic end generated code: output=a785b35421358983 input=4df930e019f32878]*/ { assert(name != NULL); assert(constraints != NULL); @@ -1295,9 +1377,9 @@ clone_with_conv_f2_impl(PyObject *module, custom_t path) /*[clinic input] -class _testclinic.TestClass "PyObject *" "PyObject" +class _testclinic.TestClass "PyObject *" "&PyBaseObject_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=668a591c65bec947]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c991635bb3c91f1a]*/ /*[clinic input] _testclinic.TestClass.get_defining_class @@ -1330,13 +1412,13 @@ _testclinic_TestClass_get_defining_class_arg_impl(PyObject *self, /*[clinic input] _testclinic.TestClass.defclass_varpos cls: defining_class - *args: object + *args: tuple [clinic start generated code]*/ static PyObject * _testclinic_TestClass_defclass_varpos_impl(PyObject *self, PyTypeObject *cls, PyObject *args) -/*[clinic end generated code: output=fad33f2d3a8d778d input=47071dcda393a7e1]*/ +/*[clinic end generated code: output=fad33f2d3a8d778d input=332043286e393d38]*/ { return PyTuple_Pack(2, cls, args); } @@ -1347,7 +1429,7 @@ _testclinic.TestClass.defclass_posonly_varpos a: object b: object / - *args: object + *args: tuple [clinic start generated code]*/ static PyObject * @@ -1355,16 +1437,197 @@ _testclinic_TestClass_defclass_posonly_varpos_impl(PyObject *self, PyTypeObject *cls, PyObject *a, PyObject *b, PyObject *args) -/*[clinic end generated code: output=1740fcf48d230b07 input=40f2e56286d4a7ef]*/ +/*[clinic end generated code: output=1740fcf48d230b07 input=191da4557203c413]*/ { return pack_arguments_newref(4, cls, a, b, args); } + +/*[clinic input] +@classmethod +_testclinic.TestClass.__new__ as varpos_no_fastcall + + *args: tuple + +[clinic start generated code]*/ + +static PyObject * +varpos_no_fastcall_impl(PyTypeObject *type, PyObject *args) +/*[clinic end generated code: output=04e94f2898bb2dde input=c5d3d30a6589f97f]*/ +{ + return Py_NewRef(args); +} + + +/*[clinic input] +@classmethod +_testclinic.TestClass.__new__ as posonly_varpos_no_fastcall + + a: object + b: object + / + *args: tuple + +[clinic start generated code]*/ + +static PyObject * +posonly_varpos_no_fastcall_impl(PyTypeObject *type, PyObject *a, PyObject *b, + PyObject *args) +/*[clinic end generated code: output=b0a0425719f69f5a input=10f29f2c2c6bfdc4]*/ +{ + return pack_arguments_newref(3, a, b, args); +} + + +/*[clinic input] +@classmethod +_testclinic.TestClass.__new__ as posonly_req_opt_varpos_no_fastcall + + a: object + b: object = False + / + *args: tuple + +[clinic start generated code]*/ + +static PyObject * +posonly_req_opt_varpos_no_fastcall_impl(PyTypeObject *type, PyObject *a, + PyObject *b, PyObject *args) +/*[clinic end generated code: output=3c44915b1a554e2d input=d319302a8748147c]*/ +{ + return pack_arguments_newref(3, a, b, args); +} + + +/*[clinic input] +@classmethod +_testclinic.TestClass.__new__ as posonly_poskw_varpos_no_fastcall + + a: object + / + b: object + *args: tuple + +[clinic start generated code]*/ + +static PyObject * +posonly_poskw_varpos_no_fastcall_impl(PyTypeObject *type, PyObject *a, + PyObject *b, PyObject *args) +/*[clinic end generated code: output=6ad74bed4bdc7f96 input=1f8c113e749414a3]*/ +{ + return pack_arguments_newref(3, a, b, args); +} + + +/*[clinic input] +@classmethod +_testclinic.TestClass.__new__ as varpos_array_no_fastcall + + *args: array + +[clinic start generated code]*/ + +static PyObject * +varpos_array_no_fastcall_impl(PyTypeObject *type, PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=f99d984346c60d42 input=368d8eea6de48c12]*/ +{ + return _PyTuple_FromArray(args, args_length); +} + + +/*[clinic input] +@classmethod +_testclinic.TestClass.__new__ as posonly_varpos_array_no_fastcall + + a: object + b: object + / + *args: array + +[clinic start generated code]*/ + +static PyObject * +posonly_varpos_array_no_fastcall_impl(PyTypeObject *type, PyObject *a, + PyObject *b, PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=1eec4da1fb5b5978 input=7330c8d819a23548]*/ +{ + return pack_arguments_2pos_varpos(a, b, args, args_length); +} + + +/*[clinic input] +@classmethod +_testclinic.TestClass.__new__ as posonly_req_opt_varpos_array_no_fastcall + + a: object + b: object = False + / + *args: array + +[clinic start generated code]*/ + +static PyObject * +posonly_req_opt_varpos_array_no_fastcall_impl(PyTypeObject *type, + PyObject *a, PyObject *b, + PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=88041c2176135218 input=7f5fd34ee5f9e0bf]*/ +{ + return pack_arguments_2pos_varpos(a, b, args, args_length); +} + + +/*[clinic input] +@classmethod +_testclinic.TestClass.__new__ as posonly_poskw_varpos_array_no_fastcall + + a: object + / + b: object + *args: array + +[clinic start generated code]*/ + +static PyObject * +posonly_poskw_varpos_array_no_fastcall_impl(PyTypeObject *type, PyObject *a, + PyObject *b, + PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=70eda18c3667681e input=2b0fcd7bd9bb865c]*/ +{ + return pack_arguments_2pos_varpos(a, b, args, args_length); +} + static struct PyMethodDef test_class_methods[] = { _TESTCLINIC_TESTCLASS_GET_DEFINING_CLASS_METHODDEF _TESTCLINIC_TESTCLASS_GET_DEFINING_CLASS_ARG_METHODDEF _TESTCLINIC_TESTCLASS_DEFCLASS_VARPOS_METHODDEF _TESTCLINIC_TESTCLASS_DEFCLASS_POSONLY_VARPOS_METHODDEF + + {"varpos_no_fastcall", _PyCFunction_CAST(varpos_no_fastcall), + METH_VARARGS|METH_KEYWORDS|METH_CLASS, ""}, + {"posonly_varpos_no_fastcall", _PyCFunction_CAST(posonly_varpos_no_fastcall), + METH_VARARGS|METH_KEYWORDS|METH_CLASS, ""}, + {"posonly_req_opt_varpos_no_fastcall", _PyCFunction_CAST(posonly_req_opt_varpos_no_fastcall), + METH_VARARGS|METH_KEYWORDS|METH_CLASS, ""}, + {"posonly_poskw_varpos_no_fastcall", _PyCFunction_CAST(posonly_poskw_varpos_no_fastcall), + METH_VARARGS|METH_KEYWORDS|METH_CLASS, ""}, + + {"varpos_array_no_fastcall", + _PyCFunction_CAST(varpos_array_no_fastcall), + METH_VARARGS|METH_KEYWORDS|METH_CLASS, ""}, + {"posonly_varpos_array_no_fastcall", + _PyCFunction_CAST(posonly_varpos_array_no_fastcall), + METH_VARARGS|METH_KEYWORDS|METH_CLASS, ""}, + {"posonly_req_opt_varpos_array_no_fastcall", + _PyCFunction_CAST(posonly_req_opt_varpos_array_no_fastcall), + METH_VARARGS|METH_KEYWORDS|METH_CLASS, ""}, + {"posonly_poskw_varpos_array_no_fastcall", + _PyCFunction_CAST(posonly_poskw_varpos_array_no_fastcall), + METH_VARARGS|METH_KEYWORDS|METH_CLASS, ""}, + {NULL, NULL} }; @@ -2023,12 +2286,19 @@ static PyMethodDef tester_methods[] = { VARPOS_METHODDEF POSONLY_VARPOS_METHODDEF + POSONLY_REQ_OPT_VARPOS_METHODDEF POSONLY_POSKW_VARPOS_METHODDEF POSKW_VARPOS_METHODDEF POSKW_VARPOS_KWONLY_OPT_METHODDEF POSKW_VARPOS_KWONLY_OPT2_METHODDEF VARPOS_KWONLY_OPT_METHODDEF VARPOS_KWONLY_REQ_OPT_METHODDEF + + VARPOS_ARRAY_METHODDEF + POSONLY_VARPOS_ARRAY_METHODDEF + POSONLY_REQ_OPT_VARPOS_ARRAY_METHODDEF + POSONLY_POSKW_VARPOS_ARRAY_METHODDEF + GH_32092_OOB_METHODDEF GH_32092_KW_PASS_METHODDEF GH_99233_REFCOUNT_METHODDEF diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index eb98b433c6c6af..5e21d96876e1d5 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -14,6 +14,7 @@ #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_ceval.h" // _PyEval_AddPendingCall() +#include "pycore_code.h" // _PyCode_GetTLBCFast() #include "pycore_compile.h" // _PyCompile_CodeGen() #include "pycore_context.h" // _PyContext_NewHamtForTests() #include "pycore_dict.h" // _PyManagedDictPointer_GetValues() @@ -1786,34 +1787,39 @@ interpreter_refcount_linked(PyObject *self, PyObject *idobj) static void _xid_capsule_destructor(PyObject *capsule) { - _PyCrossInterpreterData *data = \ - (_PyCrossInterpreterData *)PyCapsule_GetPointer(capsule, NULL); + _PyXIData_t *data = (_PyXIData_t *)PyCapsule_GetPointer(capsule, NULL); if (data != NULL) { - assert(_PyCrossInterpreterData_Release(data) == 0); - _PyCrossInterpreterData_Free(data); + assert(_PyXIData_Release(data) == 0); + _PyXIData_Free(data); } } static PyObject * get_crossinterp_data(PyObject *self, PyObject *args) { + PyInterpreterState *interp = PyInterpreterState_Get(); + _PyXIData_lookup_context_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return NULL; + } + PyObject *obj = NULL; if (!PyArg_ParseTuple(args, "O:get_crossinterp_data", &obj)) { return NULL; } - _PyCrossInterpreterData *data = _PyCrossInterpreterData_New(); + _PyXIData_t *data = _PyXIData_New(); if (data == NULL) { return NULL; } - if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { - _PyCrossInterpreterData_Free(data); + if (_PyObject_GetXIData(&ctx, obj, data) != 0) { + _PyXIData_Free(data); return NULL; } PyObject *capsule = PyCapsule_New(data, NULL, _xid_capsule_destructor); if (capsule == NULL) { - assert(_PyCrossInterpreterData_Release(data) == 0); - _PyCrossInterpreterData_Free(data); + assert(_PyXIData_Release(data) == 0); + _PyXIData_Free(data); } return capsule; } @@ -1826,12 +1832,11 @@ restore_crossinterp_data(PyObject *self, PyObject *args) return NULL; } - _PyCrossInterpreterData *data = \ - (_PyCrossInterpreterData *)PyCapsule_GetPointer(capsule, NULL); + _PyXIData_t *data = (_PyXIData_t *)PyCapsule_GetPointer(capsule, NULL); if (data == NULL) { return NULL; } - return _PyCrossInterpreterData_NewObject(data); + return _PyXIData_NewObject(data); } @@ -1963,6 +1968,48 @@ get_py_thread_id(PyObject *self, PyObject *Py_UNUSED(ignored)) Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(tid)); return PyLong_FromUnsignedLongLong(tid); } + +static PyCodeObject * +get_code(PyObject *obj) +{ + if (PyCode_Check(obj)) { + return (PyCodeObject *)obj; + } + else if (PyFunction_Check(obj)) { + return (PyCodeObject *)PyFunction_GetCode(obj); + } + return (PyCodeObject *)PyErr_Format( + PyExc_TypeError, "expected function or code object, got %s", + Py_TYPE(obj)->tp_name); +} + +static PyObject * +get_tlbc(PyObject *Py_UNUSED(module), PyObject *obj) +{ + PyCodeObject *code = get_code(obj); + if (code == NULL) { + return NULL; + } + _Py_CODEUNIT *bc = _PyCode_GetTLBCFast(PyThreadState_GET(), code); + if (bc == NULL) { + Py_RETURN_NONE; + } + return PyBytes_FromStringAndSize((const char *)bc, _PyCode_NBYTES(code)); +} + +static PyObject * +get_tlbc_id(PyObject *Py_UNUSED(module), PyObject *obj) +{ + PyCodeObject *code = get_code(obj); + if (code == NULL) { + return NULL; + } + _Py_CODEUNIT *bc = _PyCode_GetTLBCFast(PyThreadState_GET(), code); + if (bc == NULL) { + Py_RETURN_NONE; + } + return PyLong_FromVoidPtr(bc); +} #endif static PyObject * @@ -2023,6 +2070,19 @@ identify_type_slot_wrappers(PyObject *self, PyObject *Py_UNUSED(ignored)) } +static PyObject * +has_deferred_refcount(PyObject *self, PyObject *op) +{ + return PyBool_FromLong(_PyObject_HasDeferredRefcount(op)); +} + + +static PyObject * +get_heap_size(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyLong_FromInt64(PyInterpreterState_Get()->gc.heap_size); +} + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -2110,6 +2170,8 @@ static PyMethodDef module_functions[] = { #ifdef Py_GIL_DISABLED {"py_thread_id", get_py_thread_id, METH_NOARGS}, + {"get_tlbc", get_tlbc, METH_O, NULL}, + {"get_tlbc_id", get_tlbc_id, METH_O, NULL}, #endif #ifdef _Py_TIER2 {"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS}, @@ -2117,6 +2179,8 @@ static PyMethodDef module_functions[] = { GH_119213_GETARGS_METHODDEF {"get_static_builtin_types", get_static_builtin_types, METH_NOARGS}, {"identify_type_slot_wrappers", identify_type_slot_wrappers, METH_NOARGS}, + {"has_deferred_refcount", has_deferred_refcount, METH_O}, + {"get_heap_size", get_heap_size, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; @@ -2165,7 +2229,7 @@ module_exec(PyObject *module) } if (PyModule_Add(module, "TIER2_THRESHOLD", - PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE)) < 0) { + PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE + 1)) < 0) { return 1; } diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index b0b70ccb8cc3d3..45897817a56051 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -325,6 +325,7 @@ typedef struct { const Tcl_ObjType *ListType; const Tcl_ObjType *StringType; const Tcl_ObjType *UTF32StringType; + const Tcl_ObjType *PixelType; } TkappObject; #define Tkapp_Interp(v) (((TkappObject *) (v))->interp) @@ -637,6 +638,7 @@ Tkapp_New(const char *screenName, const char *className, v->ListType = Tcl_GetObjType("list"); v->StringType = Tcl_GetObjType("string"); v->UTF32StringType = Tcl_GetObjType("utf32string"); + v->PixelType = Tcl_GetObjType("pixel"); /* Delete the 'exit' command, which can screw things up */ Tcl_DeleteCommand(v->interp, "exit"); @@ -1236,7 +1238,8 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) } if (value->typePtr == tkapp->StringType || - value->typePtr == tkapp->UTF32StringType) + value->typePtr == tkapp->UTF32StringType || + value->typePtr == tkapp->PixelType) { return unicodeFromTclObj(tkapp, value); } diff --git a/Modules/_winapi.c b/Modules/_winapi.c index a330b3ff68db62..4ce689fe30e6df 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -2317,7 +2317,7 @@ _winapi_BatchedWaitForMultipleObjects_impl(PyObject *module, BOOL wait_all, DWORD milliseconds) /*[clinic end generated code: output=d21c1a4ad0a252fd input=7e196f29005dc77b]*/ { - Py_ssize_t thread_count = 0, handle_count = 0, i, j; + Py_ssize_t thread_count = 0, handle_count = 0, i; Py_ssize_t nhandles; BatchedWaitData *thread_data[MAXIMUM_WAIT_OBJECTS]; HANDLE handles[MAXIMUM_WAIT_OBJECTS]; @@ -2378,7 +2378,7 @@ _winapi_BatchedWaitForMultipleObjects_impl(PyObject *module, if (data->handle_count > MAXIMUM_WAIT_OBJECTS - 1) { data->handle_count = MAXIMUM_WAIT_OBJECTS - 1; } - for (j = 0; j < data->handle_count; ++i, ++j) { + for (DWORD j = 0; j < data->handle_count; ++i, ++j) { PyObject *v = PySequence_GetItem(handle_seq, i); if (!v || !PyArg_Parse(v, F_HANDLE, &data->handles[j])) { Py_XDECREF(v); @@ -2526,7 +2526,7 @@ _winapi_BatchedWaitForMultipleObjects_impl(PyObject *module, if (triggered_indices) { for (i = 0; i < thread_count; ++i) { Py_ssize_t triggered = (Py_ssize_t)thread_data[i]->result - WAIT_OBJECT_0; - if (triggered >= 0 && triggered < thread_data[i]->handle_count - 1) { + if (triggered >= 0 && (size_t)triggered < thread_data[i]->handle_count - 1) { PyObject *v = PyLong_FromSsize_t(thread_data[i]->handle_base + triggered); if (!v || PyList_Append(triggered_indices, v) < 0) { Py_XDECREF(v); diff --git a/Modules/blake2module.c b/Modules/blake2module.c index 1ec676c34c6128..94cdfe7fd2e962 100644 --- a/Modules/blake2module.c +++ b/Modules/blake2module.c @@ -474,7 +474,7 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, /* Validate salt parameter. */ if ((salt->obj != NULL) && salt->len) { - if (salt->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_SALT_BYTES : HACL_HASH_BLAKE2S_SALT_BYTES)) { + if ((size_t)salt->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_SALT_BYTES : HACL_HASH_BLAKE2S_SALT_BYTES)) { PyErr_Format(PyExc_ValueError, "maximum salt length is %d bytes", (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_SALT_BYTES : HACL_HASH_BLAKE2S_SALT_BYTES)); @@ -485,7 +485,7 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, /* Validate personalization parameter. */ if ((person->obj != NULL) && person->len) { - if (person->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_PERSONAL_BYTES : HACL_HASH_BLAKE2S_PERSONAL_BYTES)) { + if ((size_t)person->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_PERSONAL_BYTES : HACL_HASH_BLAKE2S_PERSONAL_BYTES)) { PyErr_Format(PyExc_ValueError, "maximum person length is %d bytes", (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_PERSONAL_BYTES : HACL_HASH_BLAKE2S_PERSONAL_BYTES)); @@ -534,7 +534,7 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, /* Set key length. */ if ((key->obj != NULL) && key->len) { - if (key->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_KEY_BYTES : HACL_HASH_BLAKE2S_KEY_BYTES)) { + if ((size_t)key->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_KEY_BYTES : HACL_HASH_BLAKE2S_KEY_BYTES)) { PyErr_Format(PyExc_ValueError, "maximum key length is %d bytes", (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_KEY_BYTES : HACL_HASH_BLAKE2S_KEY_BYTES)); diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index 73edd5c3b25553..7e7ea9e0fdfa8e 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -61,7 +61,8 @@ _multibytecodec_MultibyteCodec_encode(MultibyteCodecObject *self, PyObject *cons PyObject *input; const char *errors = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -147,7 +148,8 @@ _multibytecodec_MultibyteCodec_decode(MultibyteCodecObject *self, PyObject *cons Py_buffer input = {NULL, NULL}; const char *errors = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -234,7 +236,8 @@ _multibytecodec_MultibyteIncrementalEncoder_encode(MultibyteIncrementalEncoderOb PyObject *input; int final = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -363,7 +366,8 @@ _multibytecodec_MultibyteIncrementalDecoder_decode(MultibyteIncrementalDecoderOb Py_buffer input = {NULL, NULL}; int final = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -598,7 +602,8 @@ _multibytecodec_MultibyteStreamWriter_write(MultibyteStreamWriterObject *self, P PyObject *argsbuf[1]; PyObject *strobj; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -642,7 +647,8 @@ _multibytecodec_MultibyteStreamWriter_writelines(MultibyteStreamWriterObject *se PyObject *argsbuf[1]; PyObject *lines; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -682,4 +688,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=f09052c5a28cc6e6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=60e1fa3a7615c148 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index d619a124ccead5..32045804c35004 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -63,7 +63,8 @@ _asyncio_Future___init__(PyObject *self, PyObject *args, PyObject *kwargs) Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; PyObject *loop = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 0, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -163,7 +164,8 @@ _asyncio_Future_set_result(FutureObj *self, PyTypeObject *cls, PyObject *const * PyObject *argsbuf[1]; PyObject *result; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -210,7 +212,8 @@ _asyncio_Future_set_exception(FutureObj *self, PyTypeObject *cls, PyObject *cons PyObject *argsbuf[1]; PyObject *exception; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -272,7 +275,8 @@ _asyncio_Future_add_done_callback(FutureObj *self, PyTypeObject *cls, PyObject * PyObject *fn; PyObject *context = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -323,7 +327,8 @@ _asyncio_Future_remove_done_callback(FutureObj *self, PyTypeObject *cls, PyObjec PyObject *argsbuf[1]; PyObject *fn; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -384,7 +389,8 @@ _asyncio_Future_cancel(FutureObj *self, PyTypeObject *cls, PyObject *const *args Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *msg = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -531,7 +537,8 @@ _asyncio_Task___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *context = Py_None; int eager_start = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -653,7 +660,8 @@ _asyncio_Task_cancel(TaskObj *self, PyObject *const *args, Py_ssize_t nargs, PyO Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *msg = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -776,7 +784,8 @@ _asyncio_Task_get_stack(TaskObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *limit = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -844,7 +853,8 @@ _asyncio_Task_print_stack(TaskObj *self, PyTypeObject *cls, PyObject *const *arg PyObject *limit = Py_None; PyObject *file = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1064,7 +1074,8 @@ _asyncio__register_task(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *argsbuf[1]; PyObject *task; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1121,7 +1132,8 @@ _asyncio__register_eager_task(PyObject *module, PyObject *const *args, Py_ssize_ PyObject *argsbuf[1]; PyObject *task; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1178,7 +1190,8 @@ _asyncio__unregister_task(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *argsbuf[1]; PyObject *task; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1235,7 +1248,8 @@ _asyncio__unregister_eager_task(PyObject *module, PyObject *const *args, Py_ssiz PyObject *argsbuf[1]; PyObject *task; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1295,7 +1309,8 @@ _asyncio__enter_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *loop; PyObject *task; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1356,7 +1371,8 @@ _asyncio__leave_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *loop; PyObject *task; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1416,7 +1432,8 @@ _asyncio__swap_current_task(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *loop; PyObject *task; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1473,7 +1490,8 @@ _asyncio_current_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *loop = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1533,7 +1551,8 @@ _asyncio_all_tasks(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *loop = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1547,4 +1566,4 @@ _asyncio_all_tasks(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py exit: return return_value; } -/*[clinic end generated code: output=ffe9b71bc65888b3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e5d95a0ec229ffcd input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bisectmodule.c.h b/Modules/clinic/_bisectmodule.c.h index 528602c84a9b23..910fd52c07dd30 100644 --- a/Modules/clinic/_bisectmodule.c.h +++ b/Modules/clinic/_bisectmodule.c.h @@ -69,7 +69,8 @@ _bisect_bisect_right(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *key = Py_None; Py_ssize_t _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -176,7 +177,8 @@ _bisect_insort_right(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py_ssize_t hi = -1; PyObject *key = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -282,7 +284,8 @@ _bisect_bisect_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *key = Py_None; Py_ssize_t _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -389,7 +392,8 @@ _bisect_insort_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P Py_ssize_t hi = -1; PyObject *key = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -434,4 +438,4 @@ _bisect_insort_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P exit: return return_value; } -/*[clinic end generated code: output=0a8d5a32dd0a3f04 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=972e8938ba5a0eac input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bz2module.c.h b/Modules/clinic/_bz2module.c.h index de7b3993596446..93988bf48a1b00 100644 --- a/Modules/clinic/_bz2module.c.h +++ b/Modules/clinic/_bz2module.c.h @@ -170,7 +170,8 @@ _bz2_BZ2Decompressor_decompress(BZ2Decompressor *self, PyObject *const *args, Py Py_buffer data = {NULL, NULL}; Py_ssize_t max_length = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -234,4 +235,4 @@ _bz2_BZ2Decompressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=8daa62f47cc4853d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=701a383434374c36 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 01855aec5e123e..46476a948b66d9 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -123,7 +123,8 @@ _codecs_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje const char *encoding = NULL; const char *errors = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -223,7 +224,8 @@ _codecs_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje const char *encoding = NULL; const char *errors = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2796,4 +2798,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=b3013d4709d96ffe input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f8a7fdd0b7edc0c4 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_collectionsmodule.c.h b/Modules/clinic/_collectionsmodule.c.h index fa0ff2133a05d2..b4e3325e89502b 100644 --- a/Modules/clinic/_collectionsmodule.c.h +++ b/Modules/clinic/_collectionsmodule.c.h @@ -497,7 +497,8 @@ deque_init(PyObject *deque, PyObject *args, PyObject *kwargs) PyObject *iterable = NULL; PyObject *maxlenobj = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -629,4 +630,4 @@ tuplegetter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=64c32b16df7be07a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=65f896fb13902f6d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_csv.c.h b/Modules/clinic/_csv.c.h index 2442bdcde5679f..b5608c850b6a97 100644 --- a/Modules/clinic/_csv.c.h +++ b/Modules/clinic/_csv.c.h @@ -74,7 +74,8 @@ _csv_unregister_dialect(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *argsbuf[1]; PyObject *name; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -131,7 +132,8 @@ _csv_get_dialect(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb PyObject *argsbuf[1]; PyObject *name; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -192,7 +194,8 @@ _csv_field_size_limit(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *new_limit = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -206,4 +209,4 @@ _csv_field_size_limit(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=9ec59717f5414d8b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9c7314c2434156b3 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_curses_panel.c.h b/Modules/clinic/_curses_panel.c.h index c8788c461f745c..b6bff5274a3a91 100644 --- a/Modules/clinic/_curses_panel.c.h +++ b/Modules/clinic/_curses_panel.c.h @@ -185,7 +185,8 @@ _curses_panel_panel_move(PyCursesPanelObject *self, PyTypeObject *cls, PyObject int y; int x; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -255,7 +256,8 @@ _curses_panel_panel_replace(PyCursesPanelObject *self, PyTypeObject *cls, PyObje PyObject *argsbuf[1]; PyCursesWindowObject *win; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -303,7 +305,8 @@ _curses_panel_panel_set_userptr(PyCursesPanelObject *self, PyTypeObject *cls, Py PyObject *argsbuf[1]; PyObject *obj; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -421,4 +424,4 @@ _curses_panel_update_panels(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _curses_panel_update_panels_impl(module); } -/*[clinic end generated code: output=18dc5571174c7189 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=298e49d54c0b14a0 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index 8399c5620f125b..524a114aba98bc 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -2714,7 +2714,8 @@ _curses_setupterm(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO const char *term = NULL; int fd = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -4378,4 +4379,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_USE_DEFAULT_COLORS_METHODDEF #define _CURSES_USE_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_USE_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=cd1273354b08948f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=26fe38c09ff8ca44 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_datetimemodule.c.h b/Modules/clinic/_datetimemodule.c.h index 48499e0aaf7783..72c230fc8aee68 100644 --- a/Modules/clinic/_datetimemodule.c.h +++ b/Modules/clinic/_datetimemodule.c.h @@ -60,7 +60,8 @@ iso_calendar_date_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int week; int weekday; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 3, 3, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -130,7 +131,8 @@ datetime_date_replace(PyDateTime_Date *self, PyObject *const *args, Py_ssize_t n int month = GET_MONTH(self); int day = GET_DAY(self); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -219,7 +221,8 @@ datetime_time_replace(PyDateTime_Time *self, PyObject *const *args, Py_ssize_t n PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; int fold = TIME_GET_FOLD(self); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 5, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -333,7 +336,8 @@ datetime_datetime_now(PyTypeObject *type, PyObject *const *args, Py_ssize_t narg Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *tz = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -406,7 +410,8 @@ datetime_datetime_replace(PyDateTime_DateTime *self, PyObject *const *args, Py_s PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; int fold = DATE_GET_FOLD(self); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 8, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 8, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -496,4 +501,4 @@ datetime_datetime_replace(PyDateTime_DateTime *self, PyObject *const *args, Py_s exit: return return_value; } -/*[clinic end generated code: output=c7a04b865b1e0890 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=203217a61ea14171 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index 1a5a820d1f00b5..07045e72040664 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -41,7 +41,8 @@ _elementtree_Element_append(ElementObject *self, PyTypeObject *cls, PyObject *co PyObject *argsbuf[1]; PyObject *subelement; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -198,7 +199,8 @@ _elementtree_Element___setstate__(ElementObject *self, PyTypeObject *cls, PyObje PyObject *argsbuf[1]; PyObject *state; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -241,7 +243,8 @@ _elementtree_Element_extend(ElementObject *self, PyTypeObject *cls, PyObject *co PyObject *argsbuf[1]; PyObject *elements; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -298,7 +301,8 @@ _elementtree_Element_find(ElementObject *self, PyTypeObject *cls, PyObject *cons PyObject *path; PyObject *namespaces = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -362,7 +366,8 @@ _elementtree_Element_findtext(ElementObject *self, PyTypeObject *cls, PyObject * PyObject *default_value = Py_None; PyObject *namespaces = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -430,7 +435,8 @@ _elementtree_Element_findall(ElementObject *self, PyTypeObject *cls, PyObject *c PyObject *path; PyObject *namespaces = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -492,7 +498,8 @@ _elementtree_Element_iterfind(ElementObject *self, PyTypeObject *cls, PyObject * PyObject *path; PyObject *namespaces = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -554,7 +561,8 @@ _elementtree_Element_get(ElementObject *self, PyObject *const *args, Py_ssize_t PyObject *key; PyObject *default_value = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -615,7 +623,8 @@ _elementtree_Element_iter(ElementObject *self, PyTypeObject *cls, PyObject *cons Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *tag = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -763,7 +772,8 @@ _elementtree_Element_makeelement(ElementObject *self, PyTypeObject *cls, PyObjec PyObject *tag; PyObject *attrib; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -883,7 +893,8 @@ _elementtree_TreeBuilder___init__(PyObject *self, PyObject *args, PyObject *kwar int insert_comments = 0; int insert_pis = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -1114,7 +1125,8 @@ _elementtree_XMLParser___init__(PyObject *self, PyObject *args, PyObject *kwargs PyObject *target = Py_None; const char *encoding = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 0, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -1236,4 +1248,4 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=bd28eba33d9c1f25 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b713bf59fd0fef9b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_functoolsmodule.c.h b/Modules/clinic/_functoolsmodule.c.h index e98984dc4d3a09..afd5eb4eb12b78 100644 --- a/Modules/clinic/_functoolsmodule.c.h +++ b/Modules/clinic/_functoolsmodule.c.h @@ -56,7 +56,8 @@ _functools_cmp_to_key(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *argsbuf[1]; PyObject *mycmp; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -67,6 +68,79 @@ _functools_cmp_to_key(PyObject *module, PyObject *const *args, Py_ssize_t nargs, return return_value; } +PyDoc_STRVAR(_functools_reduce__doc__, +"reduce($module, function, iterable, /, initial=)\n" +"--\n" +"\n" +"Apply a function of two arguments cumulatively to the items of an iterable, from left to right.\n" +"\n" +"This effectively reduces the iterable to a single value. If initial is present,\n" +"it is placed before the items of the iterable in the calculation, and serves as\n" +"a default when the iterable is empty.\n" +"\n" +"For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])\n" +"calculates ((((1 + 2) + 3) + 4) + 5)."); + +#define _FUNCTOOLS_REDUCE_METHODDEF \ + {"reduce", _PyCFunction_CAST(_functools_reduce), METH_FASTCALL|METH_KEYWORDS, _functools_reduce__doc__}, + +static PyObject * +_functools_reduce_impl(PyObject *module, PyObject *func, PyObject *seq, + PyObject *result); + +static PyObject * +_functools_reduce(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(initial), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "", "initial", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "reduce", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *func; + PyObject *seq; + PyObject *result = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + func = args[0]; + seq = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + result = args[2]; +skip_optional_pos: + return_value = _functools_reduce_impl(module, func, seq, result); + +exit: + return return_value; +} + PyDoc_STRVAR(_functools__lru_cache_wrapper_cache_info__doc__, "cache_info($self, /)\n" "--\n" @@ -114,4 +188,4 @@ _functools__lru_cache_wrapper_cache_clear(PyObject *self, PyObject *Py_UNUSED(ig return return_value; } -/*[clinic end generated code: output=755265bb6d5ea751 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e6edcc01f0720daf input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index 1d85ab1c524082..f54f065f7d2b71 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -118,7 +118,8 @@ EVPXOF_digest(EVPobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *argsbuf[1]; Py_ssize_t length; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -188,7 +189,8 @@ EVPXOF_hexdigest(EVPobject *self, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *argsbuf[1]; Py_ssize_t length; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -265,7 +267,8 @@ EVP_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -341,7 +344,8 @@ _hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -416,7 +420,8 @@ _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -491,7 +496,8 @@ _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -566,7 +572,8 @@ _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -641,7 +648,8 @@ _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -716,7 +724,8 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -793,7 +802,8 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -872,7 +882,8 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -951,7 +962,8 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1030,7 +1042,8 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1109,7 +1122,8 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1188,7 +1202,8 @@ _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1270,7 +1285,8 @@ pbkdf2_hmac(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject long iterations; PyObject *dklen_obj = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 5, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 4, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1373,7 +1389,8 @@ _hashlib_scrypt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj long maxmem = 0; long dklen = 64; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1499,7 +1516,8 @@ _hashlib_hmac_singleshot(PyObject *module, PyObject *const *args, Py_ssize_t nar Py_buffer msg = {NULL, NULL}; PyObject *digest; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1573,7 +1591,8 @@ _hashlib_hmac_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO PyObject *msg_obj = NULL; PyObject *digestmod = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1664,7 +1683,8 @@ _hashlib_HMAC_update(HMACobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *argsbuf[1]; PyObject *msg; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1824,4 +1844,4 @@ _hashlib_compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef _HASHLIB_SCRYPT_METHODDEF #define _HASHLIB_SCRYPT_METHODDEF #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */ -/*[clinic end generated code: output=fef43fd9f4dbea49 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c3ef67e4a573cc7a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lsprof.c.h b/Modules/clinic/_lsprof.c.h index b3b7fda5660bfd..e19840f97e5c69 100644 --- a/Modules/clinic/_lsprof.c.h +++ b/Modules/clinic/_lsprof.c.h @@ -2,6 +2,12 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(_lsprof_Profiler_getstats__doc__, "getstats($self, /)\n" "--\n" @@ -45,4 +51,360 @@ _lsprof_Profiler_getstats(ProfilerObject *self, PyTypeObject *cls, PyObject *con } return _lsprof_Profiler_getstats_impl(self, cls); } -/*[clinic end generated code: output=5c9d87d89863dc83 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_lsprof_Profiler__pystart_callback__doc__, +"_pystart_callback($self, code, instruction_offset, /)\n" +"--\n" +"\n"); + +#define _LSPROF_PROFILER__PYSTART_CALLBACK_METHODDEF \ + {"_pystart_callback", _PyCFunction_CAST(_lsprof_Profiler__pystart_callback), METH_FASTCALL, _lsprof_Profiler__pystart_callback__doc__}, + +static PyObject * +_lsprof_Profiler__pystart_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset); + +static PyObject * +_lsprof_Profiler__pystart_callback(ProfilerObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *code; + PyObject *instruction_offset; + + if (!_PyArg_CheckPositional("_pystart_callback", nargs, 2, 2)) { + goto exit; + } + code = args[0]; + instruction_offset = args[1]; + return_value = _lsprof_Profiler__pystart_callback_impl(self, code, instruction_offset); + +exit: + return return_value; +} + +PyDoc_STRVAR(_lsprof_Profiler__pyreturn_callback__doc__, +"_pyreturn_callback($self, code, instruction_offset, retval, /)\n" +"--\n" +"\n"); + +#define _LSPROF_PROFILER__PYRETURN_CALLBACK_METHODDEF \ + {"_pyreturn_callback", _PyCFunction_CAST(_lsprof_Profiler__pyreturn_callback), METH_FASTCALL, _lsprof_Profiler__pyreturn_callback__doc__}, + +static PyObject * +_lsprof_Profiler__pyreturn_callback_impl(ProfilerObject *self, + PyObject *code, + PyObject *instruction_offset, + PyObject *retval); + +static PyObject * +_lsprof_Profiler__pyreturn_callback(ProfilerObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *code; + PyObject *instruction_offset; + PyObject *retval; + + if (!_PyArg_CheckPositional("_pyreturn_callback", nargs, 3, 3)) { + goto exit; + } + code = args[0]; + instruction_offset = args[1]; + retval = args[2]; + return_value = _lsprof_Profiler__pyreturn_callback_impl(self, code, instruction_offset, retval); + +exit: + return return_value; +} + +PyDoc_STRVAR(_lsprof_Profiler__ccall_callback__doc__, +"_ccall_callback($self, code, instruction_offset, callable, self_arg, /)\n" +"--\n" +"\n"); + +#define _LSPROF_PROFILER__CCALL_CALLBACK_METHODDEF \ + {"_ccall_callback", _PyCFunction_CAST(_lsprof_Profiler__ccall_callback), METH_FASTCALL, _lsprof_Profiler__ccall_callback__doc__}, + +static PyObject * +_lsprof_Profiler__ccall_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset, + PyObject *callable, PyObject *self_arg); + +static PyObject * +_lsprof_Profiler__ccall_callback(ProfilerObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *code; + PyObject *instruction_offset; + PyObject *callable; + PyObject *self_arg; + + if (!_PyArg_CheckPositional("_ccall_callback", nargs, 4, 4)) { + goto exit; + } + code = args[0]; + instruction_offset = args[1]; + callable = args[2]; + self_arg = args[3]; + return_value = _lsprof_Profiler__ccall_callback_impl(self, code, instruction_offset, callable, self_arg); + +exit: + return return_value; +} + +PyDoc_STRVAR(_lsprof_Profiler__creturn_callback__doc__, +"_creturn_callback($self, code, instruction_offset, callable, self_arg,\n" +" /)\n" +"--\n" +"\n"); + +#define _LSPROF_PROFILER__CRETURN_CALLBACK_METHODDEF \ + {"_creturn_callback", _PyCFunction_CAST(_lsprof_Profiler__creturn_callback), METH_FASTCALL, _lsprof_Profiler__creturn_callback__doc__}, + +static PyObject * +_lsprof_Profiler__creturn_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset, + PyObject *callable, + PyObject *self_arg); + +static PyObject * +_lsprof_Profiler__creturn_callback(ProfilerObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *code; + PyObject *instruction_offset; + PyObject *callable; + PyObject *self_arg; + + if (!_PyArg_CheckPositional("_creturn_callback", nargs, 4, 4)) { + goto exit; + } + code = args[0]; + instruction_offset = args[1]; + callable = args[2]; + self_arg = args[3]; + return_value = _lsprof_Profiler__creturn_callback_impl(self, code, instruction_offset, callable, self_arg); + +exit: + return return_value; +} + +PyDoc_STRVAR(_lsprof_Profiler_enable__doc__, +"enable($self, /, subcalls=True, builtins=True)\n" +"--\n" +"\n" +"Start collecting profiling information.\n" +"\n" +" subcalls\n" +" If True, also records for each function\n" +" statistics separated according to its current caller.\n" +" builtins\n" +" If True, records the time spent in\n" +" built-in functions separately from their caller."); + +#define _LSPROF_PROFILER_ENABLE_METHODDEF \ + {"enable", _PyCFunction_CAST(_lsprof_Profiler_enable), METH_FASTCALL|METH_KEYWORDS, _lsprof_Profiler_enable__doc__}, + +static PyObject * +_lsprof_Profiler_enable_impl(ProfilerObject *self, int subcalls, + int builtins); + +static PyObject * +_lsprof_Profiler_enable(ProfilerObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(subcalls), &_Py_ID(builtins), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"subcalls", "builtins", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "enable", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int subcalls = 1; + int builtins = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + subcalls = PyObject_IsTrue(args[0]); + if (subcalls < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + builtins = PyObject_IsTrue(args[1]); + if (builtins < 0) { + goto exit; + } +skip_optional_pos: + return_value = _lsprof_Profiler_enable_impl(self, subcalls, builtins); + +exit: + return return_value; +} + +PyDoc_STRVAR(_lsprof_Profiler_disable__doc__, +"disable($self, /)\n" +"--\n" +"\n" +"Stop collecting profiling information."); + +#define _LSPROF_PROFILER_DISABLE_METHODDEF \ + {"disable", (PyCFunction)_lsprof_Profiler_disable, METH_NOARGS, _lsprof_Profiler_disable__doc__}, + +static PyObject * +_lsprof_Profiler_disable_impl(ProfilerObject *self); + +static PyObject * +_lsprof_Profiler_disable(ProfilerObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _lsprof_Profiler_disable_impl(self); +} + +PyDoc_STRVAR(_lsprof_Profiler_clear__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Clear all profiling information collected so far."); + +#define _LSPROF_PROFILER_CLEAR_METHODDEF \ + {"clear", (PyCFunction)_lsprof_Profiler_clear, METH_NOARGS, _lsprof_Profiler_clear__doc__}, + +static PyObject * +_lsprof_Profiler_clear_impl(ProfilerObject *self); + +static PyObject * +_lsprof_Profiler_clear(ProfilerObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _lsprof_Profiler_clear_impl(self); +} + +PyDoc_STRVAR(profiler_init__doc__, +"Profiler(timer=None, timeunit=0.0, subcalls=True, builtins=True)\n" +"--\n" +"\n" +"Build a profiler object using the specified timer function.\n" +"\n" +"The default timer is a fast built-in one based on real time.\n" +"For custom timer functions returning integers, \'timeunit\' can\n" +"be a float specifying a scale (that is, how long each integer unit\n" +"is, in seconds)."); + +static int +profiler_init_impl(ProfilerObject *self, PyObject *timer, double timeunit, + int subcalls, int builtins); + +static int +profiler_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(timer), &_Py_ID(timeunit), &_Py_ID(subcalls), &_Py_ID(builtins), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"timer", "timeunit", "subcalls", "builtins", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "Profiler", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *timer = NULL; + double timeunit = 0.0; + int subcalls = 1; + int builtins = 1; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[0]) { + timer = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[1]) { + if (PyFloat_CheckExact(fastargs[1])) { + timeunit = PyFloat_AS_DOUBLE(fastargs[1]); + } + else + { + timeunit = PyFloat_AsDouble(fastargs[1]); + if (timeunit == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[2]) { + subcalls = PyObject_IsTrue(fastargs[2]); + if (subcalls < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + builtins = PyObject_IsTrue(fastargs[3]); + if (builtins < 0) { + goto exit; + } +skip_optional_pos: + return_value = profiler_init_impl((ProfilerObject *)self, timer, timeunit, subcalls, builtins); + +exit: + return return_value; +} +/*[clinic end generated code: output=e56d849e35d005a5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lzmamodule.c.h b/Modules/clinic/_lzmamodule.c.h index 51fab5eab3f7dc..187f7b183dca84 100644 --- a/Modules/clinic/_lzmamodule.c.h +++ b/Modules/clinic/_lzmamodule.c.h @@ -128,7 +128,8 @@ _lzma_LZMADecompressor_decompress(Decompressor *self, PyObject *const *args, Py_ Py_buffer data = {NULL, NULL}; Py_ssize_t max_length = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -226,7 +227,8 @@ _lzma_LZMADecompressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *memlimit = Py_None; PyObject *filters = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 3, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -327,4 +329,4 @@ _lzma__decode_filter_properties(PyObject *module, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=5e79c05ace76dc96 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=52e1b68d0886cebb input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_opcode.c.h b/Modules/clinic/_opcode.c.h index 32ac9521a2b5cf..e61762e124ad72 100644 --- a/Modules/clinic/_opcode.c.h +++ b/Modules/clinic/_opcode.c.h @@ -57,7 +57,8 @@ _opcode_stack_effect(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *jump = Py_None; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -131,7 +132,8 @@ _opcode_is_valid(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb int opcode; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -194,7 +196,8 @@ _opcode_has_arg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj int opcode; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -257,7 +260,8 @@ _opcode_has_const(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO int opcode; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -320,7 +324,8 @@ _opcode_has_name(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb int opcode; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -383,7 +388,8 @@ _opcode_has_jump(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb int opcode; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -451,7 +457,8 @@ _opcode_has_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb int opcode; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -514,7 +521,8 @@ _opcode_has_local(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO int opcode; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -577,7 +585,8 @@ _opcode_has_exc(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj int opcode; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -732,7 +741,8 @@ _opcode_get_executor(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *code; int offset; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -746,4 +756,4 @@ _opcode_get_executor(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=3b4d4f32eedd636e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2e0c39286d0253e7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_pickle.c.h b/Modules/clinic/_pickle.c.h index 40f1309b6aa03c..2e84bb83e21ca6 100644 --- a/Modules/clinic/_pickle.c.h +++ b/Modules/clinic/_pickle.c.h @@ -64,7 +64,8 @@ _pickle_Pickler_dump(PicklerObject *self, PyTypeObject *cls, PyObject *const *ar PyObject *argsbuf[1]; PyObject *obj; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -181,7 +182,8 @@ _pickle_Pickler___init__(PyObject *self, PyObject *args, PyObject *kwargs) int fix_imports = 1; PyObject *buffer_callback = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 4, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -298,7 +300,8 @@ _pickle_Unpickler_persistent_load(UnpicklerObject *self, PyTypeObject *cls, PyOb PyObject *argsbuf[1]; PyObject *pid; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -377,7 +380,8 @@ _pickle_Unpickler_find_class(UnpicklerObject *self, PyTypeObject *cls, PyObject PyObject *module_name; PyObject *global_name; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -487,7 +491,8 @@ _pickle_Unpickler___init__(PyObject *self, PyObject *args, PyObject *kwargs) const char *errors = "strict"; PyObject *buffers = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -679,7 +684,8 @@ _pickle_dump(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject int fix_imports = 1; PyObject *buffer_callback = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -782,7 +788,8 @@ _pickle_dumps(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec int fix_imports = 1; PyObject *buffer_callback = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -891,7 +898,8 @@ _pickle_load(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject const char *errors = "strict"; PyObject *buffers = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1017,7 +1025,8 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec const char *errors = "strict"; PyObject *buffers = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1077,4 +1086,4 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=a9452cf1219f2e7a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=48ceb6687a8e716c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index 6f4c715c722965..f0d4a3a164cd5f 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -89,7 +89,8 @@ _queue_SimpleQueue_put(simplequeueobject *self, PyObject *const *args, Py_ssize_ int block = 1; PyObject *timeout = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -163,7 +164,8 @@ _queue_SimpleQueue_put_nowait(simplequeueobject *self, PyObject *const *args, Py PyObject *argsbuf[1]; PyObject *item; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -231,7 +233,8 @@ _queue_SimpleQueue_get(simplequeueobject *self, PyTypeObject *cls, PyObject *con int block = 1; PyObject *timeout_obj = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -349,4 +352,4 @@ _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=44a718f40072018a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=07b5742dca7692d9 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 9d5b70dfad553d..582eef16c13244 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -610,7 +610,8 @@ _ssl__SSLSocket_get_channel_binding(PySSLSocket *self, PyObject *const *args, Py Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; const char *cb_type = "tls-unique"; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1407,7 +1408,8 @@ _ssl__SSLContext_load_cert_chain(PySSLContext *self, PyObject *const *args, Py_s PyObject *keyfile = Py_None; PyObject *password = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1480,7 +1482,8 @@ _ssl__SSLContext_load_verify_locations(PySSLContext *self, PyObject *const *args PyObject *capath = Py_None; PyObject *cadata = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1583,7 +1586,8 @@ _ssl__SSLContext__wrap_socket(PySSLContext *self, PyObject *const *args, Py_ssiz PyObject *owner = Py_None; PyObject *session = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1678,7 +1682,8 @@ _ssl__SSLContext__wrap_bio(PySSLContext *self, PyObject *const *args, Py_ssize_t PyObject *owner = Py_None; PyObject *session = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1929,7 +1934,8 @@ _ssl__SSLContext_get_ca_certs(PySSLContext *self, PyObject *const *args, Py_ssiz Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int binary_form = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1993,7 +1999,8 @@ _ssl__SSLContext_set_psk_client_callback(PySSLContext *self, PyObject *const *ar PyObject *argsbuf[1]; PyObject *callback; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2053,7 +2060,8 @@ _ssl__SSLContext_set_psk_server_callback(PySSLContext *self, PyObject *const *ar PyObject *callback; const char *identity_hint = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2601,7 +2609,8 @@ _ssl_txt2obj(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject const char *txt; int name = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2716,7 +2725,8 @@ _ssl_enum_certificates(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyObject *argsbuf[1]; const char *store_name; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2794,7 +2804,8 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *argsbuf[1]; const char *store_name; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2828,4 +2839,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=c1489122072a9f5e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4c2af0c8fab7ec4e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_struct.c.h b/Modules/clinic/_struct.c.h index 1a07532bdd75ad..cfc2fe7fc1dd58 100644 --- a/Modules/clinic/_struct.c.h +++ b/Modules/clinic/_struct.c.h @@ -57,7 +57,8 @@ Struct___init__(PyObject *self, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *format; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -159,7 +160,8 @@ Struct_unpack_from(PyStructObject *self, PyObject *const *args, Py_ssize_t nargs Py_buffer buffer = {NULL, NULL}; Py_ssize_t offset = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -357,7 +359,8 @@ unpack_from(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject Py_buffer buffer = {NULL, NULL}; Py_ssize_t offset = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -436,4 +439,4 @@ iter_unpack(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -/*[clinic end generated code: output=67bd299e5d72fee0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=faff90f99c6bd09f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 7e29998c7db520..434488adbbb77c 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -9,6 +9,7 @@ preserve #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_tuple.h" // _PyTuple_FromArray() PyDoc_STRVAR(test_empty_function__doc__, "test_empty_function($module, /)\n" @@ -1478,7 +1479,8 @@ keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *a; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1534,7 +1536,8 @@ keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *a; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1592,7 +1595,8 @@ keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *b = Py_None; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1662,7 +1666,8 @@ keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1741,7 +1746,8 @@ keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *b = Py_None; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1807,7 +1813,8 @@ posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb PyObject *a; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1863,7 +1870,8 @@ posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *a; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1921,7 +1929,8 @@ posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *b; PyObject *c; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1982,7 +1991,8 @@ posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2053,7 +2063,8 @@ posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2129,7 +2140,8 @@ posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2200,7 +2212,8 @@ posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2277,7 +2290,8 @@ posonly_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *d = Py_None; PyObject *e = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2352,7 +2366,8 @@ posonly_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssiz PyObject *d = Py_None; PyObject *e = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2435,7 +2450,8 @@ posonly_opt_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2510,7 +2526,8 @@ keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyObject *argsbuf[1]; PyObject *a; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2530,22 +2547,24 @@ PyDoc_STRVAR(varpos__doc__, {"varpos", _PyCFunction_CAST(varpos), METH_FASTCALL, varpos__doc__}, static PyObject * -varpos_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args); +varpos_impl(PyObject *module, PyObject *args); static PyObject * varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t nvararg = nargs - 0; - PyObject *const *__clinic_args = NULL; + PyObject *__clinic_args = NULL; - if (!_PyArg_CheckPositional("varpos", nargs, 0, PY_SSIZE_T_MAX)) { + __clinic_args = _PyTuple_FromArray(args, nargs); + if (__clinic_args == NULL) { goto exit; } - __clinic_args = args + 0; - return_value = varpos_impl(module, nvararg, __clinic_args); + return_value = varpos_impl(module, __clinic_args); exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + return return_value; } @@ -2559,26 +2578,75 @@ PyDoc_STRVAR(posonly_varpos__doc__, static PyObject * posonly_varpos_impl(PyObject *module, PyObject *a, PyObject *b, - Py_ssize_t nargs, PyObject *const *args); + PyObject *args); static PyObject * posonly_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t nvararg = nargs - 2; PyObject *a; PyObject *b; - PyObject *const *__clinic_args = NULL; + PyObject *__clinic_args = NULL; if (!_PyArg_CheckPositional("posonly_varpos", nargs, 2, PY_SSIZE_T_MAX)) { goto exit; } a = args[0]; b = args[1]; - __clinic_args = args + 2; - return_value = posonly_varpos_impl(module, a, b, nvararg, __clinic_args); + __clinic_args = _PyTuple_FromArray(args + 2, nargs - 2); + if (__clinic_args == NULL) { + goto exit; + } + return_value = posonly_varpos_impl(module, a, b, __clinic_args); exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +PyDoc_STRVAR(posonly_req_opt_varpos__doc__, +"posonly_req_opt_varpos($module, a, b=False, /, *args)\n" +"--\n" +"\n"); + +#define POSONLY_REQ_OPT_VARPOS_METHODDEF \ + {"posonly_req_opt_varpos", _PyCFunction_CAST(posonly_req_opt_varpos), METH_FASTCALL, posonly_req_opt_varpos__doc__}, + +static PyObject * +posonly_req_opt_varpos_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *args); + +static PyObject * +posonly_req_opt_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *b = Py_False; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("posonly_req_opt_varpos", nargs, 1, PY_SSIZE_T_MAX)) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional; + } + b = args[1]; +skip_optional: + __clinic_args = nargs > 2 + ? _PyTuple_FromArray(args + 2, nargs - 2) + : PyTuple_New(0); + if (__clinic_args == NULL) { + goto exit; + } + return_value = posonly_req_opt_varpos_impl(module, a, b, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + return return_value; } @@ -2623,22 +2691,31 @@ posonly_poskw_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs, .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[2]; + PyObject * const *fastargs; PyObject *a; PyObject *b; PyObject *__clinic_args = NULL; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, 2, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + b = fastargs[1]; + __clinic_args = nargs > 2 + ? _PyTuple_FromArray(args + 2, nargs - 2) + : PyTuple_New(0); + if (__clinic_args == NULL) { goto exit; } - a = args[0]; - b = args[1]; - __clinic_args = args[2]; return_value = posonly_poskw_varpos_impl(module, a, b, __clinic_args); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } @@ -2682,20 +2759,29 @@ poskw_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[1]; + PyObject * const *fastargs; PyObject *a; PyObject *__clinic_args = NULL; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + __clinic_args = nargs > 1 + ? _PyTuple_FromArray(args + 1, nargs - 1) + : PyTuple_New(0); + if (__clinic_args == NULL) { goto exit; } - a = args[0]; - __clinic_args = args[1]; return_value = poskw_varpos_impl(module, a, __clinic_args); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } @@ -2740,30 +2826,39 @@ poskw_varpos_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[2]; + PyObject * const *fastargs; Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *a; PyObject *__clinic_args = NULL; int b = 0; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - a = args[0]; - __clinic_args = args[1]; + a = fastargs[0]; if (!noptargs) { goto skip_optional_kwonly; } - b = PyObject_IsTrue(args[2]); + b = PyObject_IsTrue(fastargs[1]); if (b < 0) { goto exit; } skip_optional_kwonly: + __clinic_args = nargs > 1 + ? _PyTuple_FromArray(args + 1, nargs - 1) + : PyTuple_New(0); + if (__clinic_args == NULL) { + goto exit; + } return_value = poskw_varpos_kwonly_opt_impl(module, a, __clinic_args, b); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } @@ -2808,34 +2903,43 @@ poskw_varpos_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ssize_t nar .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[4]; + PyObject *argsbuf[3]; + PyObject * const *fastargs; Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *a; PyObject *__clinic_args = NULL; PyObject *b = Py_False; PyObject *c = Py_False; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - a = args[0]; - __clinic_args = args[1]; + a = fastargs[0]; if (!noptargs) { goto skip_optional_kwonly; } - if (args[2]) { - b = args[2]; + if (fastargs[1]) { + b = fastargs[1]; if (!--noptargs) { goto skip_optional_kwonly; } } - c = args[3]; + c = fastargs[2]; skip_optional_kwonly: + __clinic_args = nargs > 1 + ? _PyTuple_FromArray(args + 1, nargs - 1) + : PyTuple_New(0); + if (__clinic_args == NULL) { + goto exit; + } return_value = poskw_varpos_kwonly_opt2_impl(module, a, __clinic_args, b, c); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } @@ -2879,25 +2983,32 @@ varpos_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[1]; + PyObject * const *fastargs; Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *__clinic_args = NULL; PyObject *b = Py_False; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - __clinic_args = args[0]; if (!noptargs) { goto skip_optional_kwonly; } - b = args[1]; + b = fastargs[0]; skip_optional_kwonly: + __clinic_args = _PyTuple_FromArray(args, nargs); + if (__clinic_args == NULL) { + goto exit; + } return_value = varpos_kwonly_opt_impl(module, __clinic_args, b); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } @@ -2942,34 +3053,204 @@ varpos_kwonly_req_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[4]; + PyObject *argsbuf[3]; + PyObject * const *fastargs; Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *__clinic_args = NULL; PyObject *a; PyObject *b = Py_False; PyObject *c = Py_False; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 1, 0, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 1, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - __clinic_args = args[0]; - a = args[1]; + a = fastargs[0]; if (!noptargs) { goto skip_optional_kwonly; } - if (args[2]) { - b = args[2]; + if (fastargs[1]) { + b = fastargs[1]; if (!--noptargs) { goto skip_optional_kwonly; } } - c = args[3]; + c = fastargs[2]; skip_optional_kwonly: + __clinic_args = _PyTuple_FromArray(args, nargs); + if (__clinic_args == NULL) { + goto exit; + } return_value = varpos_kwonly_req_opt_impl(module, __clinic_args, a, b, c); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + + return return_value; +} + +PyDoc_STRVAR(varpos_array__doc__, +"varpos_array($module, /, *args)\n" +"--\n" +"\n"); + +#define VARPOS_ARRAY_METHODDEF \ + {"varpos_array", _PyCFunction_CAST(varpos_array), METH_FASTCALL, varpos_array__doc__}, + +static PyObject * +varpos_array_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +varpos_array(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + __clinic_args = args; + args_length = nargs; + return_value = varpos_array_impl(module, __clinic_args, args_length); + + return return_value; +} + +PyDoc_STRVAR(posonly_varpos_array__doc__, +"posonly_varpos_array($module, a, b, /, *args)\n" +"--\n" +"\n"); + +#define POSONLY_VARPOS_ARRAY_METHODDEF \ + {"posonly_varpos_array", _PyCFunction_CAST(posonly_varpos_array), METH_FASTCALL, posonly_varpos_array__doc__}, + +static PyObject * +posonly_varpos_array_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject * const *args, Py_ssize_t args_length); + +static PyObject * +posonly_varpos_array(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *b; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + if (!_PyArg_CheckPositional("posonly_varpos_array", nargs, 2, PY_SSIZE_T_MAX)) { + goto exit; + } + a = args[0]; + b = args[1]; + __clinic_args = args + 2; + args_length = nargs - 2; + return_value = posonly_varpos_array_impl(module, a, b, __clinic_args, args_length); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_req_opt_varpos_array__doc__, +"posonly_req_opt_varpos_array($module, a, b=False, /, *args)\n" +"--\n" +"\n"); + +#define POSONLY_REQ_OPT_VARPOS_ARRAY_METHODDEF \ + {"posonly_req_opt_varpos_array", _PyCFunction_CAST(posonly_req_opt_varpos_array), METH_FASTCALL, posonly_req_opt_varpos_array__doc__}, + +static PyObject * +posonly_req_opt_varpos_array_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +posonly_req_opt_varpos_array(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *b = Py_False; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + if (!_PyArg_CheckPositional("posonly_req_opt_varpos_array", nargs, 1, PY_SSIZE_T_MAX)) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional; + } + b = args[1]; +skip_optional: + __clinic_args = nargs > 2 ? args + 2 : args; + args_length = Py_MAX(0, nargs - 2); + return_value = posonly_req_opt_varpos_array_impl(module, a, b, __clinic_args, args_length); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_poskw_varpos_array__doc__, +"posonly_poskw_varpos_array($module, a, /, b, *args)\n" +"--\n" +"\n"); + +#define POSONLY_POSKW_VARPOS_ARRAY_METHODDEF \ + {"posonly_poskw_varpos_array", _PyCFunction_CAST(posonly_poskw_varpos_array), METH_FASTCALL|METH_KEYWORDS, posonly_poskw_varpos_array__doc__}, + +static PyObject * +posonly_poskw_varpos_array_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +posonly_poskw_varpos_array(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { _Py_LATIN1_CHR('b'), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "posonly_poskw_varpos_array", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + PyObject *a; + PyObject *b; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + b = fastargs[1]; + __clinic_args = nargs > 2 ? args + 2 : args; + args_length = Py_MAX(0, nargs - 2); + return_value = posonly_poskw_varpos_array_impl(module, a, b, __clinic_args, args_length); + +exit: return return_value; } @@ -3015,7 +3296,8 @@ gh_32092_oob(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[5]; + PyObject *argsbuf[4]; + PyObject * const *fastargs; Py_ssize_t noptargs = Py_MIN(nargs, 2) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; PyObject *pos1; PyObject *pos2; @@ -3023,28 +3305,36 @@ gh_32092_oob(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *kw1 = Py_None; PyObject *kw2 = Py_None; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, 2, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - pos1 = args[0]; - pos2 = args[1]; - varargs = args[2]; + pos1 = fastargs[0]; + pos2 = fastargs[1]; if (!noptargs) { goto skip_optional_kwonly; } - if (args[3]) { - kw1 = args[3]; + if (fastargs[2]) { + kw1 = fastargs[2]; if (!--noptargs) { goto skip_optional_kwonly; } } - kw2 = args[4]; + kw2 = fastargs[3]; skip_optional_kwonly: + varargs = nargs > 2 + ? _PyTuple_FromArray(args + 2, nargs - 2) + : PyTuple_New(0); + if (varargs == NULL) { + goto exit; + } return_value = gh_32092_oob_impl(module, pos1, pos2, varargs, kw1, kw2); exit: + /* Cleanup for varargs */ Py_XDECREF(varargs); + return return_value; } @@ -3090,27 +3380,36 @@ gh_32092_kw_pass(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[2]; + PyObject * const *fastargs; Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *pos; PyObject *__clinic_args = NULL; PyObject *kw = Py_None; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - pos = args[0]; - __clinic_args = args[1]; + pos = fastargs[0]; if (!noptargs) { goto skip_optional_kwonly; } - kw = args[2]; + kw = fastargs[1]; skip_optional_kwonly: + __clinic_args = nargs > 1 + ? _PyTuple_FromArray(args + 1, nargs - 1) + : PyTuple_New(0); + if (__clinic_args == NULL) { + goto exit; + } return_value = gh_32092_kw_pass_impl(module, pos, __clinic_args, kw); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } @@ -3124,23 +3423,24 @@ PyDoc_STRVAR(gh_99233_refcount__doc__, {"gh_99233_refcount", _PyCFunction_CAST(gh_99233_refcount), METH_FASTCALL, gh_99233_refcount__doc__}, static PyObject * -gh_99233_refcount_impl(PyObject *module, Py_ssize_t nargs, - PyObject *const *args); +gh_99233_refcount_impl(PyObject *module, PyObject *args); static PyObject * gh_99233_refcount(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t nvararg = nargs - 0; - PyObject *const *__clinic_args = NULL; + PyObject *__clinic_args = NULL; - if (!_PyArg_CheckPositional("gh_99233_refcount", nargs, 0, PY_SSIZE_T_MAX)) { + __clinic_args = _PyTuple_FromArray(args, nargs); + if (__clinic_args == NULL) { goto exit; } - __clinic_args = args + 0; - return_value = gh_99233_refcount_impl(module, nvararg, __clinic_args); + return_value = gh_99233_refcount_impl(module, __clinic_args); exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + return return_value; } @@ -3220,30 +3520,39 @@ null_or_tuple_for_varargs(PyObject *module, PyObject *const *args, Py_ssize_t na .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[2]; + PyObject * const *fastargs; Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *name; PyObject *constraints = NULL; int covariant = 0; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - name = args[0]; - constraints = args[1]; + name = fastargs[0]; if (!noptargs) { goto skip_optional_kwonly; } - covariant = PyObject_IsTrue(args[2]); + covariant = PyObject_IsTrue(fastargs[1]); if (covariant < 0) { goto exit; } skip_optional_kwonly: + constraints = nargs > 1 + ? _PyTuple_FromArray(args + 1, nargs - 1) + : PyTuple_New(0); + if (constraints == NULL) { + goto exit; + } return_value = null_or_tuple_for_varargs_impl(module, name, constraints, covariant); exit: + /* Cleanup for constraints */ Py_XDECREF(constraints); + return return_value; } @@ -3290,7 +3599,8 @@ clone_f1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *argsbuf[1]; const char *path; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3356,7 +3666,8 @@ clone_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *argsbuf[1]; const char *path; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3425,7 +3736,8 @@ clone_with_conv_f1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py .name = "clone_with_conv_f1", }; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3488,7 +3800,8 @@ clone_with_conv_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py .name = "clone_with_conv_f2", }; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3572,7 +3885,8 @@ _testclinic_TestClass_get_defining_class_arg(PyObject *self, PyTypeObject *cls, PyObject *argsbuf[1]; PyObject *arg; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3613,17 +3927,24 @@ _testclinic_TestClass_defclass_varpos(PyObject *self, PyTypeObject *cls, PyObjec }; #undef KWTUPLE PyObject *argsbuf[1]; + PyObject * const *fastargs; PyObject *__clinic_args = NULL; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { + goto exit; + } + __clinic_args = _PyTuple_FromArray(args, nargs); + if (__clinic_args == NULL) { goto exit; } - __clinic_args = args[0]; return_value = _testclinic_TestClass_defclass_varpos_impl(self, cls, __clinic_args); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } @@ -3658,22 +3979,338 @@ _testclinic_TestClass_defclass_posonly_varpos(PyObject *self, PyTypeObject *cls, .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[2]; + PyObject * const *fastargs; PyObject *a; PyObject *b; PyObject *__clinic_args = NULL; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, 2, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + b = fastargs[1]; + __clinic_args = _PyTuple_FromArray(args + 2, nargs - 2); + if (__clinic_args == NULL) { goto exit; } - a = args[0]; - b = args[1]; - __clinic_args = args[2]; return_value = _testclinic_TestClass_defclass_posonly_varpos_impl(self, cls, a, b, __clinic_args); exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +static PyObject * +varpos_no_fastcall_impl(PyTypeObject *type, PyObject *args); + +static PyObject * +varpos_no_fastcall(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &PyBaseObject_Type; + PyObject *__clinic_args = NULL; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("TestClass", kwargs)) { + goto exit; + } + __clinic_args = Py_NewRef(args); + return_value = varpos_no_fastcall_impl(type, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +static PyObject * +posonly_varpos_no_fastcall_impl(PyTypeObject *type, PyObject *a, PyObject *b, + PyObject *args); + +static PyObject * +posonly_varpos_no_fastcall(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &PyBaseObject_Type; + PyObject *a; + PyObject *b; + PyObject *__clinic_args = NULL; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("TestClass", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("TestClass", PyTuple_GET_SIZE(args), 2, PY_SSIZE_T_MAX)) { + goto exit; + } + a = PyTuple_GET_ITEM(args, 0); + b = PyTuple_GET_ITEM(args, 1); + __clinic_args = PyTuple_GetSlice(args, 2, PY_SSIZE_T_MAX); + if (!__clinic_args) { + goto exit; + } + return_value = posonly_varpos_no_fastcall_impl(type, a, b, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +static PyObject * +posonly_req_opt_varpos_no_fastcall_impl(PyTypeObject *type, PyObject *a, + PyObject *b, PyObject *args); + +static PyObject * +posonly_req_opt_varpos_no_fastcall(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &PyBaseObject_Type; + PyObject *a; + PyObject *b = Py_False; + PyObject *__clinic_args = NULL; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("TestClass", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("TestClass", PyTuple_GET_SIZE(args), 1, PY_SSIZE_T_MAX)) { + goto exit; + } + a = PyTuple_GET_ITEM(args, 0); + if (PyTuple_GET_SIZE(args) < 2) { + goto skip_optional; + } + b = PyTuple_GET_ITEM(args, 1); +skip_optional: + __clinic_args = PyTuple_GetSlice(args, 2, PY_SSIZE_T_MAX); + if (!__clinic_args) { + goto exit; + } + return_value = posonly_req_opt_varpos_no_fastcall_impl(type, a, b, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +static PyObject * +posonly_poskw_varpos_no_fastcall_impl(PyTypeObject *type, PyObject *a, + PyObject *b, PyObject *args); + +static PyObject * +posonly_poskw_varpos_no_fastcall(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { _Py_LATIN1_CHR('b'), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "TestClass", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + PyObject *b; + PyObject *__clinic_args = NULL; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + b = fastargs[1]; + __clinic_args = PyTuple_GetSlice(args, 2, PY_SSIZE_T_MAX); + if (!__clinic_args) { + goto exit; + } + return_value = posonly_poskw_varpos_no_fastcall_impl(type, a, b, __clinic_args); + +exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + + return return_value; +} + +static PyObject * +varpos_array_no_fastcall_impl(PyTypeObject *type, PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +varpos_array_no_fastcall(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &PyBaseObject_Type; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("TestClass", kwargs)) { + goto exit; + } + __clinic_args = _PyTuple_ITEMS(args); + args_length = PyTuple_GET_SIZE(args); + return_value = varpos_array_no_fastcall_impl(type, __clinic_args, args_length); + +exit: + return return_value; +} + +static PyObject * +posonly_varpos_array_no_fastcall_impl(PyTypeObject *type, PyObject *a, + PyObject *b, PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +posonly_varpos_array_no_fastcall(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &PyBaseObject_Type; + PyObject *a; + PyObject *b; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("TestClass", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("TestClass", PyTuple_GET_SIZE(args), 2, PY_SSIZE_T_MAX)) { + goto exit; + } + a = PyTuple_GET_ITEM(args, 0); + b = PyTuple_GET_ITEM(args, 1); + __clinic_args = _PyTuple_ITEMS(args) + 2; + args_length = PyTuple_GET_SIZE(args) - 2; + return_value = posonly_varpos_array_no_fastcall_impl(type, a, b, __clinic_args, args_length); + +exit: + return return_value; +} + +static PyObject * +posonly_req_opt_varpos_array_no_fastcall_impl(PyTypeObject *type, + PyObject *a, PyObject *b, + PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +posonly_req_opt_varpos_array_no_fastcall(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &PyBaseObject_Type; + PyObject *a; + PyObject *b = Py_False; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("TestClass", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("TestClass", PyTuple_GET_SIZE(args), 1, PY_SSIZE_T_MAX)) { + goto exit; + } + a = PyTuple_GET_ITEM(args, 0); + if (PyTuple_GET_SIZE(args) < 2) { + goto skip_optional; + } + b = PyTuple_GET_ITEM(args, 1); +skip_optional: + __clinic_args = PyTuple_GET_SIZE(args) > 2 ? _PyTuple_ITEMS(args) + 2 : _PyTuple_ITEMS(args); + args_length = Py_MAX(0, PyTuple_GET_SIZE(args) - 2); + return_value = posonly_req_opt_varpos_array_no_fastcall_impl(type, a, b, __clinic_args, args_length); + +exit: + return return_value; +} + +static PyObject * +posonly_poskw_varpos_array_no_fastcall_impl(PyTypeObject *type, PyObject *a, + PyObject *b, + PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +posonly_poskw_varpos_array_no_fastcall(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { _Py_LATIN1_CHR('b'), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "TestClass", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + PyObject *b; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + b = fastargs[1]; + __clinic_args = PyTuple_GET_SIZE(args) > 2 ? _PyTuple_ITEMS(args) + 2 : _PyTuple_ITEMS(args); + args_length = Py_MAX(0, PyTuple_GET_SIZE(args) - 2); + return_value = posonly_poskw_varpos_array_no_fastcall_impl(type, a, b, __clinic_args, args_length); + +exit: return return_value; } -/*[clinic end generated code: output=7662d07e7d29cbeb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=00085e5637450fe6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_depr.c.h b/Modules/clinic/_testclinic_depr.c.h index 95a2cc4cb5ed6d..9378038e6b9549 100644 --- a/Modules/clinic/_testclinic_depr.c.h +++ b/Modules/clinic/_testclinic_depr.c.h @@ -9,6 +9,7 @@ preserve #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_tuple.h" // _PyTuple_FromArray() PyDoc_STRVAR(depr_star_new__doc__, "DeprStarNew(a=None)\n" @@ -79,7 +80,8 @@ depr_star_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto exit; } } - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -162,7 +164,8 @@ depr_star_new_clone(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyO goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -246,7 +249,8 @@ depr_star_init(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } } - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -329,7 +333,8 @@ depr_star_init_clone(PyObject *self, PyObject *const *args, Py_ssize_t nargs, Py goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -473,7 +478,8 @@ depr_kwd_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; PyObject *a = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -556,7 +562,8 @@ depr_kwd_init(PyObject *self, PyObject *args, PyObject *kwargs) Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; PyObject *a = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -720,7 +727,8 @@ depr_star_pos0_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -799,7 +807,8 @@ depr_star_pos0_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -882,7 +891,8 @@ depr_star_pos0_len3_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -965,7 +975,8 @@ depr_star_pos1_len1_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1049,7 +1060,8 @@ depr_star_pos1_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1132,7 +1144,8 @@ depr_star_pos1_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1216,7 +1229,8 @@ depr_star_pos2_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1300,7 +1314,8 @@ depr_star_pos2_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 4, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1386,7 +1401,8 @@ depr_star_pos2_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 4, /*maxpos*/ 4, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1562,7 +1578,8 @@ depr_star_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 7, 7, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 7, /*maxpos*/ 7, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1638,7 +1655,8 @@ depr_kwd_required_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *a; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1720,7 +1738,8 @@ depr_kwd_required_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *b; PyObject *c; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1801,7 +1820,8 @@ depr_kwd_optional_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *a; PyObject *b = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1888,7 +1908,8 @@ depr_kwd_optional_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *b = Py_None; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1981,7 +2002,8 @@ depr_kwd_optional_3(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *b = Py_None; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2079,7 +2101,8 @@ depr_kwd_required_optional(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *b; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2258,7 +2281,8 @@ depr_kwd_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *g; PyObject *h; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 8, 8, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 8, /*maxpos*/ 8, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2368,7 +2392,8 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 6, 6, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 6, /*maxpos*/ 6, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2393,4 +2418,4 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=ca6da2c7137554be input=a9049054013a1b77]*/ +/*[clinic end generated code: output=836affc1a13d67a1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index dcca9ecf735723..d98d69df22f982 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -52,7 +52,8 @@ _testinternalcapi_compiler_cleandoc(PyObject *module, PyObject *const *args, Py_ PyObject *argsbuf[1]; PyObject *doc; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -135,7 +136,8 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s int optimize; int compile_mode = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -206,7 +208,8 @@ _testinternalcapi_optimize_cfg(PyObject *module, PyObject *const *args, Py_ssize PyObject *consts; int nlocals; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -271,7 +274,8 @@ _testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, PyObject *instructions; PyObject *metadata; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -346,7 +350,8 @@ gh_119213_getargs(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *spam = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -360,4 +365,4 @@ gh_119213_getargs(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO exit: return return_value; } -/*[clinic end generated code: output=4d0770a1c20fbf40 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ec77971c6c2663da input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testmultiphase.c.h b/Modules/clinic/_testmultiphase.c.h index 452897b3fae99e..5a432a6f70386d 100644 --- a/Modules/clinic/_testmultiphase.c.h +++ b/Modules/clinic/_testmultiphase.c.h @@ -109,7 +109,8 @@ _testmultiphase_StateAccessType_increment_count_clinic(StateAccessTypeObject *se int n = 1; int twice = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -162,4 +163,4 @@ _testmultiphase_StateAccessType_get_count(StateAccessTypeObject *self, PyTypeObj } return _testmultiphase_StateAccessType_get_count_impl(self, cls); } -/*[clinic end generated code: output=59cb50dae2d11dc1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c1aa0af3572bf059 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index b0c54fc809f4c1..8bbecc44dc9c11 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -793,7 +793,8 @@ _winapi_GetLongPathName(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *argsbuf[1]; LPCWSTR path = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -897,7 +898,8 @@ _winapi_GetShortPathName(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *argsbuf[1]; LPCWSTR path = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2004,7 +2006,8 @@ _winapi__mimetypes_read_windows_registry(PyObject *module, PyObject *const *args PyObject *argsbuf[1]; PyObject *on_type_read; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2124,4 +2127,4 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -/*[clinic end generated code: output=2304c62187a90140 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b2a178bde6868e88 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_zoneinfo.c.h b/Modules/clinic/_zoneinfo.c.h index bde88b5c4fa65b..dd72b5e663aea6 100644 --- a/Modules/clinic/_zoneinfo.c.h +++ b/Modules/clinic/_zoneinfo.c.h @@ -52,7 +52,8 @@ zoneinfo_ZoneInfo(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *key; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -112,7 +113,8 @@ zoneinfo_ZoneInfo_from_file(PyTypeObject *type, PyTypeObject *cls, PyObject *con PyObject *file_obj; PyObject *key = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -173,7 +175,8 @@ zoneinfo_ZoneInfo_no_cache(PyTypeObject *type, PyTypeObject *cls, PyObject *cons PyObject *argsbuf[1]; PyObject *key; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -230,7 +233,8 @@ zoneinfo_ZoneInfo_clear_cache(PyTypeObject *type, PyTypeObject *cls, PyObject *c Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *only_keys = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -280,7 +284,8 @@ zoneinfo_ZoneInfo_utcoffset(PyObject *self, PyTypeObject *cls, PyObject *const * PyObject *argsbuf[1]; PyObject *dt; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -323,7 +328,8 @@ zoneinfo_ZoneInfo_dst(PyObject *self, PyTypeObject *cls, PyObject *const *args, PyObject *argsbuf[1]; PyObject *dt; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -367,7 +373,8 @@ zoneinfo_ZoneInfo_tzname(PyObject *self, PyTypeObject *cls, PyObject *const *arg PyObject *argsbuf[1]; PyObject *dt; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -412,7 +419,8 @@ zoneinfo_ZoneInfo__unpickle(PyTypeObject *type, PyTypeObject *cls, PyObject *con PyObject *key; unsigned char from_cache; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -431,4 +439,4 @@ zoneinfo_ZoneInfo__unpickle(PyTypeObject *type, PyTypeObject *cls, PyObject *con exit: return return_value; } -/*[clinic end generated code: output=b4fdc0b30247110a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f8a4fb4ff634d6c9 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index 2ed7eaa6abf7af..4a7266ecb8b84f 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -194,7 +194,8 @@ array_array_extend(arrayobject *self, PyTypeObject *cls, PyObject *const *args, PyObject *argsbuf[1]; PyObject *bb; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -349,7 +350,8 @@ array_array_fromfile(arrayobject *self, PyTypeObject *cls, PyObject *const *args PyObject *f; Py_ssize_t n; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -404,7 +406,8 @@ array_array_tofile(arrayobject *self, PyTypeObject *cls, PyObject *const *args, PyObject *argsbuf[1]; PyObject *f; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -650,7 +653,8 @@ array_array___reduce_ex__(arrayobject *self, PyTypeObject *cls, PyObject *const PyObject *argsbuf[1]; PyObject *value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -691,4 +695,4 @@ PyDoc_STRVAR(array_arrayiterator___setstate____doc__, #define ARRAY_ARRAYITERATOR___SETSTATE___METHODDEF \ {"__setstate__", (PyCFunction)array_arrayiterator___setstate__, METH_O, array_arrayiterator___setstate____doc__}, -/*[clinic end generated code: output=ecd63acd7924c223 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=22dbe12826bfa86f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index 1adca415dfee12..f81f12c388f373 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -85,7 +85,8 @@ binascii_b2a_uu(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj Py_buffer data = {NULL, NULL}; int backtick = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -161,7 +162,8 @@ binascii_a2b_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P Py_buffer data = {NULL, NULL}; int strict_mode = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -232,7 +234,8 @@ binascii_b2a_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P Py_buffer data = {NULL, NULL}; int newline = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -412,7 +415,8 @@ binascii_b2a_hex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb PyObject *sep = NULL; int bytes_per_sep = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -501,7 +505,8 @@ binascii_hexlify(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb PyObject *sep = NULL; int bytes_per_sep = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -646,7 +651,8 @@ binascii_a2b_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj Py_buffer data = {NULL, NULL}; int header = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -724,7 +730,8 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj int istext = 1; int header = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -767,4 +774,4 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -/*[clinic end generated code: output=968767b663ed889d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9ed7fbeec13c6606 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/blake2module.c.h b/Modules/clinic/blake2module.c.h index 50478bcbecf8e3..f695f27e9e6c42 100644 --- a/Modules/clinic/blake2module.c.h +++ b/Modules/clinic/blake2module.c.h @@ -72,7 +72,8 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int last_node = 0; int usedforsecurity = 1; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -266,7 +267,8 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int last_node = 0; int usedforsecurity = 1; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -459,4 +461,4 @@ _blake2_blake2b_hexdigest(Blake2Object *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2b_hexdigest_impl(self); } -/*[clinic end generated code: output=d1a351f44e20e273 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e0aaaf112d023b79 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/cmathmodule.c.h b/Modules/clinic/cmathmodule.c.h index 50745fd4f407a3..16eaba7aa7ee39 100644 --- a/Modules/clinic/cmathmodule.c.h +++ b/Modules/clinic/cmathmodule.c.h @@ -932,7 +932,8 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec double abs_tol = 0.0; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -982,4 +983,4 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=454309b21cfa9bf6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5fda69f15dc9dfc9 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index be3bd35b4ffd44..0147020bd483f9 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -8,6 +8,7 @@ preserve #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_tuple.h" // _PyTuple_FromArray() PyDoc_STRVAR(gc_enable__doc__, "enable($module, /)\n" @@ -125,7 +126,8 @@ gc_collect(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * int generation = NUM_GENERATIONS - 1; Py_ssize_t _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -312,23 +314,24 @@ PyDoc_STRVAR(gc_get_referrers__doc__, {"get_referrers", _PyCFunction_CAST(gc_get_referrers), METH_FASTCALL, gc_get_referrers__doc__}, static PyObject * -gc_get_referrers_impl(PyObject *module, Py_ssize_t nargs, - PyObject *const *args); +gc_get_referrers_impl(PyObject *module, PyObject *objs); static PyObject * gc_get_referrers(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t nvararg = nargs - 0; - PyObject *const *__clinic_args = NULL; + PyObject *objs = NULL; - if (!_PyArg_CheckPositional("get_referrers", nargs, 0, PY_SSIZE_T_MAX)) { + objs = _PyTuple_FromArray(args, nargs); + if (objs == NULL) { goto exit; } - __clinic_args = args + 0; - return_value = gc_get_referrers_impl(module, nvararg, __clinic_args); + return_value = gc_get_referrers_impl(module, objs); exit: + /* Cleanup for objs */ + Py_XDECREF(objs); + return return_value; } @@ -342,23 +345,24 @@ PyDoc_STRVAR(gc_get_referents__doc__, {"get_referents", _PyCFunction_CAST(gc_get_referents), METH_FASTCALL, gc_get_referents__doc__}, static PyObject * -gc_get_referents_impl(PyObject *module, Py_ssize_t nargs, - PyObject *const *args); +gc_get_referents_impl(PyObject *module, PyObject *objs); static PyObject * gc_get_referents(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t nvararg = nargs - 0; - PyObject *const *__clinic_args = NULL; + PyObject *objs = NULL; - if (!_PyArg_CheckPositional("get_referents", nargs, 0, PY_SSIZE_T_MAX)) { + objs = _PyTuple_FromArray(args, nargs); + if (objs == NULL) { goto exit; } - __clinic_args = args + 0; - return_value = gc_get_referents_impl(module, nvararg, __clinic_args); + return_value = gc_get_referents_impl(module, objs); exit: + /* Cleanup for objs */ + Py_XDECREF(objs); + return return_value; } @@ -413,7 +417,8 @@ gc_get_objects(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; Py_ssize_t generation = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -575,4 +580,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=f488a0d4d6bd3687 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3e33248997e06c34 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index 050c21460d79d7..ea0cc495ffb969 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -71,7 +71,8 @@ batched_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t n; int strict = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -186,7 +187,8 @@ itertools_groupby(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *it; PyObject *keyfunc = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -531,7 +533,8 @@ itertools_combinations(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *iterable; Py_ssize_t r; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -602,7 +605,8 @@ itertools_combinations_with_replacement(PyTypeObject *type, PyObject *args, PyOb PyObject *iterable; Py_ssize_t r; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -673,7 +677,8 @@ itertools_permutations(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *iterable; PyObject *robj = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -736,7 +741,8 @@ itertools_accumulate(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *binop = Py_None; PyObject *initial = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -809,7 +815,8 @@ itertools_compress(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *seq1; PyObject *seq2; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -908,7 +915,8 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *long_cnt = NULL; PyObject *long_step = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -928,4 +936,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=7b13be3075f2d6d3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=199eeac2d17e8f23 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index 81eec310ddb21d..a76dde1eb4350e 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -8,6 +8,60 @@ preserve #endif #include "pycore_modsupport.h" // _PyArg_CheckPositional() +PyDoc_STRVAR(math_gcd__doc__, +"gcd($module, /, *integers)\n" +"--\n" +"\n" +"Greatest Common Divisor."); + +#define MATH_GCD_METHODDEF \ + {"gcd", _PyCFunction_CAST(math_gcd), METH_FASTCALL, math_gcd__doc__}, + +static PyObject * +math_gcd_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +math_gcd(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + __clinic_args = args; + args_length = nargs; + return_value = math_gcd_impl(module, __clinic_args, args_length); + + return return_value; +} + +PyDoc_STRVAR(math_lcm__doc__, +"lcm($module, /, *integers)\n" +"--\n" +"\n" +"Least Common Multiple."); + +#define MATH_LCM_METHODDEF \ + {"lcm", _PyCFunction_CAST(math_lcm), METH_FASTCALL, math_lcm__doc__}, + +static PyObject * +math_lcm_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +math_lcm(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + __clinic_args = args; + args_length = nargs; + return_value = math_lcm_impl(module, __clinic_args, args_length); + + return return_value; +} + PyDoc_STRVAR(math_ceil__doc__, "ceil($module, x, /)\n" "--\n" @@ -351,6 +405,44 @@ math_dist(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(math_hypot__doc__, +"hypot($module, /, *coordinates)\n" +"--\n" +"\n" +"Multidimensional Euclidean distance from the origin to a point.\n" +"\n" +"Roughly equivalent to:\n" +" sqrt(sum(x**2 for x in coordinates))\n" +"\n" +"For a two dimensional point (x, y), gives the hypotenuse\n" +"using the Pythagorean theorem: sqrt(x*x + y*y).\n" +"\n" +"For example, the hypotenuse of a 3/4/5 right triangle is:\n" +"\n" +" >>> hypot(3.0, 4.0)\n" +" 5.0"); + +#define MATH_HYPOT_METHODDEF \ + {"hypot", _PyCFunction_CAST(math_hypot), METH_FASTCALL, math_hypot__doc__}, + +static PyObject * +math_hypot_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +math_hypot(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + __clinic_args = args; + args_length = nargs; + return_value = math_hypot_impl(module, __clinic_args, args_length); + + return return_value; +} + PyDoc_STRVAR(math_sumprod__doc__, "sumprod($module, p, q, /)\n" "--\n" @@ -359,7 +451,7 @@ PyDoc_STRVAR(math_sumprod__doc__, "\n" "Roughly equivalent to:\n" "\n" -" sum(itertools.starmap(operator.mul, zip(p, q, strict=True)))\n" +" sum(map(operator.mul, p, q, strict=True))\n" "\n" "For float and mixed int/float inputs, the intermediate products\n" "and sums are computed with extended precision."); @@ -672,7 +764,8 @@ math_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject double abs_tol = 0.0; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -787,7 +880,8 @@ math_prod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k PyObject *iterable; PyObject *start = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -938,7 +1032,8 @@ math_nextafter(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje double y; PyObject *steps = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1011,4 +1106,4 @@ math_ulp(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=755da3b1dbd9e45f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1ccb4b9f570d6dad input=a9049054013a1b77]*/ diff --git a/Modules/clinic/md5module.c.h b/Modules/clinic/md5module.c.h index ee7fb3d7c613f2..7721616862ed0d 100644 --- a/Modules/clinic/md5module.c.h +++ b/Modules/clinic/md5module.c.h @@ -121,7 +121,8 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *string = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -148,4 +149,4 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw exit: return return_value; } -/*[clinic end generated code: output=4dbca39332d3f52f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=62ebf28802ae8b5f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/overlapped.c.h b/Modules/clinic/overlapped.c.h index 8b285e4a8f0a72..9d5adb5193f297 100644 --- a/Modules/clinic/overlapped.c.h +++ b/Modules/clinic/overlapped.c.h @@ -484,7 +484,8 @@ _overlapped_Overlapped(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; HANDLE event = INVALID_HANDLE_VALUE; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -1239,4 +1240,4 @@ _overlapped_Overlapped_WSARecvFromInto(OverlappedObject *self, PyObject *const * return return_value; } -/*[clinic end generated code: output=958cbddbcc355f47 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=14c4f87906f28dc5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 1857fca736ef20..dce0ea100ec435 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -76,7 +76,8 @@ os_stat(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -157,7 +158,8 @@ os_lstat(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw path_t path = PATH_T_INITIALIZE_P("lstat", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -257,7 +259,8 @@ os_access(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k int follow_symlinks = 1; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -411,7 +414,8 @@ os_chdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE_P("chdir", "path", 0, 0, 0, PATH_HAVE_FCHDIR); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -476,7 +480,8 @@ os_fchdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k PyObject *argsbuf[1]; int fd; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -565,7 +570,8 @@ os_chmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = CHMOD_DEFAULT_FOLLOW_SYMLINKS; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -658,7 +664,8 @@ os_fchmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k int fd; int mode; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -728,7 +735,8 @@ os_lchmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k path_t path = PATH_T_INITIALIZE_P("lchmod", "path", 0, 0, 0, 0); int mode; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -806,7 +814,8 @@ os_chflags(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * unsigned long flags; int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -887,7 +896,8 @@ os_lchflags(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject path_t path = PATH_T_INITIALIZE_P("lchflags", "path", 0, 0, 0, 0); unsigned long flags; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -956,7 +966,8 @@ os_chroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE_P("chroot", "path", 0, 0, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1020,7 +1031,8 @@ os_fsync(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *argsbuf[1]; int fd; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1104,7 +1116,8 @@ os_fdatasync(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *argsbuf[1]; int fd; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1196,7 +1209,8 @@ os_chown(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1286,7 +1300,8 @@ os_fchown(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k uid_t uid; gid_t gid; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1359,7 +1374,8 @@ os_lchown(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k uid_t uid; gid_t gid; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1482,7 +1498,8 @@ os_link(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn int dst_dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1585,7 +1602,8 @@ os_listdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; path_t path = PATH_T_INITIALIZE_P("listdir", "path", 1, 0, 0, PATH_HAVE_FDOPENDIR); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1701,7 +1719,8 @@ os_listmounts(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec PyObject *argsbuf[1]; path_t volume = PATH_T_INITIALIZE_P("listmounts", "volume", 0, 0, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1765,7 +1784,8 @@ os__path_isdevdrive(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE_P("_path_isdevdrive", "path", 0, 0, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1930,7 +1950,8 @@ os__getvolumepathname(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE_P("_getvolumepathname", "path", 0, 0, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1994,7 +2015,8 @@ os__path_splitroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE_P("_path_splitroot", "path", 0, 0, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2059,7 +2081,8 @@ os__path_exists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj path_t path = PATH_T_INITIALIZE_P("_path_exists", "path", 0, 0, 1, 1); int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2128,7 +2151,8 @@ os__path_lexists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb path_t path = PATH_T_INITIALIZE_P("_path_lexists", "path", 0, 0, 1, 1); int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2197,7 +2221,8 @@ os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje path_t path = PATH_T_INITIALIZE_P("_path_isdir", "path", 0, 0, 1, 1); int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2266,7 +2291,8 @@ os__path_isfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj path_t path = PATH_T_INITIALIZE_P("_path_isfile", "path", 0, 0, 1, 1); int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2335,7 +2361,8 @@ os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj path_t path = PATH_T_INITIALIZE_P("_path_islink", "path", 0, 0, 1, 1); int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2404,7 +2431,8 @@ os__path_isjunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P path_t path = PATH_T_INITIALIZE_P("_path_isjunction", "path", 0, 0, 1, 1); int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2472,7 +2500,8 @@ os__path_splitroot_ex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE("_path_splitroot_ex", "path", 0, 1, 1, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2532,7 +2561,8 @@ os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE("_path_normpath", "path", 0, 1, 1, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2603,7 +2633,8 @@ os_mkdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw int mode = 511; int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2718,7 +2749,8 @@ os_getpriority(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje int which; int who; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2786,7 +2818,8 @@ os_setpriority(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje int who; int priority; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2865,7 +2898,8 @@ os_rename(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k int src_dir_fd = DEFAULT_DIR_FD; int dst_dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2956,7 +2990,8 @@ os_replace(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * int src_dir_fd = DEFAULT_DIR_FD; int dst_dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3043,7 +3078,8 @@ os_rmdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw path_t path = PATH_T_INITIALIZE_P("rmdir", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3113,7 +3149,8 @@ os_system(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k const wchar_t *command = NULL; long _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3187,7 +3224,8 @@ os_system(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k PyObject *command = NULL; long _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3292,7 +3330,8 @@ os_unlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k path_t path = PATH_T_INITIALIZE_P("unlink", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3366,7 +3405,8 @@ os_remove(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k path_t path = PATH_T_INITIALIZE_P("remove", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3487,7 +3527,8 @@ os_utime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3579,7 +3620,8 @@ os__exit(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *argsbuf[1]; int status; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3692,7 +3734,8 @@ os_execve(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k PyObject *argv; PyObject *env; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3795,7 +3838,8 @@ os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *setsigdef = NULL; PyObject *scheduler = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3945,7 +3989,8 @@ os_posix_spawnp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *setsigdef = NULL; PyObject *scheduler = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -4181,7 +4226,8 @@ os_register_at_fork(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *after_in_child = NULL; PyObject *after_in_parent = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -4304,7 +4350,8 @@ os_sched_get_priority_max(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *argsbuf[1]; int policy; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -4366,7 +4413,8 @@ os_sched_get_priority_min(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *argsbuf[1]; int policy; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -4464,7 +4512,8 @@ os_sched_param(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *sched_priority; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -5327,7 +5376,8 @@ os_getpgid(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * PyObject *argsbuf[1]; pid_t pid; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -5838,7 +5888,8 @@ os_wait3(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *argsbuf[1]; int options; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -5904,7 +5955,8 @@ os_wait4(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw pid_t pid; int options; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -6145,7 +6197,8 @@ os_pidfd_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec pid_t pid; unsigned int flags = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -6221,7 +6274,8 @@ os_setns(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw int fd; int nstype = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -6294,7 +6348,8 @@ os_unshare(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * PyObject *argsbuf[1]; int flags; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -6364,7 +6419,8 @@ os_readlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject path_t path = PATH_T_INITIALIZE_P("readlink", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -6450,7 +6506,8 @@ os_symlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * int target_is_directory = 0; int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -6579,7 +6636,8 @@ os_timerfd_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO int clockid; int flags = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -6663,7 +6721,8 @@ os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py double initial_double = 0.0; double interval_double = 0.0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -6777,7 +6836,8 @@ os_timerfd_settime_ns(PyObject *module, PyObject *const *args, Py_ssize_t nargs, long long initial = 0; long long interval = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -7109,7 +7169,8 @@ os_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn int dir_fd = DEFAULT_DIR_FD; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -7197,7 +7258,8 @@ os_close(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *argsbuf[1]; int fd; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -7330,7 +7392,8 @@ os_dup2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn int inheritable = 1; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -7790,7 +7853,8 @@ os_sendfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *trailers = NULL; int flags = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 7, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 4, /*maxpos*/ 7, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -7892,7 +7956,8 @@ os_sendfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *trailers = NULL; int flags = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 7, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 4, /*maxpos*/ 7, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -7997,7 +8062,8 @@ os_sendfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *offobj; Py_ssize_t count; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 4, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -8122,7 +8188,8 @@ os_fstat(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *argsbuf[1]; int fd; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -8475,7 +8542,8 @@ os_copy_file_range(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py PyObject *offset_src = Py_None; PyObject *offset_dst = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 5, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -8590,7 +8658,8 @@ os_splice(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k PyObject *offset_dst = Py_None; unsigned int flags = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 6, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 6, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -8695,7 +8764,8 @@ os_mkfifo(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k int mode = 438; int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -8796,7 +8866,8 @@ os_mknod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw dev_t device = 0; int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9036,7 +9107,8 @@ os_truncate(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject path_t path = PATH_T_INITIALIZE_P("truncate", "path", 0, 0, 0, PATH_HAVE_FTRUNCATE); Py_off_t length; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9428,7 +9500,8 @@ os_WIFCONTINUED(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj int status; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9495,7 +9568,8 @@ os_WIFSTOPPED(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec int status; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9562,7 +9636,8 @@ os_WIFSIGNALED(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje int status; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9629,7 +9704,8 @@ os_WIFEXITED(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject int status; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9696,7 +9772,8 @@ os_WEXITSTATUS(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje int status; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9763,7 +9840,8 @@ os_WTERMSIG(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject int status; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9830,7 +9908,8 @@ os_WSTOPSIG(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject int status; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9934,7 +10013,8 @@ os_statvfs(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE_P("statvfs", "path", 0, 0, 0, PATH_HAVE_FSTATVFS); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9998,7 +10078,8 @@ os__getdiskusage(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE_P("_getdiskusage", "path", 0, 0, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -10114,7 +10195,8 @@ os_pathconf(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject int name; long _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -10306,7 +10388,8 @@ os_startfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject path_t cwd = PATH_T_INITIALIZE_P("startfile", "cwd", 1, 0, 0, 0); int show_cmd = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 5, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -10445,7 +10528,8 @@ os_device_encoding(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py PyObject *argsbuf[1]; int fd; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -10642,7 +10726,8 @@ os_getxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject path_t attribute = PATH_T_INITIALIZE_P("getxattr", "attribute", 0, 0, 0, 0); int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -10731,7 +10816,8 @@ os_setxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject int flags = 0; int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -10837,7 +10923,8 @@ os_removexattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje path_t attribute = PATH_T_INITIALIZE_P("removexattr", "attribute", 0, 0, 0, 0); int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -10922,7 +11009,8 @@ os_listxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject path_t path = PATH_T_INITIALIZE_P("listxattr", "path", 1, 0, 0, 1); int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11040,7 +11128,8 @@ os_memfd_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *name = NULL; unsigned int flags = MFD_CLOEXEC; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11114,7 +11203,8 @@ os_eventfd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * unsigned int initval; int flags = EFD_CLOEXEC; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11183,7 +11273,8 @@ os_eventfd_read(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *argsbuf[1]; int fd; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11246,7 +11337,8 @@ os_eventfd_write(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb int fd; unsigned long long value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11663,7 +11755,8 @@ os_DirEntry_stat(DirEntry *self, PyTypeObject *defining_class, PyObject *const * Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11728,7 +11821,8 @@ os_DirEntry_is_dir(DirEntry *self, PyTypeObject *defining_class, PyObject *const int follow_symlinks = 1; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11797,7 +11891,8 @@ os_DirEntry_is_file(DirEntry *self, PyTypeObject *defining_class, PyObject *cons int follow_symlinks = 1; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11906,7 +12001,8 @@ os_scandir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; path_t path = PATH_T_INITIALIZE_P("scandir", "path", 1, 0, 0, PATH_HAVE_FDOPENDIR); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11974,7 +12070,8 @@ os_fspath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k PyObject *argsbuf[1]; PyObject *path; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -12033,7 +12130,8 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject Py_ssize_t size; int flags = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -12118,7 +12216,8 @@ os__add_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE_P("_add_dll_directory", "path", 0, 0, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -12186,7 +12285,8 @@ os__remove_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *argsbuf[1]; PyObject *cookie; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -12257,7 +12357,8 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *argsbuf[1]; PyObject *status_obj; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -13013,4 +13114,4 @@ os__create_environ(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=9756767bdbdabe94 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5358a13b4ce6148b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index 682d8481a2a2f4..e57aa8a07d78c7 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -91,7 +91,8 @@ pyexpat_xmlparser_Parse(xmlparseobject *self, PyTypeObject *cls, PyObject *const PyObject *data; int isfinal = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -143,7 +144,8 @@ pyexpat_xmlparser_ParseFile(xmlparseobject *self, PyTypeObject *cls, PyObject *c PyObject *argsbuf[1]; PyObject *file; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -267,7 +269,8 @@ pyexpat_xmlparser_ExternalEntityParserCreate(xmlparseobject *self, PyTypeObject const char *context; const char *encoding = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -384,7 +387,8 @@ pyexpat_xmlparser_UseForeignDTD(xmlparseobject *self, PyTypeObject *cls, PyObjec PyObject *argsbuf[1]; int flag = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -453,7 +457,8 @@ pyexpat_ParserCreate(PyObject *module, PyObject *const *args, Py_ssize_t nargs, const char *namespace_separator = NULL; PyObject *intern = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -545,4 +550,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */ -/*[clinic end generated code: output=9f1e9a7192d29976 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=63be65cb1823b5f8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index 49c0e48d2e0eac..806a888d6b8cd9 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -596,7 +596,8 @@ select_epoll(PyTypeObject *type, PyObject *args, PyObject *kwargs) int sizehint = -1; int flags = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -766,7 +767,8 @@ select_epoll_register(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t na int fd; unsigned int eventmask = EPOLLIN | EPOLLPRI | EPOLLOUT; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -843,7 +845,8 @@ select_epoll_modify(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t narg int fd; unsigned int eventmask; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -912,7 +915,8 @@ select_epoll_unregister(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t PyObject *argsbuf[1]; int fd; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -986,7 +990,8 @@ select_epoll_poll(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t nargs, PyObject *timeout_obj = Py_None; int maxevents = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1360,4 +1365,4 @@ select_kqueue_control(kqueue_queue_Object *self, PyObject *const *args, Py_ssize #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=f99427b75cbe6d44 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=78b4e67f7d401b5e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha1module.c.h b/Modules/clinic/sha1module.c.h index b89c7e505c788e..6af77ba64ecce6 100644 --- a/Modules/clinic/sha1module.c.h +++ b/Modules/clinic/sha1module.c.h @@ -121,7 +121,8 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * PyObject *string = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -148,4 +149,4 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=af5a640df662066f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=917e2789f1f5ebf9 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha2module.c.h b/Modules/clinic/sha2module.c.h index cf4b88d52856b8..fec655a0dfaa58 100644 --- a/Modules/clinic/sha2module.c.h +++ b/Modules/clinic/sha2module.c.h @@ -188,7 +188,8 @@ _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *string = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -262,7 +263,8 @@ _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *string = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -336,7 +338,8 @@ _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *string = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -410,7 +413,8 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *string = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -437,4 +441,4 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject exit: return return_value; } -/*[clinic end generated code: output=b46da764024b1764 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=602a6939b8ec0927 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha3module.c.h b/Modules/clinic/sha3module.c.h index 738e9589503900..d9f4b66f81a038 100644 --- a/Modules/clinic/sha3module.c.h +++ b/Modules/clinic/sha3module.c.h @@ -54,7 +54,8 @@ py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *data = NULL; int usedforsecurity = 1; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -194,4 +195,4 @@ _sha3_shake_128_hexdigest(SHA3object *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=a8e76a880d1421ec input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5c644eb0ed42b993 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index 986c0289f2bfcb..dd8da456ce08ce 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -332,7 +332,8 @@ signal_set_wakeup_fd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *fdobj; int warn_on_full_buffer = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -776,4 +777,4 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=c57b4b98fad6f4b8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=356e1acc44a4f377 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/socketmodule.c.h b/Modules/clinic/socketmodule.c.h index 7b0a3f8d4b1cc6..db1a28b86c8773 100644 --- a/Modules/clinic/socketmodule.c.h +++ b/Modules/clinic/socketmodule.c.h @@ -77,7 +77,8 @@ sock_initobj(PyObject *self, PyObject *args, PyObject *kwargs) int proto = -1; PyObject *fdobj = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 4, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -286,4 +287,4 @@ _socket_socket_if_nametoindex(PySocketSockObject *self, PyObject *arg) #ifndef _SOCKET_SOCKET_IF_NAMETOINDEX_METHODDEF #define _SOCKET_SOCKET_IF_NAMETOINDEX_METHODDEF #endif /* !defined(_SOCKET_SOCKET_IF_NAMETOINDEX_METHODDEF) */ -/*[clinic end generated code: output=6037e47b012911c5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=59c36bb31b05de68 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/syslogmodule.c.h b/Modules/clinic/syslogmodule.c.h index 77cf24ea5ba9ca..7c3fcd64191500 100644 --- a/Modules/clinic/syslogmodule.c.h +++ b/Modules/clinic/syslogmodule.c.h @@ -58,7 +58,8 @@ syslog_openlog(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje long logopt = 0; long facility = LOG_USER; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -262,4 +263,4 @@ syslog_LOG_UPTO(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=8d25899bd31969d3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f867a4f7a9267de1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index 7ff3edf5a557f8..19906dc328d897 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -63,7 +63,8 @@ zlib_compress(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec int level = Z_DEFAULT_COMPRESSION; int wbits = MAX_WBITS; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -153,7 +154,8 @@ zlib_decompress(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj int wbits = MAX_WBITS; Py_ssize_t bufsize = DEF_BUF_SIZE; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -271,7 +273,8 @@ zlib_compressobj(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb int strategy = Z_DEFAULT_STRATEGY; Py_buffer zdict = {NULL, NULL}; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 6, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 6, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -390,7 +393,8 @@ zlib_decompressobj(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py int wbits = MAX_WBITS; PyObject *zdict = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -454,7 +458,8 @@ zlib_Compress_compress(compobject *self, PyTypeObject *cls, PyObject *const *arg PyObject *argsbuf[1]; Py_buffer data = {NULL, NULL}; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -530,7 +535,8 @@ zlib_Decompress_decompress(compobject *self, PyTypeObject *cls, PyObject *const Py_buffer data = {NULL, NULL}; Py_ssize_t max_length = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -602,7 +608,8 @@ zlib_Compress_flush(compobject *self, PyTypeObject *cls, PyObject *const *args, PyObject *argsbuf[1]; int mode = Z_FINISH; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -705,7 +712,8 @@ zlib_Compress___deepcopy__(compobject *self, PyTypeObject *cls, PyObject *const PyObject *argsbuf[1]; PyObject *memo; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -803,7 +811,8 @@ zlib_Decompress___deepcopy__(compobject *self, PyTypeObject *cls, PyObject *cons PyObject *argsbuf[1]; PyObject *memo; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -852,7 +861,8 @@ zlib_Decompress_flush(compobject *self, PyTypeObject *cls, PyObject *const *args PyObject *argsbuf[1]; Py_ssize_t length = DEF_BUF_SIZE; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -938,7 +948,8 @@ zlib_ZlibDecompressor_decompress(ZlibDecompressor *self, PyObject *const *args, Py_buffer data = {NULL, NULL}; Py_ssize_t max_length = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1098,4 +1109,4 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=8bb840fb6af43dd4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2fef49f168842b17 input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index e07c2dbd262354..81cbf0d554de3c 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -150,7 +150,7 @@ special_type(double d) #define P14 0.25*Py_MATH_PI #define P12 0.5*Py_MATH_PI #define P34 0.75*Py_MATH_PI -#define INF Py_HUGE_VAL +#define INF Py_INFINITY #define N Py_NAN #define U -9.5426319407711027e33 /* unlikely value, used as placeholder */ diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index d0d6015a66283f..523b37d8d5787d 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -130,7 +130,9 @@ enum XML_Error { /* Added in 2.3.0. */ XML_ERROR_NO_BUFFER, /* Added in 2.4.0. */ - XML_ERROR_AMPLIFICATION_LIMIT_BREACH + XML_ERROR_AMPLIFICATION_LIMIT_BREACH, + /* Added in 2.6.4. */ + XML_ERROR_NOT_STARTED, }; enum XML_Content_Type { @@ -1066,7 +1068,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); */ #define XML_MAJOR_VERSION 2 #define XML_MINOR_VERSION 6 -#define XML_MICRO_VERSION 3 +#define XML_MICRO_VERSION 4 #ifdef __cplusplus } diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index 12c560e14716ff..567872b09836e1 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -40,6 +40,10 @@ #ifndef Expat_External_INCLUDED #define Expat_External_INCLUDED 1 +/* Namespace external symbols to allow multiple libexpat version to + co-exist. */ +#include "pyexpatns.h" + /* External API definitions */ /* Expat tries very hard to make the API boundary very specifically @@ -64,11 +68,6 @@ compiled with the cdecl calling convention as the default since system headers may assume the cdecl convention. */ - -/* Namespace external symbols to allow multiple libexpat version to - co-exist. */ -#include "pyexpatns.h" - #ifndef XMLCALL # if defined(_MSC_VER) # define XMLCALL __cdecl diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh new file mode 100755 index 00000000000000..82a9dbc23ad26b --- /dev/null +++ b/Modules/expat/refresh.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +# Use this script to update libexpat + +set -e +set -o pipefail + +if [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then + echo "A bash version >= 4 required. Got: $BASH_VERSION" >&2 + exit 1 +fi + +# Update this when updating to a new version after verifying that the changes +# the update brings in are good. These values are used for verifying the SBOM, too. +expected_libexpat_tag="R_2_6_4" +expected_libexpat_version="2.6.4" +expected_libexpat_sha256="fd03b7172b3bd7427a3e7a812063f74754f24542429b634e0db6511b53fb2278" + +expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" +cd ${expat_dir} + +# Step 1: download and copy files +curl --location "https://github.com/libexpat/libexpat/releases/download/${expected_libexpat_tag}/expat-${expected_libexpat_version}.tar.gz" > libexpat.tar.gz +echo "${expected_libexpat_sha256} libexpat.tar.gz" | sha256sum --check + +# Step 2: Pull files from the libexpat distribution +declare -a lib_files +lib_files=( + ascii.h + asciitab.h + expat.h + expat_external.h + iasciitab.h + internal.h + latin1tab.h + nametab.h + siphash.h + utf8tab.h + winconfig.h + xmlparse.c + xmlrole.c + xmlrole.h + xmltok.c + xmltok.h + xmltok_impl.c + xmltok_impl.h + xmltok_ns.c +) +for f in "${lib_files[@]}"; do + tar xzvf libexpat.tar.gz "expat-${expected_libexpat_version}/lib/${f}" --strip-components 2 +done +rm libexpat.tar.gz + +# Step 3: Add the namespacing include to expat_external.h +sed -i 's/#define Expat_External_INCLUDED 1/&\n\n\/* Namespace external symbols to allow multiple libexpat version to\n co-exist. \*\/\n#include "pyexpatns.h"/' expat_external.h + +echo "Updated; verify all is okay using git diff and git status." diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index d9285b213b38bd..a4e091e7c33c0a 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* ba4cdf9bdb534f355a9def4c9e25d20ee8e72f95b0a4d930be52e563f5080196 (2.6.3+) +/* c5625880f4bf417c1463deee4eb92d86ff413f802048621c57e25fe483eb59e4 (2.6.4+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -40,6 +40,7 @@ Copyright (c) 2023 Owain Davies Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow Copyright (c) 2024 Berkay Eren Ürün + Copyright (c) 2024 Hanno Böck Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -2234,6 +2235,9 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable) { if (parser == NULL) return XML_STATUS_ERROR; switch (parser->m_parsingStatus.parsing) { + case XML_INITIALIZED: + parser->m_errorCode = XML_ERROR_NOT_STARTED; + return XML_STATUS_ERROR; case XML_SUSPENDED: if (resumable) { parser->m_errorCode = XML_ERROR_SUSPENDED; @@ -2244,7 +2248,7 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable) { case XML_FINISHED: parser->m_errorCode = XML_ERROR_FINISHED; return XML_STATUS_ERROR; - default: + case XML_PARSING: if (resumable) { #ifdef XML_DTD if (parser->m_isParamEntity) { @@ -2255,6 +2259,9 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable) { parser->m_parsingStatus.parsing = XML_SUSPENDED; } else parser->m_parsingStatus.parsing = XML_FINISHED; + break; + default: + assert(0); } return XML_STATUS_OK; } @@ -2519,6 +2526,9 @@ XML_ErrorString(enum XML_Error code) { case XML_ERROR_AMPLIFICATION_LIMIT_BREACH: return XML_L( "limit on input amplification factor (from DTD and entities) breached"); + /* Added in 2.6.4. */ + case XML_ERROR_NOT_STARTED: + return XML_L("parser not started"); } return NULL; } @@ -7856,7 +7866,7 @@ accountingReportDiff(XML_Parser rootParser, assert(! rootParser->m_parentParser); fprintf(stderr, - " (+" EXPAT_FMT_PTRDIFF_T("6") " bytes %s|%d, xmlparse.c:%d) %*s\"", + " (+" EXPAT_FMT_PTRDIFF_T("6") " bytes %s|%u, xmlparse.c:%d) %*s\"", bytesMore, (account == XML_ACCOUNT_DIRECT) ? "DIR" : "EXP", levelsAwayFromRootParser, source_line, 10, ""); @@ -7969,7 +7979,7 @@ entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity, fprintf( stderr, - "expat: Entities(%p): Count %9d, depth %2d/%2d %*s%s%s; %s length %d (xmlparse.c:%d)\n", + "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s; %s length %d (xmlparse.c:%d)\n", (void *)rootParser, rootParser->m_entity_stats.countEverOpened, rootParser->m_entity_stats.currentDepth, rootParser->m_entity_stats.maximumDepthSeen, diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index f5fea5aa4dde08..ad13496b06deaf 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -216,31 +216,21 @@ gc_get_count_impl(PyObject *module) /*[clinic input] gc.get_referrers - *objs as args: object + *objs: tuple Return the list of objects that directly refer to any of 'objs'. [clinic start generated code]*/ static PyObject * -gc_get_referrers_impl(PyObject *module, Py_ssize_t nargs, - PyObject *const *args) -/*[clinic end generated code: output=1d44a7695ea25c40 input=bae96961b14a0922]*/ +gc_get_referrers_impl(PyObject *module, PyObject *objs) +/*[clinic end generated code: output=929d6dff26f609b9 input=9102be7ebee69ee3]*/ { - PyObject *varargs = _PyTuple_FromArray(args, nargs); - - if (!varargs) { - return NULL; - } - if (PySys_Audit("gc.get_referrers", "(O)", varargs) < 0) { - Py_DECREF(varargs); + if (PySys_Audit("gc.get_referrers", "(O)", objs) < 0) { return NULL; } PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *result = _PyGC_GetReferrers(interp, varargs); - - Py_DECREF(varargs); - return result; + return _PyGC_GetReferrers(interp, objs); } /* Append obj to list; return true if error (out of memory), false if OK. */ @@ -274,43 +264,34 @@ append_referrents(PyObject *result, PyObject *args) /*[clinic input] gc.get_referents - *objs as args: object + *objs: tuple Return the list of objects that are directly referred to by 'objs'. [clinic start generated code]*/ static PyObject * -gc_get_referents_impl(PyObject *module, Py_ssize_t nargs, - PyObject *const *args) -/*[clinic end generated code: output=e459f3e8c0d19311 input=b3ceab0c34038cbf]*/ +gc_get_referents_impl(PyObject *module, PyObject *objs) +/*[clinic end generated code: output=6dfde40cd1588e1d input=55c078a6d0248fe0]*/ { - PyObject *varargs = _PyTuple_FromArray(args, nargs); - - if (!varargs) { - return NULL; - } - if (PySys_Audit("gc.get_referents", "(O)", varargs) < 0) { - Py_DECREF(varargs); + if (PySys_Audit("gc.get_referents", "(O)", objs) < 0) { return NULL; } PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *result = PyList_New(0); if (result == NULL) { - Py_DECREF(varargs); return NULL; } // NOTE: stop the world is a no-op in default build _PyEval_StopTheWorld(interp); - int err = append_referrents(result, varargs); + int err = append_referrents(result, objs); _PyEval_StartTheWorld(interp); if (err < 0) { Py_CLEAR(result); } - Py_DECREF(varargs); return result; } diff --git a/Modules/getpath.py b/Modules/getpath.py index 1f1bfcb4f64dd4..1c1eb6cbf3ee22 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -420,7 +420,7 @@ def search_up(prefix, *landmarks, test=isfile): # Only warn if the file actually exists and was unresolvable # Otherwise users who specify a fake executable may get spurious warnings. if isfile(real_executable): - warn(f'Failed to find real location of {base_executable}') + warn(f'Failed to find real location of {real_executable}') if not executable_dir and os_name == 'darwin' and library: # QUIRK: macOS checks adjacent to its library early diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 1201fa094902d7..78fbdcdf77a923 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -3291,6 +3291,9 @@ itertools_count_impl(PyTypeObject *type, PyObject *long_cnt, PyErr_Clear(); fast_mode = 0; } + else if (cnt == PY_SSIZE_T_MAX) { + fast_mode = 0; + } } } else { cnt = 0; diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 058f57770755aa..29638114dd94a9 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -438,7 +438,7 @@ m_tgamma(double x) } else { errno = ERANGE; - return Py_HUGE_VAL; + return Py_INFINITY; } } @@ -502,14 +502,14 @@ m_lgamma(double x) if (isnan(x)) return x; /* lgamma(nan) = nan */ else - return Py_HUGE_VAL; /* lgamma(+-inf) = +inf */ + return Py_INFINITY; /* lgamma(+-inf) = +inf */ } /* integer arguments */ if (x == floor(x) && x <= 2.0) { if (x <= 0.0) { errno = EDOM; /* lgamma(n) = inf, divide-by-zero for */ - return Py_HUGE_VAL; /* integers n <= 0 */ + return Py_INFINITY; /* integers n <= 0 */ } else { return 0.0; /* lgamma(1) = lgamma(2) = 0.0 */ @@ -645,7 +645,7 @@ m_log(double x) return log(x); errno = EDOM; if (x == 0.0) - return -Py_HUGE_VAL; /* log(0) = -inf */ + return -Py_INFINITY; /* log(0) = -inf */ else return Py_NAN; /* log(-ve) = nan */ } @@ -688,7 +688,7 @@ m_log2(double x) } else if (x == 0.0) { errno = EDOM; - return -Py_HUGE_VAL; /* log2(0) = -inf, divide-by-zero */ + return -Py_INFINITY; /* log2(0) = -inf, divide-by-zero */ } else { errno = EDOM; @@ -704,7 +704,7 @@ m_log10(double x) return log10(x); errno = EDOM; if (x == 0.0) - return -Py_HUGE_VAL; /* log10(0) = -inf */ + return -Py_INFINITY; /* log10(0) = -inf */ else return Py_NAN; /* log10(-ve) = nan */ } @@ -719,16 +719,26 @@ m_log10(double x) } +/*[clinic input] +math.gcd + + *integers as args: array + +Greatest Common Divisor. +[clinic start generated code]*/ + static PyObject * -math_gcd(PyObject *module, PyObject * const *args, Py_ssize_t nargs) +math_gcd_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=a26c95907374ffb4 input=ded7f0ea3850c05c]*/ { // Fast-path for the common case: gcd(int, int) - if (nargs == 2 && PyLong_CheckExact(args[0]) && PyLong_CheckExact(args[1])) + if (args_length == 2 && PyLong_CheckExact(args[0]) && PyLong_CheckExact(args[1])) { return _PyLong_GCD(args[0], args[1]); } - if (nargs == 0) { + if (args_length == 0) { return PyLong_FromLong(0); } @@ -736,13 +746,13 @@ math_gcd(PyObject *module, PyObject * const *args, Py_ssize_t nargs) if (res == NULL) { return NULL; } - if (nargs == 1) { + if (args_length == 1) { Py_SETREF(res, PyNumber_Absolute(res)); return res; } PyObject *one = _PyLong_GetOne(); // borrowed ref - for (Py_ssize_t i = 1; i < nargs; i++) { + for (Py_ssize_t i = 1; i < args_length; i++) { PyObject *x = _PyNumber_Index(args[i]); if (x == NULL) { Py_DECREF(res); @@ -763,12 +773,6 @@ math_gcd(PyObject *module, PyObject * const *args, Py_ssize_t nargs) return res; } -PyDoc_STRVAR(math_gcd_doc, -"gcd($module, *integers)\n" -"--\n" -"\n" -"Greatest Common Divisor."); - static PyObject * long_lcm(PyObject *a, PyObject *b) @@ -798,26 +802,36 @@ long_lcm(PyObject *a, PyObject *b) } +/*[clinic input] +math.lcm + + *integers as args: array + +Least Common Multiple. +[clinic start generated code]*/ + static PyObject * -math_lcm(PyObject *module, PyObject * const *args, Py_ssize_t nargs) +math_lcm_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=c8a59a5c2e55c816 input=3e4f4b7cdf948a98]*/ { PyObject *res, *x; Py_ssize_t i; - if (nargs == 0) { + if (args_length == 0) { return PyLong_FromLong(1); } res = PyNumber_Index(args[0]); if (res == NULL) { return NULL; } - if (nargs == 1) { + if (args_length == 1) { Py_SETREF(res, PyNumber_Absolute(res)); return res; } PyObject *zero = _PyLong_GetZero(); // borrowed ref - for (i = 1; i < nargs; i++) { + for (i = 1; i < args_length; i++) { x = PyNumber_Index(args[i]); if (x == NULL) { Py_DECREF(res); @@ -839,13 +853,6 @@ math_lcm(PyObject *module, PyObject * const *args, Py_ssize_t nargs) } -PyDoc_STRVAR(math_lcm_doc, -"lcm($module, *integers)\n" -"--\n" -"\n" -"Least Common Multiple."); - - /* Call is_error when errno != 0, and where x is the result libm * returned. is_error will usually set up an exception and return * true (1), but may return false (0) without setting up an exception. @@ -2121,7 +2128,7 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) errno = 0; } else if (exp > INT_MAX) { /* overflow */ - r = copysign(Py_HUGE_VAL, x); + r = copysign(Py_INFINITY, x); errno = ERANGE; } else if (exp < INT_MIN) { /* underflow to +-0 */ @@ -2621,9 +2628,29 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q) return NULL; } -/* AC: cannot convert yet, waiting for *args support */ +/*[clinic input] +math.hypot + + *coordinates as args: array + +Multidimensional Euclidean distance from the origin to a point. + +Roughly equivalent to: + sqrt(sum(x**2 for x in coordinates)) + +For a two dimensional point (x, y), gives the hypotenuse +using the Pythagorean theorem: sqrt(x*x + y*y). + +For example, the hypotenuse of a 3/4/5 right triangle is: + + >>> hypot(3.0, 4.0) + 5.0 +[clinic start generated code]*/ + static PyObject * -math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +math_hypot_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=c9de404e24370068 input=1bceaf7d4fdcd9c2]*/ { Py_ssize_t i; PyObject *item; @@ -2633,13 +2660,13 @@ math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) double coord_on_stack[NUM_STACK_ELEMS]; double *coordinates = coord_on_stack; - if (nargs > NUM_STACK_ELEMS) { - coordinates = (double *) PyMem_Malloc(nargs * sizeof(double)); + if (args_length > NUM_STACK_ELEMS) { + coordinates = (double *) PyMem_Malloc(args_length * sizeof(double)); if (coordinates == NULL) { return PyErr_NoMemory(); } } - for (i = 0; i < nargs; i++) { + for (i = 0; i < args_length; i++) { item = args[i]; ASSIGN_DOUBLE(x, item, error_exit); x = fabs(x); @@ -2649,7 +2676,7 @@ math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) max = x; } } - result = vector_norm(nargs, coordinates, max, found_nan); + result = vector_norm(args_length, coordinates, max, found_nan); if (coordinates != coord_on_stack) { PyMem_Free(coordinates); } @@ -2664,22 +2691,6 @@ math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) #undef NUM_STACK_ELEMS -PyDoc_STRVAR(math_hypot_doc, - "hypot(*coordinates) -> value\n\n\ -Multidimensional Euclidean distance from the origin to a point.\n\ -\n\ -Roughly equivalent to:\n\ - sqrt(sum(x**2 for x in coordinates))\n\ -\n\ -For a two dimensional point (x, y), gives the hypotenuse\n\ -using the Pythagorean theorem: sqrt(x*x + y*y).\n\ -\n\ -For example, the hypotenuse of a 3/4/5 right triangle is:\n\ -\n\ - >>> hypot(3.0, 4.0)\n\ - 5.0\n\ -"); - /** sumprod() ***************************************************************/ /* Forward declaration */ @@ -2702,7 +2713,7 @@ Return the sum of products of values from two iterables p and q. Roughly equivalent to: - sum(itertools.starmap(operator.mul, zip(p, q, strict=True))) + sum(map(operator.mul, p, q, strict=True)) For float and mixed int/float inputs, the intermediate products and sums are computed with extended precision. @@ -2710,7 +2721,7 @@ and sums are computed with extended precision. static PyObject * math_sumprod_impl(PyObject *module, PyObject *p, PyObject *q) -/*[clinic end generated code: output=6722dbfe60664554 input=82be54fe26f87e30]*/ +/*[clinic end generated code: output=6722dbfe60664554 input=a2880317828c61d2]*/ { PyObject *p_i = NULL, *q_i = NULL, *term_i = NULL, *new_total = NULL; PyObject *p_it, *q_it, *total; @@ -4112,14 +4123,14 @@ static PyMethodDef math_methods[] = { MATH_FREXP_METHODDEF MATH_FSUM_METHODDEF {"gamma", math_gamma, METH_O, math_gamma_doc}, - {"gcd", _PyCFunction_CAST(math_gcd), METH_FASTCALL, math_gcd_doc}, - {"hypot", _PyCFunction_CAST(math_hypot), METH_FASTCALL, math_hypot_doc}, + MATH_GCD_METHODDEF + MATH_HYPOT_METHODDEF MATH_ISCLOSE_METHODDEF MATH_ISFINITE_METHODDEF MATH_ISINF_METHODDEF MATH_ISNAN_METHODDEF MATH_ISQRT_METHODDEF - {"lcm", _PyCFunction_CAST(math_lcm), METH_FASTCALL, math_lcm_doc}, + MATH_LCM_METHODDEF MATH_LDEXP_METHODDEF {"lgamma", math_lgamma, METH_O, math_lgamma_doc}, {"log", _PyCFunction_CAST(math_log), METH_FASTCALL, math_log_doc}, diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index bb5077cc7f0f09..da7399de86f213 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -24,6 +24,7 @@ #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_signal.h" // Py_NSIG #include "pycore_time.h" // _PyLong_FromTime_t() +#include "pycore_typeobject.h" // _PyType_AddMethod() #ifdef HAVE_UNISTD_H # include // symlink() @@ -677,6 +678,7 @@ PyOS_AfterFork_Child(void) _PyEval_StartTheWorldAll(&_PyRuntime); _PyThreadState_DeleteList(list); + _PyImport_ReInitLock(tstate->interp); _PyImport_ReleaseLock(tstate->interp); _PySignal_AfterFork(); @@ -8210,6 +8212,16 @@ os_sched_param_impl(PyTypeObject *type, PyObject *sched_priority) return res; } +static PyObject * +os_sched_param_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return Py_BuildValue("(O(N))", Py_TYPE(self), PyStructSequence_GetItem(self, 0)); +} + +static PyMethodDef os_sched_param_reduce_method = { + "__reduce__", (PyCFunction)os_sched_param_reduce, METH_NOARGS|METH_COEXIST, NULL, +}; + PyDoc_VAR(os_sched_param__doc__); static PyStructSequence_Field sched_param_fields[] = { @@ -18033,6 +18045,12 @@ posixmodule_exec(PyObject *m) return -1; } ((PyTypeObject *)state->SchedParamType)->tp_new = os_sched_param; + if (_PyType_AddMethod((PyTypeObject *)state->SchedParamType, + &os_sched_param_reduce_method) < 0) + { + return -1; + } + PyType_Modified((PyTypeObject *)state->SchedParamType); #endif /* initialize TerminalSize_info */ diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index fd2a85a3fe0a61..5a52b2f702ad0b 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2452,6 +2452,7 @@ PyTypeObject PyByteArray_Type = { PyType_GenericAlloc, /* tp_alloc */ PyType_GenericNew, /* tp_new */ PyObject_Free, /* tp_free */ + .tp_version_tag = _Py_TYPE_VERSION_BYTEARRAY, }; /*********************** Bytearray Iterator ****************************/ diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index dcc1aba76abbed..ac02cfe7cf01c5 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3080,6 +3080,7 @@ PyTypeObject PyBytes_Type = { bytes_alloc, /* tp_alloc */ bytes_new, /* tp_new */ PyObject_Free, /* tp_free */ + .tp_version_tag = _Py_TYPE_VERSION_BYTES, }; void diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index c748c53e1c0a75..dee7c1e8bffd25 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -50,7 +50,8 @@ bytearray___init__(PyObject *self, PyObject *args, PyObject *kwargs) const char *encoding = NULL; const char *errors = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 3, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -617,7 +618,8 @@ bytearray_translate(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t n PyObject *table; PyObject *deletechars = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -804,7 +806,8 @@ bytearray_split(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs PyObject *sep = Py_None; Py_ssize_t maxsplit = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -926,7 +929,8 @@ bytearray_rsplit(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t narg PyObject *sep = Py_None; Py_ssize_t maxsplit = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1303,7 +1307,8 @@ bytearray_decode(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t narg const char *encoding = NULL; const char *errors = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1409,7 +1414,8 @@ bytearray_splitlines(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int keepends = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1522,7 +1528,8 @@ bytearray_hex(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *sep = NULL; int bytes_per_sep = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1616,4 +1623,4 @@ bytearray_sizeof(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl(self); } -/*[clinic end generated code: output=5f861b02e3fa278b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4488e38e7ffcc6ec input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index 0b4b37501735c1..d2c6cc88770999 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -81,7 +81,8 @@ bytes_split(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *sep = Py_None; Py_ssize_t maxsplit = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -247,7 +248,8 @@ bytes_rsplit(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *sep = Py_None; Py_ssize_t maxsplit = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -702,7 +704,8 @@ bytes_translate(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, Py PyObject *table; PyObject *deletechars = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1063,7 +1066,8 @@ bytes_decode(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObj const char *encoding = NULL; const char *errors = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1156,7 +1160,8 @@ bytes_splitlines(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, P Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int keepends = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1269,7 +1274,8 @@ bytes_hex(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *sep = NULL; int bytes_per_sep = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1334,7 +1340,8 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) const char *encoding = NULL; const char *errors = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 3, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -1384,4 +1391,4 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=d6801c6001e57f91 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fb7939a1983e463a input=a9049054013a1b77]*/ diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h index 68e2d7f528c061..45738f767df50f 100644 --- a/Objects/clinic/codeobject.c.h +++ b/Objects/clinic/codeobject.c.h @@ -223,7 +223,8 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *co_linetable = self->co_linetable; PyObject *co_exceptiontable = self->co_exceptiontable; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -451,7 +452,8 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n PyObject *argsbuf[1]; int oparg; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -464,4 +466,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n exit: return return_value; } -/*[clinic end generated code: output=d604263a3ca72a0f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e919ea67a1bcf524 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/complexobject.c.h b/Objects/clinic/complexobject.c.h index 58fd4e26871b4d..3c3d1071b6eec3 100644 --- a/Objects/clinic/complexobject.c.h +++ b/Objects/clinic/complexobject.c.h @@ -140,7 +140,8 @@ complex_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *r = NULL; PyObject *i = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -169,4 +170,4 @@ PyDoc_STRVAR(complex_from_number__doc__, #define COMPLEX_FROM_NUMBER_METHODDEF \ {"from_number", (PyCFunction)complex_from_number, METH_O|METH_CLASS, complex_from_number__doc__}, -/*[clinic end generated code: output=188438cc9ae167f7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8c49a41c5a7f0aee input=a9049054013a1b77]*/ diff --git a/Objects/clinic/descrobject.c.h b/Objects/clinic/descrobject.c.h index d79be80d3ec165..2f96adb5094fe3 100644 --- a/Objects/clinic/descrobject.c.h +++ b/Objects/clinic/descrobject.c.h @@ -51,7 +51,8 @@ mappingproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *mapping; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -141,7 +142,8 @@ property_init(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *fdel = NULL; PyObject *doc = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 4, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -173,4 +175,4 @@ property_init(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=050e331316a04207 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2f43f9297a36aa40 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/enumobject.c.h b/Objects/clinic/enumobject.c.h index 09774a73c8145c..998b2eb121c6d9 100644 --- a/Objects/clinic/enumobject.c.h +++ b/Objects/clinic/enumobject.c.h @@ -62,7 +62,8 @@ enum_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *iterable; PyObject *start = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -107,4 +108,4 @@ reversed_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=5c48a9a482a52e91 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2e339ade8bedb3a0 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/funcobject.c.h b/Objects/clinic/funcobject.c.h index 8f20bda26438cf..3f95a1db7b2788 100644 --- a/Objects/clinic/funcobject.c.h +++ b/Objects/clinic/funcobject.c.h @@ -73,7 +73,8 @@ func_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *closure = Py_None; PyObject *kwdefaults = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 6, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 6, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -115,4 +116,4 @@ func_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=10947342188f38a9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bad4e19757dd26c3 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/listobject.c.h index 588e021fb71fd3..975f253c096275 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -235,7 +235,8 @@ list_sort(PyListObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *keyfunc = Py_None; int reverse = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -439,4 +440,4 @@ list___reversed__(PyListObject *self, PyObject *Py_UNUSED(ignored)) { return list___reversed___impl(self); } -/*[clinic end generated code: output=854957a1d4a89bbd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9357151278d77ea1 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/longobject.c.h b/Objects/clinic/longobject.c.h index 90375b9a082cca..c025f74af5cb58 100644 --- a/Objects/clinic/longobject.c.h +++ b/Objects/clinic/longobject.c.h @@ -48,7 +48,8 @@ long_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *x = NULL; PyObject *obase = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -315,7 +316,8 @@ int_to_bytes(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject * PyObject *byteorder = NULL; int is_signed = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -426,7 +428,8 @@ int_from_bytes(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs, PyOb PyObject *byteorder = NULL; int is_signed = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -476,4 +479,4 @@ int_is_integer(PyObject *self, PyObject *Py_UNUSED(ignored)) { return int_is_integer_impl(self); } -/*[clinic end generated code: output=a53f5ba9a6c16737 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fb96bd642a643f75 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index f199434dacb9e8..185d7819b6b84a 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -51,7 +51,8 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *object; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -107,7 +108,8 @@ memoryview__from_flags(PyTypeObject *type, PyObject *const *args, Py_ssize_t nar PyObject *object; int flags; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -187,7 +189,8 @@ memoryview_cast(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t narg PyObject *format; PyObject *shape = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -294,7 +297,8 @@ memoryview_tobytes(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t n Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; const char *order = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -390,7 +394,8 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs PyObject *sep = NULL; int bytes_per_sep = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -413,4 +418,4 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=7e76a09106921ba2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0a93f08110630633 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/moduleobject.c.h b/Objects/clinic/moduleobject.c.h index 3c0bbe22d5ebab..5535a2f1178b9f 100644 --- a/Objects/clinic/moduleobject.c.h +++ b/Objects/clinic/moduleobject.c.h @@ -55,7 +55,8 @@ module___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *name; PyObject *doc = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -74,4 +75,4 @@ module___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=e8a71bfbed774c15 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d7da56d5c2eb6d30 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/odictobject.c.h b/Objects/clinic/odictobject.c.h index 5ef5380656905a..4b97596e5dbd2f 100644 --- a/Objects/clinic/odictobject.c.h +++ b/Objects/clinic/odictobject.c.h @@ -54,7 +54,8 @@ OrderedDict_fromkeys(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs PyObject *seq; PyObject *value = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -119,7 +120,8 @@ OrderedDict_setdefault(PyODictObject *self, PyObject *const *args, Py_ssize_t na PyObject *key; PyObject *default_value = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -185,7 +187,8 @@ OrderedDict_pop(PyODictObject *self, PyObject *const *args, Py_ssize_t nargs, Py PyObject *key; PyObject *default_value = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -248,7 +251,8 @@ OrderedDict_popitem(PyODictObject *self, PyObject *const *args, Py_ssize_t nargs Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int last = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -314,7 +318,8 @@ OrderedDict_move_to_end(PyODictObject *self, PyObject *const *args, Py_ssize_t n PyObject *key; int last = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -332,4 +337,4 @@ OrderedDict_move_to_end(PyODictObject *self, PyObject *const *args, Py_ssize_t n exit: return return_value; } -/*[clinic end generated code: output=eff78d2a3f9379bd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2aa6fc0567c9252c input=a9049054013a1b77]*/ diff --git a/Objects/clinic/setobject.c.h b/Objects/clinic/setobject.c.h index d6e381a9975050..986993b4aa9bda 100644 --- a/Objects/clinic/setobject.c.h +++ b/Objects/clinic/setobject.c.h @@ -3,7 +3,6 @@ preserve [clinic start generated code]*/ #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() -#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(set_pop__doc__, "pop($self, /)\n" @@ -41,22 +40,20 @@ PyDoc_STRVAR(set_update__doc__, {"update", _PyCFunction_CAST(set_update), METH_FASTCALL, set_update__doc__}, static PyObject * -set_update_impl(PySetObject *so, Py_ssize_t nargs, PyObject *const *args); +set_update_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length); static PyObject * set_update(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t nvararg = nargs - 0; - PyObject *const *__clinic_args = NULL; + PyObject * const *others; + Py_ssize_t others_length; - if (!_PyArg_CheckPositional("update", nargs, 0, PY_SSIZE_T_MAX)) { - goto exit; - } - __clinic_args = args + 0; - return_value = set_update_impl(so, nvararg, __clinic_args); + others = args; + others_length = nargs; + return_value = set_update_impl(so, others, others_length); -exit: return return_value; } @@ -142,22 +139,20 @@ PyDoc_STRVAR(set_union__doc__, {"union", _PyCFunction_CAST(set_union), METH_FASTCALL, set_union__doc__}, static PyObject * -set_union_impl(PySetObject *so, Py_ssize_t nargs, PyObject *const *args); +set_union_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length); static PyObject * set_union(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t nvararg = nargs - 0; - PyObject *const *__clinic_args = NULL; + PyObject * const *others; + Py_ssize_t others_length; - if (!_PyArg_CheckPositional("union", nargs, 0, PY_SSIZE_T_MAX)) { - goto exit; - } - __clinic_args = args + 0; - return_value = set_union_impl(so, nvararg, __clinic_args); + others = args; + others_length = nargs; + return_value = set_union_impl(so, others, others_length); -exit: return return_value; } @@ -171,23 +166,20 @@ PyDoc_STRVAR(set_intersection_multi__doc__, {"intersection", _PyCFunction_CAST(set_intersection_multi), METH_FASTCALL, set_intersection_multi__doc__}, static PyObject * -set_intersection_multi_impl(PySetObject *so, Py_ssize_t nargs, - PyObject *const *args); +set_intersection_multi_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length); static PyObject * set_intersection_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t nvararg = nargs - 0; - PyObject *const *__clinic_args = NULL; + PyObject * const *others; + Py_ssize_t others_length; - if (!_PyArg_CheckPositional("intersection", nargs, 0, PY_SSIZE_T_MAX)) { - goto exit; - } - __clinic_args = args + 0; - return_value = set_intersection_multi_impl(so, nvararg, __clinic_args); + others = args; + others_length = nargs; + return_value = set_intersection_multi_impl(so, others, others_length); -exit: return return_value; } @@ -201,23 +193,20 @@ PyDoc_STRVAR(set_intersection_update_multi__doc__, {"intersection_update", _PyCFunction_CAST(set_intersection_update_multi), METH_FASTCALL, set_intersection_update_multi__doc__}, static PyObject * -set_intersection_update_multi_impl(PySetObject *so, Py_ssize_t nargs, - PyObject *const *args); +set_intersection_update_multi_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length); static PyObject * set_intersection_update_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t nvararg = nargs - 0; - PyObject *const *__clinic_args = NULL; + PyObject * const *others; + Py_ssize_t others_length; - if (!_PyArg_CheckPositional("intersection_update", nargs, 0, PY_SSIZE_T_MAX)) { - goto exit; - } - __clinic_args = args + 0; - return_value = set_intersection_update_multi_impl(so, nvararg, __clinic_args); + others = args; + others_length = nargs; + return_value = set_intersection_update_multi_impl(so, others, others_length); -exit: return return_value; } @@ -255,23 +244,20 @@ PyDoc_STRVAR(set_difference_update__doc__, {"difference_update", _PyCFunction_CAST(set_difference_update), METH_FASTCALL, set_difference_update__doc__}, static PyObject * -set_difference_update_impl(PySetObject *so, Py_ssize_t nargs, - PyObject *const *args); +set_difference_update_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length); static PyObject * set_difference_update(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t nvararg = nargs - 0; - PyObject *const *__clinic_args = NULL; + PyObject * const *others; + Py_ssize_t others_length; - if (!_PyArg_CheckPositional("difference_update", nargs, 0, PY_SSIZE_T_MAX)) { - goto exit; - } - __clinic_args = args + 0; - return_value = set_difference_update_impl(so, nvararg, __clinic_args); + others = args; + others_length = nargs; + return_value = set_difference_update_impl(so, others, others_length); -exit: return return_value; } @@ -285,23 +271,20 @@ PyDoc_STRVAR(set_difference_multi__doc__, {"difference", _PyCFunction_CAST(set_difference_multi), METH_FASTCALL, set_difference_multi__doc__}, static PyObject * -set_difference_multi_impl(PySetObject *so, Py_ssize_t nargs, - PyObject *const *args); +set_difference_multi_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length); static PyObject * set_difference_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t nvararg = nargs - 0; - PyObject *const *__clinic_args = NULL; + PyObject * const *others; + Py_ssize_t others_length; - if (!_PyArg_CheckPositional("difference", nargs, 0, PY_SSIZE_T_MAX)) { - goto exit; - } - __clinic_args = args + 0; - return_value = set_difference_multi_impl(so, nvararg, __clinic_args); + others = args; + others_length = nargs; + return_value = set_difference_multi_impl(so, others, others_length); -exit: return return_value; } @@ -536,4 +519,4 @@ set___sizeof__(PySetObject *so, PyObject *Py_UNUSED(ignored)) return return_value; } -/*[clinic end generated code: output=9d4b41191b2c602f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4b65e7709927f31f input=a9049054013a1b77]*/ diff --git a/Objects/clinic/structseq.c.h b/Objects/clinic/structseq.c.h index cec49ebccccb24..0afb21ad918669 100644 --- a/Objects/clinic/structseq.c.h +++ b/Objects/clinic/structseq.c.h @@ -47,7 +47,8 @@ structseq_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *arg; PyObject *dict = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -62,4 +63,4 @@ structseq_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=5bf39b3f06a34ce4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ef2406240ce0ad3f input=a9049054013a1b77]*/ diff --git a/Objects/clinic/typevarobject.c.h b/Objects/clinic/typevarobject.c.h index 0ba4ff48bc8804..c17998830b02eb 100644 --- a/Objects/clinic/typevarobject.c.h +++ b/Objects/clinic/typevarobject.c.h @@ -6,7 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_modsupport.h" // _PyArg_UnpackKeywordsWithVararg() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(typevar_new__doc__, "typevar(name, *constraints, bound=None, default=typing.NoDefault,\n" @@ -49,7 +49,7 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[7]; + PyObject *argsbuf[6]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; @@ -61,7 +61,8 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int contravariant = 0; int infer_variance = 0; - fastargs = _PyArg_UnpackKeywordsWithVararg(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, 1, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 1, argsbuf); if (!fastargs) { goto exit; } @@ -70,24 +71,23 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto exit; } name = fastargs[0]; - constraints = fastargs[1]; if (!noptargs) { goto skip_optional_kwonly; } - if (fastargs[2]) { - bound = fastargs[2]; + if (fastargs[1]) { + bound = fastargs[1]; if (!--noptargs) { goto skip_optional_kwonly; } } - if (fastargs[3]) { - default_value = fastargs[3]; + if (fastargs[2]) { + default_value = fastargs[2]; if (!--noptargs) { goto skip_optional_kwonly; } } - if (fastargs[4]) { - covariant = PyObject_IsTrue(fastargs[4]); + if (fastargs[3]) { + covariant = PyObject_IsTrue(fastargs[3]); if (covariant < 0) { goto exit; } @@ -95,8 +95,8 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - if (fastargs[5]) { - contravariant = PyObject_IsTrue(fastargs[5]); + if (fastargs[4]) { + contravariant = PyObject_IsTrue(fastargs[4]); if (contravariant < 0) { goto exit; } @@ -104,15 +104,21 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - infer_variance = PyObject_IsTrue(fastargs[6]); + infer_variance = PyObject_IsTrue(fastargs[5]); if (infer_variance < 0) { goto exit; } skip_optional_kwonly: + constraints = PyTuple_GetSlice(args, 1, PY_SSIZE_T_MAX); + if (!constraints) { + goto exit; + } return_value = typevar_new_impl(type, name, constraints, bound, default_value, covariant, contravariant, infer_variance); exit: + /* Cleanup for constraints */ Py_XDECREF(constraints); + return return_value; } @@ -231,7 +237,8 @@ paramspecargs_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *origin; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -285,7 +292,8 @@ paramspeckwargs_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *origin; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -348,7 +356,8 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int contravariant = 0; int infer_variance = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -519,7 +528,8 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *name; PyObject *default_value = &_Py_NoDefaultStruct; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -675,7 +685,8 @@ typealias_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *value; PyObject *type_params = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -695,4 +706,4 @@ typealias_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=73b39e550e4e336c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=26351c3549f5ad83 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index 78e14b0021d006..361f20c0fa4c1d 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -246,7 +246,8 @@ unicode_encode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject const char *encoding = NULL; const char *errors = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -338,7 +339,8 @@ unicode_expandtabs(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int tabsize = 8; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -958,7 +960,8 @@ unicode_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObjec PyObject *new; Py_ssize_t count = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1282,7 +1285,8 @@ unicode_split(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *sep = Py_None; Py_ssize_t maxsplit = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1404,7 +1408,8 @@ unicode_rsplit(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *sep = Py_None; Py_ssize_t maxsplit = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1484,7 +1489,8 @@ unicode_splitlines(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int keepends = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1838,7 +1844,8 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) const char *encoding = NULL; const char *errors = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 3, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -1888,4 +1895,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=9fee62bd337f809b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=30efbf79c5a07dd2 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 775ea7aca824c4..dba43d5911da95 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -6,17 +6,22 @@ #include "pycore_code.h" // _PyCodeConstructor #include "pycore_frame.h" // FRAME_SPECIALS_SIZE #include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_index_pool.h" // _PyIndexPool #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // PyInterpreterState.co_extra_freefuncs #include "pycore_object.h" // _PyObject_SetDeferredRefcount +#include "pycore_object_stack.h" #include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches #include "pycore_opcode_utils.h" // RESUME_AT_FUNC_START +#include "pycore_pymem.h" // _PyMem_FreeDelayed #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_uniqueid.h" // _PyObject_AssignUniqueId() #include "clinic/codeobject.c.h" +#define INITIAL_SPECIALIZED_CODE_SIZE 16 + static const char * code_event_name(PyCodeEvent event) { switch (event) { @@ -440,9 +445,15 @@ _PyCode_Validate(struct _PyCodeConstructor *con) return 0; } -extern void _PyCode_Quicken(PyCodeObject *code); +extern void +_PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, PyObject *consts, + int enable_counters); -static void +#ifdef Py_GIL_DISABLED +static _PyCodeArray * _PyCodeArray_New(Py_ssize_t size); +#endif + +static int init_code(PyCodeObject *co, struct _PyCodeConstructor *con) { int nlocalsplus = (int)PyTuple_GET_SIZE(con->localsplusnames); @@ -505,14 +516,27 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) memcpy(_PyCode_CODE(co), PyBytes_AS_STRING(con->code), PyBytes_GET_SIZE(con->code)); +#ifdef Py_GIL_DISABLED + co->co_tlbc = _PyCodeArray_New(INITIAL_SPECIALIZED_CODE_SIZE); + if (co->co_tlbc == NULL) { + return -1; + } + co->co_tlbc->entries[0] = co->co_code_adaptive; +#endif int entry_point = 0; while (entry_point < Py_SIZE(co) && _PyCode_CODE(co)[entry_point].op.code != RESUME) { entry_point++; } co->_co_firsttraceable = entry_point; - _PyCode_Quicken(co); +#ifdef Py_GIL_DISABLED + _PyCode_Quicken(_PyCode_CODE(co), Py_SIZE(co), co->co_consts, + interp->config.tlbc_enabled); +#else + _PyCode_Quicken(_PyCode_CODE(co), Py_SIZE(co), co->co_consts, 1); +#endif notify_code_watchers(PY_CODE_EVENT_CREATE, co); + return 0; } static int @@ -667,7 +691,12 @@ _PyCode_New(struct _PyCodeConstructor *con) PyErr_NoMemory(); return NULL; } - init_code(co, con); + + if (init_code(co, con) < 0) { + Py_DECREF(co); + return NULL; + } + #ifdef Py_GIL_DISABLED co->_co_unique_id = _PyObject_AssignUniqueId((PyObject *)co); _PyObject_GC_TRACK(co); @@ -1871,6 +1900,17 @@ code_dealloc(PyCodeObject *co) PyObject_ClearWeakRefs((PyObject*)co); } free_monitoring_data(co->_co_monitoring); +#ifdef Py_GIL_DISABLED + // The first element always points to the mutable bytecode at the end of + // the code object, which will be freed when the code object is freed. + for (Py_ssize_t i = 1; i < co->co_tlbc->size; i++) { + char *entry = co->co_tlbc->entries[i]; + if (entry != NULL) { + PyMem_Free(entry); + } + } + PyMem_Free(co->co_tlbc); +#endif PyObject_Free(co); } @@ -2348,7 +2388,6 @@ _PyCode_ConstantKey(PyObject *op) if (op == Py_None || op == Py_Ellipsis || PyLong_CheckExact(op) || PyUnicode_CheckExact(op) - || PySlice_Check(op) /* code_richcompare() uses _PyCode_ConstantKey() internally */ || PyCode_Check(op)) { @@ -2456,6 +2495,40 @@ _PyCode_ConstantKey(PyObject *op) Py_DECREF(set); return key; } + else if (PySlice_Check(op)) { + PySliceObject *slice = (PySliceObject *)op; + PyObject *start_key = NULL; + PyObject *stop_key = NULL; + PyObject *step_key = NULL; + key = NULL; + + start_key = _PyCode_ConstantKey(slice->start); + if (start_key == NULL) { + goto slice_exit; + } + + stop_key = _PyCode_ConstantKey(slice->stop); + if (stop_key == NULL) { + goto slice_exit; + } + + step_key = _PyCode_ConstantKey(slice->step); + if (step_key == NULL) { + goto slice_exit; + } + + PyObject *slice_key = PySlice_New(start_key, stop_key, step_key); + if (slice_key == NULL) { + goto slice_exit; + } + + key = PyTuple_Pack(2, slice_key, op); + Py_DECREF(slice_key); + slice_exit: + Py_XDECREF(start_key); + Py_XDECREF(stop_key); + Py_XDECREF(step_key); + } else { /* for other types, use the object identifier as a unique identifier * to ensure that they are seen as unequal. */ @@ -2646,5 +2719,270 @@ _PyCode_Fini(PyInterpreterState *interp) _Py_hashtable_destroy(state->constants); state->constants = NULL; } + _PyIndexPool_Fini(&interp->tlbc_indices); #endif } + +#ifdef Py_GIL_DISABLED + +// Thread-local bytecode (TLBC) +// +// Each thread specializes a thread-local copy of the bytecode, created on the +// first RESUME, in free-threaded builds. All copies of the bytecode for a code +// object are stored in the `co_tlbc` array. Threads reserve a globally unique +// index identifying its copy of the bytecode in all `co_tlbc` arrays at thread +// creation and release the index at thread destruction. The first entry in +// every `co_tlbc` array always points to the "main" copy of the bytecode that +// is stored at the end of the code object. This ensures that no bytecode is +// copied for programs that do not use threads. +// +// Thread-local bytecode can be disabled at runtime by providing either `-X +// tlbc=0` or `PYTHON_TLBC=0`. Disabling thread-local bytecode also disables +// specialization. All threads share the main copy of the bytecode when +// thread-local bytecode is disabled. +// +// Concurrent modifications to the bytecode made by the specializing +// interpreter and instrumentation use atomics, with specialization taking care +// not to overwrite an instruction that was instrumented concurrently. + +int32_t +_Py_ReserveTLBCIndex(PyInterpreterState *interp) +{ + if (interp->config.tlbc_enabled) { + return _PyIndexPool_AllocIndex(&interp->tlbc_indices); + } + // All threads share the main copy of the bytecode when TLBC is disabled + return 0; +} + +void +_Py_ClearTLBCIndex(_PyThreadStateImpl *tstate) +{ + PyInterpreterState *interp = ((PyThreadState *)tstate)->interp; + if (interp->config.tlbc_enabled) { + _PyIndexPool_FreeIndex(&interp->tlbc_indices, tstate->tlbc_index); + } +} + +static _PyCodeArray * +_PyCodeArray_New(Py_ssize_t size) +{ + _PyCodeArray *arr = PyMem_Calloc( + 1, offsetof(_PyCodeArray, entries) + sizeof(void *) * size); + if (arr == NULL) { + PyErr_NoMemory(); + return NULL; + } + arr->size = size; + return arr; +} + +static void +copy_code(_Py_CODEUNIT *dst, PyCodeObject *co) +{ + int code_len = (int) Py_SIZE(co); + for (int i = 0; i < code_len; i += _PyInstruction_GetLength(co, i)) { + dst[i] = _Py_GetBaseCodeUnit(co, i); + } + _PyCode_Quicken(dst, code_len, co->co_consts, 1); +} + +static Py_ssize_t +get_pow2_greater(Py_ssize_t initial, Py_ssize_t limit) +{ + // initial must be a power of two + assert(!(initial & (initial - 1))); + Py_ssize_t res = initial; + while (res && res < limit) { + res <<= 1; + } + return res; +} + +static _Py_CODEUNIT * +create_tlbc_lock_held(PyCodeObject *co, Py_ssize_t idx) +{ + _PyCodeArray *tlbc = co->co_tlbc; + if (idx >= tlbc->size) { + Py_ssize_t new_size = get_pow2_greater(tlbc->size, idx + 1); + if (!new_size) { + PyErr_NoMemory(); + return NULL; + } + _PyCodeArray *new_tlbc = _PyCodeArray_New(new_size); + if (new_tlbc == NULL) { + return NULL; + } + memcpy(new_tlbc->entries, tlbc->entries, tlbc->size * sizeof(void *)); + _Py_atomic_store_ptr_release(&co->co_tlbc, new_tlbc); + _PyMem_FreeDelayed(tlbc); + tlbc = new_tlbc; + } + char *bc = PyMem_Calloc(1, _PyCode_NBYTES(co)); + if (bc == NULL) { + PyErr_NoMemory(); + return NULL; + } + copy_code((_Py_CODEUNIT *) bc, co); + assert(tlbc->entries[idx] == NULL); + tlbc->entries[idx] = bc; + return (_Py_CODEUNIT *) bc; +} + +static _Py_CODEUNIT * +get_tlbc_lock_held(PyCodeObject *co) +{ + _PyCodeArray *tlbc = co->co_tlbc; + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)PyThreadState_GET(); + int32_t idx = tstate->tlbc_index; + if (idx < tlbc->size && tlbc->entries[idx] != NULL) { + return (_Py_CODEUNIT *)tlbc->entries[idx]; + } + return create_tlbc_lock_held(co, idx); +} + +_Py_CODEUNIT * +_PyCode_GetTLBC(PyCodeObject *co) +{ + _Py_CODEUNIT *result; + Py_BEGIN_CRITICAL_SECTION(co); + result = get_tlbc_lock_held(co); + Py_END_CRITICAL_SECTION(); + return result; +} + +// My kingdom for a bitset +struct flag_set { + uint8_t *flags; + Py_ssize_t size; +}; + +static inline int +flag_is_set(struct flag_set *flags, Py_ssize_t idx) +{ + assert(idx >= 0); + return (idx < flags->size) && flags->flags[idx]; +} + +// Set the flag for each tlbc index in use +static int +get_indices_in_use(PyInterpreterState *interp, struct flag_set *in_use) +{ + assert(interp->stoptheworld.world_stopped); + assert(in_use->flags == NULL); + int32_t max_index = 0; + for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + int32_t idx = ((_PyThreadStateImpl *) p)->tlbc_index; + if (idx > max_index) { + max_index = idx; + } + } + in_use->size = (size_t) max_index + 1; + in_use->flags = PyMem_Calloc(in_use->size, sizeof(*in_use->flags)); + if (in_use->flags == NULL) { + return -1; + } + for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + in_use->flags[((_PyThreadStateImpl *) p)->tlbc_index] = 1; + } + return 0; +} + +struct get_code_args { + _PyObjectStack code_objs; + struct flag_set indices_in_use; + int err; +}; + +static void +clear_get_code_args(struct get_code_args *args) +{ + if (args->indices_in_use.flags != NULL) { + PyMem_Free(args->indices_in_use.flags); + args->indices_in_use.flags = NULL; + } + _PyObjectStack_Clear(&args->code_objs); +} + +static inline int +is_bytecode_unused(_PyCodeArray *tlbc, Py_ssize_t idx, + struct flag_set *indices_in_use) +{ + assert(idx > 0 && idx < tlbc->size); + return tlbc->entries[idx] != NULL && !flag_is_set(indices_in_use, idx); +} + +static int +get_code_with_unused_tlbc(PyObject *obj, struct get_code_args *args) +{ + if (!PyCode_Check(obj)) { + return 1; + } + PyCodeObject *co = (PyCodeObject *) obj; + _PyCodeArray *tlbc = co->co_tlbc; + // The first index always points at the main copy of the bytecode embedded + // in the code object. + for (Py_ssize_t i = 1; i < tlbc->size; i++) { + if (is_bytecode_unused(tlbc, i, &args->indices_in_use)) { + if (_PyObjectStack_Push(&args->code_objs, obj) < 0) { + args->err = -1; + return 0; + } + return 1; + } + } + return 1; +} + +static void +free_unused_bytecode(PyCodeObject *co, struct flag_set *indices_in_use) +{ + _PyCodeArray *tlbc = co->co_tlbc; + // The first index always points at the main copy of the bytecode embedded + // in the code object. + for (Py_ssize_t i = 1; i < tlbc->size; i++) { + if (is_bytecode_unused(tlbc, i, indices_in_use)) { + PyMem_Free(tlbc->entries[i]); + tlbc->entries[i] = NULL; + } + } +} + +int +_Py_ClearUnusedTLBC(PyInterpreterState *interp) +{ + struct get_code_args args = { + .code_objs = {NULL}, + .indices_in_use = {NULL, 0}, + .err = 0, + }; + _PyEval_StopTheWorld(interp); + // Collect in-use tlbc indices + if (get_indices_in_use(interp, &args.indices_in_use) < 0) { + goto err; + } + // Collect code objects that have bytecode not in use by any thread + _PyGC_VisitObjectsWorldStopped( + interp, (gcvisitobjects_t)get_code_with_unused_tlbc, &args); + if (args.err < 0) { + goto err; + } + // Free unused bytecode. This must happen outside of gc_visit_heaps; it is + // unsafe to allocate or free any mimalloc managed memory when it's + // running. + PyObject *obj; + while ((obj = _PyObjectStack_Pop(&args.code_objs)) != NULL) { + free_unused_bytecode((PyCodeObject*) obj, &args.indices_in_use); + } + _PyEval_StartTheWorld(interp); + clear_get_code_args(&args); + return 0; + +err: + _PyEval_StartTheWorld(interp); + clear_get_code_args(&args); + PyErr_NoMemory(); + return -1; +} + +#endif diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 108e5d0b085e65..7975d0ba0ca80a 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -1362,4 +1362,5 @@ PyTypeObject PyComplex_Type = { PyType_GenericAlloc, /* tp_alloc */ actual_complex_new, /* tp_new */ PyObject_Free, /* tp_free */ + .tp_version_tag = _Py_TYPE_VERSION_COMPLEX, }; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index f28a92657834be..19a3ba9e13e5cc 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -883,6 +883,7 @@ new_dict(PyInterpreterState *interp, mp->ma_used = used; mp->_ma_watcher_tag = 0; ASSERT_CONSISTENT(mp); + _PyObject_GC_TRACK(mp); return (PyObject *)mp; } @@ -1578,64 +1579,6 @@ _PyDict_HasOnlyStringKeys(PyObject *dict) return 1; } -#define MAINTAIN_TRACKING(mp, key, value) \ - do { \ - if (!_PyObject_GC_IS_TRACKED(mp)) { \ - if (_PyObject_GC_MAY_BE_TRACKED(key) || \ - _PyObject_GC_MAY_BE_TRACKED(value)) { \ - _PyObject_GC_TRACK(mp); \ - } \ - } \ - } while(0) - -void -_PyDict_MaybeUntrack(PyObject *op) -{ - PyDictObject *mp; - PyObject *value; - Py_ssize_t i, numentries; - - ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op); - - if (!PyDict_CheckExact(op) || !_PyObject_GC_IS_TRACKED(op)) - return; - - mp = (PyDictObject *) op; - ASSERT_CONSISTENT(mp); - numentries = mp->ma_keys->dk_nentries; - if (_PyDict_HasSplitTable(mp)) { - for (i = 0; i < numentries; i++) { - if ((value = mp->ma_values->values[i]) == NULL) - continue; - if (_PyObject_GC_MAY_BE_TRACKED(value)) { - return; - } - } - } - else { - if (DK_IS_UNICODE(mp->ma_keys)) { - PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(mp->ma_keys); - for (i = 0; i < numentries; i++) { - if ((value = ep0[i].me_value) == NULL) - continue; - if (_PyObject_GC_MAY_BE_TRACKED(value)) - return; - } - } - else { - PyDictKeyEntry *ep0 = DK_ENTRIES(mp->ma_keys); - for (i = 0; i < numentries; i++) { - if ((value = ep0[i].me_value) == NULL) - continue; - if (_PyObject_GC_MAY_BE_TRACKED(value) || - _PyObject_GC_MAY_BE_TRACKED(ep0[i].me_key)) - return; - } - } - } - _PyObject_GC_UNTRACK(op); -} - void _PyDict_EnablePerThreadRefcounting(PyObject *op) { @@ -1761,7 +1704,6 @@ insert_split_value(PyInterpreterState *interp, PyDictObject *mp, PyObject *key, { assert(PyUnicode_CheckExact(key)); ASSERT_DICT_LOCKED(mp); - MAINTAIN_TRACKING(mp, key, value); PyObject *old_value = mp->ma_values->values[ix]; if (old_value == NULL) { _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); @@ -1818,8 +1760,6 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, if (ix == DKIX_ERROR) goto Fail; - MAINTAIN_TRACKING(mp, key, value); - if (ix == DKIX_EMPTY) { assert(!_PyDict_HasSplitTable(mp)); /* Insert into new slot. */ @@ -1878,8 +1818,6 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, /* We don't decref Py_EMPTY_KEYS here because it is immortal. */ assert(mp->ma_values == NULL); - MAINTAIN_TRACKING(mp, key, value); - size_t hashpos = (size_t)hash & (PyDict_MINSIZE-1); dictkeys_set_index(newkeys, hashpos, 0); if (unicode) { @@ -3770,11 +3708,6 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe STORE_USED(mp, other->ma_used); ASSERT_CONSISTENT(mp); - if (_PyObject_GC_IS_TRACKED(other) && !_PyObject_GC_IS_TRACKED(mp)) { - /* Maintain tracking. */ - _PyObject_GC_TRACK(mp); - } - return 0; } } @@ -4024,8 +3957,7 @@ copy_lock_held(PyObject *o) split_copy->ma_used = mp->ma_used; split_copy->_ma_watcher_tag = 0; dictkeys_incref(mp->ma_keys); - if (_PyObject_GC_IS_TRACKED(mp)) - _PyObject_GC_TRACK(split_copy); + _PyObject_GC_TRACK(split_copy); return (PyObject *)split_copy; } @@ -4060,10 +3992,6 @@ copy_lock_held(PyObject *o) new->ma_used = mp->ma_used; ASSERT_CONSISTENT(new); - if (_PyObject_GC_IS_TRACKED(mp)) { - /* Maintain tracking. */ - _PyObject_GC_TRACK(new); - } return (PyObject *)new; } @@ -4350,8 +4278,6 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu *result = NULL; } } - - MAINTAIN_TRACKING(mp, key, value); STORE_USED(mp, mp->ma_used + 1); assert(mp->ma_keys->dk_usable >= 0); ASSERT_CONSISTENT(mp); @@ -4801,15 +4727,8 @@ dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) d->ma_values = NULL; ASSERT_CONSISTENT(d); - if (type != &PyDict_Type) { - // Don't track if a subclass tp_alloc is PyType_GenericAlloc() - if (!_PyObject_GC_IS_TRACKED(d)) { - _PyObject_GC_TRACK(d); - } - } - else { - // _PyType_AllocNoTrack() does not track the created object - assert(!_PyObject_GC_IS_TRACKED(d)); + if (!_PyObject_GC_IS_TRACKED(d)) { + _PyObject_GC_TRACK(d); } return self; } @@ -4912,6 +4831,7 @@ PyTypeObject PyDict_Type = { dict_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ .tp_vectorcall = dict_vectorcall, + .tp_version_tag = _Py_TYPE_VERSION_DICT, }; /* For backward compatibility with old dictionary interface */ @@ -6745,19 +6665,14 @@ make_dict_from_instance_attributes(PyInterpreterState *interp, { dictkeys_incref(keys); Py_ssize_t used = 0; - Py_ssize_t track = 0; size_t size = shared_keys_usable_size(keys); for (size_t i = 0; i < size; i++) { PyObject *val = values->values[i]; if (val != NULL) { used += 1; - track += _PyObject_GC_MAY_BE_TRACKED(val); } } PyDictObject *res = (PyDictObject *)new_dict(interp, keys, values, used, 0); - if (track && res) { - _PyObject_GC_TRACK(res); - } return res; } @@ -7203,6 +7118,7 @@ _PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict) // since we locked it. dict = _PyObject_ManagedDictPointer(obj)->dict; err = _PyDict_DetachFromObject(dict, obj); + assert(err == 0 || new_dict == NULL); if (err == 0) { FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, (PyDictObject *)Py_XNewRef(new_dict)); @@ -7235,7 +7151,21 @@ void PyObject_ClearManagedDict(PyObject *obj) { if (_PyObject_SetManagedDict(obj, NULL) < 0) { + /* Must be out of memory */ + assert(PyErr_Occurred() == PyExc_MemoryError); PyErr_WriteUnraisable(NULL); + /* Clear the dict */ + PyDictObject *dict = _PyObject_GetManagedDict(obj); + Py_BEGIN_CRITICAL_SECTION2(dict, obj); + dict = _PyObject_ManagedDictPointer(obj)->dict; + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDictKeysObject *oldkeys = dict->ma_keys; + set_keys(dict, Py_EMPTY_KEYS); + dict->ma_values = NULL; + dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(dict)); + STORE_USED(dict, 0); + set_dict_inline_values(obj, NULL); + Py_END_CRITICAL_SECTION2(); } } @@ -7260,12 +7190,6 @@ _PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) PyDictValues *values = copy_values(mp->ma_values); if (values == NULL) { - /* Out of memory. Clear the dict */ - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyDictKeysObject *oldkeys = mp->ma_keys; - set_keys(mp, Py_EMPTY_KEYS); - dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); - STORE_USED(mp, 0); PyErr_NoMemory(); return -1; } diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 5eb50a65d9c969..bcc77287454768 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1916,6 +1916,7 @@ PyTypeObject PyFloat_Type = { 0, /* tp_alloc */ float_new, /* tp_new */ .tp_vectorcall = (vectorcallfunc)float_vectorcall, + .tp_version_tag = _Py_TYPE_VERSION_FLOAT, }; static void @@ -2390,7 +2391,7 @@ PyFloat_Unpack2(const char *data, int le) if (e == 0x1f) { if (f == 0) { /* Infinity */ - return sign ? -Py_HUGE_VAL : Py_HUGE_VAL; + return sign ? -Py_INFINITY : Py_INFINITY; } else { /* NaN */ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 55394afa523213..c743c254848d3a 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1651,7 +1651,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore } /* Finally set the new lasti and return OK. */ f->f_lineno = 0; - f->f_frame->instr_ptr = _PyCode_CODE(code) + best_addr; + f->f_frame->instr_ptr = _PyFrame_GetBytecode(f->f_frame) + best_addr; return 0; } @@ -1867,10 +1867,11 @@ PyTypeObject PyFrame_Type = { }; static void -init_frame(_PyInterpreterFrame *frame, PyFunctionObject *func, PyObject *locals) +init_frame(PyThreadState *tstate, _PyInterpreterFrame *frame, + PyFunctionObject *func, PyObject *locals) { PyCodeObject *code = (PyCodeObject *)func->func_code; - _PyFrame_Initialize(frame, PyStackRef_FromPyObjectNew(func), + _PyFrame_Initialize(tstate, frame, PyStackRef_FromPyObjectNew(func), Py_XNewRef(locals), code, 0, NULL); } @@ -1922,7 +1923,7 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, Py_DECREF(func); return NULL; } - init_frame((_PyInterpreterFrame *)f->_f_frame_data, func, locals); + init_frame(tstate, (_PyInterpreterFrame *)f->_f_frame_data, func, locals); f->f_frame = (_PyInterpreterFrame *)f->_f_frame_data; f->f_frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; // This frame needs to be "complete", so pretend that the first RESUME ran: @@ -1941,7 +1942,8 @@ frame_init_get_vars(_PyInterpreterFrame *frame) // here: PyCodeObject *co = _PyFrame_GetCode(frame); int lasti = _PyInterpreterFrame_LASTI(frame); - if (!(lasti < 0 && _PyCode_CODE(co)->op.code == COPY_FREE_VARS + if (!(lasti < 0 + && _PyFrame_GetBytecode(frame)->op.code == COPY_FREE_VARS && PyStackRef_FunctionCheck(frame->f_funcobj))) { /* Free vars are initialized */ @@ -1957,7 +1959,7 @@ frame_init_get_vars(_PyInterpreterFrame *frame) frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } // COPY_FREE_VARS doesn't have inline CACHEs, either: - frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)); + frame->instr_ptr = _PyFrame_GetBytecode(frame); } diff --git a/Objects/listobject.c b/Objects/listobject.c index 930aefde325a7c..bb0040cbe9f272 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3774,6 +3774,7 @@ PyTypeObject PyList_Type = { PyType_GenericNew, /* tp_new */ PyObject_GC_Del, /* tp_free */ .tp_vectorcall = list_vectorcall, + .tp_version_tag = _Py_TYPE_VERSION_LIST, }; /*********************** List Iterator **************************/ diff --git a/Objects/longobject.c b/Objects/longobject.c index 4e948940485730..4aa35685b509f2 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -784,6 +784,39 @@ PyLong_AsUnsignedLongMask(PyObject *op) return val; } +int +PyLong_IsPositive(PyObject *obj) +{ + assert(obj != NULL); + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expected int, got %T", obj); + return -1; + } + return _PyLong_IsPositive((PyLongObject *)obj); +} + +int +PyLong_IsNegative(PyObject *obj) +{ + assert(obj != NULL); + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expected int, got %T", obj); + return -1; + } + return _PyLong_IsNegative((PyLongObject *)obj); +} + +int +PyLong_IsZero(PyObject *obj) +{ + assert(obj != NULL); + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expected int, got %T", obj); + return -1; + } + return _PyLong_IsZero((PyLongObject *)obj); +} + int _PyLong_Sign(PyObject *vv) { @@ -6584,6 +6617,7 @@ PyTypeObject PyLong_Type = { long_new, /* tp_new */ PyObject_Free, /* tp_free */ .tp_vectorcall = long_vectorcall, + .tp_version_tag = _Py_TYPE_VERSION_INT, }; static PyTypeObject Int_InfoType; diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index d4672e8198cb24..25634f997ac66b 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -3356,6 +3356,7 @@ memory_iter(PyObject *seq) PyErr_BadInternalCall(); return NULL; } + CHECK_RELEASED(seq); PyMemoryViewObject *obj = (PyMemoryViewObject *)seq; int ndims = obj->view.ndim; if (ndims == 0) { diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 535b0d068f064f..a8d64c9aefae6b 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -107,8 +107,6 @@ static void track_module(PyModuleObject *m) { _PyDict_EnablePerThreadRefcounting(m->md_dict); - PyObject_GC_Track(m->md_dict); - _PyObject_SetDeferredRefcount((PyObject *)m); PyObject_GC_Track(m); } diff --git a/Objects/object.c b/Objects/object.c index 7cc74a8dc0d8eb..052dea9ad1feff 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2519,6 +2519,35 @@ _PyObject_SetDeferredRefcount(PyObject *op) #endif } +int +PyUnstable_Object_EnableDeferredRefcount(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + if (!PyType_IS_GC(Py_TYPE(op))) { + // Deferred reference counting doesn't work + // on untracked types. + return 0; + } + + uint8_t bits = _Py_atomic_load_uint8(&op->ob_gc_bits); + if ((bits & _PyGC_BITS_DEFERRED) != 0) + { + // Nothing to do. + return 0; + } + + if (_Py_atomic_compare_exchange_uint8(&op->ob_gc_bits, &bits, bits | _PyGC_BITS_DEFERRED) == 0) + { + // Someone beat us to it! + return 0; + } + _Py_atomic_add_ssize(&op->ob_ref_shared, _Py_REF_SHARED(_Py_REF_DEFERRED, 0)); + return 1; +#else + return 0; +#endif +} + void _Py_ResurrectReference(PyObject *op) { diff --git a/Objects/setobject.c b/Objects/setobject.c index 66d7fc730c555c..955ccbebf74b54 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1054,19 +1054,20 @@ set_update_internal(PySetObject *so, PyObject *other) /*[clinic input] set.update so: setobject - *others as args: object + *others: array Update the set, adding elements from all others. [clinic start generated code]*/ static PyObject * -set_update_impl(PySetObject *so, Py_ssize_t nargs, PyObject *const *args) -/*[clinic end generated code: output=050e2a21f8d7d16a input=df4fe486e38cd337]*/ +set_update_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length) +/*[clinic end generated code: output=017c781c992d5c23 input=ed5d78885b076636]*/ { Py_ssize_t i; - for (i = 0; i < nargs; i++) { - PyObject *other = args[i]; + for (i = 0; i < others_length; i++) { + PyObject *other = others[i]; if (set_update_internal(so, other)) return NULL; } @@ -1283,14 +1284,15 @@ set_clear_impl(PySetObject *so) /*[clinic input] set.union so: setobject - *others as args: object + *others: array Return a new set with elements from the set and all others. [clinic start generated code]*/ static PyObject * -set_union_impl(PySetObject *so, Py_ssize_t nargs, PyObject *const *args) -/*[clinic end generated code: output=f68ec24d5c19d404 input=ddf088706e9577b2]*/ +set_union_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length) +/*[clinic end generated code: output=b1bfa3d74065f27e input=55a2e81db6347a4f]*/ { PySetObject *result; PyObject *other; @@ -1300,8 +1302,8 @@ set_union_impl(PySetObject *so, Py_ssize_t nargs, PyObject *const *args) if (result == NULL) return NULL; - for (i = 0; i < nargs; i++) { - other = args[i]; + for (i = 0; i < others_length; i++) { + other = others[i]; if ((PyObject *)so == other) continue; if (set_update_local(result, other)) { @@ -1434,25 +1436,25 @@ set_intersection(PySetObject *so, PyObject *other) /*[clinic input] set.intersection as set_intersection_multi so: setobject - *others as args: object + *others: array Return a new set with elements common to the set and all others. [clinic start generated code]*/ static PyObject * -set_intersection_multi_impl(PySetObject *so, Py_ssize_t nargs, - PyObject *const *args) -/*[clinic end generated code: output=ef0756ddb5f2dee9 input=0d9f3805ccbba6a4]*/ +set_intersection_multi_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length) +/*[clinic end generated code: output=db9ff9f875132b6b input=36c7b615694cadae]*/ { Py_ssize_t i; - if (nargs == 0) { + if (others_length == 0) { return set_copy(so, NULL); } PyObject *result = Py_NewRef(so); - for (i = 0; i < nargs; i++) { - PyObject *other = args[i]; + for (i = 0; i < others_length; i++) { + PyObject *other = others[i]; PyObject *newresult; Py_BEGIN_CRITICAL_SECTION2(result, other); newresult = set_intersection((PySetObject *)result, other); @@ -1482,19 +1484,19 @@ set_intersection_update(PySetObject *so, PyObject *other) /*[clinic input] set.intersection_update as set_intersection_update_multi so: setobject - *others as args: object + *others: array Update the set, keeping only elements found in it and all others. [clinic start generated code]*/ static PyObject * -set_intersection_update_multi_impl(PySetObject *so, Py_ssize_t nargs, - PyObject *const *args) -/*[clinic end generated code: output=808d7ad1935b1dfe input=223c1e086aa669a9]*/ +set_intersection_update_multi_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length) +/*[clinic end generated code: output=d768b5584675b48d input=782e422fc370e4fc]*/ { PyObject *tmp; - tmp = set_intersection_multi_impl(so, nargs, args); + tmp = set_intersection_multi_impl(so, others, others_length); if (tmp == NULL) return NULL; Py_BEGIN_CRITICAL_SECTION(so); @@ -1672,20 +1674,20 @@ set_difference_update_internal(PySetObject *so, PyObject *other) /*[clinic input] set.difference_update so: setobject - *others as args: object + *others: array Update the set, removing elements found in others. [clinic start generated code]*/ static PyObject * -set_difference_update_impl(PySetObject *so, Py_ssize_t nargs, - PyObject *const *args) -/*[clinic end generated code: output=55f850c27748d312 input=024e6baa6fbcbb3d]*/ +set_difference_update_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length) +/*[clinic end generated code: output=04a22179b322cfe6 input=93ac28ba5b233696]*/ { Py_ssize_t i; - for (i = 0; i < nargs; i++) { - PyObject *other = args[i]; + for (i = 0; i < others_length; i++) { + PyObject *other = others[i]; int rv; Py_BEGIN_CRITICAL_SECTION2(so, other); rv = set_difference_update_internal(so, other); @@ -1790,32 +1792,32 @@ set_difference(PySetObject *so, PyObject *other) /*[clinic input] set.difference as set_difference_multi so: setobject - *others as args: object + *others: array Return a new set with elements in the set that are not in the others. [clinic start generated code]*/ static PyObject * -set_difference_multi_impl(PySetObject *so, Py_ssize_t nargs, - PyObject *const *args) -/*[clinic end generated code: output=8150d008c00523f3 input=ba78ea5f099e58df]*/ +set_difference_multi_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length) +/*[clinic end generated code: output=b0d33fb05d5477a7 input=c1eb448d483416ad]*/ { Py_ssize_t i; PyObject *result, *other; - if (nargs == 0) { + if (others_length == 0) { return set_copy(so, NULL); } - other = args[0]; + other = others[0]; Py_BEGIN_CRITICAL_SECTION2(so, other); result = set_difference(so, other); Py_END_CRITICAL_SECTION2(); if (result == NULL) return NULL; - for (i = 1; i < nargs; i++) { - other = args[i]; + for (i = 1; i < others_length; i++) { + other = others[i]; int rv; Py_BEGIN_CRITICAL_SECTION(other); rv = set_difference_update_internal((PySetObject *)result, other); @@ -2518,6 +2520,7 @@ PyTypeObject PySet_Type = { set_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ .tp_vectorcall = set_vectorcall, + .tp_version_tag = _Py_TYPE_VERSION_SET, }; /* frozenset object ********************************************************/ @@ -2608,6 +2611,7 @@ PyTypeObject PyFrozenSet_Type = { frozenset_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ .tp_vectorcall = frozenset_vectorcall, + .tp_version_tag = _Py_TYPE_VERSION_FROZEN_SET, }; diff --git a/Objects/stringlib/clinic/transmogrify.h.h b/Objects/stringlib/clinic/transmogrify.h.h index cef7a9496fa874..5b82c2de737249 100644 --- a/Objects/stringlib/clinic/transmogrify.h.h +++ b/Objects/stringlib/clinic/transmogrify.h.h @@ -56,7 +56,8 @@ stringlib_expandtabs(PyObject *self, PyObject *const *args, Py_ssize_t nargs, Py Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int tabsize = 8; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -321,4 +322,4 @@ stringlib_zfill(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=06dd79019356b6bb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0e24a10bac3ec053 input=a9049054013a1b77]*/ diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index f3132e0933ac30..193914d54bd90e 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -909,6 +909,7 @@ PyTypeObject PyTuple_Type = { tuple_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ .tp_vectorcall = tuple_vectorcall, + .tp_version_tag = _Py_TYPE_VERSION_TUPLE, }; /* The following function breaks the notion that tuples are immutable: diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b4a11195613d74..a6cf3da542b691 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4761,10 +4761,10 @@ PyType_FromMetaclass( if (strcmp(memb->name, "__weaklistoffset__") == 0) { weaklistoffset_member = memb; } - if (strcmp(memb->name, "__dictoffset__") == 0) { + else if (strcmp(memb->name, "__dictoffset__") == 0) { dictoffset_member = memb; } - if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + else if (strcmp(memb->name, "__vectorcalloffset__") == 0) { vectorcalloffset_member = memb; } } @@ -7656,6 +7656,12 @@ type_add_method(PyTypeObject *type, PyMethodDef *meth) return 0; } +int +_PyType_AddMethod(PyTypeObject *type, PyMethodDef *meth) +{ + return type_add_method(type, meth); +} + /* Add the methods from tp_methods to the __dict__ in a type object */ static int @@ -8607,7 +8613,9 @@ init_static_type(PyInterpreterState *interp, PyTypeObject *self, self->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; assert(NEXT_GLOBAL_VERSION_TAG <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); - _PyType_SetVersion(self, NEXT_GLOBAL_VERSION_TAG++); + if (self->tp_version_tag == 0) { + _PyType_SetVersion(self, NEXT_GLOBAL_VERSION_TAG++); + } } else { assert(!initial); @@ -11638,9 +11646,10 @@ super_descr_get(PyObject *self, PyObject *obj, PyObject *type) } static int -super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co, - PyTypeObject **type_p, PyObject **obj_p) +super_init_without_args(_PyInterpreterFrame *cframe, PyTypeObject **type_p, + PyObject **obj_p) { + PyCodeObject *co = _PyFrame_GetCode(cframe); if (co->co_argcount == 0) { PyErr_SetString(PyExc_RuntimeError, "super(): no arguments"); @@ -11740,7 +11749,7 @@ super_init_impl(PyObject *self, PyTypeObject *type, PyObject *obj) { "super(): no current frame"); return -1; } - int res = super_init_without_args(frame, _PyFrame_GetCode(frame), &type, &obj); + int res = super_init_without_args(frame, &type, &obj); if (res < 0) { return -1; diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 91cc37c9a72636..bacb858978c5d7 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -651,7 +651,7 @@ typevar_alloc(PyObject *name, PyObject *bound, PyObject *evaluate_bound, typevar.__new__ as typevar_new name: object(subclass_of="&PyUnicode_Type") - *constraints: object + *constraints: tuple bound: object = None default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault covariant: bool = False @@ -665,7 +665,7 @@ static PyObject * typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, PyObject *bound, PyObject *default_value, int covariant, int contravariant, int infer_variance) -/*[clinic end generated code: output=d2b248ff074eaab6 input=836f97f631d7293a]*/ +/*[clinic end generated code: output=d2b248ff074eaab6 input=1b5b62e40c92c167]*/ { if (covariant && contravariant) { PyErr_SetString(PyExc_ValueError, diff --git a/PC/clinic/_wmimodule.cpp.h b/PC/clinic/_wmimodule.cpp.h index fccf3d795d5340..f0be691173ad64 100644 --- a/PC/clinic/_wmimodule.cpp.h +++ b/PC/clinic/_wmimodule.cpp.h @@ -55,7 +55,8 @@ _wmi_exec_query(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *argsbuf[1]; PyObject *query; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -69,4 +70,4 @@ _wmi_exec_query(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj exit: return return_value; } -/*[clinic end generated code: output=ba04920d127f3ceb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9ac5fd7faf889e52 input=a9049054013a1b77]*/ diff --git a/PC/clinic/winreg.c.h b/PC/clinic/winreg.c.h index 7e5545e113e718..b14913366b77eb 100644 --- a/PC/clinic/winreg.c.h +++ b/PC/clinic/winreg.c.h @@ -351,7 +351,8 @@ winreg_CreateKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py REGSAM access = KEY_WRITE; HKEY _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -534,7 +535,8 @@ winreg_DeleteKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py REGSAM access = KEY_WOW64_64KEY; int reserved = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -966,7 +968,8 @@ winreg_OpenKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje REGSAM access = KEY_READ; HKEY _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1083,7 +1086,8 @@ winreg_OpenKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb REGSAM access = KEY_READ; HKEY _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1762,4 +1766,4 @@ winreg_QueryReflectionKey(PyObject *module, PyObject *arg) #ifndef WINREG_QUERYREFLECTIONKEY_METHODDEF #define WINREG_QUERYREFLECTIONKEY_METHODDEF #endif /* !defined(WINREG_QUERYREFLECTIONKEY_METHODDEF) */ -/*[clinic end generated code: output=1ee4098b2f143b6a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=aef4aa8ab8ddf38f input=a9049054013a1b77]*/ diff --git a/PC/layout/main.py b/PC/layout/main.py index 0350ed7af3f9b5..8bd435456c635a 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -31,11 +31,13 @@ from .support.nuspec import * TEST_PYDS_ONLY = FileStemSet("xxlimited", "xxlimited_35", "_ctypes_test", "_test*") +TEST_DLLS_ONLY = set() TEST_DIRS_ONLY = FileNameSet("test", "tests") IDLE_DIRS_ONLY = FileNameSet("idlelib") -TCLTK_PYDS_ONLY = FileStemSet("tcl*", "tk*", "_tkinter", "zlib1") +TCLTK_PYDS_ONLY = FileStemSet("_tkinter") +TCLTK_DLLS_ONLY = FileStemSet("tcl*", "tk*", "zlib1") TCLTK_DIRS_ONLY = FileNameSet("tkinter", "turtledemo") TCLTK_FILES_ONLY = FileNameSet("turtle.py") @@ -226,6 +228,10 @@ def in_build(f, dest="", new_name=None, no_lib=False): continue if src in EXCLUDE_FROM_DLLS: continue + if src in TEST_DLLS_ONLY and not ns.include_tests: + continue + if src in TCLTK_DLLS_ONLY and not ns.include_tcltk: + continue yield from in_build(src.name, dest=dest, no_lib=True) if ns.zip_lib: diff --git a/PC/python_ver_rc.h b/PC/python_ver_rc.h index 08509f96ed1db8..ee867fe41224c3 100644 --- a/PC/python_ver_rc.h +++ b/PC/python_ver_rc.h @@ -5,7 +5,7 @@ #include "winver.h" #define PYTHON_COMPANY "Python Software Foundation" -#define PYTHON_COPYRIGHT "Copyright \xA9 2001-2024 Python Software Foundation. Copyright \xA9 2000 BeOpen.com. Copyright \xA9 1995-2001 CNRI. Copyright \xA9 1991-1995 SMC." +#define PYTHON_COPYRIGHT "Copyright \xA9 2001 Python Software Foundation. Copyright \xA9 2000 BeOpen.com. Copyright \xA9 1995-2001 CNRI. Copyright \xA9 1991-1995 SMC." #define MS_WINDOWS #include "modsupport.h" diff --git a/PC/store_info.txt b/PC/store_info.txt index f6a85cb8ebec1f..d150ba17cbe62d 100644 --- a/PC/store_info.txt +++ b/PC/store_info.txt @@ -109,7 +109,7 @@ PSF LICENSE AGREEMENT FOR PYTHON 3.9 analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python 3.9 alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of - copyright, i.e., "Copyright © 2001-2018 Python Software Foundation; All Rights + copyright, i.e., "Copyright © 2001 Python Software Foundation; All Rights Reserved" are retained in Python 3.9 alone or in any derivative version prepared by Licensee. diff --git a/PC/venvlauncher.c b/PC/venvlauncher.c index b1c8d0763d8c76..b6bb0218236ae9 100644 --- a/PC/venvlauncher.c +++ b/PC/venvlauncher.c @@ -223,7 +223,7 @@ find_home_value(const char *buffer, DWORD maxlen, const char **start, DWORD *len return 0; } for (const char *s = strstr(buffer, "home"); - s && ((ptrdiff_t)s - (ptrdiff_t)buffer) < maxlen; + s && (size_t)((ptrdiff_t)s - (ptrdiff_t)buffer) < maxlen; s = strstr(s + 1, "\nhome") ) { if (*s == '\n') { diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index a3c2d32c454e04..51b493f8a84c6f 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -222,6 +222,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 91b1d75fb8df5e..09a5f4d30ef490 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -232,6 +232,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/_tkinter.vcxproj b/PCbuild/_tkinter.vcxproj index 117488a01621cc..87f6005fffc7c5 100644 --- a/PCbuild/_tkinter.vcxproj +++ b/PCbuild/_tkinter.vcxproj @@ -94,6 +94,7 @@ $(tcltkDir)include;%(AdditionalIncludeDirectories) + TCL_WITH_EXTERNAL_TOMMATH;%(PreprocessorDefinitions) WITH_APPINIT;%(PreprocessorDefinitions) Py_TCLTK_DIR="$(tcltkDir.TrimEnd('\').Replace('\', '\\'))";%(PreprocessorDefinitions) @@ -109,9 +110,10 @@ - <_TclTkDLL Include="$(tcltkdir)\bin\$(tclDllName)" /> - <_TclTkDLL Include="$(tcltkdir)\bin\$(tkDllName)" /> - <_TclTkDLL Include="$(tcltkdir)\bin\$(tclZlibDllName)" /> + <_TclTkDLL Include="$(tcltkdir)\bin\$(tclDLLName)" /> + <_TclTkDLL Include="$(tcltkdir)\bin\$(tkDLLName)" /> + <_TclTkDLL Include="$(tcltkdir)\bin\$(tclZlibDLLName)" /> + <_TclTkDLL Include="$(tcltkdir)\bin\$(tommathDLLName)" Condition="$(tommathDLLName) != ''"/> @@ -134,4 +136,4 @@ - \ No newline at end of file + diff --git a/PCbuild/build.bat b/PCbuild/build.bat index abe649553756a7..6d3ce81651ade5 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -11,7 +11,7 @@ echo.directly to MSBuild may be passed. If the argument contains an '=', the echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v141"`). echo.Alternatively you can put extra flags for MSBuild in a file named echo.`msbuild.rsp` in the `PCbuild` directory, one flag per line. This file -echo.will be picked automatically by MSBuild. Flags put in this file does not +echo.will be picked automatically by MSBuild. Flags put in this file do not echo.need to be quoted. You can still use environment variables inside the echo.response file. echo. @@ -196,4 +196,4 @@ rem Display the current build version information call "%dir%find_msbuild.bat" %MSBUILD% if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) %MSBUILD% "%dir%pythoncore.vcxproj" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9 -if ERRORLEVEL 1 exit /b 3 \ No newline at end of file +if ERRORLEVEL 1 exit /b 3 diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index a4881e9256e4dd..95552cade52b75 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -229,6 +229,7 @@ + @@ -255,6 +256,7 @@ + @@ -614,6 +616,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 6b294683320a73..1708cf6e0b3a52 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -609,6 +609,9 @@ Include\internal + + Include\internal + Include\internal @@ -687,6 +690,9 @@ Include\internal + + Include\internal + Include\internal @@ -917,15 +923,6 @@ Modules - - Modules - - - Modules - - - Modules - Modules @@ -1382,6 +1379,9 @@ Python + + Python + Python diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index b4cb401609d409..d26b36ba98e493 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -17,15 +17,21 @@ $(ExternalsDir)tcltk-$(TclVersion)\$(ArchName)\ $(tcltkDir)\bin\tclsh$(TclMajorVersion)$(TclMinorVersion)t.exe $(tcltkDir)\..\win32\bin\tclsh$(TclMajorVersion)$(TclMinorVersion)t.exe + TCL_WITH_EXTERNAL_TOMMATH; - tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).dll - tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).lib - tclsh$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).exe - tk$(TkMajorVersion)$(TkMinorVersion)t$(TclDebugExt).dll - tk$(TkMajorVersion)$(TkMinorVersion)t$(TclDebugExt).lib + t + tcl9 + tcl$(TclMajorVersion)$(TclMinorVersion)$(tcltkSuffix)$(TclDebugExt).dll + tcl$(TclMajorVersion)$(TclMinorVersion)$(tcltkSuffix)$(TclDebugExt).lib + tclsh$(TclMajorVersion)$(TclMinorVersion)$(tcltkSuffix)$(TclDebugExt).exe + $(tkPrefix)tk$(TkMajorVersion)$(TkMinorVersion)$(tcltkSuffix)$(TclDebugExt).dll + $(tkPrefix)tk$(TclMajorVersion)$(TclMinorVersion)$(tcltkSuffix)$(TclDebugExt).lib zlib1.dll - $(tcltkDir)lib\tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).lib;$(tcltkDir)lib\tk$(TkMajorVersion)$(TkMinorVersion)t$(TclDebugExt).lib + libtommath.dll + tommath.lib + $(tcltkDir)lib\$(TclLibName);$(tcltkDir)lib\$(TkLibName); + $(tcltkLib);$(tcltkDir)lib\$(tommathLibName) IX86 AMD64 ARM64 diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index cb21777f566189..5ac1dd7813689c 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1128,6 +1128,9 @@ expr_ty _PyPegen_collect_call_seqs(Parser *p, asdl_expr_seq *a, asdl_seq *b, } asdl_expr_seq *args = _Py_asdl_expr_seq_new(total_len, arena); + if (args == NULL) { + return NULL; + } Py_ssize_t i = 0; for (i = 0; i < args_len; i++) { @@ -1298,6 +1301,9 @@ unpack_top_level_joined_strs(Parser *p, asdl_expr_seq *raw_expressions) } asdl_expr_seq *expressions = _Py_asdl_expr_seq_new(req_size, p->arena); + if (expressions == NULL) { + return NULL; + } Py_ssize_t raw_index, req_index = 0; for (raw_index = 0; raw_index < raw_size; raw_index++) { @@ -1490,6 +1496,9 @@ expr_ty _PyPegen_formatted_value(Parser *p, expr_ty expression, Token *debug, Re } asdl_expr_seq *values = _Py_asdl_expr_seq_new(2, arena); + if (values == NULL) { + return NULL; + } asdl_seq_SET(values, 0, debug_text); asdl_seq_SET(values, 1, formatted_value); return _PyAST_JoinedStr(values, lineno, col_offset, debug_end_line, debug_end_offset, p->arena); diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index 891e4256e897ab..06d1ee016dc2a8 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -121,6 +121,7 @@ compile_and_marshal(const char *name, const char *text) return NULL; } + assert(Py_MARSHAL_VERSION >= 5); PyObject *marshalled = PyMarshal_WriteObjectToString(code, Py_MARSHAL_VERSION); Py_CLEAR(code); if (marshalled == NULL) { diff --git a/Python/_warnings.c b/Python/_warnings.c index 3f9e73b5376223..e05ba99e8eaec4 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -803,7 +803,8 @@ is_filename_to_skip(PyObject *filename, PyTupleObject *skip_file_prefixes) for (Py_ssize_t idx = 0; idx < prefixes; ++idx) { PyObject *prefix = PyTuple_GET_ITEM(skip_file_prefixes, idx); - Py_ssize_t found = PyUnicode_Tailmatch(filename, prefix, 0, -1, -1); + Py_ssize_t found = PyUnicode_Tailmatch(filename, prefix, + 0, PY_SSIZE_T_MAX, -1); if (found == 1) { return true; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 8fd3ce29d6b3b1..17df9208f224f4 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1311,6 +1311,7 @@ typedef struct { PyObject_HEAD PyObject *iters; PyObject *func; + int strict; } mapobject; static PyObject * @@ -1319,10 +1320,21 @@ map_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyObject *it, *iters, *func; mapobject *lz; Py_ssize_t numargs, i; + int strict = 0; - if ((type == &PyMap_Type || type->tp_init == PyMap_Type.tp_init) && - !_PyArg_NoKeywords("map", kwds)) - return NULL; + if (kwds) { + PyObject *empty = PyTuple_New(0); + if (empty == NULL) { + return NULL; + } + static char *kwlist[] = {"strict", NULL}; + int parsed = PyArg_ParseTupleAndKeywords( + empty, kwds, "|$p:map", kwlist, &strict); + Py_DECREF(empty); + if (!parsed) { + return NULL; + } + } numargs = PyTuple_Size(args); if (numargs < 2) { @@ -1354,6 +1366,7 @@ map_new(PyTypeObject *type, PyObject *args, PyObject *kwds) lz->iters = iters; func = PyTuple_GET_ITEM(args, 0); lz->func = Py_NewRef(func); + lz->strict = strict; return (PyObject *)lz; } @@ -1363,11 +1376,14 @@ map_vectorcall(PyObject *type, PyObject * const*args, size_t nargsf, PyObject *kwnames) { PyTypeObject *tp = _PyType_CAST(type); - if (tp == &PyMap_Type && !_PyArg_NoKwnames("map", kwnames)) { - return NULL; - } Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + if (kwnames != NULL && PyTuple_GET_SIZE(kwnames) != 0) { + // Fallback to map_new() + PyThreadState *tstate = _PyThreadState_GET(); + return _PyObject_MakeTpCall(tstate, type, args, nargs, kwnames); + } + if (nargs < 2) { PyErr_SetString(PyExc_TypeError, "map() must have at least two arguments."); @@ -1395,6 +1411,7 @@ map_vectorcall(PyObject *type, PyObject * const*args, } lz->iters = iters; lz->func = Py_NewRef(args[0]); + lz->strict = 0; return (PyObject *)lz; } @@ -1419,6 +1436,7 @@ map_traverse(mapobject *lz, visitproc visit, void *arg) static PyObject * map_next(mapobject *lz) { + Py_ssize_t i; PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; PyObject **stack; PyObject *result = NULL; @@ -1437,10 +1455,13 @@ map_next(mapobject *lz) } Py_ssize_t nargs = 0; - for (Py_ssize_t i=0; i < niters; i++) { + for (i=0; i < niters; i++) { PyObject *it = PyTuple_GET_ITEM(lz->iters, i); PyObject *val = Py_TYPE(it)->tp_iternext(it); if (val == NULL) { + if (lz->strict) { + goto check; + } goto exit; } stack[i] = val; @@ -1450,13 +1471,50 @@ map_next(mapobject *lz) result = _PyObject_VectorcallTstate(tstate, lz->func, stack, nargs, NULL); exit: - for (Py_ssize_t i=0; i < nargs; i++) { + for (i=0; i < nargs; i++) { Py_DECREF(stack[i]); } if (stack != small_stack) { PyMem_Free(stack); } return result; +check: + if (PyErr_Occurred()) { + if (!PyErr_ExceptionMatches(PyExc_StopIteration)) { + // next() on argument i raised an exception (not StopIteration) + return NULL; + } + PyErr_Clear(); + } + if (i) { + // ValueError: map() argument 2 is shorter than argument 1 + // ValueError: map() argument 3 is shorter than arguments 1-2 + const char* plural = i == 1 ? " " : "s 1-"; + return PyErr_Format(PyExc_ValueError, + "map() argument %d is shorter than argument%s%d", + i + 1, plural, i); + } + for (i = 1; i < niters; i++) { + PyObject *it = PyTuple_GET_ITEM(lz->iters, i); + PyObject *val = (*Py_TYPE(it)->tp_iternext)(it); + if (val) { + Py_DECREF(val); + const char* plural = i == 1 ? " " : "s 1-"; + return PyErr_Format(PyExc_ValueError, + "map() argument %d is longer than argument%s%d", + i + 1, plural, i); + } + if (PyErr_Occurred()) { + if (!PyErr_ExceptionMatches(PyExc_StopIteration)) { + // next() on argument i raised an exception (not StopIteration) + return NULL; + } + PyErr_Clear(); + } + // Argument i is exhausted. So far so good... + } + // All arguments are exhausted. Success! + goto exit; } static PyObject * @@ -1473,21 +1531,41 @@ map_reduce(mapobject *lz, PyObject *Py_UNUSED(ignored)) PyTuple_SET_ITEM(args, i+1, Py_NewRef(it)); } + if (lz->strict) { + return Py_BuildValue("ONO", Py_TYPE(lz), args, Py_True); + } return Py_BuildValue("ON", Py_TYPE(lz), args); } +PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); + +static PyObject * +map_setstate(mapobject *lz, PyObject *state) +{ + int strict = PyObject_IsTrue(state); + if (strict < 0) { + return NULL; + } + lz->strict = strict; + Py_RETURN_NONE; +} + static PyMethodDef map_methods[] = { {"__reduce__", _PyCFunction_CAST(map_reduce), METH_NOARGS, reduce_doc}, + {"__setstate__", _PyCFunction_CAST(map_setstate), METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; PyDoc_STRVAR(map_doc, -"map(function, iterable, /, *iterables)\n\ +"map(function, iterable, /, *iterables, strict=False)\n\ --\n\ \n\ Make an iterator that computes the function using arguments from\n\ -each of the iterables. Stops when the shortest iterable is exhausted."); +each of the iterables. Stops when the shortest iterable is exhausted.\n\ +\n\ +If strict is true and one of the arguments is exhausted before the others,\n\ +raise a ValueError."); PyTypeObject PyMap_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -2049,7 +2127,7 @@ builtin_pow_impl(PyObject *module, PyObject *base, PyObject *exp, /*[clinic input] print as builtin_print - *args: object + *args: array sep: object(c_default="Py_None") = ' ' string inserted between values, default a space. end: object(c_default="Py_None") = '\n' @@ -2064,9 +2142,10 @@ Prints the values to a stream, or to sys.stdout by default. [clinic start generated code]*/ static PyObject * -builtin_print_impl(PyObject *module, PyObject *args, PyObject *sep, - PyObject *end, PyObject *file, int flush) -/*[clinic end generated code: output=3cfc0940f5bc237b input=c143c575d24fe665]*/ +builtin_print_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length, PyObject *sep, PyObject *end, + PyObject *file, int flush) +/*[clinic end generated code: output=3cb7e5b66f1a8547 input=66ea4de1605a2437]*/ { int i, err; @@ -2103,7 +2182,7 @@ builtin_print_impl(PyObject *module, PyObject *args, PyObject *sep, return NULL; } - for (i = 0; i < PyTuple_GET_SIZE(args); i++) { + for (i = 0; i < args_length; i++) { if (i > 0) { if (sep == NULL) { err = PyFile_WriteString(" ", file); @@ -2115,7 +2194,7 @@ builtin_print_impl(PyObject *module, PyObject *args, PyObject *sep, return NULL; } } - err = PyFile_WriteObject(PyTuple_GET_ITEM(args, i), file, Py_PRINT_RAW); + err = PyFile_WriteObject(args[i], file, Py_PRINT_RAW); if (err) { return NULL; } @@ -3066,8 +3145,6 @@ zip_reduce(zipobject *lz, PyObject *Py_UNUSED(ignored)) return PyTuple_Pack(2, Py_TYPE(lz), lz->ittuple); } -PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); - static PyObject * zip_setstate(zipobject *lz, PyObject *state) { @@ -3257,7 +3334,7 @@ _PyBuiltin_Init(PyInterpreterState *interp) SETBUILTIN("False", Py_False); SETBUILTIN("True", Py_True); SETBUILTIN("bool", &PyBool_Type); - SETBUILTIN("memoryview", &PyMemoryView_Type); + SETBUILTIN("memoryview", &PyMemoryView_Type); SETBUILTIN("bytearray", &PyByteArray_Type); SETBUILTIN("bytes", &PyBytes_Type); SETBUILTIN("classmethod", &PyClassMethod_Type); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fa98af12c69aef..4479bd5dc56cb1 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -168,11 +168,11 @@ dummy_func( } op(_QUICKEN_RESUME, (--)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (tstate->tracing == 0 && this_instr->op.code == RESUME) { FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); } - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } tier1 op(_MAYBE_INSTRUMENT, (--)) { @@ -190,7 +190,26 @@ dummy_func( } } + op(_LOAD_BYTECODE, (--)) { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + ERROR_IF(bytecode == NULL, error); + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + // Make sure this_instr gets reset correctley for any uops that + // follow + next_instr = frame->instr_ptr; + DISPATCH(); + } + #endif + } + macro(RESUME) = + _LOAD_BYTECODE + _MAYBE_INSTRUMENT + _QUICKEN_RESUME + _CHECK_PERIODIC_IF_NOT_YIELD_FROM; @@ -204,6 +223,10 @@ dummy_func( uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); DEOPT_IF(eval_breaker != version); + #ifdef Py_GIL_DISABLED + DEOPT_IF(frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index); + #endif } op(_MONITOR_RESUME, (--)) { @@ -217,6 +240,7 @@ dummy_func( } macro(INSTRUMENTED_RESUME) = + _LOAD_BYTECODE + _MAYBE_INSTRUMENT + _CHECK_PERIODIC_IF_NOT_YIELD_FROM + _MONITOR_RESUME; @@ -637,7 +661,7 @@ dummy_func( assert(next_instr->op.code == STORE_FAST); next_oparg = next_instr->op.arg; #else - next_oparg = CURRENT_OPERAND(); + next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); DEOPT_IF(!PyStackRef_Is(*target_local, left)); @@ -682,8 +706,8 @@ dummy_func( }; specializing op(_SPECIALIZE_BINARY_SUBSCR, (counter/1, container, sub -- container, sub)) { - assert(frame->stackpointer == NULL); #if ENABLE_SPECIALIZATION + assert(frame->stackpointer == NULL); if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_BinarySubscr(container, sub, next_instr); @@ -1236,7 +1260,7 @@ dummy_func( if (oparg) { PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); if (PyLong_Check(lasti)) { - frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); + frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti); assert(!_PyErr_Occurred(tstate)); } else { @@ -2108,7 +2132,7 @@ dummy_func( op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyModule_CheckExact(owner_o)); + DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); DEOPT_IF(dict->ma_keys->dk_version != dict_version); @@ -2316,10 +2340,6 @@ dummy_func( DEOPT_IF(ep->me_key != name); PyObject *old_value = ep->me_value; DEOPT_IF(old_value == NULL); - /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(PyStackRef_AsPyObjectBorrow(value))) { - _PyObject_GC_TRACK(dict); - } _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); ep->me_value = PyStackRef_AsPyObjectSteal(value); // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, @@ -2484,7 +2504,7 @@ dummy_func( } specializing op(_SPECIALIZE_CONTAINS_OP, (counter/1, left, right -- left, right)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_ContainsOp(right, next_instr); @@ -2671,9 +2691,7 @@ dummy_func( assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_False); DEAD(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(oparg * flag); } @@ -2681,9 +2699,7 @@ dummy_func( assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_True); DEAD(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(oparg * flag); } @@ -2791,11 +2807,12 @@ dummy_func( } else { /* `iterable` is not a generator. */ - iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o)); + PyObject *iter_o = PyObject_GetIter(iterable_o); DEAD(iterable); - if (PyStackRef_IsNull(iter)) { + if (iter_o == NULL) { ERROR_NO_POP(); } + iter = PyStackRef_FromPyObjectSteal(iter_o); DECREF_INPUTS(); } } @@ -3442,6 +3459,12 @@ dummy_func( EXIT_IF(func->func_version != func_version); } + tier2 op(_CHECK_FUNCTION_VERSION_INLINE, (func_version/2, callable_o/4 --)) { + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + EXIT_IF(func->func_version != func_version); + } + macro(CALL_PY_GENERAL) = unused/1 + // Skip over the counter _CHECK_PEP_523 + @@ -3697,7 +3720,7 @@ dummy_func( op(_CREATE_INIT_FRAME, (init[1], self[1], args[oparg] -- init_frame: _PyInterpreterFrame *)) { _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - assert(_PyCode_CODE(_PyFrame_GetCode(shim))[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self[0]); DEAD(init); @@ -4593,7 +4616,7 @@ dummy_func( } specializing op(_SPECIALIZE_BINARY_OP, (counter/1, lhs, rhs -- lhs, rhs)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); @@ -4601,7 +4624,7 @@ dummy_func( } OPCODE_DEFERRED_INC(BINARY_OP); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); } @@ -4632,7 +4655,7 @@ dummy_func( int original_opcode = 0; if (tstate->tracing) { PyCodeObject *code = _PyFrame_GetCode(frame); - original_opcode = code->_co_monitoring->lines[(int)(this_instr - _PyCode_CODE(code))].original_opcode; + original_opcode = code->_co_monitoring->lines[(int)(this_instr - _PyFrame_GetBytecode(frame))].original_opcode; next_instr = this_instr; } else { original_opcode = _Py_call_instrumentation_line( @@ -4687,9 +4710,7 @@ dummy_func( assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_True); int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } @@ -4698,9 +4719,7 @@ dummy_func( assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_False); int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } @@ -4715,9 +4734,7 @@ dummy_func( PyStackRef_CLOSE(value_stackref); offset = 0; } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } @@ -4815,7 +4832,7 @@ dummy_func( tier2 op(_EXIT_TRACE, (exit_p/4 --)) { _PyExitData *exit = (_PyExitData *)exit_p; PyCodeObject *code = _PyFrame_GetCode(frame); - _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; #if defined(Py_DEBUG) && !defined(_Py_JIT) OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (lltrace >= 2) { @@ -4823,7 +4840,7 @@ dummy_func( _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyCode_CODE(code)), + (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); } #endif @@ -4933,7 +4950,7 @@ dummy_func( _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), + (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); } #endif @@ -4995,7 +5012,7 @@ dummy_func( } tier2 op(_ERROR_POP_N, (target/2, unused[oparg] --)) { - frame->instr_ptr = ((_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive) + target; + frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; SYNC_SP(); GOTO_UNWIND(); } diff --git a/Python/ceval.c b/Python/ceval.c index beee5325cd6259..3360e50d57fbcb 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -189,7 +189,7 @@ lltrace_instruction(_PyInterpreterFrame *frame, dump_stack(frame, stack_pointer); const char *opname = _PyOpcode_OpName[opcode]; assert(opname != NULL); - int offset = (int)(next_instr - _PyCode_CODE(_PyFrame_GetCode(frame))); + int offset = (int)(next_instr - _PyFrame_GetBytecode(frame)); if (OPCODE_HAS_ARG((int)_PyOpcode_Deopt[opcode])) { printf("%d: %s %d\n", offset * 2, opname, oparg); } @@ -821,6 +821,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; entry_frame.stackpointer = entry_frame.localsplus; entry_frame.owner = FRAME_OWNED_BY_CSTACK; + entry_frame.visited = 0; entry_frame.return_offset = 0; /* Push frame */ entry_frame.previous = tstate->current_frame; @@ -841,6 +842,19 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } /* Because this avoids the RESUME, * we need to update instrumentation */ +#ifdef Py_GIL_DISABLED + /* Load thread-local bytecode */ + if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + if (bytecode == NULL) { + goto error; + } + ptrdiff_t off = frame->instr_ptr - _PyFrame_GetBytecode(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + } +#endif _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); monitor_throw(tstate, frame, frame->instr_ptr); /* TO DO -- Monitor throw entry. */ @@ -983,7 +997,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int Python main loop. */ PyObject *exc = _PyErr_GetRaisedException(tstate); PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + handler; + next_instr = _PyFrame_GetBytecode(frame) + handler; if (monitor_handled(tstate, frame, next_instr, exc) < 0) { goto exception_unwind; @@ -1045,6 +1059,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #undef ENABLE_SPECIALIZATION #define ENABLE_SPECIALIZATION 0 +#undef ENABLE_SPECIALIZATION_FT +#define ENABLE_SPECIALIZATION_FT 0 #ifdef Py_DEBUG #define DPRINTF(level, ...) \ @@ -1139,7 +1155,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int goto goto_to_tier1; exit_to_tier1: assert(next_uop[-1].format == UOP_FORMAT_TARGET); - next_instr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); + next_instr = next_uop[-1].target + _PyFrame_GetBytecode(frame); goto_to_tier1: #ifdef Py_DEBUG if (lltrace >= 2) { @@ -1764,7 +1780,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, if (frame == NULL) { goto fail; } - _PyFrame_Initialize(frame, func, locals, code, 0, previous); + _PyFrame_Initialize(tstate, frame, func, locals, code, 0, previous); if (initialize_locals(tstate, func_obj, frame->localsplus, args, argcount, kwnames)) { assert(frame->owner == FRAME_OWNED_BY_THREAD); clear_thread_frame(tstate, frame); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 6674c4ccf9f693..603b71ea938cde 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -151,7 +151,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* Code access macros */ /* The integer overflow is checked by an assertion below. */ -#define INSTR_OFFSET() ((int)(next_instr - _PyCode_CODE(_PyFrame_GetCode(frame)))) +#define INSTR_OFFSET() ((int)(next_instr - _PyFrame_GetBytecode(frame))) #define NEXTOPARG() do { \ _Py_CODEUNIT word = {.cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t*)next_instr)}; \ opcode = word.op.code; \ @@ -301,14 +301,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ backoff_counter_triggers(forge_backoff_counter((COUNTER))) -#ifdef Py_GIL_DISABLED -#define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \ - do { \ - /* gh-115999 tracks progress on addressing this. */ \ - static_assert(0, "The specializing interpreter is not yet thread-safe"); \ - } while (0); -#define PAUSE_ADAPTIVE_COUNTER(COUNTER) ((void)COUNTER) -#else #define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \ do { \ (COUNTER) = advance_backoff_counter((COUNTER)); \ @@ -318,6 +310,18 @@ GETITEM(PyObject *v, Py_ssize_t i) { do { \ (COUNTER) = pause_backoff_counter((COUNTER)); \ } while (0); + +#ifdef ENABLE_SPECIALIZATION_FT +/* Multiple threads may execute these concurrently if thread-local bytecode is + * disabled and they all execute the main copy of the bytecode. Specialization + * is disabled in that case so the value is unused, but the RMW cycle should be + * free of data races. + */ +#define RECORD_BRANCH_TAKEN(bitset, flag) \ + FT_ATOMIC_STORE_UINT16_RELAXED( \ + bitset, (FT_ATOMIC_LOAD_UINT16_RELAXED(bitset) << 1) | (flag)) +#else +#define RECORD_BRANCH_TAKEN(bitset, flag) #endif #define UNBOUNDLOCAL_ERROR_MSG \ @@ -409,7 +413,8 @@ do { \ #define CURRENT_OPARG() (next_uop[-1].oparg) -#define CURRENT_OPERAND() (next_uop[-1].operand) +#define CURRENT_OPERAND0() (next_uop[-1].operand0) +#define CURRENT_OPERAND1() (next_uop[-1].operand1) #define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target #define JUMP_TO_ERROR() goto jump_to_error_target diff --git a/Python/clinic/Python-tokenize.c.h b/Python/clinic/Python-tokenize.c.h index 730fa8ef2a2154..2a0d50bda65f00 100644 --- a/Python/clinic/Python-tokenize.c.h +++ b/Python/clinic/Python-tokenize.c.h @@ -49,7 +49,8 @@ tokenizeriter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int extra_tokens; const char *encoding = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 1, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -80,4 +81,4 @@ tokenizeriter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=dcd6ec48f06a092e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=831a75133d4a5034 input=a9049054013a1b77]*/ diff --git a/Python/clinic/_warnings.c.h b/Python/clinic/_warnings.c.h index 98dc49db3059b1..9a2c33f2ea8169 100644 --- a/Python/clinic/_warnings.c.h +++ b/Python/clinic/_warnings.c.h @@ -74,7 +74,8 @@ warnings_warn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec PyObject *source = Py_None; PyTupleObject *skip_file_prefixes = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -184,7 +185,8 @@ warnings_warn_explicit(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyObject *module_globals = Py_None; PyObject *sourceobj = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 8, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 4, /*maxpos*/ 8, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -244,4 +246,4 @@ warnings_filters_mutated(PyObject *module, PyObject *Py_UNUSED(ignored)) { return warnings_filters_mutated_impl(module); } -/*[clinic end generated code: output=f2d4214c382717a6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ed02c0f521a03a37 input=a9049054013a1b77]*/ diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index f75a8d4ac0ccd4..7d348dbebed385 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -73,7 +73,8 @@ builtin___import__(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py PyObject *fromlist = NULL; int level = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 5, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -298,7 +299,8 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj int optimize = -1; int feature_version = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 6, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 6, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -448,7 +450,8 @@ builtin_eval(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *globals = Py_None; PyObject *locals = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -527,7 +530,8 @@ builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *locals = Py_None; PyObject *closure = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -865,7 +869,8 @@ builtin_pow(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *exp; PyObject *mod = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -901,8 +906,9 @@ PyDoc_STRVAR(builtin_print__doc__, {"print", _PyCFunction_CAST(builtin_print), METH_FASTCALL|METH_KEYWORDS, builtin_print__doc__}, static PyObject * -builtin_print_impl(PyObject *module, PyObject *args, PyObject *sep, - PyObject *end, PyObject *file, int flush); +builtin_print_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length, PyObject *sep, PyObject *end, + PyObject *file, int flush); static PyObject * builtin_print(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -933,49 +939,52 @@ builtin_print(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[5]; + PyObject *argsbuf[4]; + PyObject * const *fastargs; Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *__clinic_args = NULL; + PyObject * const *__clinic_args; + Py_ssize_t args_length; PyObject *sep = Py_None; PyObject *end = Py_None; PyObject *file = Py_None; int flush = 0; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - __clinic_args = args[0]; if (!noptargs) { goto skip_optional_kwonly; } - if (args[1]) { - sep = args[1]; + if (fastargs[0]) { + sep = fastargs[0]; if (!--noptargs) { goto skip_optional_kwonly; } } - if (args[2]) { - end = args[2]; + if (fastargs[1]) { + end = fastargs[1]; if (!--noptargs) { goto skip_optional_kwonly; } } - if (args[3]) { - file = args[3]; + if (fastargs[2]) { + file = fastargs[2]; if (!--noptargs) { goto skip_optional_kwonly; } } - flush = PyObject_IsTrue(args[4]); + flush = PyObject_IsTrue(fastargs[3]); if (flush < 0) { goto exit; } skip_optional_kwonly: - return_value = builtin_print_impl(module, __clinic_args, sep, end, file, flush); + __clinic_args = args; + args_length = nargs; + return_value = builtin_print_impl(module, __clinic_args, args_length, sep, end, file, flush); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -1077,7 +1086,8 @@ builtin_round(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec PyObject *number; PyObject *ndigits = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1143,7 +1153,8 @@ builtin_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *iterable; PyObject *start = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1228,4 +1239,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=435d3f286a863c49 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b0178189d13e8bf8 input=a9049054013a1b77]*/ diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index 5edeaef656ef62..0e6a560d57c8ba 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -223,7 +223,8 @@ _imp_find_frozen(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb PyObject *name; int withdata = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -594,7 +595,8 @@ _imp_source_hash(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb long key; Py_buffer source = {NULL, NULL}; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -623,4 +625,4 @@ _imp_source_hash(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=dbd63707bd40b07c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d0e278351d6adbb1 input=a9049054013a1b77]*/ diff --git a/Python/clinic/instruction_sequence.c.h b/Python/clinic/instruction_sequence.c.h index 66c2ccadfa03c9..45693e5856f8a7 100644 --- a/Python/clinic/instruction_sequence.c.h +++ b/Python/clinic/instruction_sequence.c.h @@ -82,7 +82,8 @@ InstructionSequenceType_use_label(_PyInstructionSequence *self, PyObject *const PyObject *argsbuf[1]; int label; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -148,7 +149,8 @@ InstructionSequenceType_addop(_PyInstructionSequence *self, PyObject *const *arg int end_lineno; int end_col_offset; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 6, 6, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 6, /*maxpos*/ 6, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -255,7 +257,8 @@ InstructionSequenceType_add_nested(_PyInstructionSequence *self, PyObject *const PyObject *argsbuf[1]; PyObject *nested; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -301,4 +304,4 @@ InstructionSequenceType_get_instructions(_PyInstructionSequence *self, PyObject { return InstructionSequenceType_get_instructions_impl(self); } -/*[clinic end generated code: output=8809d7aa11d9b2bb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=35163e5b589b4446 input=a9049054013a1b77]*/ diff --git a/Python/clinic/marshal.c.h b/Python/clinic/marshal.c.h index c19a3ed5050ed3..d9bbb0527416c6 100644 --- a/Python/clinic/marshal.c.h +++ b/Python/clinic/marshal.c.h @@ -70,7 +70,8 @@ marshal_dump(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject int version = Py_MARSHAL_VERSION; int allow_code = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -157,7 +158,8 @@ marshal_load(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *file; int allow_code = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -234,7 +236,8 @@ marshal_dumps(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec int version = Py_MARSHAL_VERSION; int allow_code = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -314,7 +317,8 @@ marshal_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec Py_buffer bytes = {NULL, NULL}; int allow_code = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -339,4 +343,4 @@ marshal_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec return return_value; } -/*[clinic end generated code: output=1575b9a3ae48ad3d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c7ef4f599658d8ab input=a9049054013a1b77]*/ diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 8277d286cf51ef..86c42ceffc5e31 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_tuple.h" // _PyTuple_FromArray() PyDoc_STRVAR(sys_addaudithook__doc__, "addaudithook($module, /, hook)\n" @@ -52,7 +53,8 @@ sys_addaudithook(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb PyObject *argsbuf[1]; PyObject *hook; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -63,6 +65,54 @@ sys_addaudithook(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb return return_value; } +PyDoc_STRVAR(sys_audit__doc__, +"audit($module, event, /, *args)\n" +"--\n" +"\n" +"Passes the event to any audit hooks that are attached."); + +#define SYS_AUDIT_METHODDEF \ + {"audit", _PyCFunction_CAST(sys_audit), METH_FASTCALL, sys_audit__doc__}, + +static PyObject * +sys_audit_impl(PyObject *module, const char *event, PyObject *args); + +static PyObject * +sys_audit(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + const char *event; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("audit", nargs, 1, PY_SSIZE_T_MAX)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("audit", "argument 1", "str", args[0]); + goto exit; + } + Py_ssize_t event_length; + event = PyUnicode_AsUTF8AndSize(args[0], &event_length); + if (event == NULL) { + goto exit; + } + if (strlen(event) != (size_t)event_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + __clinic_args = _PyTuple_FromArray(args + 1, nargs - 1); + if (__clinic_args == NULL) { + goto exit; + } + return_value = sys_audit_impl(module, event, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + PyDoc_STRVAR(sys_displayhook__doc__, "displayhook($module, object, /)\n" "--\n" @@ -561,7 +611,8 @@ sys_set_coroutine_origin_tracking_depth(PyObject *module, PyObject *const *args, PyObject *argsbuf[1]; int depth; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -862,7 +913,8 @@ sys_set_int_max_str_digits(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *argsbuf[1]; int maxdigits; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1013,7 +1065,8 @@ sys_getunicodeinternedsize(PyObject *module, PyObject *const *args, Py_ssize_t n int _only_immortal = 0; Py_ssize_t _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1479,7 +1532,8 @@ sys__getframemodulename(PyObject *module, PyObject *const *args, Py_ssize_t narg Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int depth = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1614,4 +1668,4 @@ sys__is_gil_enabled(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=9cc9069aef1482bc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6d4f6cd20419b675 input=a9049054013a1b77]*/ diff --git a/Python/clinic/traceback.c.h b/Python/clinic/traceback.c.h index fe53a2786d1ad6..9607e773f4be88 100644 --- a/Python/clinic/traceback.c.h +++ b/Python/clinic/traceback.c.h @@ -55,7 +55,8 @@ tb_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int tb_lasti; int tb_lineno; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 4, 4, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 4, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -78,4 +79,4 @@ tb_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=916a759875507c5a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=62ebc0196940f663 input=a9049054013a1b77]*/ diff --git a/Python/codegen.c b/Python/codegen.c index d79aee4859e51b..bce3b94b27a45d 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -51,16 +51,19 @@ #define ERROR -1 #define RETURN_IF_ERROR(X) \ - if ((X) == -1) { \ - return ERROR; \ - } - -#define RETURN_IF_ERROR_IN_SCOPE(C, CALL) { \ - if ((CALL) < 0) { \ - _PyCompile_ExitScope((C)); \ - return ERROR; \ - } \ -} + do { \ + if ((X) == -1) { \ + return ERROR; \ + } \ + } while (0) + +#define RETURN_IF_ERROR_IN_SCOPE(C, CALL) \ + do { \ + if ((CALL) < 0) { \ + _PyCompile_ExitScope((C)); \ + return ERROR; \ + } \ + } while (0) struct _PyCompiler; typedef struct _PyCompiler compiler; @@ -261,7 +264,7 @@ codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) #define ADDOP_I_IN_SCOPE(C, LOC, OP, O) \ - RETURN_IF_ERROR_IN_SCOPE(C, codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))); + RETURN_IF_ERROR_IN_SCOPE(C, codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) static int codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) @@ -303,17 +306,18 @@ codegen_addop_load_const(compiler *c, location loc, PyObject *o) RETURN_IF_ERROR_IN_SCOPE((C), codegen_addop_load_const((C), (LOC), (O))) /* Same as ADDOP_LOAD_CONST, but steals a reference. */ -#define ADDOP_LOAD_CONST_NEW(C, LOC, O) { \ - PyObject *__new_const = (O); \ - if (__new_const == NULL) { \ - return ERROR; \ - } \ - if (codegen_addop_load_const((C), (LOC), __new_const) < 0) { \ - Py_DECREF(__new_const); \ - return ERROR; \ - } \ - Py_DECREF(__new_const); \ -} +#define ADDOP_LOAD_CONST_NEW(C, LOC, O) \ + do { \ + PyObject *__new_const = (O); \ + if (__new_const == NULL) { \ + return ERROR; \ + } \ + if (codegen_addop_load_const((C), (LOC), __new_const) < 0) { \ + Py_DECREF(__new_const); \ + return ERROR; \ + } \ + Py_DECREF(__new_const); \ + } while (0) static int codegen_addop_o(compiler *c, location loc, @@ -325,19 +329,23 @@ codegen_addop_o(compiler *c, location loc, return SUCCESS; } -#define ADDOP_N(C, LOC, OP, O, TYPE) { \ - assert(!OPCODE_HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \ - int ret = codegen_addop_o((C), (LOC), (OP), METADATA(C)->u_ ## TYPE, (O)); \ - Py_DECREF((O)); \ - RETURN_IF_ERROR(ret); \ -} - -#define ADDOP_N_IN_SCOPE(C, LOC, OP, O, TYPE) { \ - assert(!OPCODE_HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \ - int ret = codegen_addop_o((C), (LOC), (OP), METADATA(C)->u_ ## TYPE, (O)); \ - Py_DECREF((O)); \ - RETURN_IF_ERROR_IN_SCOPE((C), ret); \ -} +#define ADDOP_N(C, LOC, OP, O, TYPE) \ + do { \ + assert(!OPCODE_HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \ + int ret = codegen_addop_o((C), (LOC), (OP), \ + METADATA(C)->u_ ## TYPE, (O)); \ + Py_DECREF((O)); \ + RETURN_IF_ERROR(ret); \ + } while (0) + +#define ADDOP_N_IN_SCOPE(C, LOC, OP, O, TYPE) \ + do { \ + assert(!OPCODE_HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \ + int ret = codegen_addop_o((C), (LOC), (OP), \ + METADATA(C)->u_ ## TYPE, (O)); \ + Py_DECREF((O)); \ + RETURN_IF_ERROR_IN_SCOPE((C), ret); \ + } while (0) #define LOAD_METHOD -1 #define LOAD_SUPER_METHOD -2 @@ -426,31 +434,31 @@ codegen_addop_j(instr_sequence *seq, location loc, */ #define VISIT(C, TYPE, V) \ - RETURN_IF_ERROR(codegen_visit_ ## TYPE((C), (V))); + RETURN_IF_ERROR(codegen_visit_ ## TYPE((C), (V))) #define VISIT_IN_SCOPE(C, TYPE, V) \ RETURN_IF_ERROR_IN_SCOPE((C), codegen_visit_ ## TYPE((C), (V))) -#define VISIT_SEQ(C, TYPE, SEQ) { \ - int _i; \ - asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ - for (_i = 0; _i < asdl_seq_LEN(seq); _i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \ - RETURN_IF_ERROR(codegen_visit_ ## TYPE((C), elt)); \ - } \ -} - -#define VISIT_SEQ_IN_SCOPE(C, TYPE, SEQ) { \ - int _i; \ - asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ - for (_i = 0; _i < asdl_seq_LEN(seq); _i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \ - if (codegen_visit_ ## TYPE((C), elt) < 0) { \ - _PyCompile_ExitScope(C); \ - return ERROR; \ - } \ - } \ -} +#define VISIT_SEQ(C, TYPE, SEQ) \ + do { \ + asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ + for (int _i = 0; _i < asdl_seq_LEN(seq); _i++) { \ + TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \ + RETURN_IF_ERROR(codegen_visit_ ## TYPE((C), elt)); \ + } \ + } while (0) + +#define VISIT_SEQ_IN_SCOPE(C, TYPE, SEQ) \ + do { \ + asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ + for (int _i = 0; _i < asdl_seq_LEN(seq); _i++) { \ + TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \ + if (codegen_visit_ ## TYPE((C), elt) < 0) { \ + _PyCompile_ExitScope(C); \ + return ERROR; \ + } \ + } \ + } while (0) static int codegen_call_exit_with_nones(compiler *c, location loc) @@ -664,9 +672,7 @@ codegen_setup_annotations_scope(compiler *c, location loc, codegen_enter_scope(c, name, COMPILE_SCOPE_ANNOTATIONS, key, loc.lineno, NULL, &umd)); - // Insert None into consts to prevent an annotation - // appearing to be a docstring - _PyCompile_AddConst(c, Py_None); + assert(!SYMTABLE_ENTRY(c)->ste_has_docstring); // if .format != 1: raise NotImplementedError _Py_DECLARE_STR(format, ".format"); ADDOP_I(c, loc, LOAD_FAST, 0); @@ -762,7 +768,8 @@ _PyCodegen_Body(compiler *c, location loc, asdl_stmt_seq *stmts, bool is_interac /* If from __future__ import annotations is active, * every annotated class and module should have __annotations__. * Else __annotate__ is created when necessary. */ - if ((FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) && SYMTABLE_ENTRY(c)->ste_annotations_used) { + PySTEntryObject *ste = SYMTABLE_ENTRY(c); + if ((FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) && ste->ste_annotations_used) { ADDOP(c, loc, SETUP_ANNOTATIONS); } if (!asdl_seq_LEN(stmts)) { @@ -770,8 +777,9 @@ _PyCodegen_Body(compiler *c, location loc, asdl_stmt_seq *stmts, bool is_interac } Py_ssize_t first_instr = 0; if (!is_interactive) { /* A string literal on REPL prompt is not a docstring */ - PyObject *docstring = _PyAST_GetDocString(stmts); - if (docstring) { + if (ste->ste_has_docstring) { + PyObject *docstring = _PyAST_GetDocString(stmts); + assert(docstring); first_instr = 1; /* set docstring */ assert(OPTIMIZATION_LEVEL(c) < 2); @@ -1233,10 +1241,11 @@ codegen_function_body(compiler *c, stmt_ty s, int is_async, Py_ssize_t funcflags RETURN_IF_ERROR( codegen_enter_scope(c, name, scope_type, (void *)s, firstlineno, NULL, &umd)); + PySTEntryObject *ste = SYMTABLE_ENTRY(c); Py_ssize_t first_instr = 0; - PyObject *docstring = _PyAST_GetDocString(body); - assert(OPTIMIZATION_LEVEL(c) < 2 || docstring == NULL); - if (docstring) { + if (ste->ste_has_docstring) { + PyObject *docstring = _PyAST_GetDocString(body); + assert(docstring); first_instr = 1; docstring = _PyCompile_CleanDoc(docstring); if (docstring == NULL) { @@ -1250,7 +1259,6 @@ codegen_function_body(compiler *c, stmt_ty s, int is_async, Py_ssize_t funcflags NEW_JUMP_TARGET_LABEL(c, start); USE_LABEL(c, start); - PySTEntryObject *ste = SYMTABLE_ENTRY(c); bool add_stopiteration_handler = ste->ste_coroutine || ste->ste_generator; if (add_stopiteration_handler) { /* codegen_wrap_in_stopiteration_handler will push a block, so we need to account for that */ @@ -1592,9 +1600,8 @@ codegen_typealias_body(compiler *c, stmt_ty s) ADDOP_LOAD_CONST_NEW(c, loc, defaults); RETURN_IF_ERROR( codegen_setup_annotations_scope(c, LOC(s), s, name)); - /* Make None the first constant, so the evaluate function can't have a - docstring. */ - RETURN_IF_ERROR(_PyCompile_AddConst(c, Py_None)); + + assert(!SYMTABLE_ENTRY(c)->ste_has_docstring); VISIT_IN_SCOPE(c, expr, s->v.TypeAlias.value); ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 0); @@ -1890,9 +1897,7 @@ codegen_lambda(compiler *c, expr_ty e) codegen_enter_scope(c, &_Py_STR(anon_lambda), COMPILE_SCOPE_LAMBDA, (void *)e, e->lineno, NULL, &umd)); - /* Make None the first constant, so the lambda can't have a - docstring. */ - RETURN_IF_ERROR(_PyCompile_AddConst(c, Py_None)); + assert(!SYMTABLE_ENTRY(c)->ste_has_docstring); VISIT_IN_SCOPE(c, expr, e->v.Lambda.body); if (SYMTABLE_ENTRY(c)->ste_generator) { @@ -2866,7 +2871,7 @@ codegen_visit_stmt(compiler *c, stmt_ty s) case Return_kind: return codegen_return(c, s); case Delete_kind: - VISIT_SEQ(c, expr, s->v.Delete.targets) + VISIT_SEQ(c, expr, s->v.Delete.targets); break; case Assign_kind: { @@ -4087,9 +4092,12 @@ codegen_call_helper(compiler *c, location loc, return codegen_call_helper_impl(c, loc, n, args, NULL, keywords); } -/* List and set comprehensions and generator expressions work by creating a - nested function to perform the actual iteration. This means that the - iteration variables don't leak into the current scope. +/* List and set comprehensions work by being inlined at the location where + they are defined. The isolation of iteration variables is provided by + pushing/popping clashing locals on the stack. Generator expressions work + by creating a nested function to perform the actual iteration. + This means that the iteration variables don't leak into the current scope. + See https://peps.python.org/pep-0709/ for additional information. The defined function is called immediately following its definition, with the result of that call being the result of the expression. The LC/SC version returns the populated container, while the GE version is @@ -4756,7 +4764,7 @@ codegen_async_with(compiler *c, stmt_ty s, int pos) pos++; if (pos == asdl_seq_LEN(s->v.AsyncWith.items)) { /* BLOCK code */ - VISIT_SEQ(c, stmt, s->v.AsyncWith.body) + VISIT_SEQ(c, stmt, s->v.AsyncWith.body); } else { RETURN_IF_ERROR(codegen_async_with(c, s, pos)); @@ -4855,7 +4863,7 @@ codegen_with(compiler *c, stmt_ty s, int pos) pos++; if (pos == asdl_seq_LEN(s->v.With.items)) { /* BLOCK code */ - VISIT_SEQ(c, stmt, s->v.With.body) + VISIT_SEQ(c, stmt, s->v.With.body); } else { RETURN_IF_ERROR(codegen_with(c, s, pos)); diff --git a/Python/compile.c b/Python/compile.c index 4dcb9a1b5acdb3..cbfba7f493e07d 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -31,9 +31,11 @@ #define ERROR -1 #define RETURN_IF_ERROR(X) \ - if ((X) == -1) { \ - return ERROR; \ - } + do { \ + if ((X) == -1) { \ + return ERROR; \ + } \ + } while (0) typedef _Py_SourceLocation location; typedef _PyJumpTargetLabel jump_target_label; @@ -901,7 +903,7 @@ _PyCompile_LookupArg(compiler *c, PyCodeObject *co, PyObject *name) c->u->u_metadata.u_name, co->co_name, freevars); - Py_DECREF(freevars); + Py_XDECREF(freevars); return ERROR; } return arg; diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 0aca322d987dba..7aaa045f375cf0 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -3,22 +3,21 @@ #include "Python.h" #include "pycore_ceval.h" // _Py_simple_func -#include "pycore_crossinterp.h" // struct _xid +#include "pycore_crossinterp.h" // _PyXIData_t #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_namespace.h" //_PyNamespace_New() #include "pycore_pyerrors.h" // _PyErr_Clear() -#include "pycore_weakref.h" // _PyWeakref_GET_REF() /**************/ /* exceptions */ /**************/ -static int init_exceptions(PyInterpreterState *); -static void fini_exceptions(PyInterpreterState *); -static int _init_not_shareable_error_type(PyInterpreterState *); -static void _fini_not_shareable_error_type(PyInterpreterState *); -static PyObject * _get_not_shareable_error_type(PyInterpreterState *); +typedef struct xi_exceptions exceptions_t; +static int init_static_exctypes(exceptions_t *, PyInterpreterState *); +static void fini_static_exctypes(exceptions_t *, PyInterpreterState *); +static int init_heap_exctypes(exceptions_t *); +static void fini_heap_exctypes(exceptions_t *); #include "crossinterp_exceptions.h" @@ -57,25 +56,24 @@ _Py_CallInInterpreterAndRawFree(PyInterpreterState *interp, /* cross-interpreter data */ /**************************/ -/* registry of {type -> crossinterpdatafunc} */ +/* registry of {type -> xidatafunc} */ /* For now we use a global registry of shareable classes. An alternative would be to add a tp_* slot for a class's - crossinterpdatafunc. It would be simpler and more efficient. */ + xidatafunc. It would be simpler and more efficient. */ -static void xid_lookup_init(PyInterpreterState *); -static void xid_lookup_fini(PyInterpreterState *); -static crossinterpdatafunc lookup_getdata(PyInterpreterState *, PyObject *); +static void xid_lookup_init(_PyXIData_lookup_t *); +static void xid_lookup_fini(_PyXIData_lookup_t *); +static xidatafunc lookup_getdata(_PyXIData_lookup_context_t *, PyObject *); #include "crossinterp_data_lookup.h" /* lifecycle */ -_PyCrossInterpreterData * -_PyCrossInterpreterData_New(void) +_PyXIData_t * +_PyXIData_New(void) { - _PyCrossInterpreterData *xid = PyMem_RawMalloc( - sizeof(_PyCrossInterpreterData)); + _PyXIData_t *xid = PyMem_RawMalloc(sizeof(_PyXIData_t)); if (xid == NULL) { PyErr_NoMemory(); } @@ -83,10 +81,10 @@ _PyCrossInterpreterData_New(void) } void -_PyCrossInterpreterData_Free(_PyCrossInterpreterData *xid) +_PyXIData_Free(_PyXIData_t *xid) { PyInterpreterState *interp = PyInterpreterState_Get(); - _PyCrossInterpreterData_Clear(interp, xid); + _PyXIData_Clear(interp, xid); PyMem_RawFree(xid); } @@ -94,20 +92,20 @@ _PyCrossInterpreterData_Free(_PyCrossInterpreterData *xid) /* defining cross-interpreter data */ static inline void -_xidata_init(_PyCrossInterpreterData *data) +_xidata_init(_PyXIData_t *data) { // If the value is being reused // then _xidata_clear() should have been called already. assert(data->data == NULL); assert(data->obj == NULL); - *data = (_PyCrossInterpreterData){0}; - _PyCrossInterpreterData_INTERPID(data) = -1; + *data = (_PyXIData_t){0}; + _PyXIData_INTERPID(data) = -1; } static inline void -_xidata_clear(_PyCrossInterpreterData *data) +_xidata_clear(_PyXIData_t *data) { - // _PyCrossInterpreterData only has two members that need to be + // _PyXIData_t only has two members that need to be // cleaned up, if set: "data" must be freed and "obj" must be decref'ed. // In both cases the original (owning) interpreter must be used, // which is the caller's responsibility to ensure. @@ -121,10 +119,10 @@ _xidata_clear(_PyCrossInterpreterData *data) } void -_PyCrossInterpreterData_Init(_PyCrossInterpreterData *data, - PyInterpreterState *interp, - void *shared, PyObject *obj, - xid_newobjectfunc new_object) +_PyXIData_Init(_PyXIData_t *data, + PyInterpreterState *interp, + void *shared, PyObject *obj, + xid_newobjfunc new_object) { assert(data != NULL); assert(new_object != NULL); @@ -132,29 +130,29 @@ _PyCrossInterpreterData_Init(_PyCrossInterpreterData *data, data->data = shared; if (obj != NULL) { assert(interp != NULL); - // released in _PyCrossInterpreterData_Clear() + // released in _PyXIData_Clear() data->obj = Py_NewRef(obj); } // Ideally every object would know its owning interpreter. // Until then, we have to rely on the caller to identify it // (but we don't need it in all cases). - _PyCrossInterpreterData_INTERPID(data) = (interp != NULL) + _PyXIData_INTERPID(data) = (interp != NULL) ? PyInterpreterState_GetID(interp) : -1; data->new_object = new_object; } int -_PyCrossInterpreterData_InitWithSize(_PyCrossInterpreterData *data, - PyInterpreterState *interp, - const size_t size, PyObject *obj, - xid_newobjectfunc new_object) +_PyXIData_InitWithSize(_PyXIData_t *data, + PyInterpreterState *interp, + const size_t size, PyObject *obj, + xid_newobjfunc new_object) { assert(size > 0); // For now we always free the shared data in the same interpreter // where it was allocated, so the interpreter is required. assert(interp != NULL); - _PyCrossInterpreterData_Init(data, interp, NULL, obj, new_object); + _PyXIData_Init(data, interp, NULL, obj, new_object); data->data = PyMem_RawMalloc(size); if (data->data == NULL) { return -1; @@ -164,14 +162,13 @@ _PyCrossInterpreterData_InitWithSize(_PyCrossInterpreterData *data, } void -_PyCrossInterpreterData_Clear(PyInterpreterState *interp, - _PyCrossInterpreterData *data) +_PyXIData_Clear(PyInterpreterState *interp, _PyXIData_t *data) { assert(data != NULL); // This must be called in the owning interpreter. assert(interp == NULL - || _PyCrossInterpreterData_INTERPID(data) == -1 - || _PyCrossInterpreterData_INTERPID(data) == PyInterpreterState_GetID(interp)); + || _PyXIData_INTERPID(data) == -1 + || _PyXIData_INTERPID(data) == PyInterpreterState_GetID(interp)); _xidata_clear(data); } @@ -179,13 +176,13 @@ _PyCrossInterpreterData_Clear(PyInterpreterState *interp, /* using cross-interpreter data */ static int -_check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) +_check_xidata(PyThreadState *tstate, _PyXIData_t *data) { // data->data can be anything, including NULL, so we don't check it. // data->obj may be NULL, so we don't check it. - if (_PyCrossInterpreterData_INTERPID(data) < 0) { + if (_PyXIData_INTERPID(data) < 0) { PyErr_SetString(PyExc_SystemError, "missing interp"); return -1; } @@ -201,10 +198,9 @@ _check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) } static inline void -_set_xid_lookup_failure(PyInterpreterState *interp, - PyObject *obj, const char *msg) +_set_xid_lookup_failure(dlcontext_t *ctx, PyObject *obj, const char *msg) { - PyObject *exctype = _get_not_shareable_error_type(interp); + PyObject *exctype = ctx->PyExc_NotShareableError; assert(exctype != NULL); if (msg != NULL) { assert(obj == NULL); @@ -221,13 +217,12 @@ _set_xid_lookup_failure(PyInterpreterState *interp, } int -_PyObject_CheckCrossInterpreterData(PyObject *obj) +_PyObject_CheckXIData(_PyXIData_lookup_context_t *ctx, PyObject *obj) { - PyInterpreterState *interp = PyInterpreterState_Get(); - crossinterpdatafunc getdata = lookup_getdata(interp, obj); + xidatafunc getdata = lookup_getdata(ctx, obj); if (getdata == NULL) { if (!PyErr_Occurred()) { - _set_xid_lookup_failure(interp, obj, NULL); + _set_xid_lookup_failure(ctx, obj, NULL); } return -1; } @@ -235,22 +230,23 @@ _PyObject_CheckCrossInterpreterData(PyObject *obj) } int -_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) +_PyObject_GetXIData(_PyXIData_lookup_context_t *ctx, + PyObject *obj, _PyXIData_t *data) { PyThreadState *tstate = PyThreadState_Get(); PyInterpreterState *interp = tstate->interp; // Reset data before re-populating. - *data = (_PyCrossInterpreterData){0}; - _PyCrossInterpreterData_INTERPID(data) = -1; + *data = (_PyXIData_t){0}; + _PyXIData_INTERPID(data) = -1; // Call the "getdata" func for the object. Py_INCREF(obj); - crossinterpdatafunc getdata = lookup_getdata(interp, obj); + xidatafunc getdata = lookup_getdata(ctx, obj); if (getdata == NULL) { Py_DECREF(obj); if (!PyErr_Occurred()) { - _set_xid_lookup_failure(interp, obj, NULL); + _set_xid_lookup_failure(ctx, obj, NULL); } return -1; } @@ -261,9 +257,9 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) } // Fill in the blanks and validate the result. - _PyCrossInterpreterData_INTERPID(data) = PyInterpreterState_GetID(interp); + _PyXIData_INTERPID(data) = PyInterpreterState_GetID(interp); if (_check_xidata(tstate, data) != 0) { - (void)_PyCrossInterpreterData_Release(data); + (void)_PyXIData_Release(data); return -1; } @@ -271,7 +267,7 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) } PyObject * -_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) +_PyXIData_NewObject(_PyXIData_t *data) { return data->new_object(data); } @@ -279,12 +275,12 @@ _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) static int _call_clear_xidata(void *data) { - _xidata_clear((_PyCrossInterpreterData *)data); + _xidata_clear((_PyXIData_t *)data); return 0; } static int -_xidata_release(_PyCrossInterpreterData *data, int rawfree) +_xidata_release(_PyXIData_t *data, int rawfree) { if ((data->data == NULL || data->free == NULL) && data->obj == NULL) { // Nothing to release! @@ -299,7 +295,7 @@ _xidata_release(_PyCrossInterpreterData *data, int rawfree) // Switch to the original interpreter. PyInterpreterState *interp = _PyInterpreterState_LookUpID( - _PyCrossInterpreterData_INTERPID(data)); + _PyXIData_INTERPID(data)); if (interp == NULL) { // The interpreter was already destroyed. // This function shouldn't have been called. @@ -321,13 +317,13 @@ _xidata_release(_PyCrossInterpreterData *data, int rawfree) } int -_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) +_PyXIData_Release(_PyXIData_t *data) { return _xidata_release(data, 0); } int -_PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *data) +_PyXIData_ReleaseAndRawFree(_PyXIData_t *data) { return _xidata_release(data, 1); } @@ -446,15 +442,15 @@ _format_TracebackException(PyObject *tbexc) static int -_release_xid_data(_PyCrossInterpreterData *data, int rawfree) +_release_xid_data(_PyXIData_t *data, int rawfree) { PyObject *exc = PyErr_GetRaisedException(); int res = rawfree - ? _PyCrossInterpreterData_Release(data) - : _PyCrossInterpreterData_ReleaseAndRawFree(data); + ? _PyXIData_Release(data) + : _PyXIData_ReleaseAndRawFree(data); if (res < 0) { /* The owning interpreter is already destroyed. */ - _PyCrossInterpreterData_Clear(NULL, data); + _PyXIData_Clear(NULL, data); // XXX Emit a warning? PyErr_Clear(); } @@ -967,6 +963,8 @@ _PyXI_ClearExcInfo(_PyXI_excinfo *info) static int _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) { + dlcontext_t ctx; + assert(!PyErr_Occurred()); switch (code) { case _PyXI_ERR_NO_ERROR: _Py_FALLTHROUGH; @@ -985,8 +983,7 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) break; case _PyXI_ERR_ALREADY_RUNNING: assert(interp != NULL); - assert(_PyInterpreterState_IsRunningMain(interp)); - _PyInterpreterState_FailIfRunningMain(interp); + _PyErr_SetInterpreterAlreadyRunning(); break; case _PyXI_ERR_MAIN_NS_FAILURE: PyErr_SetString(PyExc_InterpreterError, @@ -997,7 +994,10 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) "failed to apply namespace to __main__"); break; case _PyXI_ERR_NOT_SHAREABLE: - _set_xid_lookup_failure(interp, NULL, NULL); + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return -1; + } + _set_xid_lookup_failure(&ctx, NULL, NULL); break; default: #ifdef Py_DEBUG @@ -1059,7 +1059,11 @@ _PyXI_ApplyError(_PyXI_error *error) } else if (error->code == _PyXI_ERR_NOT_SHAREABLE) { // Propagate the exception directly. - _set_xid_lookup_failure(error->interp, NULL, error->uncaught.msg); + dlcontext_t ctx; + if (_PyXIData_GetLookupContext(error->interp, &ctx) < 0) { + return NULL; + } + _set_xid_lookup_failure(&ctx, NULL, error->uncaught.msg); } else { // Raise an exception corresponding to the code. @@ -1094,8 +1098,8 @@ _PyXI_ApplyError(_PyXI_error *error) typedef struct _sharednsitem { const char *name; - _PyCrossInterpreterData *data; - // We could have a "PyCrossInterpreterData _data" field, so it would + _PyXIData_t *data; + // We could have a "PyXIData _data" field, so it would // be allocated as part of the item and avoid an extra allocation. // However, doing so adds a bunch of complexity because we must // ensure the item isn't freed before a pending call might happen @@ -1131,7 +1135,7 @@ _sharednsitem_has_value(_PyXI_namespace_item *item, int64_t *p_interpid) return 0; } if (p_interpid != NULL) { - *p_interpid = _PyCrossInterpreterData_INTERPID(item->data); + *p_interpid = _PyXIData_INTERPID(item->data); } return 1; } @@ -1141,12 +1145,17 @@ _sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value) { assert(_sharednsitem_is_initialized(item)); assert(item->data == NULL); - item->data = PyMem_RawMalloc(sizeof(_PyCrossInterpreterData)); + item->data = PyMem_RawMalloc(sizeof(_PyXIData_t)); if (item->data == NULL) { PyErr_NoMemory(); return -1; } - if (_PyObject_GetCrossInterpreterData(value, item->data) != 0) { + PyInterpreterState *interp = PyInterpreterState_Get(); + dlcontext_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return -1; + } + if (_PyObject_GetXIData(&ctx, value, item->data) != 0) { PyMem_RawFree(item->data); item->data = NULL; // The caller may want to propagate PyExc_NotShareableError @@ -1159,7 +1168,7 @@ _sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value) static void _sharednsitem_clear_value(_PyXI_namespace_item *item) { - _PyCrossInterpreterData *data = item->data; + _PyXIData_t *data = item->data; if (data != NULL) { item->data = NULL; int rawfree = 1; @@ -1205,7 +1214,7 @@ _sharednsitem_apply(_PyXI_namespace_item *item, PyObject *ns, PyObject *dflt) } PyObject *value; if (item->data != NULL) { - value = _PyCrossInterpreterData_NewObject(item->data); + value = _PyXIData_NewObject(item->data); if (value == NULL) { Py_DECREF(name); return -1; @@ -1604,7 +1613,13 @@ _propagate_not_shareable_error(_PyXI_session *session) return; } PyInterpreterState *interp = PyInterpreterState_Get(); - if (PyErr_ExceptionMatches(_get_not_shareable_error_type(interp))) { + dlcontext_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + PyErr_FormatUnraisable( + "Exception ignored while propagating not shareable error"); + return; + } + if (PyErr_ExceptionMatches(ctx.PyExc_NotShareableError)) { // We want to propagate the exception directly. session->_error_override = _PyXI_ERR_NOT_SHAREABLE; session->error_override = &session->_error_override; @@ -1772,15 +1787,87 @@ _PyXI_Exit(_PyXI_session *session) /* runtime lifecycle */ /*********************/ +int +_Py_xi_global_state_init(_PyXI_global_state_t *state) +{ + assert(state != NULL); + xid_lookup_init(&state->data_lookup); + return 0; +} + +void +_Py_xi_global_state_fini(_PyXI_global_state_t *state) +{ + assert(state != NULL); + xid_lookup_fini(&state->data_lookup); +} + +int +_Py_xi_state_init(_PyXI_state_t *state, PyInterpreterState *interp) +{ + assert(state != NULL); + assert(interp == NULL || state == _PyXI_GET_STATE(interp)); + + xid_lookup_init(&state->data_lookup); + + // Initialize exceptions. + if (interp != NULL) { + if (init_static_exctypes(&state->exceptions, interp) < 0) { + fini_heap_exctypes(&state->exceptions); + return -1; + } + } + if (init_heap_exctypes(&state->exceptions) < 0) { + return -1; + } + + return 0; +} + +void +_Py_xi_state_fini(_PyXI_state_t *state, PyInterpreterState *interp) +{ + assert(state != NULL); + assert(interp == NULL || state == _PyXI_GET_STATE(interp)); + + fini_heap_exctypes(&state->exceptions); + if (interp != NULL) { + fini_static_exctypes(&state->exceptions, interp); + } + + xid_lookup_fini(&state->data_lookup); +} + + PyStatus _PyXI_Init(PyInterpreterState *interp) { - // Initialize the XID lookup state (e.g. registry). - xid_lookup_init(interp); + if (_Py_IsMainInterpreter(interp)) { + _PyXI_global_state_t *global_state = _PyXI_GET_GLOBAL_STATE(interp); + if (global_state == NULL) { + PyErr_PrintEx(0); + return _PyStatus_ERR( + "failed to get global cross-interpreter state"); + } + if (_Py_xi_global_state_init(global_state) < 0) { + PyErr_PrintEx(0); + return _PyStatus_ERR( + "failed to initialize global cross-interpreter state"); + } + } - // Initialize exceptions (heap types). - if (_init_not_shareable_error_type(interp) < 0) { - return _PyStatus_ERR("failed to initialize NotShareableError"); + _PyXI_state_t *state = _PyXI_GET_STATE(interp); + if (state == NULL) { + PyErr_PrintEx(0); + return _PyStatus_ERR( + "failed to get interpreter's cross-interpreter state"); + } + // The static types were already initialized in _PyXI_InitTypes(), + // so we pass in NULL here to avoid initializing them again. + if (_Py_xi_state_init(state, NULL) < 0) { + PyErr_PrintEx(0); + return _PyStatus_ERR( + "failed to initialize interpreter's cross-interpreter state"); } return _PyStatus_OK(); @@ -1792,27 +1879,42 @@ _PyXI_Init(PyInterpreterState *interp) void _PyXI_Fini(PyInterpreterState *interp) { - // Finalize exceptions (heap types). - _fini_not_shareable_error_type(interp); + _PyXI_state_t *state = _PyXI_GET_STATE(interp); +#ifndef NDEBUG + if (state == NULL) { + PyErr_PrintEx(0); + return; + } +#endif + // The static types will be finalized soon in _PyXI_FiniTypes(), + // so we pass in NULL here to avoid finalizing them right now. + _Py_xi_state_fini(state, NULL); - // Finalize the XID lookup state (e.g. registry). - xid_lookup_fini(interp); + if (_Py_IsMainInterpreter(interp)) { + _PyXI_global_state_t *global_state = _PyXI_GET_GLOBAL_STATE(interp); + _Py_xi_global_state_fini(global_state); + } } PyStatus _PyXI_InitTypes(PyInterpreterState *interp) { - if (init_exceptions(interp) < 0) { + if (init_static_exctypes(&_PyXI_GET_STATE(interp)->exceptions, interp) < 0) { PyErr_PrintEx(0); - return _PyStatus_ERR("failed to initialize an exception type"); + return _PyStatus_ERR( + "failed to initialize the cross-interpreter exception types"); } + // We would initialize heap types here too but that leads to ref leaks. + // Instead, we intialize them in _PyXI_Init(). return _PyStatus_OK(); } void _PyXI_FiniTypes(PyInterpreterState *interp) { - fini_exceptions(interp); + // We would finalize heap types here too but that leads to ref leaks. + // Instead, we finalize them in _PyXI_Fini(). + fini_static_exctypes(&_PyXI_GET_STATE(interp)->exceptions, interp); } diff --git a/Python/crossinterp_data_lookup.h b/Python/crossinterp_data_lookup.h index 863919ad42fb97..48e5d9762cd697 100644 --- a/Python/crossinterp_data_lookup.h +++ b/Python/crossinterp_data_lookup.h @@ -1,39 +1,87 @@ +#include "pycore_weakref.h" // _PyWeakref_GET_REF() -static crossinterpdatafunc _lookup_getdata_from_registry( - PyInterpreterState *, PyObject *); -static crossinterpdatafunc -lookup_getdata(PyInterpreterState *interp, PyObject *obj) +typedef _PyXIData_lookup_context_t dlcontext_t; +typedef _PyXIData_registry_t dlregistry_t; +typedef _PyXIData_regitem_t dlregitem_t; + + +// forward +static void _xidregistry_init(dlregistry_t *); +static void _xidregistry_fini(dlregistry_t *); +static xidatafunc _lookup_getdata_from_registry(dlcontext_t *, PyObject *); + + +/* used in crossinterp.c */ + +static void +xid_lookup_init(_PyXIData_lookup_t *state) +{ + _xidregistry_init(&state->registry); +} + +static void +xid_lookup_fini(_PyXIData_lookup_t *state) +{ + _xidregistry_fini(&state->registry); +} + +static xidatafunc +lookup_getdata(dlcontext_t *ctx, PyObject *obj) { /* Cross-interpreter objects are looked up by exact match on the class. We can reassess this policy when we move from a global registry to a tp_* slot. */ - return _lookup_getdata_from_registry(interp, obj); + return _lookup_getdata_from_registry(ctx, obj); } -crossinterpdatafunc -_PyCrossInterpreterData_Lookup(PyObject *obj) + +/* exported API */ + +int +_PyXIData_GetLookupContext(PyInterpreterState *interp, + _PyXIData_lookup_context_t *res) { - PyInterpreterState *interp = PyInterpreterState_Get(); - return lookup_getdata(interp, obj); + _PyXI_global_state_t *global = _PyXI_GET_GLOBAL_STATE(interp); + if (global == NULL) { + assert(PyErr_Occurred()); + return -1; + } + _PyXI_state_t *local = _PyXI_GET_STATE(interp); + if (local == NULL) { + assert(PyErr_Occurred()); + return -1; + } + *res = (dlcontext_t){ + .global = &global->data_lookup, + .local = &local->data_lookup, + .PyExc_NotShareableError = local->exceptions.PyExc_NotShareableError, + }; + return 0; +} + +xidatafunc +_PyXIData_Lookup(_PyXIData_lookup_context_t *ctx, PyObject *obj) +{ + return lookup_getdata(ctx, obj); } /***********************************************/ -/* a registry of {type -> crossinterpdatafunc} */ +/* a registry of {type -> xidatafunc} */ /***********************************************/ /* For now we use a global registry of shareable classes. An alternative would be to add a tp_* slot for a class's - crossinterpdatafunc. It would be simpler and more efficient. */ + xidatafunc. It would be simpler and more efficient. */ /* registry lifecycle */ -static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *); +static void _register_builtins_for_crossinterpreter_data(dlregistry_t *); static void -_xidregistry_init(struct _xidregistry *registry) +_xidregistry_init(dlregistry_t *registry) { if (registry->initialized) { return; @@ -47,10 +95,10 @@ _xidregistry_init(struct _xidregistry *registry) } } -static void _xidregistry_clear(struct _xidregistry *); +static void _xidregistry_clear(dlregistry_t *); static void -_xidregistry_fini(struct _xidregistry *registry) +_xidregistry_fini(dlregistry_t *registry) { if (!registry->initialized) { return; @@ -60,32 +108,11 @@ _xidregistry_fini(struct _xidregistry *registry) _xidregistry_clear(registry); } -static inline struct _xidregistry * _get_global_xidregistry(_PyRuntimeState *); -static inline struct _xidregistry * _get_xidregistry(PyInterpreterState *); - -static void -xid_lookup_init(PyInterpreterState *interp) -{ - if (_Py_IsMainInterpreter(interp)) { - _xidregistry_init(_get_global_xidregistry(interp->runtime)); - } - _xidregistry_init(_get_xidregistry(interp)); -} - -static void -xid_lookup_fini(PyInterpreterState *interp) -{ - _xidregistry_fini(_get_xidregistry(interp)); - if (_Py_IsMainInterpreter(interp)) { - _xidregistry_fini(_get_global_xidregistry(interp->runtime)); - } -} - /* registry thread safety */ static void -_xidregistry_lock(struct _xidregistry *registry) +_xidregistry_lock(dlregistry_t *registry) { if (registry->global) { PyMutex_Lock(®istry->mutex); @@ -94,7 +121,7 @@ _xidregistry_lock(struct _xidregistry *registry) } static void -_xidregistry_unlock(struct _xidregistry *registry) +_xidregistry_unlock(dlregistry_t *registry) { if (registry->global) { PyMutex_Unlock(®istry->mutex); @@ -104,35 +131,21 @@ _xidregistry_unlock(struct _xidregistry *registry) /* accessing the registry */ -static inline struct _xidregistry * -_get_global_xidregistry(_PyRuntimeState *runtime) -{ - return &runtime->xi.registry; -} - -static inline struct _xidregistry * -_get_xidregistry(PyInterpreterState *interp) +static inline dlregistry_t * +_get_xidregistry_for_type(dlcontext_t *ctx, PyTypeObject *cls) { - return &interp->xi.registry; -} - -static inline struct _xidregistry * -_get_xidregistry_for_type(PyInterpreterState *interp, PyTypeObject *cls) -{ - struct _xidregistry *registry = _get_global_xidregistry(interp->runtime); if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { - registry = _get_xidregistry(interp); + return &ctx->local->registry; } - return registry; + return &ctx->global->registry; } -static struct _xidregitem * _xidregistry_remove_entry( - struct _xidregistry *, struct _xidregitem *); +static dlregitem_t* _xidregistry_remove_entry(dlregistry_t *, dlregitem_t *); -static struct _xidregitem * -_xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls) +static dlregitem_t * +_xidregistry_find_type(dlregistry_t *xidregistry, PyTypeObject *cls) { - struct _xidregitem *cur = xidregistry->head; + dlregitem_t *cur = xidregistry->head; while (cur != NULL) { if (cur->weakref != NULL) { // cur is/was a heap type. @@ -155,16 +168,16 @@ _xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls) return NULL; } -static crossinterpdatafunc -_lookup_getdata_from_registry(PyInterpreterState *interp, PyObject *obj) +static xidatafunc +_lookup_getdata_from_registry(dlcontext_t *ctx, PyObject *obj) { PyTypeObject *cls = Py_TYPE(obj); - struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls); + dlregistry_t *xidregistry = _get_xidregistry_for_type(ctx, cls); _xidregistry_lock(xidregistry); - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); - crossinterpdatafunc func = matched != NULL ? matched->getdata : NULL; + dlregitem_t *matched = _xidregistry_find_type(xidregistry, cls); + xidatafunc func = matched != NULL ? matched->getdata : NULL; _xidregistry_unlock(xidregistry); return func; @@ -174,14 +187,14 @@ _lookup_getdata_from_registry(PyInterpreterState *interp, PyObject *obj) /* updating the registry */ static int -_xidregistry_add_type(struct _xidregistry *xidregistry, - PyTypeObject *cls, crossinterpdatafunc getdata) +_xidregistry_add_type(dlregistry_t *xidregistry, + PyTypeObject *cls, xidatafunc getdata) { - struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem)); + dlregitem_t *newhead = PyMem_RawMalloc(sizeof(dlregitem_t)); if (newhead == NULL) { return -1; } - *newhead = (struct _xidregitem){ + *newhead = (dlregitem_t){ // We do not keep a reference, to avoid keeping the class alive. .cls = cls, .refcount = 1, @@ -203,11 +216,10 @@ _xidregistry_add_type(struct _xidregistry *xidregistry, return 0; } -static struct _xidregitem * -_xidregistry_remove_entry(struct _xidregistry *xidregistry, - struct _xidregitem *entry) +static dlregitem_t * +_xidregistry_remove_entry(dlregistry_t *xidregistry, dlregitem_t *entry) { - struct _xidregitem *next = entry->next; + dlregitem_t *next = entry->next; if (entry->prev != NULL) { assert(entry->prev->next == entry); entry->prev->next = next; @@ -225,12 +237,12 @@ _xidregistry_remove_entry(struct _xidregistry *xidregistry, } static void -_xidregistry_clear(struct _xidregistry *xidregistry) +_xidregistry_clear(dlregistry_t *xidregistry) { - struct _xidregitem *cur = xidregistry->head; + dlregitem_t *cur = xidregistry->head; xidregistry->head = NULL; while (cur != NULL) { - struct _xidregitem *next = cur->next; + dlregitem_t *next = cur->next; Py_XDECREF(cur->weakref); PyMem_RawFree(cur); cur = next; @@ -238,8 +250,8 @@ _xidregistry_clear(struct _xidregistry *xidregistry) } int -_PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, - crossinterpdatafunc getdata) +_PyXIData_RegisterClass(_PyXIData_lookup_context_t *ctx, + PyTypeObject *cls, xidatafunc getdata) { if (!PyType_Check(cls)) { PyErr_Format(PyExc_ValueError, "only classes may be registered"); @@ -251,11 +263,10 @@ _PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, } int res = 0; - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls); + dlregistry_t *xidregistry = _get_xidregistry_for_type(ctx, cls); _xidregistry_lock(xidregistry); - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + dlregitem_t *matched = _xidregistry_find_type(xidregistry, cls); if (matched != NULL) { assert(matched->getdata == getdata); matched->refcount += 1; @@ -270,14 +281,13 @@ _PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, } int -_PyCrossInterpreterData_UnregisterClass(PyTypeObject *cls) +_PyXIData_UnregisterClass(_PyXIData_lookup_context_t *ctx, PyTypeObject *cls) { int res = 0; - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls); + dlregistry_t *xidregistry = _get_xidregistry_for_type(ctx, cls); _xidregistry_lock(xidregistry); - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + dlregitem_t *matched = _xidregistry_find_type(xidregistry, cls); if (matched != NULL) { assert(matched->refcount > 0); matched->refcount -= 1; @@ -304,17 +314,16 @@ struct _shared_bytes_data { }; static PyObject * -_new_bytes_object(_PyCrossInterpreterData *data) +_new_bytes_object(_PyXIData_t *data) { struct _shared_bytes_data *shared = (struct _shared_bytes_data *)(data->data); return PyBytes_FromStringAndSize(shared->bytes, shared->len); } static int -_bytes_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_bytes_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { - if (_PyCrossInterpreterData_InitWithSize( + if (_PyXIData_InitWithSize( data, tstate->interp, sizeof(struct _shared_bytes_data), obj, _new_bytes_object ) < 0) @@ -323,7 +332,7 @@ _bytes_shared(PyThreadState *tstate, PyObject *obj, } struct _shared_bytes_data *shared = (struct _shared_bytes_data *)data->data; if (PyBytes_AsStringAndSize(obj, &shared->bytes, &shared->len) < 0) { - _PyCrossInterpreterData_Clear(tstate->interp, data); + _PyXIData_Clear(tstate->interp, data); return -1; } return 0; @@ -338,17 +347,16 @@ struct _shared_str_data { }; static PyObject * -_new_str_object(_PyCrossInterpreterData *data) +_new_str_object(_PyXIData_t *data) { struct _shared_str_data *shared = (struct _shared_str_data *)(data->data); return PyUnicode_FromKindAndData(shared->kind, shared->buffer, shared->len); } static int -_str_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_str_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { - if (_PyCrossInterpreterData_InitWithSize( + if (_PyXIData_InitWithSize( data, tstate->interp, sizeof(struct _shared_str_data), obj, _new_str_object ) < 0) @@ -365,14 +373,13 @@ _str_shared(PyThreadState *tstate, PyObject *obj, // int static PyObject * -_new_long_object(_PyCrossInterpreterData *data) +_new_long_object(_PyXIData_t *data) { return PyLong_FromSsize_t((Py_ssize_t)(data->data)); } static int -_long_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_long_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { /* Note that this means the size of shareable ints is bounded by * sys.maxsize. Hence on 32-bit architectures that is half the @@ -385,8 +392,7 @@ _long_shared(PyThreadState *tstate, PyObject *obj, } return -1; } - _PyCrossInterpreterData_Init(data, tstate->interp, (void *)value, NULL, - _new_long_object); + _PyXIData_Init(data, tstate->interp, (void *)value, NULL, _new_long_object); // data->obj and data->free remain NULL return 0; } @@ -394,17 +400,16 @@ _long_shared(PyThreadState *tstate, PyObject *obj, // float static PyObject * -_new_float_object(_PyCrossInterpreterData *data) +_new_float_object(_PyXIData_t *data) { double * value_ptr = data->data; return PyFloat_FromDouble(*value_ptr); } static int -_float_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_float_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { - if (_PyCrossInterpreterData_InitWithSize( + if (_PyXIData_InitWithSize( data, tstate->interp, sizeof(double), NULL, _new_float_object ) < 0) @@ -419,18 +424,16 @@ _float_shared(PyThreadState *tstate, PyObject *obj, // None static PyObject * -_new_none_object(_PyCrossInterpreterData *data) +_new_none_object(_PyXIData_t *data) { // XXX Singleton refcounts are problematic across interpreters... return Py_NewRef(Py_None); } static int -_none_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_none_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { - _PyCrossInterpreterData_Init(data, tstate->interp, NULL, NULL, - _new_none_object); + _PyXIData_Init(data, tstate->interp, NULL, NULL, _new_none_object); // data->data, data->obj and data->free remain NULL return 0; } @@ -438,7 +441,7 @@ _none_shared(PyThreadState *tstate, PyObject *obj, // bool static PyObject * -_new_bool_object(_PyCrossInterpreterData *data) +_new_bool_object(_PyXIData_t *data) { if (data->data){ Py_RETURN_TRUE; @@ -447,10 +450,9 @@ _new_bool_object(_PyCrossInterpreterData *data) } static int -_bool_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_bool_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { - _PyCrossInterpreterData_Init(data, tstate->interp, + _PyXIData_Init(data, tstate->interp, (void *) (Py_IsTrue(obj) ? (uintptr_t) 1 : (uintptr_t) 0), NULL, _new_bool_object); // data->obj and data->free remain NULL @@ -461,11 +463,11 @@ _bool_shared(PyThreadState *tstate, PyObject *obj, struct _shared_tuple_data { Py_ssize_t len; - _PyCrossInterpreterData **data; + _PyXIData_t **data; }; static PyObject * -_new_tuple_object(_PyCrossInterpreterData *data) +_new_tuple_object(_PyXIData_t *data) { struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data->data); PyObject *tuple = PyTuple_New(shared->len); @@ -474,7 +476,7 @@ _new_tuple_object(_PyCrossInterpreterData *data) } for (Py_ssize_t i = 0; i < shared->len; i++) { - PyObject *item = _PyCrossInterpreterData_NewObject(shared->data[i]); + PyObject *item = _PyXIData_NewObject(shared->data[i]); if (item == NULL){ Py_DECREF(tuple); return NULL; @@ -493,8 +495,8 @@ _tuple_shared_free(void* data) #endif for (Py_ssize_t i = 0; i < shared->len; i++) { if (shared->data[i] != NULL) { - assert(_PyCrossInterpreterData_INTERPID(shared->data[i]) == interpid); - _PyCrossInterpreterData_Release(shared->data[i]); + assert(_PyXIData_INTERPID(shared->data[i]) == interpid); + _PyXIData_Release(shared->data[i]); PyMem_RawFree(shared->data[i]); shared->data[i] = NULL; } @@ -504,9 +506,13 @@ _tuple_shared_free(void* data) } static int -_tuple_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_tuple_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { + dlcontext_t ctx; + if (_PyXIData_GetLookupContext(tstate->interp, &ctx) < 0) { + return -1; + } + Py_ssize_t len = PyTuple_GET_SIZE(obj); if (len < 0) { return -1; @@ -518,14 +524,14 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, } shared->len = len; - shared->data = (_PyCrossInterpreterData **) PyMem_Calloc(shared->len, sizeof(_PyCrossInterpreterData *)); + shared->data = (_PyXIData_t **) PyMem_Calloc(shared->len, sizeof(_PyXIData_t *)); if (shared->data == NULL) { PyErr_NoMemory(); return -1; } for (Py_ssize_t i = 0; i < shared->len; i++) { - _PyCrossInterpreterData *data = _PyCrossInterpreterData_New(); + _PyXIData_t *data = _PyXIData_New(); if (data == NULL) { goto error; // PyErr_NoMemory already set } @@ -533,7 +539,7 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, int res = -1; if (!_Py_EnterRecursiveCallTstate(tstate, " while sharing a tuple")) { - res = _PyObject_GetCrossInterpreterData(item, data); + res = _PyObject_GetXIData(&ctx, item, data); _Py_LeaveRecursiveCallTstate(tstate); } if (res < 0) { @@ -542,8 +548,7 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, } shared->data[i] = data; } - _PyCrossInterpreterData_Init( - data, tstate->interp, shared, obj, _new_tuple_object); + _PyXIData_Init(data, tstate->interp, shared, obj, _new_tuple_object); data->free = _tuple_shared_free; return 0; @@ -555,7 +560,7 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, // registration static void -_register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) +_register_builtins_for_crossinterpreter_data(dlregistry_t *xidregistry) { // None if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { diff --git a/Python/crossinterp_exceptions.h b/Python/crossinterp_exceptions.h index 278511da615c75..3cb45d2067710b 100644 --- a/Python/crossinterp_exceptions.h +++ b/Python/crossinterp_exceptions.h @@ -25,71 +25,78 @@ static PyTypeObject _PyExc_InterpreterNotFoundError = { }; PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFoundError; -/* NotShareableError extends ValueError */ - -static int -_init_not_shareable_error_type(PyInterpreterState *interp) -{ - const char *name = "interpreters.NotShareableError"; - PyObject *base = PyExc_ValueError; - PyObject *ns = NULL; - PyObject *exctype = PyErr_NewException(name, base, ns); - if (exctype == NULL) { - return -1; - } - - _PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError = exctype; - return 0; -} - -static void -_fini_not_shareable_error_type(PyInterpreterState *interp) -{ - Py_CLEAR(_PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError); -} - -static PyObject * -_get_not_shareable_error_type(PyInterpreterState *interp) -{ - assert(_PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError != NULL); - return _PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError; -} - /* lifecycle */ static int -init_exceptions(PyInterpreterState *interp) +init_static_exctypes(exceptions_t *state, PyInterpreterState *interp) { + assert(state == &_PyXI_GET_STATE(interp)->exceptions); PyTypeObject *base = (PyTypeObject *)PyExc_Exception; - // builtin static types - + // PyExc_InterpreterError _PyExc_InterpreterError.tp_base = base; _PyExc_InterpreterError.tp_traverse = base->tp_traverse; _PyExc_InterpreterError.tp_clear = base->tp_clear; if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterError) < 0) { - return -1; + goto error; } + state->PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError; + // PyExc_InterpreterNotFoundError _PyExc_InterpreterNotFoundError.tp_traverse = base->tp_traverse; _PyExc_InterpreterNotFoundError.tp_clear = base->tp_clear; if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterNotFoundError) < 0) { - return -1; + goto error; } + state->PyExc_InterpreterNotFoundError = + (PyObject *)&_PyExc_InterpreterNotFoundError; - // heap types + return 0; - // We would call _init_not_shareable_error_type() here too, - // but that leads to ref leaks +error: + fini_static_exctypes(state, interp); + return -1; +} + +static void +fini_static_exctypes(exceptions_t *state, PyInterpreterState *interp) +{ + assert(state == &_PyXI_GET_STATE(interp)->exceptions); + if (state->PyExc_InterpreterNotFoundError != NULL) { + state->PyExc_InterpreterNotFoundError = NULL; + _PyStaticType_FiniBuiltin(interp, &_PyExc_InterpreterNotFoundError); + } + if (state->PyExc_InterpreterError != NULL) { + state->PyExc_InterpreterError = NULL; + _PyStaticType_FiniBuiltin(interp, &_PyExc_InterpreterError); + } +} + +static int +init_heap_exctypes(exceptions_t *state) +{ + PyObject *exctype; + + /* NotShareableError extends ValueError */ + const char *name = "interpreters.NotShareableError"; + PyObject *base = PyExc_ValueError; + PyObject *ns = NULL; + exctype = PyErr_NewException(name, base, ns); + if (exctype == NULL) { + goto error; + } + state->PyExc_NotShareableError = exctype; return 0; + +error: + fini_heap_exctypes(state); + return -1; } static void -fini_exceptions(PyInterpreterState *interp) +fini_heap_exctypes(exceptions_t *state) { - // Likewise with _fini_not_shareable_error_type(). - _PyStaticType_FiniBuiltin(interp, &_PyExc_InterpreterNotFoundError); - _PyStaticType_FiniBuiltin(interp, &_PyExc_InterpreterError); + Py_CLEAR(state->PyExc_NotShareableError); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index ff4a0a52a0b445..33e4c57be5628a 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -41,6 +41,8 @@ /* _QUICKEN_RESUME is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + /* _LOAD_BYTECODE is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + case _RESUME_CHECK: { #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { @@ -56,6 +58,13 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #endif break; } @@ -822,7 +831,7 @@ assert(next_instr->op.code == STORE_FAST); next_oparg = next_instr->op.arg; #else - next_oparg = CURRENT_OPERAND(); + next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); if (!PyStackRef_Is(*target_local, left)) { @@ -1855,7 +1864,7 @@ } case _GUARD_GLOBALS_VERSION: { - uint16_t version = (uint16_t)CURRENT_OPERAND(); + uint16_t version = (uint16_t)CURRENT_OPERAND0(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -1871,7 +1880,7 @@ case _GUARD_GLOBALS_VERSION_PUSH_KEYS: { PyDictKeysObject *globals_keys; - uint16_t version = (uint16_t)CURRENT_OPERAND(); + uint16_t version = (uint16_t)CURRENT_OPERAND0(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -1891,7 +1900,7 @@ case _GUARD_BUILTINS_VERSION_PUSH_KEYS: { PyDictKeysObject *builtins_keys; - uint16_t version = (uint16_t)CURRENT_OPERAND(); + uint16_t version = (uint16_t)CURRENT_OPERAND0(); PyDictObject *dict = (PyDictObject *)BUILTINS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -1915,7 +1924,7 @@ _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); globals_keys = (PyDictKeysObject *)stack_pointer[-1].bits; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = entries[index].me_value; stack_pointer += -1; @@ -1941,7 +1950,7 @@ _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); builtins_keys = (PyDictKeysObject *)stack_pointer[-1].bits; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = entries[index].me_value; stack_pointer += -1; @@ -2514,7 +2523,7 @@ case _GUARD_TYPE_VERSION: { _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (tp->tp_version_tag != type_version) { @@ -2543,7 +2552,7 @@ _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; - uint16_t offset = (uint16_t)CURRENT_OPERAND(); + uint16_t offset = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = *value_ptr; @@ -2566,7 +2575,7 @@ _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; - uint16_t offset = (uint16_t)CURRENT_OPERAND(); + uint16_t offset = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = *value_ptr; @@ -2591,9 +2600,9 @@ case _CHECK_ATTR_MODULE: { _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t dict_version = (uint32_t)CURRENT_OPERAND(); + uint32_t dict_version = (uint32_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyModule_CheckExact(owner_o)) { + if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -2612,7 +2621,7 @@ _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); @@ -2655,7 +2664,7 @@ _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - uint16_t hint = (uint16_t)CURRENT_OPERAND(); + uint16_t hint = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject *attr_o; PyDictObject *dict = _PyObject_GetManagedDict(owner_o); @@ -2696,7 +2705,7 @@ _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); char *addr = (char *)owner_o + index; PyObject *attr_o = *(PyObject **)addr; @@ -2718,7 +2727,7 @@ _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); char *addr = (char *)owner_o + index; PyObject *attr_o = *(PyObject **)addr; @@ -2742,7 +2751,7 @@ case _CHECK_ATTR_CLASS: { _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UOP_STAT_INC(uopcode, miss); @@ -2762,7 +2771,7 @@ _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); @@ -2778,7 +2787,7 @@ _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); @@ -2798,7 +2807,7 @@ _PyInterpreterFrame *new_frame; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *fget = (PyObject *)CURRENT_OPERAND(); + PyObject *fget = (PyObject *)CURRENT_OPERAND0(); assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; @@ -2850,7 +2859,7 @@ _PyStackRef value; owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint16_t offset = (uint16_t)CURRENT_OPERAND(); + uint16_t offset = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); assert(_PyObject_GetManagedDict(owner_o) == NULL); @@ -2877,7 +2886,7 @@ oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint16_t hint = (uint16_t)CURRENT_OPERAND(); + uint16_t hint = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); @@ -2905,10 +2914,6 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(PyStackRef_AsPyObjectBorrow(value))) { - _PyObject_GC_TRACK(dict); - } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2928,7 +2933,7 @@ _PyStackRef value; owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); @@ -3428,11 +3433,12 @@ else { /* `iterable` is not a generator. */ _PyFrame_SetStackPointer(frame, stack_pointer); - iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o)); + PyObject *iter_o = PyObject_GetIter(iterable_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(iter)) { + if (iter_o == NULL) { JUMP_TO_ERROR(); } + iter = PyStackRef_FromPyObjectSteal(iter_o); PyStackRef_CLOSE(iterable); } } @@ -3770,7 +3776,7 @@ case _GUARD_KEYS_VERSION: { _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND(); + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; if (owner_heap_type->ht_cached_keys->dk_version != keys_version) { @@ -3786,7 +3792,7 @@ _PyStackRef self = PyStackRef_NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); @@ -3807,7 +3813,7 @@ _PyStackRef self = PyStackRef_NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert(oparg & 1); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -3827,7 +3833,7 @@ _PyStackRef attr; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3842,7 +3848,7 @@ _PyStackRef attr; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert((oparg & 1) == 0); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -3856,7 +3862,7 @@ case _CHECK_ATTR_METHOD_LAZY_DICT: { _PyStackRef owner; owner = stack_pointer[-1]; - uint16_t dictoffset = (uint16_t)CURRENT_OPERAND(); + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ @@ -3873,7 +3879,7 @@ _PyStackRef self = PyStackRef_NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3957,7 +3963,7 @@ _PyStackRef *callable; oparg = CURRENT_OPARG(); callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (!PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -3971,13 +3977,25 @@ break; } + case _CHECK_FUNCTION_VERSION_INLINE: { + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + case _CHECK_METHOD_VERSION: { _PyStackRef *null; _PyStackRef *callable; oparg = CURRENT_OPARG(); null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (Py_TYPE(callable_o) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); @@ -4433,7 +4451,7 @@ callable = &stack_pointer[-2 - oparg]; init = &stack_pointer[-2 - oparg]; self = &stack_pointer[-1 - oparg]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (!PyStackRef_IsNull(null[0])) { UOP_STAT_INC(uopcode, miss); @@ -4480,8 +4498,8 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); stack_pointer = _PyFrame_GetStackPointer(frame); - assert(_PyCode_CODE(_PyFrame_GetCode(shim))[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self[0]); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5191,7 +5209,7 @@ _PyStackRef *callable; oparg = CURRENT_OPARG(); callable = &stack_pointer[-3 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (!PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -5211,7 +5229,7 @@ oparg = CURRENT_OPARG(); null = &stack_pointer[-2 - oparg]; callable = &stack_pointer[-3 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (Py_TYPE(callable_o) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); @@ -5649,13 +5667,13 @@ } case _SET_IP: { - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; break; } case _CHECK_STACK_SPACE_OPERAND: { - uint32_t framesize = (uint32_t)CURRENT_OPERAND(); + uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); assert(framesize <= INT_MAX); if (!_PyThreadState_HasStackSpace(tstate, framesize)) { UOP_STAT_INC(uopcode, miss); @@ -5680,10 +5698,12 @@ } case _EXIT_TRACE: { - PyObject *exit_p = (PyObject *)CURRENT_OPERAND(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _PyExitData *exit = (_PyExitData *)exit_p; PyCodeObject *code = _PyFrame_GetCode(frame); - _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + stack_pointer = _PyFrame_GetStackPointer(frame); #if defined(Py_DEBUG) && !defined(_Py_JIT) OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (lltrace >= 2) { @@ -5692,7 +5712,7 @@ _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyCode_CODE(code)), + (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); stack_pointer = _PyFrame_GetStackPointer(frame); } @@ -5745,7 +5765,7 @@ case _LOAD_CONST_INLINE: { _PyStackRef value; - PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); value = PyStackRef_FromPyObjectNew(ptr); stack_pointer[0] = value; stack_pointer += 1; @@ -5755,7 +5775,7 @@ case _LOAD_CONST_INLINE_BORROW: { _PyStackRef value; - PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); value = PyStackRef_FromPyObjectImmortal(ptr); stack_pointer[0] = value; stack_pointer += 1; @@ -5767,7 +5787,7 @@ _PyStackRef pop; _PyStackRef value; pop = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); PyStackRef_CLOSE(pop); value = PyStackRef_FromPyObjectImmortal(ptr); stack_pointer[-1] = value; @@ -5777,7 +5797,7 @@ case _LOAD_CONST_INLINE_WITH_NULL: { _PyStackRef value; _PyStackRef null; - PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); value = PyStackRef_FromPyObjectNew(ptr); null = PyStackRef_NULL; stack_pointer[0] = value; @@ -5790,7 +5810,7 @@ case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { _PyStackRef value; _PyStackRef null; - PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); value = PyStackRef_FromPyObjectImmortal(ptr); null = PyStackRef_NULL; stack_pointer[0] = value; @@ -5801,7 +5821,7 @@ } case _CHECK_FUNCTION: { - uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); if (func->func_version != func_version) { @@ -5815,7 +5835,7 @@ _PyStackRef res; _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); PyObject *res_o = entries[index].me_value; @@ -5837,7 +5857,7 @@ _PyStackRef res; _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyDictObject *dict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); PyObject *res_o = entries[index].me_value; @@ -5866,7 +5886,7 @@ } case _DYNAMIC_EXIT: { - PyObject *exit_p = (PyObject *)CURRENT_OPERAND(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); tstate->previous_executor = (PyObject *)current_executor; _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; @@ -5878,7 +5898,7 @@ _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), + (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); stack_pointer = _PyFrame_GetStackPointer(frame); } @@ -5913,7 +5933,7 @@ } case _START_EXECUTOR: { - PyObject *executor = (PyObject *)CURRENT_OPERAND(); + PyObject *executor = (PyObject *)CURRENT_OPERAND0(); Py_DECREF(tstate->previous_executor); tstate->previous_executor = NULL; #ifndef _Py_JIT @@ -5939,7 +5959,7 @@ } case _CHECK_VALIDITY_AND_SET_IP: { - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); if (!current_executor->vm_data.valid) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -5955,10 +5975,12 @@ case _ERROR_POP_N: { oparg = CURRENT_OPARG(); - uint32_t target = (uint32_t)CURRENT_OPERAND(); - frame->instr_ptr = ((_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive) + target; + uint32_t target = (uint32_t)CURRENT_OPERAND0(); stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; + stack_pointer = _PyFrame_GetStackPointer(frame); GOTO_UNWIND(); break; } diff --git a/Python/frame.c b/Python/frame.c index 35e6c2d0a93333..9a865e57d97cc6 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -63,7 +63,8 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) // This may be a newly-created generator or coroutine frame. Since it's // dead anyways, just pretend that the first RESUME ran: PyCodeObject *code = _PyFrame_GetCode(frame); - frame->instr_ptr = _PyCode_CODE(code) + code->_co_firsttraceable + 1; + frame->instr_ptr = + _PyFrame_GetBytecode(frame) + code->_co_firsttraceable + 1; } assert(!_PyFrame_IsIncomplete(frame)); assert(f->f_back == NULL); diff --git a/Python/gc.c b/Python/gc.c index 028657eb8999c1..60b41377654af4 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -5,7 +5,7 @@ #include "Python.h" #include "pycore_ceval.h" // _Py_set_eval_breaker_bit() #include "pycore_context.h" -#include "pycore_dict.h" // _PyDict_MaybeUntrack() +#include "pycore_dict.h" // _PyInlineValuesSize() #include "pycore_initconfig.h" #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_object.h" @@ -23,6 +23,10 @@ typedef struct _gc_runtime_state GCState; # define GC_DEBUG #endif +// Define this when debugging the GC +// #define GC_EXTRA_DEBUG + + #define GC_NEXT _PyGCHead_NEXT #define GC_PREV _PyGCHead_PREV @@ -181,6 +185,7 @@ _PyGC_Init(PyInterpreterState *interp) if (gcstate->callbacks == NULL) { return _PyStatus_NO_MEMORY(); } + gcstate->prior_heap_size = 0; gcstate->heap_size = 0; return _PyStatus_OK(); @@ -421,6 +426,11 @@ validate_list(PyGC_Head *head, enum flagstates flags) assert(prev == GC_PREV(head)); } +#else +#define validate_list(x, y) do{}while(0) +#endif + +#ifdef GC_EXTRA_DEBUG static void validate_old(GCState *gcstate) { @@ -464,7 +474,6 @@ gc_list_validate_space(PyGC_Head *head, int space) { } #else -#define validate_list(x, y) do{}while(0) #define validate_old(g) do{}while(0) #define validate_consistent_old_space(l) do{}while(0) #define gc_list_validate_space(l, s) do{}while(0) @@ -739,21 +748,6 @@ untrack_tuples(PyGC_Head *head) } } -/* Try to untrack all currently tracked dictionaries */ -static void -untrack_dicts(PyGC_Head *head) -{ - PyGC_Head *next, *gc = GC_NEXT(head); - while (gc != head) { - PyObject *op = FROM_GC(gc); - next = GC_NEXT(gc); - if (PyDict_CheckExact(op)) { - _PyDict_MaybeUntrack(op); - } - gc = next; - } -} - /* Return true if object has a pre-PEP 442 finalization method. */ static int has_legacy_finalizer(PyObject *op) @@ -1250,15 +1244,10 @@ handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable, gc_list_merge(resurrected, old_generation); } - -#define UNTRACK_TUPLES 1 -#define UNTRACK_DICTS 2 - static void gc_collect_region(PyThreadState *tstate, PyGC_Head *from, PyGC_Head *to, - int untrack, struct gc_collection_stats *stats); static inline Py_ssize_t @@ -1307,6 +1296,7 @@ gc_collect_young(PyThreadState *tstate, GCState *gcstate = &tstate->interp->gc; PyGC_Head *young = &gcstate->young.head; PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; + untrack_tuples(&gcstate->young.head); GC_STAT_ADD(0, collections, 1); #ifdef Py_STATS { @@ -1320,7 +1310,8 @@ gc_collect_young(PyThreadState *tstate, PyGC_Head survivors; gc_list_init(&survivors); - gc_collect_region(tstate, young, &survivors, UNTRACK_TUPLES, stats); + gc_list_set_space(young, gcstate->visited_space); + gc_collect_region(tstate, young, &survivors, stats); Py_ssize_t survivor_count = 0; if (gcstate->visited_space) { /* objects in visited space have bit set, so we set it here */ @@ -1335,16 +1326,11 @@ gc_collect_young(PyThreadState *tstate, survivor_count++; } } - (void)survivor_count; // Silence compiler warning gc_list_merge(&survivors, visited); validate_old(gcstate); gcstate->young.count = 0; gcstate->old[gcstate->visited_space].count++; - Py_ssize_t scale_factor = gcstate->old[0].threshold; - if (scale_factor < 1) { - scale_factor = 1; - } - gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; + gcstate->work_to_do += survivor_count * 4; add_stats(gcstate, 0, stats); } @@ -1360,15 +1346,15 @@ IS_IN_VISITED(PyGC_Head *gc, int visited_space) struct container_and_flag { PyGC_Head *container; int visited_space; - uintptr_t size; + Py_ssize_t size; }; /* A traversal callback for adding to container) */ static int visit_add_to_container(PyObject *op, void *arg) { - OBJECT_STAT_INC(object_visits); struct container_and_flag *cf = (struct container_and_flag *)arg; + OBJECT_STAT_INC(object_visits); int visited = cf->visited_space; assert(visited == get_gc_state()->visited_space); if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { @@ -1383,10 +1369,9 @@ visit_add_to_container(PyObject *op, void *arg) return 0; } -static uintptr_t +static Py_ssize_t expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCState *gcstate) { - validate_list(container, collecting_clear_unreachable_clear); struct container_and_flag arg = { .container = container, .visited_space = gcstate->visited_space, @@ -1398,6 +1383,7 @@ expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCStat * have been marked as visited */ assert(IS_IN_VISITED(gc, gcstate->visited_space)); PyObject *op = FROM_GC(gc); + assert(_PyObject_GC_IS_TRACKED(op)); if (_Py_IsImmortal(op)) { PyGC_Head *next = GC_NEXT(gc); gc_list_move(gc, &get_gc_state()->permanent_generation.head); @@ -1417,20 +1403,187 @@ expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCStat static void completed_cycle(GCState *gcstate) { -#ifdef Py_DEBUG - PyGC_Head *not_visited = &gcstate->old[gcstate->visited_space^1].head; - assert(gc_list_is_empty(not_visited)); -#endif - gcstate->visited_space = flip_old_space(gcstate->visited_space); + assert(gc_list_is_empty(&gcstate->old[gcstate->visited_space^1].head)); + int not_visited = gcstate->visited_space; + gcstate->visited_space = flip_old_space(not_visited); /* Make sure all young objects have old space bit set correctly */ PyGC_Head *young = &gcstate->young.head; PyGC_Head *gc = GC_NEXT(young); while (gc != young) { PyGC_Head *next = GC_NEXT(gc); - gc_set_old_space(gc, gcstate->visited_space); + gc_set_old_space(gc, not_visited); gc = next; } gcstate->work_to_do = 0; + gcstate->phase = GC_PHASE_MARK; +} + +static Py_ssize_t +move_to_reachable(PyObject *op, PyGC_Head *reachable, int visited_space) +{ + if (op != NULL && !_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + if (_PyObject_GC_IS_TRACKED(op) && + gc_old_space(gc) != visited_space) { + gc_flip_old_space(gc); + gc_list_move(gc, reachable); + return 1; + } + } + return 0; +} + +static Py_ssize_t +mark_all_reachable(PyGC_Head *reachable, PyGC_Head *visited, int visited_space) +{ + // Transitively traverse all objects from reachable, until empty + struct container_and_flag arg = { + .container = reachable, + .visited_space = visited_space, + .size = 0 + }; + while (!gc_list_is_empty(reachable)) { + PyGC_Head *gc = _PyGCHead_NEXT(reachable); + assert(gc_old_space(gc) == visited_space); + gc_list_move(gc, visited); + PyObject *op = FROM_GC(gc); + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void) traverse(op, + visit_add_to_container, + &arg); + } + gc_list_validate_space(visited, visited_space); + return arg.size; +} + +static Py_ssize_t +mark_global_roots(PyInterpreterState *interp, PyGC_Head *visited, int visited_space) +{ + PyGC_Head reachable; + gc_list_init(&reachable); + Py_ssize_t objects_marked = 0; + objects_marked += move_to_reachable(interp->sysdict, &reachable, visited_space); + objects_marked += move_to_reachable(interp->builtins, &reachable, visited_space); + objects_marked += move_to_reachable(interp->dict, &reachable, visited_space); + struct types_state *types = &interp->types; + for (int i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) { + objects_marked += move_to_reachable(types->builtins.initialized[i].tp_dict, &reachable, visited_space); + objects_marked += move_to_reachable(types->builtins.initialized[i].tp_subclasses, &reachable, visited_space); + } + for (int i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) { + objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_dict, &reachable, visited_space); + objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_subclasses, &reachable, visited_space); + } + objects_marked += mark_all_reachable(&reachable, visited, visited_space); + assert(gc_list_is_empty(&reachable)); + return objects_marked; +} + +static Py_ssize_t +mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, bool start) +{ + PyGC_Head reachable; + gc_list_init(&reachable); + Py_ssize_t objects_marked = 0; + // Move all objects on stacks to reachable + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); + PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + HEAD_UNLOCK(runtime); + while (ts) { + _PyInterpreterFrame *frame = ts->current_frame; + while (frame) { + if (frame->owner == FRAME_OWNED_BY_CSTACK) { + frame = frame->previous; + continue; + } + _PyStackRef *locals = frame->localsplus; + _PyStackRef *sp = frame->stackpointer; + objects_marked += move_to_reachable(frame->f_locals, &reachable, visited_space); + PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + objects_marked += move_to_reachable(func, &reachable, visited_space); + while (sp > locals) { + sp--; + if (PyStackRef_IsNull(*sp)) { + continue; + } + PyObject *op = PyStackRef_AsPyObjectBorrow(*sp); + if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + if (_PyObject_GC_IS_TRACKED(op) && + gc_old_space(gc) != visited_space) { + gc_flip_old_space(gc); + objects_marked++; + gc_list_move(gc, &reachable); + } + } + } + if (!start && frame->visited) { + // If this frame has already been visited, then the lower frames + // will have already been visited and will not have changed + break; + } + frame->visited = 1; + frame = frame->previous; + } + HEAD_LOCK(runtime); + ts = PyThreadState_Next(ts); + HEAD_UNLOCK(runtime); + } + objects_marked += mark_all_reachable(&reachable, visited, visited_space); + assert(gc_list_is_empty(&reachable)); + return objects_marked; +} + +static Py_ssize_t +mark_at_start(PyThreadState *tstate) +{ + // TO DO -- Make this incremental + GCState *gcstate = &tstate->interp->gc; + validate_old(gcstate); + PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; + Py_ssize_t objects_marked = mark_global_roots(tstate->interp, visited, gcstate->visited_space); + objects_marked += mark_stacks(tstate->interp, visited, gcstate->visited_space, true); + gcstate->work_to_do -= objects_marked; + gcstate->phase = GC_PHASE_COLLECT; + return objects_marked; +} + +static Py_ssize_t +assess_work_to_do(GCState *gcstate) +{ + /* The amount of work we want to do depends on three things. + * 1. The number of new objects created + * 2. The growth in heap size since the last collection + * 3. The heap size (up to the number of new objects, to avoid quadratic effects) + * + * For a steady state heap, the amount of work to do is three times the number + * of new objects added to the heap. This ensures that we stay ahead in the + * worst case of all new objects being garbage. + * + * This could be improved by tracking survival rates, but it is still a + * large improvement on the non-marking approach. + */ + Py_ssize_t scale_factor = gcstate->old[0].threshold; + if (scale_factor < 2) { + scale_factor = 2; + } + Py_ssize_t new_objects = gcstate->young.count; + Py_ssize_t growth = gcstate->heap_size - gcstate->prior_heap_size; + if (growth < 0) { + growth = 0; + } + if (gcstate->heap_size < new_objects * scale_factor) { + // Small heap: ignore growth + growth = 0; + } + Py_ssize_t heap_fraction = gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; + if (heap_fraction > new_objects) { + heap_fraction = new_objects; + } + gcstate->young.count = 0; + gcstate->prior_heap_size = gcstate->heap_size; + return new_objects*3/2 + growth*2 + heap_fraction*3/2; } static void @@ -1438,16 +1591,24 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) { GC_STAT_ADD(1, collections, 1); GCState *gcstate = &tstate->interp->gc; + + gcstate->work_to_do += assess_work_to_do(gcstate); + untrack_tuples(&gcstate->young.head); + if (gcstate->phase == GC_PHASE_MARK) { + Py_ssize_t objects_marked = mark_at_start(tstate); + GC_STAT_ADD(1, objects_transitively_reachable, objects_marked); + gcstate->work_to_do -= objects_marked; + return; + } PyGC_Head *not_visited = &gcstate->old[gcstate->visited_space^1].head; PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; PyGC_Head increment; gc_list_init(&increment); - Py_ssize_t scale_factor = gcstate->old[0].threshold; - if (scale_factor < 1) { - scale_factor = 1; - } + Py_ssize_t objects_marked = mark_stacks(tstate->interp, visited, gcstate->visited_space, false); + GC_STAT_ADD(1, objects_transitively_reachable, objects_marked); + gcstate->work_to_do -= objects_marked; + gc_list_set_space(&gcstate->young.head, gcstate->visited_space); gc_list_merge(&gcstate->young.head, &increment); - gcstate->young.count = 0; gc_list_validate_space(&increment, gcstate->visited_space); Py_ssize_t increment_size = 0; while (increment_size < gcstate->work_to_do) { @@ -1457,17 +1618,18 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) PyGC_Head *gc = _PyGCHead_NEXT(not_visited); gc_list_move(gc, &increment); increment_size++; + assert(!_Py_IsImmortal(FROM_GC(gc))); gc_set_old_space(gc, gcstate->visited_space); increment_size += expand_region_transitively_reachable(&increment, gc, gcstate); } + GC_STAT_ADD(1, objects_not_transitively_reachable, increment_size); gc_list_validate_space(&increment, gcstate->visited_space); PyGC_Head survivors; gc_list_init(&survivors); - gc_collect_region(tstate, &increment, &survivors, UNTRACK_TUPLES, stats); + gc_collect_region(tstate, &increment, &survivors, stats); gc_list_validate_space(&survivors, gcstate->visited_space); gc_list_merge(&survivors, visited); assert(gc_list_is_empty(&increment)); - gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; gcstate->work_to_do -= increment_size; validate_old(gcstate); @@ -1488,20 +1650,25 @@ gc_collect_full(PyThreadState *tstate, PyGC_Head *young = &gcstate->young.head; PyGC_Head *pending = &gcstate->old[gcstate->visited_space^1].head; PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; - /* merge all generations into visited */ - gc_list_validate_space(young, gcstate->visited_space); - gc_list_set_space(pending, gcstate->visited_space); + untrack_tuples(&gcstate->young.head); + /* merge all generations into pending */ + gc_list_validate_space(young, 1-gcstate->visited_space); gc_list_merge(young, pending); + gc_list_set_space(visited, 1-gcstate->visited_space); + gc_list_merge(visited, pending); + /* Mark reachable */ + Py_ssize_t reachable = mark_global_roots(tstate->interp, visited, gcstate->visited_space); + reachable += mark_stacks(tstate->interp, visited, gcstate->visited_space, true); + (void)reachable; + GC_STAT_ADD(2, objects_transitively_reachable, reachable); + GC_STAT_ADD(2, objects_not_transitively_reachable, gc_list_size(pending)); gcstate->young.count = 0; - gc_list_merge(pending, visited); - - gc_collect_region(tstate, visited, visited, - UNTRACK_TUPLES | UNTRACK_DICTS, - stats); + gc_list_set_space(pending, gcstate->visited_space); + gc_collect_region(tstate, pending, visited, stats); gcstate->young.count = 0; gcstate->old[0].count = 0; gcstate->old[1].count = 0; - + completed_cycle(gcstate); gcstate->work_to_do = - gcstate->young.threshold * 2; _PyGC_ClearAllFreeLists(tstate->interp); validate_old(gcstate); @@ -1514,7 +1681,6 @@ static void gc_collect_region(PyThreadState *tstate, PyGC_Head *from, PyGC_Head *to, - int untrack, struct gc_collection_stats *stats) { PyGC_Head unreachable; /* non-problematic unreachable trash */ @@ -1528,12 +1694,6 @@ gc_collect_region(PyThreadState *tstate, gc_list_init(&unreachable); deduce_unreachable(from, &unreachable); validate_consistent_old_space(from); - if (untrack & UNTRACK_TUPLES) { - untrack_tuples(from); - } - if (untrack & UNTRACK_DICTS) { - untrack_dicts(from); - } validate_consistent_old_space(to); if (from != to) { gc_list_merge(from, to); @@ -1753,9 +1913,10 @@ _PyGC_Freeze(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; /* The permanent_generation has its old space bit set to zero */ - if (gcstate->visited_space) { + if (!gcstate->visited_space) { gc_list_set_space(&gcstate->young.head, 0); } + gc_list_validate_space(&gcstate->young.head, 0); gc_list_merge(&gcstate->young.head, &gcstate->permanent_generation.head); gcstate->young.count = 0; PyGC_Head*old0 = &gcstate->old[0].head; diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 1969ed608ea524..a6e0022340b7fe 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -3,7 +3,7 @@ #include "pycore_brc.h" // struct _brc_thread_state #include "pycore_ceval.h" // _Py_set_eval_breaker_bit() #include "pycore_context.h" -#include "pycore_dict.h" // _PyDict_MaybeUntrack() +#include "pycore_dict.h" // _PyInlineValuesSize() #include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" #include "pycore_interp.h" // PyInterpreterState.gc @@ -113,6 +113,12 @@ worklist_remove(struct worklist_iter *iter) iter->next = iter->ptr; } +static inline int +gc_is_frozen(PyObject *op) +{ + return (op->ob_gc_bits & _PyGC_BITS_FROZEN) != 0; +} + static inline int gc_is_unreachable(PyObject *op) { @@ -277,7 +283,7 @@ op_from_block(void *block, void *arg, bool include_frozen) if (!_PyObject_GC_IS_TRACKED(op)) { return NULL; } - if (!include_frozen && (op->ob_gc_bits & _PyGC_BITS_FROZEN) != 0) { + if (!include_frozen && gc_is_frozen(op)) { return NULL; } return op; @@ -358,7 +364,7 @@ gc_visit_stackref(_PyStackRef stackref) // being dead already. if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsNull(stackref)) { PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref); - if (_PyObject_GC_IS_TRACKED(obj)) { + if (_PyObject_GC_IS_TRACKED(obj) && !gc_is_frozen(obj)) { gc_add_refs(obj, 1); } } @@ -439,7 +445,10 @@ process_delayed_frees(PyInterpreterState *interp) static int visit_decref(PyObject *op, void *arg) { - if (_PyObject_GC_IS_TRACKED(op) && !_Py_IsImmortal(op)) { + if (_PyObject_GC_IS_TRACKED(op) + && !_Py_IsImmortal(op) + && !gc_is_frozen(op)) + { // If update_refs hasn't reached this object yet, mark it // as (tentatively) unreachable and initialize ob_tid to zero. gc_maybe_init_refs(op); @@ -484,13 +493,6 @@ update_refs(const mi_heap_t *heap, const mi_heap_area_t *area, return true; } } - else if (PyDict_CheckExact(op)) { - _PyDict_MaybeUntrack(op); - if (!_PyObject_GC_IS_TRACKED(op)) { - gc_restore_refs(op); - return true; - } - } } // We repurpose ob_tid to compute "gc_refs", the number of external @@ -1539,7 +1541,7 @@ visit_freeze(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *args) { PyObject *op = op_from_block(block, args, true); - if (op != NULL) { + if (op != NULL && !gc_is_unreachable(op)) { op->ob_gc_bits |= _PyGC_BITS_FROZEN; } return true; @@ -1584,7 +1586,7 @@ visit_count_frozen(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *args) { PyObject *op = op_from_block(block, args, true); - if (op != NULL && (op->ob_gc_bits & _PyGC_BITS_FROZEN) != 0) { + if (op != NULL && gc_is_frozen(op)) { struct count_frozen_args *arg = (struct count_frozen_args *)args; arg->count++; } @@ -1953,16 +1955,22 @@ custom_visitor_wrapper(const mi_heap_t *heap, const mi_heap_area_t *area, } void -PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) +_PyGC_VisitObjectsWorldStopped(PyInterpreterState *interp, + gcvisitobjects_t callback, void *arg) { - PyInterpreterState *interp = _PyInterpreterState_GET(); struct custom_visitor_args wrapper = { .callback = callback, .arg = arg, }; + gc_visit_heaps(interp, &custom_visitor_wrapper, &wrapper.base); +} +void +PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); _PyEval_StopTheWorld(interp); - gc_visit_heaps(interp, &custom_visitor_wrapper, &wrapper.base); + _PyGC_VisitObjectsWorldStopped(interp, callback, arg); _PyEval_StartTheWorld(interp); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 632cbc7790a4d8..f00e10f40d689a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -25,7 +25,7 @@ lhs = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -35,7 +35,7 @@ } OPCODE_DEFERRED_INC(BINARY_OP); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); } @@ -192,7 +192,7 @@ assert(next_instr->op.code == STORE_FAST); next_oparg = next_instr->op.arg; #else - next_oparg = CURRENT_OPERAND(); + next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); DEOPT_IF(!PyStackRef_Is(*target_local, left), BINARY_OP); @@ -435,8 +435,8 @@ container = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - assert(frame->stackpointer == NULL); #if ENABLE_SPECIALIZATION + assert(frame->stackpointer == NULL); if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1066,8 +1066,8 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); stack_pointer = _PyFrame_GetStackPointer(frame); - assert(_PyCode_CODE(_PyFrame_GetCode(shim))[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self[0]); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3395,7 +3395,7 @@ right = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4304,11 +4304,12 @@ else { /* `iterable` is not a generator. */ _PyFrame_SetStackPointer(frame, stack_pointer); - iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o)); + PyObject *iter_o = PyObject_GetIter(iterable_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(iter)) { + if (iter_o == NULL) { goto error; } + iter = PyStackRef_FromPyObjectSteal(iter_o); PyStackRef_CLOSE(iterable); } } @@ -4711,7 +4712,9 @@ int original_opcode = 0; if (tstate->tracing) { PyCodeObject *code = _PyFrame_GetCode(frame); - original_opcode = code->_co_monitoring->lines[(int)(this_instr - _PyCode_CODE(code))].original_opcode; + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = code->_co_monitoring->lines[(int)(this_instr - _PyFrame_GetBytecode(frame))].original_opcode; + stack_pointer = _PyFrame_GetStackPointer(frame); next_instr = this_instr; } else { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4759,9 +4762,7 @@ assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_False); int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -4782,9 +4783,7 @@ PyStackRef_CLOSE(value_stackref); offset = 0; } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -4822,9 +4821,7 @@ assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_True); int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -4834,6 +4831,28 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RESUME); + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + // Make sure this_instr gets reset correctley for any uops that + // follow + next_instr = frame->instr_ptr; + DISPATCH(); + } + #endif + } // _MAYBE_INSTRUMENT { if (tstate->tracing == 0) { @@ -5542,7 +5561,7 @@ owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyModule_CheckExact(owner_o), LOAD_ATTR); + DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); DEOPT_IF(dict->ma_keys->dk_version != dict_version, LOAD_ATTR); @@ -6646,9 +6665,7 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_False); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(oparg * flag); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -6680,9 +6697,7 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_True); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(oparg * flag); } stack_pointer += -1; @@ -6715,9 +6730,7 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_False); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(oparg * flag); } stack_pointer += -1; @@ -6735,9 +6748,7 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_True); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(oparg * flag); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -6832,7 +6843,11 @@ if (oparg) { PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); if (PyLong_Check(lasti)) { - frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti); + stack_pointer = _PyFrame_GetStackPointer(frame); assert(!_PyErr_Occurred(tstate)); } else { @@ -6844,6 +6859,8 @@ Py_DECREF(exc); goto error; } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } assert(exc && PyExceptionInstance_Check(exc)); stack_pointer += -1; @@ -6871,6 +6888,28 @@ PREDICTED(RESUME); _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + // Make sure this_instr gets reset correctley for any uops that + // follow + next_instr = frame->instr_ptr; + DISPATCH(); + } + #endif + } // _MAYBE_INSTRUMENT { if (tstate->tracing == 0) { @@ -6890,11 +6929,11 @@ } // _QUICKEN_RESUME { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (tstate->tracing == 0 && this_instr->op.code == RESUME) { FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); } - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } // _CHECK_PERIODIC_IF_NOT_YIELD_FROM { @@ -6925,6 +6964,10 @@ uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); DEOPT_IF(eval_breaker != version, RESUME); + #ifdef Py_GIL_DISABLED + DEOPT_IF(frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); + #endif DISPATCH(); } @@ -7389,10 +7432,6 @@ DEOPT_IF(ep->me_key != name, STORE_ATTR); PyObject *old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(PyStackRef_AsPyObjectBorrow(value))) { - _PyObject_GC_TRACK(dict); - } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); diff --git a/Python/getargs.c b/Python/getargs.c index a764343ea9ee3f..6485a714a4e3fe 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -2314,7 +2314,7 @@ PyObject * const * _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, PyObject *kwnames, struct _PyArg_Parser *parser, - int minpos, int maxpos, int minkw, + int minpos, int maxpos, int minkw, int varpos, PyObject **buf) { PyObject *kwtuple; @@ -2360,11 +2360,11 @@ _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, else { nkwargs = 0; } - if (nkwargs == 0 && minkw == 0 && minpos <= nargs && nargs <= maxpos) { + if (nkwargs == 0 && minkw == 0 && minpos <= nargs && (varpos || nargs <= maxpos)) { /* Fast path. */ return args; } - if (nargs + nkwargs > maxargs) { + if (!varpos && nargs + nkwargs > maxargs) { /* Adding "keyword" (when nargs == 0) prevents producing wrong error messages in some special cases (see bpo-31229). */ PyErr_Format(PyExc_TypeError, @@ -2377,7 +2377,7 @@ _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, nargs + nkwargs); return NULL; } - if (nargs > maxpos) { + if (!varpos && nargs > maxpos) { if (maxpos == 0) { PyErr_Format(PyExc_TypeError, "%.200s%s takes no positional arguments", @@ -2402,13 +2402,16 @@ _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, " (%zd given)", (parser->fname == NULL) ? "function" : parser->fname, (parser->fname == NULL) ? "" : "()", - minposonly < maxpos ? "at least" : "exactly", + (varpos || minposonly < maxpos) ? "at least" : "exactly", minposonly, minposonly == 1 ? "" : "s", nargs); return NULL; } + if (varpos) { + nargs = Py_MIN(maxpos, nargs); + } /* copy tuple args */ for (i = 0; i < nargs; i++) { buf[i] = args[i]; @@ -2486,157 +2489,6 @@ _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, return buf; } -PyObject * const * -_PyArg_UnpackKeywordsWithVararg(PyObject *const *args, Py_ssize_t nargs, - PyObject *kwargs, PyObject *kwnames, - struct _PyArg_Parser *parser, - int minpos, int maxpos, int minkw, - int vararg, PyObject **buf) -{ - PyObject *kwtuple; - PyObject *keyword; - Py_ssize_t varargssize = 0; - int i, posonly, minposonly, maxargs; - int reqlimit = minkw ? maxpos + minkw : minpos; - Py_ssize_t nkwargs; - PyObject * const *kwstack = NULL; - - assert(kwargs == NULL || PyDict_Check(kwargs)); - assert(kwargs == NULL || kwnames == NULL); - - if (parser == NULL) { - PyErr_BadInternalCall(); - return NULL; - } - - if (kwnames != NULL && !PyTuple_Check(kwnames)) { - PyErr_BadInternalCall(); - return NULL; - } - - if (args == NULL && nargs == 0) { - args = buf; - } - - if (parser_init(parser) < 0) { - return NULL; - } - - kwtuple = parser->kwtuple; - posonly = parser->pos; - minposonly = Py_MIN(posonly, minpos); - maxargs = posonly + (int)PyTuple_GET_SIZE(kwtuple); - if (kwargs != NULL) { - nkwargs = PyDict_GET_SIZE(kwargs); - } - else if (kwnames != NULL) { - nkwargs = PyTuple_GET_SIZE(kwnames); - kwstack = args + nargs; - } - else { - nkwargs = 0; - } - if (nargs < minposonly) { - PyErr_Format(PyExc_TypeError, - "%.200s%s takes %s %d positional argument%s" - " (%zd given)", - (parser->fname == NULL) ? "function" : parser->fname, - (parser->fname == NULL) ? "" : "()", - minposonly < maxpos ? "at least" : "exactly", - minposonly, - minposonly == 1 ? "" : "s", - nargs); - return NULL; - } - - /* create varargs tuple */ - varargssize = nargs - maxpos; - if (varargssize < 0) { - varargssize = 0; - } - buf[vararg] = PyTuple_New(varargssize); - if (!buf[vararg]) { - return NULL; - } - - /* copy tuple args */ - for (i = 0; i < nargs; i++) { - if (i >= vararg) { - PyTuple_SET_ITEM(buf[vararg], i - vararg, Py_NewRef(args[i])); - continue; - } - else { - buf[i] = args[i]; - } - } - - /* copy keyword args using kwtuple to drive process */ - for (i = Py_MAX((int)nargs, posonly) - Py_SAFE_DOWNCAST(varargssize, Py_ssize_t, int); i < maxargs; i++) { - PyObject *current_arg; - if (nkwargs) { - keyword = PyTuple_GET_ITEM(kwtuple, i - posonly); - if (kwargs != NULL) { - if (PyDict_GetItemRef(kwargs, keyword, ¤t_arg) < 0) { - goto exit; - } - } - else { - current_arg = find_keyword(kwnames, kwstack, keyword); - } - } - else { - current_arg = NULL; - } - - /* If an arguments is passed in as a keyword argument, - * it should be placed before `buf[vararg]`. - * - * For example: - * def f(a, /, b, *args): - * pass - * f(1, b=2) - * - * This `buf` array should be: [1, 2, NULL]. - * In this case, nargs < vararg. - * - * Otherwise, we leave a place at `buf[vararg]` for vararg tuple - * so the index is `i + 1`. */ - if (i < vararg) { - buf[i] = current_arg; - } - else { - buf[i + 1] = current_arg; - } - - if (current_arg) { - Py_DECREF(current_arg); - --nkwargs; - } - else if (i < minpos || (maxpos <= i && i < reqlimit)) { - /* Less arguments than required */ - keyword = PyTuple_GET_ITEM(kwtuple, i - posonly); - PyErr_Format(PyExc_TypeError, "%.200s%s missing required " - "argument '%U' (pos %d)", - (parser->fname == NULL) ? "function" : parser->fname, - (parser->fname == NULL) ? "" : "()", - keyword, i+1); - goto exit; - } - } - - if (nkwargs > 0) { - error_unexpected_keyword_arg(kwargs, kwnames, kwtuple, parser->fname); - goto exit; - } - - return buf; - -exit: - Py_XDECREF(buf[vararg]); - return NULL; -} - - static const char * skipitem(const char **p_format, va_list *p_va, int flags) { diff --git a/Python/getcopyright.c b/Python/getcopyright.c index 066c2ed66acddf..964584ddf7998e 100644 --- a/Python/getcopyright.c +++ b/Python/getcopyright.c @@ -4,7 +4,7 @@ static const char cprt[] = "\ -Copyright (c) 2001-2024 Python Software Foundation.\n\ +Copyright (c) 2001 Python Software Foundation.\n\ All Rights Reserved.\n\ \n\ Copyright (c) 2000 BeOpen.com.\n\ diff --git a/Python/import.c b/Python/import.c index d8ad37b2422795..09fe95fa1fb647 100644 --- a/Python/import.c +++ b/Python/import.c @@ -122,6 +122,13 @@ _PyImport_ReleaseLock(PyInterpreterState *interp) _PyRecursiveMutex_Unlock(&IMPORT_LOCK(interp)); } +void +_PyImport_ReInitLock(PyInterpreterState *interp) +{ + // gh-126688: Thread id may change after fork() on some operating systems. + IMPORT_LOCK(interp).thread = PyThread_get_thread_ident_ex(); +} + /***************/ /* sys.modules */ @@ -4424,7 +4431,7 @@ _imp_find_frozen_impl(PyObject *module, PyObject *name, int withdata) if (info.origname != NULL && info.origname[0] != '\0') { origname = PyUnicode_FromString(info.origname); if (origname == NULL) { - Py_DECREF(data); + Py_XDECREF(data); return NULL; } } diff --git a/Python/index_pool.c b/Python/index_pool.c new file mode 100644 index 00000000000000..526eccff74af00 --- /dev/null +++ b/Python/index_pool.c @@ -0,0 +1,193 @@ +#include + +#include "Python.h" + +#include "pycore_index_pool.h" +#include "pycore_lock.h" + +#ifdef Py_GIL_DISABLED + +static inline void +swap(int32_t *values, Py_ssize_t i, Py_ssize_t j) +{ + int32_t tmp = values[i]; + values[i] = values[j]; + values[j] = tmp; +} + +static bool +heap_try_swap(_PyIndexHeap *heap, Py_ssize_t i, Py_ssize_t j) +{ + if (i < 0 || i >= heap->size) { + return 0; + } + if (j < 0 || j >= heap->size) { + return 0; + } + if (i <= j) { + if (heap->values[i] <= heap->values[j]) { + return 0; + } + } + else if (heap->values[j] <= heap->values[i]) { + return 0; + } + swap(heap->values, i, j); + return 1; +} + +static inline Py_ssize_t +parent(Py_ssize_t i) +{ + return (i - 1) / 2; +} + +static inline Py_ssize_t +left_child(Py_ssize_t i) +{ + return 2 * i + 1; +} + +static inline Py_ssize_t +right_child(Py_ssize_t i) +{ + return 2 * i + 2; +} + +static void +heap_add(_PyIndexHeap *heap, int32_t val) +{ + assert(heap->size < heap->capacity); + // Add val to end + heap->values[heap->size] = val; + heap->size++; + // Sift up + for (Py_ssize_t cur = heap->size - 1; cur > 0; cur = parent(cur)) { + if (!heap_try_swap(heap, cur, parent(cur))) { + break; + } + } +} + +static Py_ssize_t +heap_min_child(_PyIndexHeap *heap, Py_ssize_t i) +{ + if (left_child(i) < heap->size) { + if (right_child(i) < heap->size) { + Py_ssize_t lval = heap->values[left_child(i)]; + Py_ssize_t rval = heap->values[right_child(i)]; + return lval < rval ? left_child(i) : right_child(i); + } + return left_child(i); + } + else if (right_child(i) < heap->size) { + return right_child(i); + } + return -1; +} + +static int32_t +heap_pop(_PyIndexHeap *heap) +{ + assert(heap->size > 0); + // Pop smallest and replace with the last element + int32_t result = heap->values[0]; + heap->values[0] = heap->values[heap->size - 1]; + heap->size--; + // Sift down + for (Py_ssize_t cur = 0; cur < heap->size;) { + Py_ssize_t min_child = heap_min_child(heap, cur); + if (min_child > -1 && heap_try_swap(heap, cur, min_child)) { + cur = min_child; + } + else { + break; + } + } + return result; +} + +static int +heap_ensure_capacity(_PyIndexHeap *heap, Py_ssize_t limit) +{ + assert(limit > 0); + if (heap->capacity > limit) { + return 0; + } + Py_ssize_t new_capacity = heap->capacity ? heap->capacity : 1024; + while (new_capacity && new_capacity < limit) { + new_capacity <<= 1; + } + if (!new_capacity) { + return -1; + } + int32_t *new_values = PyMem_RawCalloc(new_capacity, sizeof(int32_t)); + if (new_values == NULL) { + return -1; + } + if (heap->values != NULL) { + memcpy(new_values, heap->values, heap->capacity); + PyMem_RawFree(heap->values); + } + heap->values = new_values; + heap->capacity = new_capacity; + return 0; +} + +static void +heap_fini(_PyIndexHeap *heap) +{ + if (heap->values != NULL) { + PyMem_RawFree(heap->values); + heap->values = NULL; + } + heap->size = -1; + heap->capacity = -1; +} + +#define LOCK_POOL(pool) PyMutex_LockFlags(&pool->mutex, _Py_LOCK_DONT_DETACH) +#define UNLOCK_POOL(pool) PyMutex_Unlock(&pool->mutex) + +int32_t +_PyIndexPool_AllocIndex(_PyIndexPool *pool) +{ + LOCK_POOL(pool); + int32_t index; + _PyIndexHeap *free_indices = &pool->free_indices; + if (free_indices->size == 0) { + // No free indices. Make sure the heap can always store all of the + // indices that have been allocated to avoid having to allocate memory + // (which can fail) when freeing an index. Freeing indices happens when + // threads are being destroyed, which makes error handling awkward / + // impossible. This arrangement shifts handling of allocation failures + // to when indices are allocated, which happens at thread creation, + // where we are better equipped to deal with failure. + if (heap_ensure_capacity(free_indices, pool->next_index + 1) < 0) { + UNLOCK_POOL(pool); + PyErr_NoMemory(); + return -1; + } + index = pool->next_index++; + } + else { + index = heap_pop(free_indices); + } + UNLOCK_POOL(pool); + return index; +} + +void +_PyIndexPool_FreeIndex(_PyIndexPool *pool, int32_t index) +{ + LOCK_POOL(pool); + heap_add(&pool->free_indices, index); + UNLOCK_POOL(pool); +} + +void +_PyIndexPool_Fini(_PyIndexPool *pool) +{ + heap_fini(&pool->free_indices); +} + +#endif // Py_GIL_DISABLED diff --git a/Python/initconfig.c b/Python/initconfig.c index c142438b02bfd9..438f8a5c1cf1ce 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -134,6 +134,7 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(dump_refs_file, WSTR_OPT, READ_ONLY, NO_SYS), #ifdef Py_GIL_DISABLED SPEC(enable_gil, INT, READ_ONLY, NO_SYS), + SPEC(tlbc_enabled, INT, READ_ONLY, NO_SYS), #endif SPEC(faulthandler, BOOL, READ_ONLY, NO_SYS), SPEC(filesystem_encoding, WSTR, READ_ONLY, NO_SYS), @@ -315,8 +316,13 @@ The following implementation-specific options are available:\n\ "\ -X showrefcount: output the total reference count and number of used\n\ memory blocks when the program finishes or after each statement in\n\ - the interactive interpreter; only works on debug builds\n\ --X tracemalloc[=N]: trace Python memory allocations; N sets a traceback limit\n\ + the interactive interpreter; only works on debug builds\n" +#ifdef Py_GIL_DISABLED +"-X tlbc=[0|1]: enable (1) or disable (0) thread-local bytecode. Also\n\ + PYTHON_TLBC\n" +#endif +"\ +-X tracemalloc[=N]: trace Python memory allocations; N sets a traceback limit\n \ of N frames (default: 1); also PYTHONTRACEMALLOC=N\n\ -X utf8[=0|1]: enable (1) or disable (0) UTF-8 mode; also PYTHONUTF8\n\ -X warn_default_encoding: enable opt-in EncodingWarning for 'encoding=None';\n\ @@ -400,6 +406,9 @@ static const char usage_envvars[] = #ifdef Py_STATS "PYTHONSTATS : turns on statistics gathering (-X pystats)\n" #endif +#ifdef Py_GIL_DISABLED +"PYTHON_TLBC : when set to 0, disables thread-local bytecode (-X tlbc)\n" +#endif "PYTHONTRACEMALLOC: trace Python memory allocations (-X tracemalloc)\n" "PYTHONUNBUFFERED: disable stdout/stderr buffering (-u)\n" "PYTHONUTF8 : control the UTF-8 mode (-X utf8)\n" @@ -979,6 +988,7 @@ _PyConfig_InitCompatConfig(PyConfig *config) config->cpu_count = -1; #ifdef Py_GIL_DISABLED config->enable_gil = _PyConfig_GIL_DEFAULT; + config->tlbc_enabled = 1; #endif } @@ -1862,6 +1872,36 @@ config_init_cpu_count(PyConfig *config) "n must be greater than 0"); } +static PyStatus +config_init_tlbc(PyConfig *config) +{ +#ifdef Py_GIL_DISABLED + const char *env = config_get_env(config, "PYTHON_TLBC"); + if (env) { + int enabled; + if (_Py_str_to_int(env, &enabled) < 0 || (enabled < 0) || (enabled > 1)) { + return _PyStatus_ERR( + "PYTHON_TLBC=N: N is missing or invalid"); + } + config->tlbc_enabled = enabled; + } + + const wchar_t *xoption = config_get_xoption(config, L"tlbc"); + if (xoption) { + int enabled; + const wchar_t *sep = wcschr(xoption, L'='); + if (!sep || (config_wstr_to_int(sep + 1, &enabled) < 0) || (enabled < 0) || (enabled > 1)) { + return _PyStatus_ERR( + "-X tlbc=n: n is missing or invalid"); + } + config->tlbc_enabled = enabled; + } + return _PyStatus_OK(); +#else + return _PyStatus_OK(); +#endif +} + static PyStatus config_init_perf_profiling(PyConfig *config) { @@ -2111,6 +2151,11 @@ config_read_complex_options(PyConfig *config) } #endif + status = config_init_tlbc(config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + return _PyStatus_OK(); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index d4568764117563..87c2addaf809eb 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -44,10 +44,24 @@ #define UNLOCK_CODE() Py_END_CRITICAL_SECTION() +#define MODIFY_BYTECODE(code, func, ...) \ + do { \ + PyCodeObject *co = (code); \ + for (Py_ssize_t i = 0; i < code->co_tlbc->size; i++) { \ + char *bc = co->co_tlbc->entries[i]; \ + if (bc == NULL) { \ + continue; \ + } \ + (func)((_Py_CODEUNIT *)bc, __VA_ARGS__); \ + } \ + } while (0) + #else #define LOCK_CODE(code) #define UNLOCK_CODE() +#define MODIFY_BYTECODE(code, func, ...) \ + (func)(_PyCode_CODE(code), __VA_ARGS__) #endif @@ -309,7 +323,8 @@ _PyInstruction_GetLength(PyCodeObject *code, int offset) { ASSERT_WORLD_STOPPED_OR_LOCKED(code); - int opcode = _PyCode_CODE(code)[offset].op.code; + int opcode = + FT_ATOMIC_LOAD_UINT8_RELAXED(_PyCode_CODE(code)[offset].op.code); assert(opcode != 0); assert(opcode != RESERVED); if (opcode == INSTRUMENTED_LINE) { @@ -578,7 +593,9 @@ sanity_check_instrumentation(PyCodeObject *code) _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int i) { - _Py_CODEUNIT inst = _PyCode_CODE(code)[i]; + _Py_CODEUNIT *src_instr = _PyCode_CODE(code) + i; + _Py_CODEUNIT inst = { + .cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t *)src_instr)}; int opcode = inst.op.code; if (opcode < MIN_INSTRUMENTED_OPCODE) { inst.op.code = _PyOpcode_Deopt[opcode]; @@ -614,21 +631,22 @@ _Py_GetBaseCodeUnit(PyCodeObject *code, int i) } static void -de_instrument(PyCodeObject *code, int i, int event) +de_instrument(_Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i, + int event) { assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(event != PY_MONITORING_EVENT_LINE); - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + _Py_CODEUNIT *instr = &bytecode[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode = *opcode_ptr; assert(opcode != ENTER_EXECUTOR); if (opcode == INSTRUMENTED_LINE) { - opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; + opcode_ptr = &monitoring->lines[i].original_opcode; opcode = *opcode_ptr; } if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode_ptr = &monitoring->per_instruction_opcodes[i]; opcode = *opcode_ptr; } int deinstrumented = DE_INSTRUMENT[opcode]; @@ -644,65 +662,68 @@ de_instrument(PyCodeObject *code, int i, int event) } static void -de_instrument_line(PyCodeObject *code, int i) +de_instrument_line(_Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, + int i) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + _Py_CODEUNIT *instr = &bytecode[i]; int opcode = instr->op.code; if (opcode != INSTRUMENTED_LINE) { return; } - _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; + _PyCoLineInstrumentationData *lines = &monitoring->lines[i]; int original_opcode = lines->original_opcode; if (original_opcode == INSTRUMENTED_INSTRUCTION) { - lines->original_opcode = code->_co_monitoring->per_instruction_opcodes[i]; + lines->original_opcode = monitoring->per_instruction_opcodes[i]; } CHECK(original_opcode != 0); CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); - instr->op.code = original_opcode; + FT_ATOMIC_STORE_UINT8(instr->op.code, original_opcode); if (_PyOpcode_Caches[original_opcode]) { - instr[1].counter = adaptive_counter_warmup(); + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff, + adaptive_counter_warmup().value_and_backoff); } assert(instr->op.code != INSTRUMENTED_LINE); } static void -de_instrument_per_instruction(PyCodeObject *code, int i) +de_instrument_per_instruction(_Py_CODEUNIT *bytecode, + _PyCoMonitoringData *monitoring, int i) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + _Py_CODEUNIT *instr = &bytecode[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode = *opcode_ptr; if (opcode == INSTRUMENTED_LINE) { - opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; + opcode_ptr = &monitoring->lines[i].original_opcode; opcode = *opcode_ptr; } if (opcode != INSTRUMENTED_INSTRUCTION) { return; } - int original_opcode = code->_co_monitoring->per_instruction_opcodes[i]; + int original_opcode = monitoring->per_instruction_opcodes[i]; CHECK(original_opcode != 0); CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); - *opcode_ptr = original_opcode; + FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, original_opcode); if (_PyOpcode_Caches[original_opcode]) { - instr[1].counter = adaptive_counter_warmup(); + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff, + adaptive_counter_warmup().value_and_backoff); } assert(*opcode_ptr != INSTRUMENTED_INSTRUCTION); assert(instr->op.code != INSTRUMENTED_INSTRUCTION); } - static void -instrument(PyCodeObject *code, int i) +instrument(_Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + _Py_CODEUNIT *instr = &bytecode[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode =*opcode_ptr; if (opcode == INSTRUMENTED_LINE) { - _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; + _PyCoLineInstrumentationData *lines = &monitoring->lines[i]; opcode_ptr = &lines->original_opcode; opcode = *opcode_ptr; } if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode_ptr = &monitoring->per_instruction_opcodes[i]; opcode = *opcode_ptr; CHECK(opcode != INSTRUMENTED_INSTRUCTION && opcode != INSTRUMENTED_LINE); CHECK(opcode == _PyOpcode_Deopt[opcode]); @@ -716,52 +737,52 @@ instrument(PyCodeObject *code, int i) if (_PyOpcode_Caches[deopt]) { FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff, adaptive_counter_warmup().value_and_backoff); - instr[1].counter = adaptive_counter_warmup(); } } } static void -instrument_line(PyCodeObject *code, int i) +instrument_line(_Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i) { - uint8_t *opcode_ptr = &_PyCode_CODE(code)[i].op.code; + uint8_t *opcode_ptr = &bytecode[i].op.code; int opcode = *opcode_ptr; if (opcode == INSTRUMENTED_LINE) { return; } - _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; + _PyCoLineInstrumentationData *lines = &monitoring->lines[i]; lines->original_opcode = _PyOpcode_Deopt[opcode]; CHECK(lines->original_opcode > 0); - *opcode_ptr = INSTRUMENTED_LINE; + FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, INSTRUMENTED_LINE); } static void -instrument_per_instruction(PyCodeObject *code, int i) +instrument_per_instruction(_Py_CODEUNIT *bytecode, + _PyCoMonitoringData *monitoring, int i) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + _Py_CODEUNIT *instr = &bytecode[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode = *opcode_ptr; if (opcode == INSTRUMENTED_LINE) { - _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; + _PyCoLineInstrumentationData *lines = &monitoring->lines[i]; opcode_ptr = &lines->original_opcode; opcode = *opcode_ptr; } if (opcode == INSTRUMENTED_INSTRUCTION) { - assert(code->_co_monitoring->per_instruction_opcodes[i] > 0); + assert(monitoring->per_instruction_opcodes[i] > 0); return; } CHECK(opcode != 0); if (is_instrumented(opcode)) { - code->_co_monitoring->per_instruction_opcodes[i] = opcode; + monitoring->per_instruction_opcodes[i] = opcode; } else { assert(opcode != 0); assert(_PyOpcode_Deopt[opcode] != 0); assert(_PyOpcode_Deopt[opcode] != RESUME); - code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; + monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; } - assert(code->_co_monitoring->per_instruction_opcodes[i] > 0); - *opcode_ptr = INSTRUMENTED_INSTRUCTION; + assert(monitoring->per_instruction_opcodes[i] > 0); + FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, INSTRUMENTED_INSTRUCTION); } static void @@ -773,19 +794,19 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event)); assert(opcode_has_event(_Py_GetBaseCodeUnit(code, offset).op.code)); _PyCoMonitoringData *monitoring = code->_co_monitoring; + bool should_de_instrument; if (monitoring && monitoring->tools) { monitoring->tools[offset] &= ~tools; - if (monitoring->tools[offset] == 0) { - de_instrument(code, offset, event); - } + should_de_instrument = (monitoring->tools[offset] == 0); } else { /* Single tool */ uint8_t single_tool = code->_co_monitoring->active_monitors.tools[event]; assert(_Py_popcount32(single_tool) <= 1); - if (((single_tool & tools) == single_tool)) { - de_instrument(code, offset, event); - } + should_de_instrument = ((single_tool & tools) == single_tool); + } + if (should_de_instrument) { + MODIFY_BYTECODE(code, de_instrument, monitoring, offset, event); } } @@ -804,22 +825,23 @@ remove_line_tools(PyCodeObject * code, int offset, int tools) { ASSERT_WORLD_STOPPED_OR_LOCKED(code); - assert(code->_co_monitoring); - if (code->_co_monitoring->line_tools) + _PyCoMonitoringData *monitoring = code->_co_monitoring; + assert(monitoring); + bool should_de_instrument; + if (monitoring->line_tools) { - uint8_t *toolsptr = &code->_co_monitoring->line_tools[offset]; + uint8_t *toolsptr = &monitoring->line_tools[offset]; *toolsptr &= ~tools; - if (*toolsptr == 0 ) { - de_instrument_line(code, offset); - } + should_de_instrument = (*toolsptr == 0); } else { /* Single tool */ - uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_LINE]; + uint8_t single_tool = monitoring->active_monitors.tools[PY_MONITORING_EVENT_LINE]; assert(_Py_popcount32(single_tool) <= 1); - if (((single_tool & tools) == single_tool)) { - de_instrument_line(code, offset); - } + should_de_instrument = ((single_tool & tools) == single_tool); + } + if (should_de_instrument) { + MODIFY_BYTECODE(code, de_instrument_line, monitoring, offset); } } @@ -841,7 +863,7 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) assert(_Py_popcount32(tools) == 1); assert(tools_is_subset_for_event(code, event, tools)); } - instrument(code, offset); + MODIFY_BYTECODE(code, instrument, code->_co_monitoring, offset); } static void @@ -858,7 +880,7 @@ add_line_tools(PyCodeObject * code, int offset, int tools) /* Single tool */ assert(_Py_popcount32(tools) == 1); } - instrument_line(code, offset); + MODIFY_BYTECODE(code, instrument_line, code->_co_monitoring, offset); } @@ -876,7 +898,7 @@ add_per_instruction_tools(PyCodeObject * code, int offset, int tools) /* Single tool */ assert(_Py_popcount32(tools) == 1); } - instrument_per_instruction(code, offset); + MODIFY_BYTECODE(code, instrument_per_instruction, code->_co_monitoring, offset); } @@ -885,21 +907,22 @@ remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) { ASSERT_WORLD_STOPPED_OR_LOCKED(code); + _PyCoMonitoringData *monitoring = code->_co_monitoring; assert(code->_co_monitoring); + bool should_de_instrument; if (code->_co_monitoring->per_instruction_tools) { uint8_t *toolsptr = &code->_co_monitoring->per_instruction_tools[offset]; *toolsptr &= ~tools; - if (*toolsptr == 0) { - de_instrument_per_instruction(code, offset); - } + should_de_instrument = (*toolsptr == 0); } else { /* Single tool */ uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION]; assert(_Py_popcount32(single_tool) <= 1); - if (((single_tool & tools) == single_tool)) { - de_instrument_per_instruction(code, offset); - } + should_de_instrument = ((single_tool & tools) == single_tool); + } + if (should_de_instrument) { + MODIFY_BYTECODE(code, de_instrument_per_instruction, monitoring, offset); } } @@ -1087,7 +1110,7 @@ call_instrumentation_vector( PyCodeObject *code = _PyFrame_GetCode(frame); assert(args[1] == NULL); args[1] = (PyObject *)code; - int offset = (int)(instr - _PyCode_CODE(code)); + int offset = (int)(instr - _PyFrame_GetBytecode(frame)); /* Offset visible to user should be the offset in bytes, as that is the * convention for APIs involving code offsets. */ int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT); @@ -1173,8 +1196,7 @@ _Py_call_instrumentation_jump( assert(event == PY_MONITORING_EVENT_JUMP || event == PY_MONITORING_EVENT_BRANCH); assert(frame->instr_ptr == instr); - PyCodeObject *code = _PyFrame_GetCode(frame); - int to = (int)(target - _PyCode_CODE(code)); + int to = (int)(target - _PyFrame_GetBytecode(frame)); PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT)); if (to_obj == NULL) { return NULL; @@ -1240,7 +1262,8 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, PyCodeObject *code = _PyFrame_GetCode(frame); assert(tstate->tracing == 0); assert(debug_check_sanity(tstate->interp, code)); - int i = (int)(instr - _PyCode_CODE(code)); + _Py_CODEUNIT *bytecode = _PyFrame_GetBytecode(frame); + int i = (int)(instr - bytecode); _PyCoMonitoringData *monitoring = code->_co_monitoring; _PyCoLineInstrumentationData *line_data = &monitoring->lines[i]; @@ -1256,10 +1279,10 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, line = compute_line(code, i, line_delta); assert(line >= 0); assert(prev != NULL); - int prev_index = (int)(prev - _PyCode_CODE(code)); + int prev_index = (int)(prev - bytecode); int prev_line = _Py_Instrumentation_GetLine(code, prev_index); if (prev_line == line) { - int prev_opcode = _PyCode_CODE(code)[prev_index].op.code; + int prev_opcode = bytecode[prev_index].op.code; /* RESUME and INSTRUMENTED_RESUME are needed for the operation of * instrumentation, so must never be hidden by an INSTRUMENTED_LINE. */ @@ -1359,7 +1382,7 @@ int _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) { PyCodeObject *code = _PyFrame_GetCode(frame); - int offset = (int)(instr - _PyCode_CODE(code)); + int offset = (int)(instr - _PyFrame_GetBytecode(frame)); _PyCoMonitoringData *instrumentation_data = code->_co_monitoring; assert(instrumentation_data->per_instruction_opcodes); int next_opcode = instrumentation_data->per_instruction_opcodes[offset]; diff --git a/Python/jit.c b/Python/jit.c index 135daeb1b1da80..7dd0da7a45055a 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -58,7 +58,12 @@ jit_alloc(size_t size) int failed = memory == NULL; #else int flags = MAP_ANONYMOUS | MAP_PRIVATE; - unsigned char *memory = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0); + int prot = PROT_READ | PROT_WRITE; +# ifdef MAP_JIT + flags |= MAP_JIT; + prot |= PROT_EXEC; +# endif + unsigned char *memory = mmap(NULL, size, prot, flags, -1, 0); int failed = memory == MAP_FAILED; #endif if (failed) { @@ -102,8 +107,11 @@ mark_executable(unsigned char *memory, size_t size) int old; int failed = !VirtualProtect(memory, size, PAGE_EXECUTE_READ, &old); #else + int failed = 0; __builtin___clear_cache((char *)memory, (char *)memory + size); - int failed = mprotect(memory, size, PROT_EXEC | PROT_READ); +#ifndef MAP_JIT + failed = mprotect(memory, size, PROT_EXEC | PROT_READ); +#endif #endif if (failed) { jit_error("unable to protect executable memory"); @@ -470,7 +478,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz size_t code_size = 0; size_t data_size = 0; jit_state state = {0}; - group = &trampoline; + group = &shim; code_size += group->code_size; data_size += group->data_size; combine_symbol_mask(group->trampoline_mask, state.trampolines.mask); @@ -499,6 +507,9 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz if (memory == NULL) { return -1; } +#ifdef MAP_JIT + pthread_jit_write_protect_np(0); +#endif // Update the offsets of each instruction: for (size_t i = 0; i < length; i++) { state.instruction_starts[i] += (uintptr_t)memory; @@ -507,12 +518,10 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz unsigned char *code = memory; unsigned char *data = memory + code_size; state.trampolines.mem = memory + code_size + data_size; - // Compile the trampoline, which handles converting between the native + // Compile the shim, which handles converting between the native // calling convention and the calling convention used by jitted code - // (which may be different for efficiency reasons). On platforms where - // we don't change calling conventions, the trampoline is empty and - // nothing is emitted here: - group = &trampoline; + // (which may be different for efficiency reasons). + group = &shim; group->emit(code, data, executor, NULL, &state); code += group->code_size; data += group->data_size; @@ -531,12 +540,15 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz data += group->data_size; assert(code == memory + code_size); assert(data == memory + code_size + data_size); +#ifdef MAP_JIT + pthread_jit_write_protect_np(1); +#endif if (mark_executable(memory, total_size)) { jit_free(memory, total_size); return -1; } executor->jit_code = memory; - executor->jit_side_entry = memory + trampoline.code_size; + executor->jit_side_entry = memory + shim.code_size; executor->jit_size = total_size; return 0; } diff --git a/Python/marshal.c b/Python/marshal.c index a280fbfd078f41..72afa4ff89432c 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -50,41 +50,52 @@ module marshal # define MAX_MARSHAL_STACK_DEPTH 2000 #endif +/* Supported types */ #define TYPE_NULL '0' #define TYPE_NONE 'N' #define TYPE_FALSE 'F' #define TYPE_TRUE 'T' #define TYPE_STOPITER 'S' #define TYPE_ELLIPSIS '.' -#define TYPE_INT 'i' -/* TYPE_INT64 is not generated anymore. - Supported for backward compatibility only. */ -#define TYPE_INT64 'I' -#define TYPE_FLOAT 'f' -#define TYPE_BINARY_FLOAT 'g' -#define TYPE_COMPLEX 'x' -#define TYPE_BINARY_COMPLEX 'y' -#define TYPE_LONG 'l' -#define TYPE_STRING 's' -#define TYPE_INTERNED 't' -#define TYPE_REF 'r' -#define TYPE_TUPLE '(' +#define TYPE_BINARY_FLOAT 'g' // Version 0 uses TYPE_FLOAT instead. +#define TYPE_BINARY_COMPLEX 'y' // Version 0 uses TYPE_COMPLEX instead. +#define TYPE_LONG 'l' // See also TYPE_INT. +#define TYPE_STRING 's' // Bytes. (Name comes from Python 2.) +#define TYPE_TUPLE '(' // See also TYPE_SMALL_TUPLE. #define TYPE_LIST '[' #define TYPE_DICT '{' #define TYPE_CODE 'c' #define TYPE_UNICODE 'u' #define TYPE_UNKNOWN '?' +// added in version 2: #define TYPE_SET '<' #define TYPE_FROZENSET '>' +// added in version 5: #define TYPE_SLICE ':' -#define FLAG_REF '\x80' /* with a type, add obj to index */ +// Remember to update the version and documentation when adding new types. +/* Special cases for unicode strings (added in version 4) */ +#define TYPE_INTERNED 't' // Version 1+ #define TYPE_ASCII 'a' #define TYPE_ASCII_INTERNED 'A' -#define TYPE_SMALL_TUPLE ')' #define TYPE_SHORT_ASCII 'z' #define TYPE_SHORT_ASCII_INTERNED 'Z' +/* Special cases for small objects */ +#define TYPE_INT 'i' // All versions. 32-bit encoding. +#define TYPE_SMALL_TUPLE ')' // Version 4+ + +/* Supported for backwards compatibility */ +#define TYPE_COMPLEX 'x' // Generated for version 0 only. +#define TYPE_FLOAT 'f' // Generated for version 0 only. +#define TYPE_INT64 'I' // Not generated any more. + +/* References (added in version 3) */ +#define TYPE_REF 'r' +#define FLAG_REF '\x80' /* with a type, add obj to index */ + + +// Error codes: #define WFERR_OK 0 #define WFERR_UNMARSHALLABLE 1 #define WFERR_NESTEDTOODEEP 2 @@ -615,6 +626,11 @@ w_complex_object(PyObject *v, char flag, WFILE *p) PyBuffer_Release(&view); } else if (PySlice_Check(v)) { + if (p->version < 5) { + w_byte(TYPE_UNKNOWN, p); + p->error = WFERR_UNMARSHALLABLE; + return; + } PySliceObject *slice = (PySliceObject *)v; W_TYPE(TYPE_SLICE, p); w_object(slice->start, p); diff --git a/Python/optimizer.c b/Python/optimizer.c index b876b6c2bd72fd..bc2ecc098b0e15 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -288,13 +288,13 @@ _PyUOpPrint(const _PyUOpInstruction *uop) printf(" (%d, target=%d, operand=%#" PRIx64, uop->oparg, uop->target, - (uint64_t)uop->operand); + (uint64_t)uop->operand0); break; case UOP_FORMAT_JUMP: printf(" (%d, jump_target=%d, operand=%#" PRIx64, uop->oparg, uop->jump_target, - (uint64_t)uop->operand); + (uint64_t)uop->operand0); break; default: printf(" (%d, Unknown format)", uop->oparg); @@ -340,7 +340,7 @@ uop_item(_PyExecutorObject *self, Py_ssize_t index) Py_DECREF(oname); return NULL; } - PyObject *operand = PyLong_FromUnsignedLongLong(self->trace[index].operand); + PyObject *operand = PyLong_FromUnsignedLongLong(self->trace[index].operand0); if (operand == NULL) { Py_DECREF(target); Py_DECREF(oparg); @@ -463,7 +463,7 @@ add_to_trace( trace[trace_length].format = UOP_FORMAT_TARGET; trace[trace_length].target = target; trace[trace_length].oparg = oparg; - trace[trace_length].operand = operand; + trace[trace_length].operand0 = operand; return trace_length + 1; } @@ -970,7 +970,7 @@ static void make_exit(_PyUOpInstruction *inst, int opcode, int target) { inst->opcode = opcode; inst->oparg = 0; - inst->operand = 0; + inst->operand0 = 0; inst->format = UOP_FORMAT_TARGET; inst->target = target; } @@ -1033,7 +1033,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) current_error_target = target; make_exit(&buffer[next_spare], _ERROR_POP_N, 0); buffer[next_spare].oparg = popped; - buffer[next_spare].operand = target; + buffer[next_spare].operand0 = target; next_spare++; } buffer[i].error_target = current_error; @@ -1150,7 +1150,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil int next_exit = exit_count-1; _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; assert(buffer[0].opcode == _START_EXECUTOR); - buffer[0].operand = (uint64_t)executor; + buffer[0].operand0 = (uint64_t)executor; for (int i = length-1; i >= 0; i--) { int opcode = buffer[i].opcode; dest--; @@ -1159,13 +1159,13 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil if (opcode == _EXIT_TRACE) { _PyExitData *exit = &executor->exits[next_exit]; exit->target = buffer[i].target; - dest->operand = (uint64_t)exit; + dest->operand0 = (uint64_t)exit; next_exit--; } if (opcode == _DYNAMIC_EXIT) { _PyExitData *exit = &executor->exits[next_exit]; exit->target = 0; - dest->operand = (uint64_t)exit; + dest->operand0 = (uint64_t)exit; next_exit--; } } @@ -1312,7 +1312,7 @@ _PyOptimizer_NewUOpOptimizer(void) static void counter_dealloc(_PyExecutorObject *self) { /* The optimizer is the operand of the second uop. */ - PyObject *opt = (PyObject *)self->trace[1].operand; + PyObject *opt = (PyObject *)self->trace[1].operand0; Py_DECREF(opt); uop_dealloc(self); } @@ -1352,7 +1352,7 @@ counter_optimize( _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[JUMP_BACKWARD] - oparg; _PyUOpInstruction buffer[4] = { { .opcode = _START_EXECUTOR, .jump_target = 3, .format=UOP_FORMAT_JUMP }, - { .opcode = _LOAD_CONST_INLINE, .operand = (uintptr_t)self }, + { .opcode = _LOAD_CONST_INLINE, .operand0 = (uintptr_t)self }, { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)), .format=UOP_FORMAT_TARGET } }; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 25166bc2dc5c02..a4a0472b64e57c 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -100,11 +100,11 @@ convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj) PyDictObject *dict = (PyDictObject *)obj; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - assert(inst->operand <= UINT16_MAX); - if ((int)inst->operand >= dict->ma_keys->dk_nentries) { + assert(inst->operand0 <= UINT16_MAX); + if ((int)inst->operand0 >= dict->ma_keys->dk_nentries) { return NULL; } - PyObject *res = entries[inst->operand].me_value; + PyObject *res = entries[inst->operand0].me_value; if (res == NULL) { return NULL; } @@ -114,7 +114,7 @@ convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj) else { inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_WITH_NULL : _LOAD_CONST_INLINE; } - inst->operand = (uint64_t)res; + inst->operand0 = (uint64_t)res; return res; } @@ -125,7 +125,7 @@ incorrect_keys(_PyUOpInstruction *inst, PyObject *obj) return 1; } PyDictObject *dict = (PyDictObject *)obj; - if (dict->ma_keys->dk_version != inst->operand) { + if (dict->ma_keys->dk_version != inst->operand0) { return 1; } return 0; @@ -215,7 +215,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, } else { buffer[pc].opcode = _CHECK_FUNCTION; - buffer[pc].operand = function_version; + buffer[pc].operand0 = function_version; function_checked |= 1; } // We're no longer pushing the builtins keys; rewrite the @@ -248,7 +248,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, } else { buffer[pc].opcode = _CHECK_FUNCTION; - buffer[pc].operand = function_version; + buffer[pc].operand0 = function_version; function_checked |= 1; } if (opcode == _GUARD_GLOBALS_VERSION_PUSH_KEYS) { @@ -273,7 +273,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, builtins_watched <<= 1; globals_watched <<= 1; function_checked <<= 1; - uint64_t operand = buffer[pc].operand; + uint64_t operand = buffer[pc].operand0; if (operand == 0 || (operand & 1)) { // It's either a code object or NULL, so bail return 1; @@ -301,7 +301,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, builtins_watched >>= 1; globals_watched >>= 1; function_checked >>= 1; - uint64_t operand = buffer[pc].operand; + uint64_t operand = buffer[pc].operand0; if (operand == 0 || (operand & 1)) { // It's either a code object or NULL, so bail return 1; @@ -317,7 +317,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, break; } case _CHECK_FUNCTION_EXACT_ARGS: - prechecked_function_version = (uint32_t)buffer[pc].operand; + prechecked_function_version = (uint32_t)buffer[pc].operand0; break; default: if (is_terminator(inst)) { @@ -343,7 +343,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define REPLACE_OP(INST, OP, ARG, OPERAND) \ INST->opcode = OP; \ INST->oparg = ARG; \ - INST->operand = OPERAND; + INST->operand0 = OPERAND; /* Shortened forms for convenience, used in optimizer_bytecodes.c */ #define sym_is_not_null _Py_uop_sym_is_not_null @@ -409,7 +409,7 @@ get_code(_PyUOpInstruction *op) { assert(op->opcode == _PUSH_FRAME || op->opcode == _RETURN_VALUE || op->opcode == _RETURN_GENERATOR); PyCodeObject *co = NULL; - uint64_t operand = op->operand; + uint64_t operand = op->operand0; if (operand == 0) { return NULL; } @@ -429,7 +429,7 @@ static PyCodeObject * get_code_with_logging(_PyUOpInstruction *op) { PyCodeObject *co = NULL; - uint64_t push_operand = op->operand; + uint64_t push_operand = op->operand0; if (push_operand & 1) { co = (PyCodeObject *)(push_operand & ~1); DPRINTF(3, "code=%p ", co); @@ -534,7 +534,7 @@ optimize_uops( assert(max_space <= INT_MAX); assert(max_space <= INT32_MAX); first_valid_check_stack->opcode = _CHECK_STACK_SPACE_OPERAND; - first_valid_check_stack->operand = max_space; + first_valid_check_stack->operand0 = max_space; } return trace_len; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 71904c1bc73f88..42bdbd9ca8d0cd 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -346,7 +346,7 @@ dummy_func(void) { res = sym_new_type(ctx, &PyUnicode_Type); } // _STORE_FAST: - GETLOCAL(this_instr->operand) = res; + GETLOCAL(this_instr->operand0) = res; } op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame: _Py_UOpsAbstractFrame *)) { @@ -589,8 +589,27 @@ dummy_func(void) { self = sym_new_not_null(ctx); } - op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + (void)self_or_null; + if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { + assert(PyFunction_Check(sym_get_const(callable))); + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + this_instr->operand1 = (uintptr_t)sym_get_const(callable); + } sym_set_type(callable, &PyFunction_Type); + } + + op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + assert(sym_matches_type(callable, &PyFunction_Type)); + if (sym_is_const(callable)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + PyFunctionObject *func = (PyFunctionObject *)sym_get_const(callable); + PyCodeObject *co = (PyCodeObject *)func->func_code; + if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + } + } (void)self_or_null; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 0a7e44ef78dda9..f77a5aa35bdf82 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -17,6 +17,8 @@ /* _QUICKEN_RESUME is not a viable micro-op for tier 2 */ + /* _LOAD_BYTECODE is not a viable micro-op for tier 2 */ + case _RESUME_CHECK: { break; } @@ -523,7 +525,7 @@ res = sym_new_type(ctx, &PyUnicode_Type); } // _STORE_FAST: - GETLOCAL(this_instr->operand) = res; + GETLOCAL(this_instr->operand0) = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; @@ -886,7 +888,7 @@ case _GUARD_GLOBALS_VERSION_PUSH_KEYS: { _Py_UopsSymbol *globals_keys; - uint16_t version = (uint16_t)this_instr->operand; + uint16_t version = (uint16_t)this_instr->operand0; globals_keys = sym_new_unknown(ctx); (void)version; stack_pointer[0] = globals_keys; @@ -897,7 +899,7 @@ case _GUARD_BUILTINS_VERSION_PUSH_KEYS: { _Py_UopsSymbol *builtins_keys; - uint16_t version = (uint16_t)this_instr->operand; + uint16_t version = (uint16_t)this_instr->operand0; builtins_keys = sym_new_unknown(ctx); (void)version; stack_pointer[0] = builtins_keys; @@ -1088,7 +1090,7 @@ case _GUARD_TYPE_VERSION: { _Py_UopsSymbol *owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)this_instr->operand; + uint32_t type_version = (uint32_t)this_instr->operand0; assert(type_version); if (sym_matches_type_version(owner, type_version)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1120,7 +1122,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; - uint16_t offset = (uint16_t)this_instr->operand; + uint16_t offset = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); (void)offset; @@ -1135,7 +1137,7 @@ case _CHECK_ATTR_MODULE: { _Py_UopsSymbol *owner; owner = stack_pointer[-1]; - uint32_t dict_version = (uint32_t)this_instr->operand; + uint32_t dict_version = (uint32_t)this_instr->operand0; (void)dict_version; if (sym_is_const(owner)) { PyObject *cnst = sym_get_const(owner); @@ -1158,7 +1160,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)this_instr->operand; + uint16_t index = (uint16_t)this_instr->operand0; (void)index; null = sym_new_null(ctx); attr = NULL; @@ -1200,7 +1202,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; - uint16_t hint = (uint16_t)this_instr->operand; + uint16_t hint = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); (void)hint; @@ -1217,7 +1219,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)this_instr->operand; + uint16_t index = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); (void)index; @@ -1238,7 +1240,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)this_instr->operand; + PyObject *descr = (PyObject *)this_instr->operand0; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); (void)descr; @@ -1254,7 +1256,7 @@ _Py_UopsSymbol *owner; _Py_UOpsAbstractFrame *new_frame; owner = stack_pointer[-1]; - PyObject *fget = (PyObject *)this_instr->operand; + PyObject *fget = (PyObject *)this_instr->operand0; (void)fget; (void)owner; new_frame = NULL; @@ -1637,7 +1639,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *self = NULL; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)this_instr->operand; + PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; attr = sym_new_not_null(ctx); self = owner; @@ -1653,7 +1655,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *self = NULL; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)this_instr->operand; + PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; attr = sym_new_not_null(ctx); self = owner; @@ -1687,7 +1689,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *self = NULL; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)this_instr->operand; + PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; attr = sym_new_not_null(ctx); self = owner; @@ -1747,6 +1749,22 @@ } case _CHECK_FUNCTION_VERSION: { + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)this_instr->operand0; + (void)self_or_null; + if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { + assert(PyFunction_Check(sym_get_const(callable))); + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + this_instr->operand1 = (uintptr_t)sym_get_const(callable); + } + sym_set_type(callable, &PyFunction_Type); + break; + } + + case _CHECK_FUNCTION_VERSION_INLINE: { break; } @@ -1814,7 +1832,16 @@ _Py_UopsSymbol *callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - sym_set_type(callable, &PyFunction_Type); + assert(sym_matches_type(callable, &PyFunction_Type)); + if (sym_is_const(callable)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + PyFunctionObject *func = (PyFunctionObject *)sym_get_const(callable); + PyCodeObject *co = (PyCodeObject *)func->func_code; + if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + } + } (void)self_or_null; break; } @@ -1937,7 +1964,7 @@ null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; args = &stack_pointer[-oparg]; - uint32_t type_version = (uint32_t)this_instr->operand; + uint32_t type_version = (uint32_t)this_instr->operand0; (void)type_version; (void)callable; (void)null; @@ -2397,7 +2424,7 @@ } case _CHECK_STACK_SPACE_OPERAND: { - uint32_t framesize = (uint32_t)this_instr->operand; + uint32_t framesize = (uint32_t)this_instr->operand0; (void)framesize; /* We should never see _CHECK_STACK_SPACE_OPERANDs. * They are only created at the end of this pass. */ @@ -2410,7 +2437,7 @@ } case _EXIT_TRACE: { - PyObject *exit_p = (PyObject *)this_instr->operand; + PyObject *exit_p = (PyObject *)this_instr->operand0; (void)exit_p; ctx->done = true; break; @@ -2422,7 +2449,7 @@ case _LOAD_CONST_INLINE: { _Py_UopsSymbol *value; - PyObject *ptr = (PyObject *)this_instr->operand; + PyObject *ptr = (PyObject *)this_instr->operand0; value = sym_new_const(ctx, ptr); stack_pointer[0] = value; stack_pointer += 1; @@ -2432,7 +2459,7 @@ case _LOAD_CONST_INLINE_BORROW: { _Py_UopsSymbol *value; - PyObject *ptr = (PyObject *)this_instr->operand; + PyObject *ptr = (PyObject *)this_instr->operand0; value = sym_new_const(ctx, ptr); stack_pointer[0] = value; stack_pointer += 1; @@ -2450,7 +2477,7 @@ case _LOAD_CONST_INLINE_WITH_NULL: { _Py_UopsSymbol *value; _Py_UopsSymbol *null; - PyObject *ptr = (PyObject *)this_instr->operand; + PyObject *ptr = (PyObject *)this_instr->operand0; value = sym_new_const(ctx, ptr); null = sym_new_null(ctx); stack_pointer[0] = value; @@ -2463,7 +2490,7 @@ case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { _Py_UopsSymbol *value; _Py_UopsSymbol *null; - PyObject *ptr = (PyObject *)this_instr->operand; + PyObject *ptr = (PyObject *)this_instr->operand0; value = sym_new_const(ctx, ptr); null = sym_new_null(ctx); stack_pointer[0] = value; diff --git a/Python/parking_lot.c b/Python/parking_lot.c index bffc959e5d0978..8edf43235942ab 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -221,8 +221,7 @@ _PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout, int detach) PyThreadState *tstate = NULL; if (detach) { tstate = _PyThreadState_GET(); - if (tstate && _Py_atomic_load_int_relaxed(&tstate->state) == - _Py_THREAD_ATTACHED) { + if (tstate && _PyThreadState_IsAttached(tstate)) { // Only detach if we are attached PyEval_ReleaseThread(tstate); } diff --git a/Python/pyarena.c b/Python/pyarena.c index 7ab370163b2b93..28970f9d0670f4 100644 --- a/Python/pyarena.c +++ b/Python/pyarena.c @@ -4,7 +4,7 @@ /* A simple arena block structure. Measurements with standard library modules suggest the average - allocation is about 20 bytes and that most compiles use a single + allocation is about 20 bytes and that most compilers use a single block. */ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 2efaa9db7d7d58..23882d083844ac 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1310,12 +1310,17 @@ init_interp_main(PyThreadState *tstate) enabled = *env != '0'; } if (enabled) { +#ifdef _Py_JIT + // perf profiler works fine with tier 2 interpreter, so + // only checking for a "real JIT". if (config->perf_profiling > 0) { (void)PyErr_WarnEx( PyExc_RuntimeWarning, "JIT deactivated as perf profiling support is active", 0); - } else { + } else +#endif + { PyObject *opt = _PyOptimizer_NewUOpOptimizer(); if (opt == NULL) { return _PyStatus_ERR("can't initialize optimizer"); diff --git a/Python/pystate.c b/Python/pystate.c index 36b31f3b9e4200..a209a26f16f840 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -396,7 +396,7 @@ _Py_COMP_DIAG_POP #define LOCKS_INIT(runtime) \ { \ &(runtime)->interpreters.mutex, \ - &(runtime)->xi.registry.mutex, \ + &(runtime)->xi.data_lookup.registry.mutex, \ &(runtime)->unicode_state.ids.mutex, \ &(runtime)->imports.extensions.mutex, \ &(runtime)->ceval.pending_mainthread.mutex, \ @@ -1047,10 +1047,17 @@ get_main_thread(PyInterpreterState *interp) return _Py_atomic_load_ptr_relaxed(&interp->threads.main); } +void +_PyErr_SetInterpreterAlreadyRunning(void) +{ + PyErr_SetString(PyExc_InterpreterError, "interpreter already running"); +} + int _PyInterpreterState_SetRunningMain(PyInterpreterState *interp) { - if (_PyInterpreterState_FailIfRunningMain(interp) < 0) { + if (get_main_thread(interp) != NULL) { + _PyErr_SetInterpreterAlreadyRunning(); return -1; } PyThreadState *tstate = current_fast_get(); @@ -1096,17 +1103,6 @@ _PyThreadState_IsRunningMain(PyThreadState *tstate) return get_main_thread(interp) == tstate; } -int -_PyInterpreterState_FailIfRunningMain(PyInterpreterState *interp) -{ - if (get_main_thread(interp) != NULL) { - PyErr_SetString(PyExc_InterpreterError, - "interpreter already running"); - return -1; - } - return 0; -} - void _PyInterpreterState_ReinitRunningMain(PyThreadState *tstate) { @@ -1513,6 +1509,11 @@ new_threadstate(PyInterpreterState *interp, int whence) PyMem_RawFree(new_tstate); return NULL; } + int32_t tlbc_idx = _Py_ReserveTLBCIndex(interp); + if (tlbc_idx < 0) { + PyMem_RawFree(new_tstate); + return NULL; + } #endif /* We serialize concurrent creation to protect global state. */ @@ -1555,6 +1556,7 @@ new_threadstate(PyInterpreterState *interp, int whence) #ifdef Py_GIL_DISABLED // Must be called with lock unlocked to avoid lock ordering deadlocks. _Py_qsbr_register(tstate, interp, qsbr_idx); + tstate->tlbc_index = tlbc_idx; #endif return (PyThreadState *)tstate; @@ -1706,6 +1708,10 @@ PyThreadState_Clear(PyThreadState *tstate) // Remove ourself from the biased reference counting table of threads. _Py_brc_remove_thread(tstate); + + // Release our thread-local copies of the bytecode for reuse by another + // thread + _Py_ClearTLBCIndex((_PyThreadStateImpl *)tstate); #endif // Merge our queue of pointers to be freed into the interpreter queue. diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 2f2b588bd147d8..7b74f613ed563b 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -43,7 +43,7 @@ _Py_parse_inf_or_nan(const char *p, char **endptr) s += 3; if (case_insensitive_match(s, "inity")) s += 5; - retval = negate ? -Py_HUGE_VAL : Py_HUGE_VAL; + retval = negate ? -Py_INFINITY : Py_INFINITY; } else if (case_insensitive_match(s, "nan")) { s += 3; @@ -286,7 +286,7 @@ _PyOS_ascii_strtod(const char *nptr, char **endptr) string, -1.0 is returned and again ValueError is raised. On overflow (e.g., when trying to convert '1e500' on an IEEE 754 machine), - if overflow_exception is NULL then +-Py_HUGE_VAL is returned, and no Python + if overflow_exception is NULL then +-Py_INFINITY is returned, and no Python exception is raised. Otherwise, overflow_exception should point to a Python exception, this exception will be raised, -1.0 will be returned, and *endptr will point just past the end of the converted value. diff --git a/Python/specialize.c b/Python/specialize.c index ae47809305a300..dbdff197ac7829 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -230,6 +230,8 @@ print_gc_stats(FILE *out, GCStats *stats) for (int i = 0; i < NUM_GENERATIONS; i++) { fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats[i].collections); fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats[i].object_visits); + fprintf(out, "GC[%d] objects reachable from roots: %" PRIu64 "\n", i, stats[i].objects_transitively_reachable); + fprintf(out, "GC[%d] objects not reachable from roots: %" PRIu64 "\n", i, stats[i].objects_not_transitively_reachable); fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats[i].objects_collected); } } @@ -436,16 +438,25 @@ do { \ # define SPECIALIZATION_FAIL(opcode, kind) ((void)0) #endif -// Initialize warmup counters and insert superinstructions. This cannot fail. +// Initialize warmup counters and optimize instructions. This cannot fail. void -_PyCode_Quicken(PyCodeObject *code) +_PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, PyObject *consts, + int enable_counters) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT + _Py_BackoffCounter jump_counter, adaptive_counter; + if (enable_counters) { + jump_counter = initial_jump_backoff_counter(); + adaptive_counter = adaptive_counter_warmup(); + } + else { + jump_counter = initial_unreachable_backoff_counter(); + adaptive_counter = initial_unreachable_backoff_counter(); + } int opcode = 0; int oparg = 0; - _Py_CODEUNIT *instructions = _PyCode_CODE(code); /* The last code unit cannot have a cache, so we don't need to check it */ - for (int i = 0; i < Py_SIZE(code)-1; i++) { + for (Py_ssize_t i = 0; i < size-1; i++) { opcode = instructions[i].op.code; int caches = _PyOpcode_Caches[opcode]; oparg = (oparg << 8) | instructions[i].op.arg; @@ -453,7 +464,7 @@ _PyCode_Quicken(PyCodeObject *code) // The initial value depends on the opcode switch (opcode) { case JUMP_BACKWARD: - instructions[i + 1].counter = initial_jump_backoff_counter(); + instructions[i + 1].counter = jump_counter; break; case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: @@ -462,7 +473,7 @@ _PyCode_Quicken(PyCodeObject *code) instructions[i + 1].cache = 0x5555; // Alternating 0, 1 bits break; default: - instructions[i + 1].counter = adaptive_counter_warmup(); + instructions[i + 1].counter = adaptive_counter; break; } i += caches; @@ -471,7 +482,7 @@ _PyCode_Quicken(PyCodeObject *code) /* We can't do this in the bytecode compiler as * marshalling can intern strings and make them immortal. */ - PyObject *obj = PyTuple_GET_ITEM(code->co_consts, oparg); + PyObject *obj = PyTuple_GET_ITEM(consts, oparg); if (_Py_IsImmortal(obj)) { instructions[i].op.code = LOAD_CONST_IMMORTAL; } @@ -480,7 +491,7 @@ _PyCode_Quicken(PyCodeObject *code) oparg = 0; } } - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } #define SIMPLE_FUNCTION 0 @@ -659,6 +670,73 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CONTAINS_OP_LIST 11 #define SPEC_FAIL_CONTAINS_OP_USER_CLASS 12 +static inline int +set_opcode(_Py_CODEUNIT *instr, uint8_t opcode) +{ +#ifdef Py_GIL_DISABLED + uint8_t old_op = _Py_atomic_load_uint8_relaxed(&instr->op.code); + if (old_op >= MIN_INSTRUMENTED_OPCODE) { + /* Lost race with instrumentation */ + return 0; + } + if (!_Py_atomic_compare_exchange_uint8(&instr->op.code, &old_op, opcode)) { + /* Lost race with instrumentation */ + assert(old_op >= MIN_INSTRUMENTED_OPCODE); + return 0; + } + return 1; +#else + instr->op.code = opcode; + return 1; +#endif +} + +static inline void +set_counter(_Py_BackoffCounter *counter, _Py_BackoffCounter value) +{ + FT_ATOMIC_STORE_UINT16_RELAXED(counter->value_and_backoff, + value.value_and_backoff); +} + +static inline _Py_BackoffCounter +load_counter(_Py_BackoffCounter *counter) +{ + _Py_BackoffCounter result = { + .value_and_backoff = + FT_ATOMIC_LOAD_UINT16_RELAXED(counter->value_and_backoff)}; + return result; +} + +static inline void +specialize(_Py_CODEUNIT *instr, uint8_t specialized_opcode) +{ + assert(!PyErr_Occurred()); + if (!set_opcode(instr, specialized_opcode)) { + STAT_INC(_PyOpcode_Deopt[specialized_opcode], failure); + SPECIALIZATION_FAIL(_PyOpcode_Deopt[specialized_opcode], + SPEC_FAIL_OTHER); + return; + } + set_counter((_Py_BackoffCounter *)instr + 1, adaptive_counter_cooldown()); +} + +static inline void +unspecialize(_Py_CODEUNIT *instr, int reason) +{ + assert(!PyErr_Occurred()); + uint8_t opcode = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.code); + uint8_t generic_opcode = _PyOpcode_Deopt[opcode]; + STAT_INC(generic_opcode, failure); + if (!set_opcode(instr, generic_opcode)) { + SPECIALIZATION_FAIL(generic_opcode, SPEC_FAIL_OTHER); + return; + } + SPECIALIZATION_FAIL(generic_opcode, reason); + _Py_BackoffCounter *counter = (_Py_BackoffCounter *)instr + 1; + _Py_BackoffCounter cur = load_counter(counter); + set_counter(counter, adaptive_counter_backoff(cur)); +} + static int function_kind(PyCodeObject *code); static bool function_check_args(PyObject *o, int expected_argcount, int opcode); static uint32_t function_get_version(PyObject *o, int opcode); @@ -1143,7 +1221,7 @@ _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *nam SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); fail = true; } - else if (PyModule_CheckExact(owner)) { + else if (Py_TYPE(owner)->tp_getattro == PyModule_Type.tp_getattro) { fail = specialize_module_load_attr(owner, instr, name); } else if (PyType_Check(owner)) { @@ -2167,7 +2245,6 @@ _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) } } -#ifdef Py_STATS static int binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) { @@ -2235,7 +2312,6 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) } Py_UNREACHABLE(); } -#endif // Py_STATS void _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr, @@ -2243,9 +2319,8 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in { PyObject *lhs = PyStackRef_AsPyObjectBorrow(lhs_st); PyObject *rhs = PyStackRef_AsPyObjectBorrow(rhs_st); - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP); - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1); switch (oparg) { case NB_ADD: case NB_INPLACE_ADD: @@ -2256,19 +2331,19 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_BINARY_OP + 1]; bool to_store = (next.op.code == STORE_FAST); if (to_store && PyStackRef_AsPyObjectBorrow(locals[next.op.arg]) == lhs) { - instr->op.code = BINARY_OP_INPLACE_ADD_UNICODE; - goto success; + specialize(instr, BINARY_OP_INPLACE_ADD_UNICODE); + return; } - instr->op.code = BINARY_OP_ADD_UNICODE; - goto success; + specialize(instr, BINARY_OP_ADD_UNICODE); + return; } if (PyLong_CheckExact(lhs)) { - instr->op.code = BINARY_OP_ADD_INT; - goto success; + specialize(instr, BINARY_OP_ADD_INT); + return; } if (PyFloat_CheckExact(lhs)) { - instr->op.code = BINARY_OP_ADD_FLOAT; - goto success; + specialize(instr, BINARY_OP_ADD_FLOAT); + return; } break; case NB_MULTIPLY: @@ -2277,12 +2352,12 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in break; } if (PyLong_CheckExact(lhs)) { - instr->op.code = BINARY_OP_MULTIPLY_INT; - goto success; + specialize(instr, BINARY_OP_MULTIPLY_INT); + return; } if (PyFloat_CheckExact(lhs)) { - instr->op.code = BINARY_OP_MULTIPLY_FLOAT; - goto success; + specialize(instr, BINARY_OP_MULTIPLY_FLOAT); + return; } break; case NB_SUBTRACT: @@ -2291,23 +2366,16 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in break; } if (PyLong_CheckExact(lhs)) { - instr->op.code = BINARY_OP_SUBTRACT_INT; - goto success; + specialize(instr, BINARY_OP_SUBTRACT_INT); + return; } if (PyFloat_CheckExact(lhs)) { - instr->op.code = BINARY_OP_SUBTRACT_FLOAT; - goto success; + specialize(instr, BINARY_OP_SUBTRACT_FLOAT); + return; } break; } - SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs)); - STAT_INC(BINARY_OP, failure); - instr->op.code = BINARY_OP; - cache->counter = adaptive_counter_backoff(cache->counter); - return; -success: - STAT_INC(BINARY_OP, success); - cache->counter = adaptive_counter_cooldown(); + unspecialize(instr, binary_op_fail_kind(oparg, lhs, rhs)); } @@ -2694,8 +2762,8 @@ _Py_Specialize_ToBool(_PyStackRef value_o, _Py_CODEUNIT *instr) cache->counter = adaptive_counter_cooldown(); } -#ifdef Py_STATS -static int containsop_fail_kind(PyObject *value) { +static int +containsop_fail_kind(PyObject *value) { if (PyUnicode_CheckExact(value)) { return SPEC_FAIL_CONTAINS_OP_STR; } @@ -2710,33 +2778,25 @@ static int containsop_fail_kind(PyObject *value) { } return SPEC_FAIL_OTHER; } -#endif // Py_STATS void _Py_Specialize_ContainsOp(_PyStackRef value_st, _Py_CODEUNIT *instr) { PyObject *value = PyStackRef_AsPyObjectBorrow(value_st); - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[CONTAINS_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); - _PyContainsOpCache *cache = (_PyContainsOpCache *)(instr + 1); if (PyDict_CheckExact(value)) { - instr->op.code = CONTAINS_OP_DICT; - goto success; + specialize(instr, CONTAINS_OP_DICT); + return; } if (PySet_CheckExact(value) || PyFrozenSet_CheckExact(value)) { - instr->op.code = CONTAINS_OP_SET; - goto success; + specialize(instr, CONTAINS_OP_SET); + return; } - SPECIALIZATION_FAIL(CONTAINS_OP, containsop_fail_kind(value)); - STAT_INC(CONTAINS_OP, failure); - instr->op.code = CONTAINS_OP; - cache->counter = adaptive_counter_backoff(cache->counter); + unspecialize(instr, containsop_fail_kind(value)); return; -success: - STAT_INC(CONTAINS_OP, success); - cache->counter = adaptive_counter_cooldown(); } /* Code init cleanup. diff --git a/Python/symtable.c b/Python/symtable.c index 32d715197c541b..ebddb0b93fca0a 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -434,6 +434,9 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future) switch (mod->kind) { case Module_kind: seq = mod->v.Module.body; + if (_PyAST_GetDocString(seq)) { + st->st_cur->ste_has_docstring = 1; + } for (i = 0; i < asdl_seq_LEN(seq); i++) if (!symtable_visit_stmt(st, (stmt_ty)asdl_seq_GET(seq, i))) @@ -1909,6 +1912,11 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) return 0; } } + + if (_PyAST_GetDocString(s->v.ClassDef.body)) { + st->st_cur->ste_has_docstring = 1; + } + VISIT_SEQ(st, stmt, s->v.ClassDef.body); if (!symtable_exit_block(st)) return 0; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index cbb73977e1aae6..aaef5aa532412b 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -499,56 +499,28 @@ sys_addaudithook_impl(PyObject *module, PyObject *hook) Py_RETURN_NONE; } -PyDoc_STRVAR(audit_doc, -"audit($module, event, /, *args)\n\ ---\n\ -\n\ -Passes the event to any audit hooks that are attached."); +/*[clinic input] +sys.audit + + event: str + / + *args: tuple + +Passes the event to any audit hooks that are attached. +[clinic start generated code]*/ static PyObject * -sys_audit(PyObject *self, PyObject *const *args, Py_ssize_t argc) +sys_audit_impl(PyObject *module, const char *event, PyObject *args) +/*[clinic end generated code: output=1d0fc82da768f49d input=ec3b688527945109]*/ { PyThreadState *tstate = _PyThreadState_GET(); _Py_EnsureTstateNotNULL(tstate); - if (argc == 0) { - _PyErr_SetString(tstate, PyExc_TypeError, - "audit() missing 1 required positional argument: " - "'event'"); - return NULL; - } - - assert(args[0] != NULL); - if (!should_audit(tstate->interp)) { Py_RETURN_NONE; } - PyObject *auditEvent = args[0]; - if (!auditEvent) { - _PyErr_SetString(tstate, PyExc_TypeError, - "expected str for argument 'event'"); - return NULL; - } - if (!PyUnicode_Check(auditEvent)) { - _PyErr_Format(tstate, PyExc_TypeError, - "expected str for argument 'event', not %.200s", - Py_TYPE(auditEvent)->tp_name); - return NULL; - } - const char *event = PyUnicode_AsUTF8(auditEvent); - if (!event) { - return NULL; - } - - PyObject *auditArgs = _PyTuple_FromArray(args + 1, argc - 1); - if (!auditArgs) { - return NULL; - } - - int res = _PySys_Audit(tstate, event, "O", auditArgs); - Py_DECREF(auditArgs); - + int res = _PySys_Audit(tstate, event, "O", args); if (res < 0) { return NULL; } @@ -2174,6 +2146,11 @@ sys__clear_internal_caches_impl(PyObject *module) #ifdef _Py_TIER2 PyInterpreterState *interp = _PyInterpreterState_GET(); _Py_Executors_InvalidateAll(interp, 0); +#endif +#ifdef Py_GIL_DISABLED + if (_Py_ClearUnusedTLBC(_PyInterpreterState_GET()) < 0) { + return NULL; + } #endif PyType_ClearCache(); Py_RETURN_NONE; @@ -2290,6 +2267,7 @@ sys_activate_stack_trampoline_impl(PyObject *module, const char *backend) #ifdef _Py_JIT _PyOptimizerObject* optimizer = _Py_GetOptimizer(); if (optimizer != NULL) { + Py_DECREF(optimizer); PyErr_SetString(PyExc_ValueError, "Cannot activate the perf trampoline if the JIT is active"); return NULL; } @@ -2558,7 +2536,7 @@ PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename) { static PyMethodDef sys_methods[] = { /* Might as well keep this in alphabetic order */ SYS_ADDAUDITHOOK_METHODDEF - {"audit", _PyCFunction_CAST(sys_audit), METH_FASTCALL, audit_doc }, + SYS_AUDIT_METHODDEF {"breakpointhook", _PyCFunction_CAST(sys_breakpointhook), METH_FASTCALL | METH_KEYWORDS, breakpointhook_doc}, SYS__CLEAR_INTERNAL_CACHES_METHODDEF diff --git a/README.rst b/README.rst index 3f694771e090cb..0134aafe2a969a 100644 --- a/README.rst +++ b/README.rst @@ -14,7 +14,7 @@ This is Python version 3.14.0 alpha 1 :target: https://discuss.python.org/ -Copyright © 2001-2024 Python Software Foundation. All rights reserved. +Copyright © 2001 Python Software Foundation. All rights reserved. See the end of this file for further copyright and license information. @@ -215,7 +215,7 @@ Copyright and License Information --------------------------------- -Copyright © 2001-2024 Python Software Foundation. All rights reserved. +Copyright © 2001 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index 020f874cffeaef..5c4a725102d79a 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -59,6 +59,8 @@ class PackageFiles(typing.NamedTuple): include=["Modules/expat/**"], exclude=[ "Modules/expat/expat_config.h", + "Modules/expat/pyexpatns.h", + "Modules/_hacl/refresh.sh", ] ), "macholib": PackageFiles( @@ -218,6 +220,32 @@ def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None: "HACL* SBOM version doesn't match value in 'Modules/_hacl/refresh.sh'" ) + # libexpat specifies its expected rev in a refresh script. + if package["name"] == "libexpat": + libexpat_refresh_sh = (CPYTHON_ROOT_DIR / "Modules/expat/refresh.sh").read_text() + libexpat_expected_version_match = re.search( + r"expected_libexpat_version=\"([0-9]+\.[0-9]+\.[0-9]+)\"", + libexpat_refresh_sh + ) + libexpat_expected_sha256_match = re.search( + r"expected_libexpat_sha256=\"[a-f0-9]{40}\"", + libexpat_refresh_sh + ) + libexpat_expected_version = libexpat_expected_version_match and libexpat_expected_version_match.group(1) + libexpat_expected_sha256 = libexpat_expected_sha256_match and libexpat_expected_sha256_match.group(1) + + error_if( + libexpat_expected_version != version, + "libexpat SBOM version doesn't match value in 'Modules/expat/refresh.sh'" + ) + error_if( + package["checksums"] != [{ + "algorithm": "SHA256", + "checksumValue": libexpat_expected_sha256 + }], + "libexpat SBOM checksum doesn't match value in 'Modules/expat/refresh.sh'" + ) + # License must be on the approved list for SPDX. license_concluded = package["licenseConcluded"] error_if( diff --git a/Tools/build/regen-configure.sh b/Tools/build/regen-configure.sh index e1ecefddeb8732..d2a613b1e40dc1 100755 --- a/Tools/build/regen-configure.sh +++ b/Tools/build/regen-configure.sh @@ -5,7 +5,7 @@ set -e -x # The check_autoconf_regen job of .github/workflows/build.yml must kept in # sync with this script. Use the same container image than the job so the job # doesn't need to run autoreconf in a container. -IMAGE="ghcr.io/python/autoconf:2024.10.16.11360930377" +IMAGE="ghcr.io/python/autoconf:2024.11.11.11786316759" AUTORECONF="autoreconf -ivf -Werror" WORK_DIR="/src" diff --git a/Tools/c-analyzer/TODO b/Tools/c-analyzer/TODO index 3d599538510bd9..e81ceb29c64bf0 100644 --- a/Tools/c-analyzer/TODO +++ b/Tools/c-analyzer/TODO @@ -562,8 +562,6 @@ Objects/unicodeobject.c:static_strings static _Py_Iden # PyTypeObject (311) Modules/_abc.c:_abc_data_type static PyTypeObject _abc_data_type -Modules/_blake2/blake2b_impl.c:PyBlake2_BLAKE2bType PyTypeObject PyBlake2_BLAKE2bType -Modules/_blake2/blake2s_impl.c:PyBlake2_BLAKE2sType PyTypeObject PyBlake2_BLAKE2sType Modules/_collectionsmodule.c:defdict_type static PyTypeObject defdict_type Modules/_collectionsmodule.c:deque_type static PyTypeObject deque_type Modules/_collectionsmodule.c:dequeiter_type static PyTypeObject dequeiter_type diff --git a/Tools/c-analyzer/c_parser/parser/__init__.py b/Tools/c-analyzer/c_parser/parser/__init__.py index 4227e938d7f8da..ff4f303c4a2bec 100644 --- a/Tools/c-analyzer/c_parser/parser/__init__.py +++ b/Tools/c-analyzer/c_parser/parser/__init__.py @@ -164,7 +164,7 @@ def _parse(srclines, anon_name, **srckwargs): # We use defaults that cover most files. Files with bigger declarations # are covered elsewhere (MAX_SIZES in cpython/_parser.py). -def _iter_source(lines, *, maxtext=10_000, maxlines=200, showtext=False): +def _iter_source(lines, *, maxtext=11_000, maxlines=200, showtext=False): maxtext = maxtext if maxtext and maxtext > 0 else None maxlines = maxlines if maxlines and maxlines > 0 else None filestack = [] diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 3a73f65f8ff7b3..21be53e78841d5 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -290,6 +290,7 @@ def clean_lines(text): Modules/_sre/sre_lib.h LOCAL(type) static inline type Modules/_sre/sre_lib.h SRE(F) sre_ucs2_##F Objects/stringlib/codecs.h STRINGLIB_IS_UNICODE 1 +Include/internal/pycore_crossinterp_data_registry.h Py_CORE_CROSSINTERP_DATA_REGISTRY_H 1 # @end=tsv@ ''')[1:] diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 2605825d3d0078..686f3935d91bda 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -629,8 +629,6 @@ Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput ELEMENTTREE_PARSEWHOLE_INITI Include/internal/pycore_importdl.h - _PyImport_DynLoadFiletab - Include/py_curses.h - PyCurses_API - Include/pydecimal.h - _decimal_api - -Modules/_blake2/blake2module.c - blake2b_type_spec - -Modules/_blake2/blake2module.c - blake2s_type_spec - Modules/_io/fileio.c - _Py_open_cloexec_works - Modules/_io/_iomodule.h - PyIOBase_Type - Modules/_io/_iomodule.h - PyRawIOBase_Type - diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 66ead741b87a2b..96b32445fb62e2 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -200,7 +200,7 @@ def why_not_viable(self) -> str | None: return "has tier 1 control flow" if self.properties.needs_this: return "uses the 'this_instr' variable" - if len([c for c in self.caches if c.name != "unused"]) > 1: + if len([c for c in self.caches if c.name != "unused"]) > 2: return "has unused cache entries" if self.properties.error_with_pop and self.properties.error_without_pop: return "has both popping and not-popping errors" @@ -637,7 +637,7 @@ def find_stmt_start(node: parser.InstDef, idx: int) -> lexer.Token: assert idx < len(node.block.tokens) while True: tkn = node.block.tokens[idx-1] - if tkn.kind in {"SEMI", "LBRACE", "RBRACE"}: + if tkn.kind in {"SEMI", "LBRACE", "RBRACE", "CMACRO"}: break idx -= 1 assert idx > 0 diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 7a1dfe1b85bf1a..d08b621aed552b 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -143,7 +143,7 @@ def write_uop( else: type = f"uint{cache.size*16}_t " cast = f"uint{cache.size*16}_t" - out.emit(f"{type}{cache.name} = ({cast})this_instr->operand;\n") + out.emit(f"{type}{cache.name} = ({cast})this_instr->operand0;\n") if override: emitter = OptimizerEmitter(out) # No reference management of inputs needed. diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index ce761495cca435..dd16a1a7eb28b5 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -181,14 +181,14 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: code_list, storage = Storage.for_uop(stack, uop) for code in code_list: emitter.emit(code) - for cache in uop.caches: + for idx, cache in enumerate(uop.caches): if cache.name != "unused": if cache.size == 4: type = cast = "PyObject *" else: type = f"uint{cache.size*16}_t " cast = f"uint{cache.size*16}_t" - emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND();\n") + emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND{idx}();\n") storage = emitter.emit_tokens(uop, storage, None) except StackError as ex: raise analysis_error(ex.args[0], uop.body[0]) from None diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index 7b3325ada4a49f..6eb022899d6cae 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -51,6 +51,8 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: if uop.is_viable() and uop.properties.tier != 1: stack = Stack() for var in reversed(uop.stack.inputs): + if var.peek: + break stack.pop(var) popped = (-stack.base_offset).to_c() out.emit(f"case {uop.name}:\n") diff --git a/Tools/clinic/libclinic/clanguage.py b/Tools/clinic/libclinic/clanguage.py index 32aba81ab8a850..32d2c045b06bca 100644 --- a/Tools/clinic/libclinic/clanguage.py +++ b/Tools/clinic/libclinic/clanguage.py @@ -15,7 +15,7 @@ Module, Class, Function, Parameter, permute_optional_groups, GETTER, SETTER, METHOD_INIT) -from libclinic.converters import defining_class_converter, self_converter +from libclinic.converters import self_converter from libclinic.parse_args import ParseArgsCodeGen if TYPE_CHECKING: from libclinic.app import Clinic @@ -396,12 +396,6 @@ def render_function( first_optional = len(selfless) positional = selfless and selfless[-1].is_positional_only() has_option_groups = False - requires_defining_class = (len(selfless) - and isinstance(selfless[0].converter, - defining_class_converter)) - pass_vararg_directly = (all(p.is_positional_only() or p.is_vararg() - for p in selfless) - and not requires_defining_class) # offset i by -1 because first_optional needs to ignore self for i, p in enumerate(parameters, -1): @@ -410,9 +404,6 @@ def render_function( if (i != -1) and (p.default is not unspecified): first_optional = min(first_optional, i) - if p.is_vararg() and not pass_vararg_directly: - data.cleanup.append(f"Py_XDECREF({c.parser_name});") - # insert group variable group = p.group if last_group != group: @@ -424,11 +415,6 @@ def render_function( data.impl_parameters.append("int " + group_name) has_option_groups = True - if p.is_vararg() and pass_vararg_directly: - data.impl_arguments.append('nvararg') - data.impl_parameters.append('Py_ssize_t nargs') - p.converter.type = 'PyObject *const *' - c.render(p, data) if has_option_groups and (not positional): diff --git a/Tools/clinic/libclinic/converter.py b/Tools/clinic/libclinic/converter.py index 86853bb4fba253..2c93dda3541030 100644 --- a/Tools/clinic/libclinic/converter.py +++ b/Tools/clinic/libclinic/converter.py @@ -312,7 +312,7 @@ def render(self, parameter: Parameter, data: CRenderData) -> None: def length_name(self) -> str: """Computes the name of the associated "length" variable.""" assert self.length is not None - return self.parser_name + "_length" + return self.name + "_length" # Why is this one broken out separately? # For "positional-only" function parsing, diff --git a/Tools/clinic/libclinic/converters.py b/Tools/clinic/libclinic/converters.py index bd5c2a2b73b94a..a65f73ba02fb5d 100644 --- a/Tools/clinic/libclinic/converters.py +++ b/Tools/clinic/libclinic/converters.py @@ -1228,3 +1228,99 @@ def set_template_dict(self, template_dict: TemplateDict) -> None: type_object = cls.type_object type_ptr = f'PyTypeObject *base_tp = {type_object};' template_dict['base_type_ptr'] = type_ptr + + +# Converters for var-positional parameter. + +class VarPosCConverter(CConverter): + format_unit = '' + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + raise AssertionError('should never be called') + + def parse_vararg(self, *, pos_only: int, min_pos: int, max_pos: int, + fastcall: bool, limited_capi: bool) -> str: + raise NotImplementedError + + +class varpos_tuple_converter(VarPosCConverter): + type = 'PyObject *' + format_unit = '' + c_default = 'NULL' + + def cleanup(self) -> str: + return f"""Py_XDECREF({self.parser_name});\n""" + + def parse_vararg(self, *, pos_only: int, min_pos: int, max_pos: int, + fastcall: bool, limited_capi: bool) -> str: + paramname = self.parser_name + if fastcall: + if limited_capi: + if min(pos_only, min_pos) < max_pos: + size = f'Py_MAX(nargs - {max_pos}, 0)' + else: + size = f'nargs - {max_pos}' if max_pos else 'nargs' + return f""" + {paramname} = PyTuple_New({size}); + if (!{paramname}) {{{{ + goto exit; + }}}} + for (Py_ssize_t i = {max_pos}; i < nargs; ++i) {{{{ + PyTuple_SET_ITEM({paramname}, i - {max_pos}, Py_NewRef(args[i])); + }}}} + """ + else: + self.add_include('pycore_tuple.h', '_PyTuple_FromArray()') + start = f'args + {max_pos}' if max_pos else 'args' + size = f'nargs - {max_pos}' if max_pos else 'nargs' + if min(pos_only, min_pos) < max_pos: + return f""" + {paramname} = nargs > {max_pos} + ? _PyTuple_FromArray({start}, {size}) + : PyTuple_New(0); + if ({paramname} == NULL) {{{{ + goto exit; + }}}} + """ + else: + return f""" + {paramname} = _PyTuple_FromArray({start}, {size}); + if ({paramname} == NULL) {{{{ + goto exit; + }}}} + """ + else: + if max_pos: + return f""" + {paramname} = PyTuple_GetSlice(args, {max_pos}, PY_SSIZE_T_MAX); + if (!{paramname}) {{{{ + goto exit; + }}}} + """ + else: + return f"{paramname} = Py_NewRef(args);\n" + + +class varpos_array_converter(VarPosCConverter): + type = 'PyObject * const *' + length = True + c_ignored_default = '' + + def parse_vararg(self, *, pos_only: int, min_pos: int, max_pos: int, + fastcall: bool, limited_capi: bool) -> str: + paramname = self.parser_name + if not fastcall: + self.add_include('pycore_tuple.h', '_PyTuple_ITEMS()') + start = 'args' if fastcall else '_PyTuple_ITEMS(args)' + size = 'nargs' if fastcall else 'PyTuple_GET_SIZE(args)' + if max_pos: + if min(pos_only, min_pos) < max_pos: + start = f'{size} > {max_pos} ? {start} + {max_pos} : {start}' + size = f'Py_MAX(0, {size} - {max_pos})' + else: + start = f'{start} + {max_pos}' + size = f'{size} - {max_pos}' + return f""" + {paramname} = {start}; + {self.length_name} = {size}; + """ diff --git a/Tools/clinic/libclinic/dsl_parser.py b/Tools/clinic/libclinic/dsl_parser.py index 5ca3bd5cb6c3f1..4b4a8b9969d142 100644 --- a/Tools/clinic/libclinic/dsl_parser.py +++ b/Tools/clinic/libclinic/dsl_parser.py @@ -925,16 +925,17 @@ def parse_parameter(self, line: str) -> None: parameter_name = parameter.arg name, legacy, kwargs = self.parse_converter(parameter.annotation) + if is_vararg: + name = 'varpos_' + name value: object if not default: - if self.parameter_state is ParamState.OPTIONAL: - fail(f"Can't have a parameter without a default ({parameter_name!r}) " - "after a parameter with a default!") if is_vararg: value = NULL - kwargs.setdefault('c_default', "NULL") else: + if self.parameter_state is ParamState.OPTIONAL: + fail(f"Can't have a parameter without a default ({parameter_name!r}) " + "after a parameter with a default!") value = unspecified if 'py_default' in kwargs: fail("You can't specify py_default without specifying a default value!") diff --git a/Tools/clinic/libclinic/parse_args.py b/Tools/clinic/libclinic/parse_args.py index 559d4fbdd09c57..fc2d9fe987096d 100644 --- a/Tools/clinic/libclinic/parse_args.py +++ b/Tools/clinic/libclinic/parse_args.py @@ -217,8 +217,7 @@ class ParseArgsCodeGen: min_pos: int = 0 max_pos: int = 0 min_kw_only: int = 0 - pseudo_args: int = 0 - vararg: int | str = NO_VARARG + varpos: Parameter | None = None docstring_prototype: str docstring_definition: str @@ -246,6 +245,13 @@ def __init__(self, func: Function, codegen: CodeGen) -> None: if self.parameters and isinstance(self.parameters[0].converter, defining_class_converter): self.requires_defining_class = True del self.parameters[0] + + for i, p in enumerate(self.parameters): + if p.is_vararg(): + self.varpos = p + del self.parameters[i] + break + self.converters = [p.converter for p in self.parameters] if self.func.critical_section: @@ -257,18 +263,13 @@ def __init__(self, func: Function, codegen: CodeGen) -> None: self.min_pos = 0 self.max_pos = 0 self.min_kw_only = 0 - self.pseudo_args = 0 for i, p in enumerate(self.parameters, 1): if p.is_keyword_only(): assert not p.is_positional_only() if not p.is_optional(): - self.min_kw_only = i - self.max_pos - int(self.vararg != NO_VARARG) - elif p.is_vararg(): - self.pseudo_args += 1 - self.vararg = i - 1 + self.min_kw_only = i - self.max_pos else: - if self.vararg == NO_VARARG: - self.max_pos = i + self.max_pos = i if p.is_positional_only(): self.pos_only = i if not p.is_optional(): @@ -285,6 +286,7 @@ def use_meth_o(self) -> bool: return (len(self.parameters) == 1 and self.parameters[0].is_positional_only() and not self.converters[0].is_optional() + and not self.varpos and not self.requires_defining_class and not self.is_new_or_init()) @@ -315,7 +317,8 @@ def select_prototypes(self) -> None: def init_limited_capi(self) -> None: self.limited_capi = self.codegen.limited_capi - if self.limited_capi and (self.pseudo_args or + if self.limited_capi and ( + (self.varpos and self.pos_only < len(self.parameters)) or (any(p.is_optional() for p in self.parameters) and any(p.is_keyword_only() and not p.is_optional() for p in self.parameters)) or any(c.broken_limited_capi for c in self.converters)): @@ -447,6 +450,16 @@ def parse_option_groups(self) -> None: parser_code = ' {option_group_parsing}' self.parser_body(parser_code) + def _parse_vararg(self) -> str: + assert self.varpos is not None + c = self.varpos.converter + assert isinstance(c, libclinic.converters.VarPosCConverter) + return c.parse_vararg(pos_only=self.pos_only, + min_pos=self.min_pos, + max_pos=self.max_pos, + fastcall=self.fastcall, + limited_capi=self.limited_capi) + def parse_pos_only(self) -> None: if self.fastcall: # positional-only, but no option groups @@ -469,14 +482,9 @@ def parse_pos_only(self) -> None: nargs = 'PyTuple_GET_SIZE(args)' argname_fmt = 'PyTuple_GET_ITEM(args, %d)' - if self.vararg != NO_VARARG: - self.declarations = f"Py_ssize_t nvararg = {nargs} - {self.max_pos};" - else: - self.declarations = "" - - max_args = NO_VARARG if (self.vararg != NO_VARARG) else self.max_pos + parser_code = [] + max_args = NO_VARARG if self.varpos else self.max_pos if self.limited_capi: - parser_code = [] if nargs != 'nargs': nargs_def = f'Py_ssize_t nargs = {nargs};' parser_code.append(libclinic.normalize_snippet(nargs_def, indent=4)) @@ -509,33 +517,27 @@ def parse_pos_only(self) -> None: }}}} """, indent=4)) - else: + elif self.min_pos or max_args != NO_VARARG: self.codegen.add_include('pycore_modsupport.h', '_PyArg_CheckPositional()') - parser_code = [libclinic.normalize_snippet(f""" + parser_code.append(libclinic.normalize_snippet(f""" if (!_PyArg_CheckPositional("{{name}}", {nargs}, {self.min_pos}, {max_args})) {{{{ goto exit; }}}} - """, indent=4)] + """, indent=4)) has_optional = False use_parser_code = True for i, p in enumerate(self.parameters): - if p.is_vararg(): - var = p.converter.parser_name - if self.fastcall: - code = f"{var} = args + {self.vararg};" - else: - code = f"{var} = _PyTuple_CAST(args)->ob_item;" - formatted_code = libclinic.normalize_snippet(code, indent=4) - parser_code.append(formatted_code) - continue - displayname = p.get_displayname(i+1) argname = argname_fmt % i parsearg: str | None parsearg = p.converter.parse_arg(argname, displayname, limited_capi=self.limited_capi) if parsearg is None: + if self.varpos: + raise ValueError( + f"Using converter {p.converter} is not supported " + f"in function with var-positional parameter") use_parser_code = False parser_code = [] break @@ -551,6 +553,8 @@ def parse_pos_only(self) -> None: if use_parser_code: if has_optional: parser_code.append("skip_optional:") + if self.varpos: + parser_code.append(libclinic.normalize_snippet(self._parse_vararg(), indent=4)) else: for parameter in self.parameters: parameter.converter.use_converter() @@ -575,7 +579,7 @@ def parse_pos_only(self) -> None: goto exit; }} """, indent=4)] - self.parser_body(*parser_code, declarations=self.declarations) + self.parser_body(*parser_code) def parse_general(self, clang: CLanguage) -> None: parsearg: str | None @@ -589,7 +593,7 @@ def parse_general(self, clang: CLanguage) -> None: has_optional_kw = ( max(self.pos_only, self.min_pos) + self.min_kw_only - < len(self.converters) - int(self.vararg != NO_VARARG) + < len(self.converters) ) use_parser_code = True @@ -598,57 +602,48 @@ def parse_general(self, clang: CLanguage) -> None: use_parser_code = False self.fastcall = False else: - if self.vararg == NO_VARARG: - self.codegen.add_include('pycore_modsupport.h', - '_PyArg_UnpackKeywords()') - args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( - self.min_pos, - self.max_pos, - self.min_kw_only - ) + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_UnpackKeywords()') + if not self.varpos: nargs = "nargs" else: - self.codegen.add_include('pycore_modsupport.h', - '_PyArg_UnpackKeywordsWithVararg()') - args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % ( - self.min_pos, - self.max_pos, - self.min_kw_only, - self.vararg - ) nargs = f"Py_MIN(nargs, {self.max_pos})" if self.max_pos else "0" if self.fastcall: self.flags = "METH_FASTCALL|METH_KEYWORDS" self.parser_prototype = PARSER_PROTOTYPE_FASTCALL_KEYWORDS - argname_fmt = 'args[%d]' self.declarations = declare_parser(self.func, codegen=self.codegen) - self.declarations += "\nPyObject *argsbuf[%s];" % len(self.converters) + self.declarations += "\nPyObject *argsbuf[%s];" % (len(self.converters) or 1) + if self.varpos: + self.declarations += "\nPyObject * const *fastargs;" + argsname = 'fastargs' + argname_fmt = 'fastargs[%d]' + else: + argsname = 'args' + argname_fmt = 'args[%d]' if has_optional_kw: self.declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, self.min_pos + self.min_kw_only) - parser_code = [libclinic.normalize_snippet(""" - args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); - if (!args) {{ - goto exit; - }} - """ % args_declaration, indent=4)] + unpack_args = 'args, nargs, NULL, kwnames' else: # positional-or-keyword arguments self.flags = "METH_VARARGS|METH_KEYWORDS" self.parser_prototype = PARSER_PROTOTYPE_KEYWORD + argsname = 'fastargs' argname_fmt = 'fastargs[%d]' self.declarations = declare_parser(self.func, codegen=self.codegen) - self.declarations += "\nPyObject *argsbuf[%s];" % len(self.converters) + self.declarations += "\nPyObject *argsbuf[%s];" % (len(self.converters) or 1) self.declarations += "\nPyObject * const *fastargs;" self.declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" if has_optional_kw: self.declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, self.min_pos + self.min_kw_only) - parser_code = [libclinic.normalize_snippet(""" - fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); - if (!fastargs) {{ - goto exit; - }} - """ % args_declaration, indent=4)] + unpack_args = '_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL' + parser_code = [libclinic.normalize_snippet(f""" + {argsname} = _PyArg_UnpackKeywords({unpack_args}, &_parser, + /*minpos*/ {self.min_pos}, /*maxpos*/ {self.max_pos}, /*minkw*/ {self.min_kw_only}, /*varpos*/ {1 if self.varpos else 0}, argsbuf); + if (!{argsname}) {{{{ + goto exit; + }}}} + """, indent=4)] if self.requires_defining_class: self.flags = 'METH_METHOD|' + self.flags @@ -697,8 +692,6 @@ def parse_general(self, clang: CLanguage) -> None: else: label = 'skip_optional_kwonly' first_opt = self.max_pos + self.min_kw_only - if self.vararg != NO_VARARG: - first_opt += 1 if i == first_opt: add_label = label parser_code.append(libclinic.normalize_snippet(""" @@ -724,6 +717,8 @@ def parse_general(self, clang: CLanguage) -> None: if use_parser_code: if add_label: parser_code.append("%s:" % add_label) + if self.varpos: + parser_code.append(libclinic.normalize_snippet(self._parse_vararg(), indent=4)) else: for parameter in self.parameters: parameter.converter.use_converter() @@ -781,7 +776,10 @@ def parse_general(self, clang: CLanguage) -> None: def copy_includes(self) -> None: # Copy includes from parameters to Clinic after parse_arg() # has been called above. - for converter in self.converters: + converters = self.converters + if self.varpos: + converters = converters + [self.varpos.converter] + for converter in converters: for include in converter.get_includes(): self.codegen.add_include( include.filename, @@ -914,14 +912,14 @@ def parse_args(self, clang: CLanguage) -> dict[str, str]: # previous call to parser_body. this is used for an awful hack. self.parser_body_fields: tuple[str, ...] = () - if not self.parameters: + if not self.parameters and not self.varpos: self.parse_no_args() elif self.use_meth_o(): self.parse_one_arg() elif self.has_option_groups(): self.parse_option_groups() elif (not self.requires_defining_class - and self.pos_only == len(self.parameters) - self.pseudo_args): + and self.pos_only == len(self.parameters)): self.parse_pos_only() else: self.parse_general(clang) diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 946af4be1a7589..698ecbd3b549aa 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -77,6 +77,14 @@ def _managed_dict_offset(): else: return -3 * _sizeof_void_p() +_INTERP_FRAME_HAS_TLBC_INDEX = None +def interp_frame_has_tlbc_index(): + global _INTERP_FRAME_HAS_TLBC_INDEX + if _INTERP_FRAME_HAS_TLBC_INDEX is None: + interp_frame = gdb.lookup_type("_PyInterpreterFrame") + _INTERP_FRAME_HAS_TLBC_INDEX = any(field.name == "tlbc_index" + for field in interp_frame.fields()) + return _INTERP_FRAME_HAS_TLBC_INDEX Py_TPFLAGS_INLINE_VALUES = (1 << 2) Py_TPFLAGS_MANAGED_DICT = (1 << 4) @@ -105,6 +113,7 @@ def _managed_dict_offset(): UNABLE_READ_INFO_PYTHON_FRAME = 'Unable to read information on python frame' EVALFRAME = '_PyEval_EvalFrameDefault' + class NullPyObjectPtr(RuntimeError): pass @@ -693,6 +702,16 @@ def parse_location_table(firstlineno, linetable): yield addr, end_addr, line addr = end_addr + +class PyCodeArrayPtr: + def __init__(self, gdbval): + self._gdbval = gdbval + + def get_entry(self, index): + assert (index >= 0) and (index < self._gdbval["size"]) + return self._gdbval["entries"][index] + + class PyCodeObjectPtr(PyObjectPtr): """ Class wrapping a gdb.Value that's a PyCodeObject* i.e. a instance @@ -1085,7 +1104,12 @@ def _f_nlocalsplus(self): def _f_lasti(self): codeunit_p = gdb.lookup_type("_Py_CODEUNIT").pointer() instr_ptr = self._gdbval["instr_ptr"] - first_instr = self._f_code().field("co_code_adaptive").cast(codeunit_p) + if interp_frame_has_tlbc_index(): + tlbc_index = self._gdbval["tlbc_index"] + code_arr = PyCodeArrayPtr(self._f_code().field("co_tlbc")) + first_instr = code_arr.get_entry(tlbc_index).cast(codeunit_p) + else: + first_instr = self._f_code().field("co_code_adaptive").cast(codeunit_p) return int(instr_ptr - first_instr) def is_shim(self): diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 3a0b27ba420e7a..0d16e8f7da0071 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -341,6 +341,9 @@ def __waiting(self, ttype, tstring, lineno): if ttype == tokenize.NAME and tstring in ('class', 'def'): self.__state = self.__suiteseen return + if ttype == tokenize.NAME and tstring in ('class', 'def'): + self.__state = self.__ignorenext + return if ttype == tokenize.NAME and tstring in opts.keywords: self.__state = self.__keywordseen return @@ -448,6 +451,9 @@ def __openseen(self, ttype, tstring, lineno): }, file=sys.stderr) self.__state = self.__waiting + def __ignorenext(self, ttype, tstring, lineno): + self.__state = self.__waiting + def __addentry(self, msg, lineno=None, isdocstring=0): if lineno is None: lineno = self.__lineno diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 2cd051b0a77b8d..61be8fd3bbdf55 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -29,11 +29,16 @@ class HoleValue(enum.Enum): GOT = enum.auto() # The current uop's oparg (exposed as _JIT_OPARG): OPARG = enum.auto() - # The current uop's operand on 64-bit platforms (exposed as _JIT_OPERAND): - OPERAND = enum.auto() - # The current uop's operand on 32-bit platforms (exposed as _JIT_OPERAND_HI/LO): - OPERAND_HI = enum.auto() - OPERAND_LO = enum.auto() + # The current uop's operand0 on 64-bit platforms (exposed as _JIT_OPERAND0): + OPERAND0 = enum.auto() + # The current uop's operand0 on 32-bit platforms (exposed as _JIT_OPERAND0_HI/LO): + OPERAND0_HI = enum.auto() + OPERAND0_LO = enum.auto() + # The current uop's operand1 on 64-bit platforms (exposed as _JIT_OPERAND1): + OPERAND1 = enum.auto() + # The current uop's operand1 on 32-bit platforms (exposed as _JIT_OPERAND1_HI/LO): + OPERAND1_HI = enum.auto() + OPERAND1_LO = enum.auto() # The current uop's target (exposed as _JIT_TARGET): TARGET = enum.auto() # The base address of the machine code for the jump target (exposed as _JIT_JUMP_TARGET): @@ -99,9 +104,12 @@ class HoleValue(enum.Enum): # These should all have been turned into DATA values by process_relocations: # HoleValue.GOT: "", HoleValue.OPARG: "instruction->oparg", - HoleValue.OPERAND: "instruction->operand", - HoleValue.OPERAND_HI: "(instruction->operand >> 32)", - HoleValue.OPERAND_LO: "(instruction->operand & UINT32_MAX)", + HoleValue.OPERAND0: "instruction->operand0", + HoleValue.OPERAND0_HI: "(instruction->operand0 >> 32)", + HoleValue.OPERAND0_LO: "(instruction->operand0 & UINT32_MAX)", + HoleValue.OPERAND1: "instruction->operand1", + HoleValue.OPERAND1_HI: "(instruction->operand1 >> 32)", + HoleValue.OPERAND1_LO: "(instruction->operand1 & UINT32_MAX)", HoleValue.TARGET: "instruction->target", HoleValue.JUMP_TARGET: "state->instruction_starts[instruction->jump_target]", HoleValue.ERROR_TARGET: "state->instruction_starts[instruction->error_target]", diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 634208da3c8157..d8dce0a905c0f8 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -154,8 +154,8 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: with tempfile.TemporaryDirectory() as tempdir: work = pathlib.Path(tempdir).resolve() async with asyncio.TaskGroup() as group: - coro = self._compile("trampoline", TOOLS_JIT / "trampoline.c", work) - tasks.append(group.create_task(coro, name="trampoline")) + coro = self._compile("shim", TOOLS_JIT / "shim.c", work) + tasks.append(group.create_task(coro, name="shim")) template = TOOLS_JIT_TEMPLATE_C.read_text() for case, opname in cases_and_opnames: # Write out a copy of the template with *only* this case diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py index f33d8ef322f073..81a9f08db31703 100644 --- a/Tools/jit/_writer.py +++ b/Tools/jit/_writer.py @@ -22,11 +22,11 @@ def _dump_footer( yield " symbol_mask trampoline_mask;" yield "} StencilGroup;" yield "" - yield f"static const StencilGroup trampoline = {groups['trampoline'].as_c('trampoline')};" + yield f"static const StencilGroup shim = {groups['shim'].as_c('shim')};" yield "" yield "static const StencilGroup stencil_groups[MAX_UOP_ID + 1] = {" for opname, group in sorted(groups.items()): - if opname == "trampoline": + if opname == "shim": continue yield f" [{opname}] = {group.as_c(opname)}," yield "};" diff --git a/Tools/jit/ignore-tests-emulated-linux.txt b/Tools/jit/ignore-tests-emulated-linux.txt index e379e39def0eaf..080a569574470c 100644 --- a/Tools/jit/ignore-tests-emulated-linux.txt +++ b/Tools/jit/ignore-tests-emulated-linux.txt @@ -71,6 +71,7 @@ test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen1 test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen2Minus1 test.test_subprocess.POSIXProcessTestCase.test_exception_bad_args_0 test.test_subprocess.POSIXProcessTestCase.test_exception_bad_executable +test.test_subprocess.POSIXProcessTestCase.test_vfork_used_when_expected test.test_subprocess.ProcessTestCase.test_cwd_with_relative_arg test.test_subprocess.ProcessTestCase.test_cwd_with_relative_executable test.test_subprocess.ProcessTestCase.test_empty_env diff --git a/Tools/jit/trampoline.c b/Tools/jit/shim.c similarity index 100% rename from Tools/jit/trampoline.c rename to Tools/jit/shim.c diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 57c1006ab423e9..95c90bda70f352 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -26,8 +26,11 @@ #undef CURRENT_OPARG #define CURRENT_OPARG() (_oparg) -#undef CURRENT_OPERAND -#define CURRENT_OPERAND() (_operand) +#undef CURRENT_OPERAND0 +#define CURRENT_OPERAND0() (_operand0) + +#undef CURRENT_OPERAND1 +#define CURRENT_OPERAND1() (_operand1) #undef DEOPT_IF #define DEOPT_IF(COND, INSTNAME) \ @@ -99,12 +102,17 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState // Other stuff we need handy: PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) #if SIZEOF_VOID_P == 8 - PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND) + PATCH_VALUE(uint64_t, _operand0, _JIT_OPERAND0) + PATCH_VALUE(uint64_t, _operand1, _JIT_OPERAND1) #else assert(SIZEOF_VOID_P == 4); - PATCH_VALUE(uint32_t, _operand_hi, _JIT_OPERAND_HI) - PATCH_VALUE(uint32_t, _operand_lo, _JIT_OPERAND_LO) - uint64_t _operand = ((uint64_t)_operand_hi << 32) | _operand_lo; + PATCH_VALUE(uint32_t, _operand0_hi, _JIT_OPERAND0_HI) + PATCH_VALUE(uint32_t, _operand0_lo, _JIT_OPERAND0_LO) + uint64_t _operand0 = ((uint64_t)_operand0_hi << 32) | _operand0_lo; + + PATCH_VALUE(uint32_t, _operand1_hi, _JIT_OPERAND1_HI) + PATCH_VALUE(uint32_t, _operand1_lo, _JIT_OPERAND1_LO) + uint64_t _operand1 = ((uint64_t)_operand1_hi << 32) | _operand1_lo; #endif PATCH_VALUE(uint32_t, _target, _JIT_TARGET) diff --git a/Tools/msi/freethreaded/freethreaded_files.wxs b/Tools/msi/freethreaded/freethreaded_files.wxs index 49ecb3429ad8f3..b3ce28e7aedc84 100644 --- a/Tools/msi/freethreaded/freethreaded_files.wxs +++ b/Tools/msi/freethreaded/freethreaded_files.wxs @@ -103,7 +103,7 @@ - + @@ -159,11 +159,13 @@ - - + + + - - + + + diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index 57f0b982b00f5d..e5badaccadda40 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,6 +1,6 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==1.12 +mypy==1.13 # needed for peg_generator: types-psutil==6.0.0.20240901 diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 5793e5c649d6b3..296a31d5916bd8 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -1118,6 +1118,8 @@ def calc_gc_stats(stats: Stats) -> Rows: Count(gen["collections"]), Count(gen["objects collected"]), Count(gen["object visits"]), + Count(gen["objects reachable from roots"]), + Count(gen["objects not reachable from roots"]), ) for (i, gen) in enumerate(gc_stats) ] @@ -1127,7 +1129,8 @@ def calc_gc_stats(stats: Stats) -> Rows: "GC collections and effectiveness", [ Table( - ("Generation:", "Collections:", "Objects collected:", "Object visits:"), + ("Generation:", "Collections:", "Objects collected:", "Object visits:", + "Reachable from roots:", "Not reachable from roots:"), calc_gc_stats, ) ], diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py index c94de7f9377b74..889ae8fc869b8a 100644 --- a/Tools/unicode/makeunicodedata.py +++ b/Tools/unicode/makeunicodedata.py @@ -35,7 +35,7 @@ from textwrap import dedent from typing import Iterator, List, Optional, Set, Tuple -SCRIPT = sys.argv[0] +SCRIPT = os.path.normpath(sys.argv[0]) VERSION = "3.3" # The Unicode Database diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index bc3e4ba8bd5b76..3f4211fb1dfb28 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -1,7 +1,7 @@ # Python WebAssembly (WASM) build **WASI support is [tier 2](https://peps.python.org/pep-0011/#tier-2).** -**Emscripten is NOT officially supported as of Python 3.13.** +**Emscripten support is [tier 3](https://peps.python.org/pep-0011/#tier-3).** This directory contains configuration and helpers to facilitate cross compilation of CPython to WebAssembly (WASM). Python supports Emscripten @@ -21,160 +21,57 @@ https://github.com/psf/webassembly for more information. ### Build -For now the build system has two target flavors. The ``Emscripten/browser`` -target (``--with-emscripten-target=browser``) is optimized for browsers. -It comes with a reduced and preloaded stdlib without tests and threading -support. The ``Emscripten/node`` target has threading enabled and can -access the file system directly. +To cross compile to the ``wasm32-emscripten`` platform you need +[the Emscripten compiler toolchain](https://emscripten.org/), +a Python interpreter, and an installation of Node version 18 or newer. Emscripten +version 3.1.42 or newer is recommended. All commands below are relative to a checkout +of the Python repository. -Cross compiling to the wasm32-emscripten platform needs the -[Emscripten](https://emscripten.org/) SDK and a build Python interpreter. -Emscripten 3.1.19 or newer are recommended. All commands below are relative -to a repository checkout. +#### Install [the Emscripten compiler toolchain](https://emscripten.org/docs/getting_started/downloads.html) -#### Toolchain - -##### Container image - -Christian Heimes maintains a container image with Emscripten SDK, Python -build dependencies, WASI-SDK, wasmtime, and several additional tools. - -From within your local CPython repo clone, run one of the following commands: - -``` -# Fedora, RHEL, CentOS -podman run --rm -ti -v $(pwd):/python-wasm/cpython:Z -w /python-wasm/cpython quay.io/tiran/cpythonbuild:emsdk3 - -# other -docker run --rm -ti -v $(pwd):/python-wasm/cpython -w /python-wasm/cpython quay.io/tiran/cpythonbuild:emsdk3 +You can install the Emscripten toolchain as follows: +```shell +git clone https://github.com/emscripten-core/emsdk.git --depth 1 +./emsdk/emsdk install latest +./emsdk/emsdk activate latest ``` - -##### Manually - -###### Install [Emscripten SDK](https://emscripten.org/docs/getting_started/downloads.html) - -**NOTE**: Follow the on-screen instructions how to add the SDK to ``PATH``. - +To add the Emscripten compiler to your path: ```shell -git clone https://github.com/emscripten-core/emsdk.git /opt/emsdk -/opt/emsdk/emsdk install latest -/opt/emsdk/emsdk activate latest +source ./emsdk/emsdk_env.sh ``` +This adds `emcc` and `emconfigure` to your path. -###### Optionally: enable ccache for EMSDK +##### Optionally: enable ccache for EMSDK The ``EM_COMPILER_WRAPPER`` must be set after the EMSDK environment is sourced. Otherwise the source script removes the environment variable. -``` -. /opt/emsdk/emsdk_env.sh -EM_COMPILER_WRAPPER=ccache -``` - -###### Optionally: pre-build and cache static libraries - -Emscripten SDK provides static builds of core libraries without PIC -(position-independent code). Python builds with ``dlopen`` support require -PIC. To populate the build cache, run: - ```shell -. /opt/emsdk/emsdk_env.sh -embuilder build zlib bzip2 MINIMAL_PIC -embuilder --pic build zlib bzip2 MINIMAL_PIC +export EM_COMPILER_WRAPPER=ccache ``` - ### Compile and build Python interpreter -From within the container, run the following command: - +You can use `python Tools/wasm/emscripten` to compile and build targetting +Emscripten. You can do everything at once with: ```shell -./Tools/wasm/wasm_build.py build +python Tools/wasm/emscripten build ``` - -The command is roughly equivalent to: - +or you can break it out into four separate steps: ```shell -mkdir -p builddir/build -pushd builddir/build -../../configure -C -make -j$(nproc) -popd +python Tools/wasm/emscripten configure-build-python +python Tools/wasm/emscripten make-build-python +python Tools/wasm/emscripten configure-host +python Tools/wasm/emscripten make-host ``` - -#### Cross-compile to wasm32-emscripten for browser - +Extra arguments to the configure steps are passed along to configure. For +instance, to do a debug build, you can use: ```shell -./Tools/wasm/wasm_build.py emscripten-browser +python Tools/wasm/emscripten build --with-py-debug ``` -The command is roughly equivalent to: - -```shell -mkdir -p builddir/emscripten-browser -pushd builddir/emscripten-browser - -CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \ - emconfigure ../../configure -C \ - --host=wasm32-unknown-emscripten \ - --build=$(../../config.guess) \ - --with-emscripten-target=browser \ - --with-build-python=$(pwd)/../build/python - -emmake make -j$(nproc) -popd -``` - -Serve `python.html` with a local webserver and open the file in a browser. -Python comes with a minimal web server script that sets necessary HTTP -headers like COOP, COEP, and mimetypes. Run the script outside the container -and from the root of the CPython checkout. - -```shell -./Tools/wasm/wasm_webserver.py -``` - -and open http://localhost:8000/builddir/emscripten-browser/python.html . This -directory structure enables the *C/C++ DevTools Support (DWARF)* to load C -and header files with debug builds. - - -#### Cross compile to wasm32-emscripten for node - -```shell -./Tools/wasm/wasm_build.py emscripten-node-dl -``` - -The command is roughly equivalent to: - -```shell -mkdir -p builddir/emscripten-node-dl -pushd builddir/emscripten-node-dl - -CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \ - emconfigure ../../configure -C \ - --host=wasm32-unknown-emscripten \ - --build=$(../../config.guess) \ - --with-emscripten-target=node \ - --enable-wasm-dynamic-linking \ - --with-build-python=$(pwd)/../build/python - -emmake make -j$(nproc) -popd -``` - -```shell -node --experimental-wasm-threads --experimental-wasm-bulk-memory --experimental-wasm-bigint builddir/emscripten-node-dl/python.js -``` - -(``--experimental-wasm-bigint`` is not needed with recent NodeJS versions) - ### Limitations and issues -Emscripten before 3.1.8 has known bugs that can cause memory corruption and -resource leaks. 3.1.8 contains several fixes for bugs in date and time -functions. - #### Network stack - Python's socket module does not work with Emscripten's emulated POSIX @@ -241,8 +138,6 @@ functions. [gh-90548](https://github.com/python/cpython/issues/90548). - Python's object allocator ``obmalloc`` is disabled by default. - ``ensurepip`` is not available. -- Some ``ctypes`` features like ``c_longlong`` and ``c_longdouble`` may need - NodeJS option ``--experimental-wasm-bigint``. #### In the browser @@ -263,15 +158,6 @@ Node builds use ``NODERAWFS``. - Node RawFS allows direct access to the host file system without need to perform ``FS.mount()`` call. -### wasm64-emscripten - -- wasm64 requires recent NodeJS and ``--experimental-wasm-memory64``. -- ``EM_JS`` functions must return ``BigInt()``. -- ``Py_BuildValue()`` format strings must match size of types. Confusing 32 - and 64 bits types leads to memory corruption, see - [gh-95876](https://github.com/python/cpython/issues/95876) and - [gh-95878](https://github.com/python/cpython/issues/95878). - ### Hosting Python WASM builds The simple REPL terminal uses SharedArrayBuffer. For security reasons diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py new file mode 100644 index 00000000000000..2015a3764ea8c8 --- /dev/null +++ b/Tools/wasm/emscripten/__main__.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 + +import argparse +import contextlib +import functools +import os + +try: + from os import process_cpu_count as cpu_count +except ImportError: + from os import cpu_count +from pathlib import Path +import shutil +import subprocess +import sys +import sysconfig +import tempfile + +WASM_DIR = Path(__file__).parent.parent +CHECKOUT = WASM_DIR.parent.parent + +CROSS_BUILD_DIR = CHECKOUT / "cross-build" +BUILD_DIR = CROSS_BUILD_DIR / "build" +HOST_TRIPLE = "wasm32-emscripten" +HOST_DIR = CROSS_BUILD_DIR / HOST_TRIPLE + +LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" +LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode("utf-8") + + +def updated_env(updates={}): + """Create a new dict representing the environment to use. + + The changes made to the execution environment are printed out. + """ + env_defaults = {} + # https://reproducible-builds.org/docs/source-date-epoch/ + git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] + try: + epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() + env_defaults["SOURCE_DATE_EPOCH"] = epoch + except subprocess.CalledProcessError: + pass # Might be building from a tarball. + # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. + environment = env_defaults | os.environ | updates + + env_diff = {} + for key, value in environment.items(): + if os.environ.get(key) != value: + env_diff[key] = value + + print("🌎 Environment changes:") + for key in sorted(env_diff.keys()): + print(f" {key}={env_diff[key]}") + + return environment + + +def subdir(working_dir, *, clean_ok=False): + """Decorator to change to a working directory.""" + + def decorator(func): + @functools.wraps(func) + def wrapper(context): + try: + tput_output = subprocess.check_output( + ["tput", "cols"], encoding="utf-8" + ) + terminal_width = int(tput_output.strip()) + except subprocess.CalledProcessError: + terminal_width = 80 + print("⎯" * terminal_width) + print("📁", working_dir) + if clean_ok and getattr(context, "clean", False) and working_dir.exists(): + print(f"🚮 Deleting directory (--clean)...") + shutil.rmtree(working_dir) + + working_dir.mkdir(parents=True, exist_ok=True) + + with contextlib.chdir(working_dir): + return func(context, working_dir) + + return wrapper + + return decorator + + +def call(command, *, quiet, **kwargs): + """Execute a command. + + If 'quiet' is true, then redirect stdout and stderr to a temporary file. + """ + print("❯", " ".join(map(str, command))) + if not quiet: + stdout = None + stderr = None + else: + stdout = tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + delete=False, + prefix="cpython-emscripten-", + suffix=".log", + ) + stderr = subprocess.STDOUT + print(f"📝 Logging output to {stdout.name} (--quiet)...") + + subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) + + +def build_platform(): + """The name of the build/host platform.""" + # Can also be found via `config.guess`.` + return sysconfig.get_config_var("BUILD_GNU_TYPE") + + +def build_python_path(): + """The path to the build Python binary.""" + binary = BUILD_DIR / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError("Unable to find `python(.exe)` in " f"{BUILD_DIR}") + + return binary + + +@subdir(BUILD_DIR, clean_ok=True) +def configure_build_python(context, working_dir): + """Configure the build/host Python.""" + if LOCAL_SETUP.exists(): + print(f"👍 {LOCAL_SETUP} exists ...") + else: + print(f"📝 Touching {LOCAL_SETUP} ...") + LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) + + configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] + if context.args: + configure.extend(context.args) + + call(configure, quiet=context.quiet) + + +@subdir(BUILD_DIR) +def make_build_python(context, working_dir): + """Make/build the build Python.""" + call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) + + binary = build_python_path() + cmd = [ + binary, + "-c", + "import sys; " "print(f'{sys.version_info.major}.{sys.version_info.minor}')", + ] + version = subprocess.check_output(cmd, encoding="utf-8").strip() + + print(f"🎉 {binary} {version}") + + +@subdir(HOST_DIR, clean_ok=True) +def configure_emscripten_python(context, working_dir): + """Configure the emscripten/host build.""" + config_site = os.fsdecode( + CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-emscripten" + ) + + emscripten_build_dir = working_dir.relative_to(CHECKOUT) + + python_build_dir = BUILD_DIR / "build" + lib_dirs = list(python_build_dir.glob("lib.*")) + assert ( + len(lib_dirs) == 1 + ), f"Expected a single lib.* directory in {python_build_dir}" + lib_dir = os.fsdecode(lib_dirs[0]) + pydebug = lib_dir.endswith("-pydebug") + python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] + sysconfig_data = ( + f"{emscripten_build_dir}/build/lib.emscripten-wasm32-{python_version}" + ) + if pydebug: + sysconfig_data += "-pydebug" + + host_runner = context.host_runner + env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} + build_python = os.fsdecode(build_python_path()) + configure = [ + "emconfigure", + os.path.relpath(CHECKOUT / "configure", working_dir), + "CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2", + f"--host={HOST_TRIPLE}", + f"--build={build_platform()}", + f"--with-build-python={build_python}", + "--without-pymalloc", + "--disable-shared", + "--disable-ipv6", + "--enable-big-digits=30", + "--enable-wasm-dynamic-linking", + f"--prefix={HOST_DIR}", + ] + if pydebug: + configure.append("--with-pydebug") + if context.args: + configure.extend(context.args) + call( + configure, + env=updated_env(env_additions), + quiet=context.quiet, + ) + + python_js = working_dir / "python.js" + exec_script = working_dir / "python.sh" + exec_script.write_text(f'#!/bin/sh\nexec {host_runner} {python_js} "$@"\n') + exec_script.chmod(0o755) + print(f"🏃‍♀️ Created {exec_script} ... ") + sys.stdout.flush() + + +@subdir(HOST_DIR) +def make_emscripten_python(context, working_dir): + """Run `make` for the emscripten/host build.""" + call( + ["make", "--jobs", str(cpu_count()), "commoninstall"], + env=updated_env(), + quiet=context.quiet, + ) + + exec_script = working_dir / "python.sh" + subprocess.check_call([exec_script, "--version"]) + + +def build_all(context): + """Build everything.""" + steps = [ + configure_build_python, + make_build_python, + configure_emscripten_python, + make_emscripten_python, + ] + for step in steps: + step(context) + + +def clean_contents(context): + """Delete all files created by this script.""" + if CROSS_BUILD_DIR.exists(): + print(f"🧹 Deleting {CROSS_BUILD_DIR} ...") + shutil.rmtree(CROSS_BUILD_DIR) + + if LOCAL_SETUP.exists(): + with LOCAL_SETUP.open("rb") as file: + if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER: + print(f"🧹 Deleting generated {LOCAL_SETUP} ...") + + +def main(): + default_host_runner = "node" + + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(dest="subcommand") + build = subcommands.add_parser("build", help="Build everything") + configure_build = subcommands.add_parser( + "configure-build-python", help="Run `configure` for the " "build Python" + ) + make_build = subcommands.add_parser( + "make-build-python", help="Run `make` for the build Python" + ) + configure_host = subcommands.add_parser( + "configure-host", + help="Run `configure` for the host/emscripten (pydebug builds are inferred from the build Python)", + ) + make_host = subcommands.add_parser("make-host", help="Run `make` for the host/emscripten") + clean = subcommands.add_parser( + "clean", help="Delete files and directories created by this script" + ) + for subcommand in build, configure_build, make_build, configure_host, make_host: + subcommand.add_argument( + "--quiet", + action="store_true", + default=False, + dest="quiet", + help="Redirect output from subprocesses to a log file", + ) + for subcommand in configure_build, configure_host: + subcommand.add_argument( + "--clean", + action="store_true", + default=False, + dest="clean", + help="Delete any relevant directories before building", + ) + for subcommand in build, configure_build, configure_host: + subcommand.add_argument( + "args", nargs="*", help="Extra arguments to pass to `configure`" + ) + for subcommand in build, configure_host: + subcommand.add_argument( + "--host-runner", + action="store", + default=default_host_runner, + dest="host_runner", + help="Command template for running the emscripten host" + f"`{default_host_runner}`)", + ) + + context = parser.parse_args() + + dispatch = { + "configure-build-python": configure_build_python, + "make-build-python": make_build_python, + "configure-host": configure_emscripten_python, + "make-host": make_emscripten_python, + "build": build_all, + "clean": clean_contents, + } + + if not context.subcommand: + # No command provided, display help and exit + print("Expected one of", ", ".join(sorted(dispatch.keys())), file=sys.stderr) + parser.print_help(sys.stderr) + sys.exit(1) + dispatch[context.subcommand](context) + + +if __name__ == "__main__": + main() diff --git a/Tools/wasm/emscripten/node_pre.js b/Tools/wasm/emscripten/node_pre.js index 3490d3ca591ef6..54b09dc08233f3 100644 --- a/Tools/wasm/emscripten/node_pre.js +++ b/Tools/wasm/emscripten/node_pre.js @@ -1,9 +1,15 @@ // If process is undefined, we're not running in the node runtime let it go I // guess? if (typeof process !== "undefined") { - const nodeVersion = Number(process.versions.node.split('.',1)[0]); + const nodeVersion = Number(process.versions.node.split(".", 1)[0]); if (nodeVersion < 18) { - process.stderr.write(`Node version must be >= 18, got version ${process.version}\n`); + process.stderr.write( + `Node version must be >= 18, got version ${process.version}\n`, + ); process.exit(1); } + Module.preRun = () => { + FS.mkdirTree("/lib/"); + FS.mount(NODEFS, { root: __dirname + "/lib/" }, "/lib/"); + }; } diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 050e3723feb247..ac36d55587a38f 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -346,7 +346,7 @@ def main(): "(default designed for wasmtime 14 or newer: " f"`{default_host_runner}`)") for subcommand in build, configure_host, make_host: - subcommand.add_argument("--host-triple", action="store", default="wasm32-wasi", + subcommand.add_argument("--host-triple", action="store", default="wasm32-wasip1", help="The target triple for the WASI host build") context = parser.parse_args() diff --git a/aclocal.m4 b/aclocal.m4 index b082a5b1bc5e07..920c2b38560faa 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -91,7 +91,7 @@ m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun # and this notice are preserved. This file is offered as-is, without any # warranty. -#serial 12 +#serial 14 AC_DEFUN([AX_C_FLOAT_WORDS_BIGENDIAN], [AC_CACHE_CHECK(whether float word ordering is bigendian, @@ -112,10 +112,10 @@ int main (int argc, char *argv[]) ]])], [ -if grep noonsees conftest$EXEEXT >/dev/null ; then +if grep noonsees conftest* > /dev/null ; then ax_cv_c_float_words_bigendian=yes fi -if grep seesnoon conftest$EXEEXT >/dev/null ; then +if grep seesnoon conftest* >/dev/null ; then if test "$ax_cv_c_float_words_bigendian" = unknown; then ax_cv_c_float_words_bigendian=no else @@ -398,7 +398,7 @@ AC_DEFUN([AX_CHECK_OPENSSL], [ AC_SUBST([OPENSSL_LDFLAGS]) ]) -# pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- +# pkg.m4 - Macros to locate and use pkg-config. -*- Autoconf -*- # serial 12 (pkg-config-0.29.2) dnl Copyright © 2004 Scott James Remnant . @@ -486,7 +486,7 @@ dnl Check to see whether a particular set of modules exists. Similar to dnl PKG_CHECK_MODULES(), but does not set variables or print errors. dnl dnl Please remember that m4 expands AC_REQUIRE([PKG_PROG_PKG_CONFIG]) -dnl only at the first occurence in configure.ac, so if the first place +dnl only at the first occurrence in configure.ac, so if the first place dnl it's called might be skipped (such as if it is within an "if", you dnl have to call PKG_CHECK_EXISTS manually AC_DEFUN([PKG_CHECK_EXISTS], @@ -555,14 +555,14 @@ if test $pkg_failed = yes; then AC_MSG_RESULT([no]) _PKG_SHORT_ERRORS_SUPPORTED if test $_pkg_short_errors_supported = yes; then - $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$2" 2>&1` + $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$2" 2>&1` else - $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$2" 2>&1` + $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$2" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$$1[]_PKG_ERRORS" >&AS_MESSAGE_LOG_FD + # Put the nasty error message in config.log where it belongs + echo "$$1[]_PKG_ERRORS" >&AS_MESSAGE_LOG_FD - m4_default([$4], [AC_MSG_ERROR( + m4_default([$4], [AC_MSG_ERROR( [Package requirements ($2) were not met: $$1_PKG_ERRORS @@ -574,7 +574,7 @@ _PKG_TEXT])[]dnl ]) elif test $pkg_failed = untried; then AC_MSG_RESULT([no]) - m4_default([$4], [AC_MSG_FAILURE( + m4_default([$4], [AC_MSG_FAILURE( [The pkg-config script could not be found or is too old. Make sure it is in your PATH or set the PKG_CONFIG environment variable to the full path to pkg-config. @@ -584,10 +584,10 @@ _PKG_TEXT To get pkg-config, see .])[]dnl ]) else - $1[]_CFLAGS=$pkg_cv_[]$1[]_CFLAGS - $1[]_LIBS=$pkg_cv_[]$1[]_LIBS + $1[]_CFLAGS=$pkg_cv_[]$1[]_CFLAGS + $1[]_LIBS=$pkg_cv_[]$1[]_LIBS AC_MSG_RESULT([yes]) - $3 + $3 fi[]dnl ])dnl PKG_CHECK_MODULES @@ -674,6 +674,74 @@ AS_VAR_COPY([$1], [pkg_cv_][$1]) AS_VAR_IF([$1], [""], [$5], [$4])dnl ])dnl PKG_CHECK_VAR +dnl PKG_WITH_MODULES(VARIABLE-PREFIX, MODULES, +dnl [ACTION-IF-FOUND],[ACTION-IF-NOT-FOUND], +dnl [DESCRIPTION], [DEFAULT]) +dnl ------------------------------------------ +dnl +dnl Prepare a "--with-" configure option using the lowercase +dnl [VARIABLE-PREFIX] name, merging the behaviour of AC_ARG_WITH and +dnl PKG_CHECK_MODULES in a single macro. +AC_DEFUN([PKG_WITH_MODULES], +[ +m4_pushdef([with_arg], m4_tolower([$1])) + +m4_pushdef([description], + [m4_default([$5], [build with ]with_arg[ support])]) + +m4_pushdef([def_arg], [m4_default([$6], [auto])]) +m4_pushdef([def_action_if_found], [AS_TR_SH([with_]with_arg)=yes]) +m4_pushdef([def_action_if_not_found], [AS_TR_SH([with_]with_arg)=no]) + +m4_case(def_arg, + [yes],[m4_pushdef([with_without], [--without-]with_arg)], + [m4_pushdef([with_without],[--with-]with_arg)]) + +AC_ARG_WITH(with_arg, + AS_HELP_STRING(with_without, description[ @<:@default=]def_arg[@:>@]),, + [AS_TR_SH([with_]with_arg)=def_arg]) + +AS_CASE([$AS_TR_SH([with_]with_arg)], + [yes],[PKG_CHECK_MODULES([$1],[$2],$3,$4)], + [auto],[PKG_CHECK_MODULES([$1],[$2], + [m4_n([def_action_if_found]) $3], + [m4_n([def_action_if_not_found]) $4])]) + +m4_popdef([with_arg]) +m4_popdef([description]) +m4_popdef([def_arg]) + +])dnl PKG_WITH_MODULES + +dnl PKG_HAVE_WITH_MODULES(VARIABLE-PREFIX, MODULES, +dnl [DESCRIPTION], [DEFAULT]) +dnl ----------------------------------------------- +dnl +dnl Convenience macro to trigger AM_CONDITIONAL after PKG_WITH_MODULES +dnl check._[VARIABLE-PREFIX] is exported as make variable. +AC_DEFUN([PKG_HAVE_WITH_MODULES], +[ +PKG_WITH_MODULES([$1],[$2],,,[$3],[$4]) + +AM_CONDITIONAL([HAVE_][$1], + [test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"]) +])dnl PKG_HAVE_WITH_MODULES + +dnl PKG_HAVE_DEFINE_WITH_MODULES(VARIABLE-PREFIX, MODULES, +dnl [DESCRIPTION], [DEFAULT]) +dnl ------------------------------------------------------ +dnl +dnl Convenience macro to run AM_CONDITIONAL and AC_DEFINE after +dnl PKG_WITH_MODULES check. HAVE_[VARIABLE-PREFIX] is exported as make +dnl and preprocessor variable. +AC_DEFUN([PKG_HAVE_DEFINE_WITH_MODULES], +[ +PKG_HAVE_WITH_MODULES([$1],[$2],[$3],[$4]) + +AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"], + [AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])]) +])dnl PKG_HAVE_DEFINE_WITH_MODULES + # AM_CONDITIONAL -*- Autoconf -*- # Copyright (C) 1997-2021 Free Software Foundation, Inc. diff --git a/configure b/configure index 3989de82a99ecd..5b44a3d69929a0 100755 --- a/configure +++ b/configure @@ -1082,7 +1082,6 @@ with_universal_archs with_framework_name enable_framework with_app_store_compliance -with_emscripten_target enable_wasm_dynamic_linking enable_wasm_pthreads with_suffix @@ -1868,8 +1867,6 @@ Optional Packages: Enable any patches required for compiliance with app stores. Optional PATCH-FILE specifies the custom patch to apply. - --with-emscripten-target=[browser|node] - Emscripten platform --with-suffix=SUFFIX set executable suffix to SUFFIX (default is empty, yes is mapped to '.exe') --without-static-libpython @@ -4062,7 +4059,7 @@ then *-*-emscripten) ac_sys_system=Emscripten ;; - *-*-wasi) + *-*-wasi*) ac_sys_system=WASI ;; *) @@ -6193,6 +6190,8 @@ printf "%s\n" "$ac_cv_path_EGREP" >&6; } +CC_BASENAME=$(expr "//$CC" : '.*/\(.*\)') + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for CC compiler name" >&5 printf %s "checking for CC compiler name... " >&6; } if test ${ac_cv_cc_name+y} @@ -6220,9 +6219,10 @@ EOF if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then ac_cv_cc_name=`grep -v '^#' conftest.out | grep -v '^ *$' | tr -d ' '` - if test $(expr "//$CC" : '.*/\(.*\)') = "mpicc"; then - ac_cv_cc_name="mpicc" - fi + if test "x$CC_BASENAME" = xmpicc +then : + ac_cv_cc_name=mpicc +fi else ac_cv_cc_name="unknown" fi @@ -6440,7 +6440,7 @@ printf "%s\n" "$ac_cv_gcc_compat" >&6; } preset_cxx="$CXX" if test -z "$CXX" then - case "$CC" in + case "$ac_cv_cc_name" in gcc) if test -n "$ac_tool_prefix"; then # Extract the first word of "${ac_tool_prefix}g++", so it can be a program name with args. set dummy ${ac_tool_prefix}g++; ac_word=$2 @@ -6657,7 +6657,7 @@ else CXX="$ac_cv_path_CXX" fi ;; - clang|*/clang) if test -n "$ac_tool_prefix"; then + clang) if test -n "$ac_tool_prefix"; then # Extract the first word of "${ac_tool_prefix}clang++", so it can be a program name with args. set dummy ${ac_tool_prefix}clang++; ac_word=$2 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 @@ -6765,7 +6765,7 @@ else CXX="$ac_cv_path_CXX" fi ;; - icc|*/icc) if test -n "$ac_tool_prefix"; then + icc) if test -n "$ac_tool_prefix"; then # Extract the first word of "${ac_tool_prefix}icpc", so it can be a program name with args. set dummy ${ac_tool_prefix}icpc; ac_word=$2 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 @@ -7083,7 +7083,7 @@ case $host/$ac_cv_cc_name in #( PY_SUPPORT_TIER=2 ;; #( powerpc64le-*-linux-gnu/gcc) : PY_SUPPORT_TIER=2 ;; #( - wasm32-unknown-wasi/clang) : + wasm32-unknown-wasip1/clang) : PY_SUPPORT_TIER=2 ;; #( x86_64-*-linux-gnu/clang) : PY_SUPPORT_TIER=2 ;; #( @@ -7220,48 +7220,6 @@ case $ac_sys_system/$ac_sys_release in #( ;; esac -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-emscripten-target" >&5 -printf %s "checking for --with-emscripten-target... " >&6; } - -# Check whether --with-emscripten-target was given. -if test ${with_emscripten_target+y} -then : - withval=$with_emscripten_target; - if test "x$ac_sys_system" = xEmscripten -then : - - case $with_emscripten_target in #( - browser) : - ac_sys_emscripten_target=browser ;; #( - node) : - ac_sys_emscripten_target=node ;; #( - browser-debug) : - ac_sys_emscripten_target=browser-debug ;; #( - node-debug) : - ac_sys_emscripten_target=node-debug ;; #( - *) : - as_fn_error $? "Invalid argument: --with-emscripten-target=browser|node" "$LINENO" 5 - ;; -esac - -else $as_nop - - as_fn_error $? "--with-emscripten-target only applies to Emscripten" "$LINENO" 5 - -fi - -else $as_nop - - if test "x$ac_sys_system" = xEmscripten -then : - ac_sys_emscripten_target=browser -fi - -fi - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_sys_emscripten_target" >&5 -printf "%s\n" "$ac_sys_emscripten_target" >&6; } - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-wasm-dynamic-linking" >&5 printf %s "checking for --enable-wasm-dynamic-linking... " >&6; } # Check whether --enable-wasm-dynamic-linking was given. @@ -7331,12 +7289,10 @@ esac else $as_nop - case $ac_sys_system/$ac_sys_emscripten_target in #( - Emscripten/browser*) : - EXEEXT=.js ;; #( - Emscripten/node*) : + case $ac_sys_system in #( + Emscripten) : EXEEXT=.js ;; #( - WASI/*) : + WASI) : EXEEXT=.wasm ;; #( *) : EXEEXT= @@ -7374,7 +7330,7 @@ rmdir CaseSensitiveTestDir case $ac_sys_system in hp*|HP*) - case $CC in + case $ac_cv_cc_name in cc|*/cc) CC="$CC -Ae";; esac;; esac @@ -7467,7 +7423,7 @@ printf "%s\n" "$EXPORTSYMS" >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for GNU ld" >&5 printf %s "checking for GNU ld... " >&6; } ac_prog=ld -if test "$GCC" = yes; then +if test "$ac_cv_cc_name" = "gcc"; then ac_prog=`$CC -print-prog-name=ld` fi case `"$ac_prog" -V 2>&1 < /dev/null` in @@ -7671,8 +7627,8 @@ printf %s "checking HOSTRUNNER... " >&6; } if test -z "$HOSTRUNNER" then - case $ac_sys_system/$ac_sys_emscripten_target in #( - Emscripten/node*) : + case $ac_sys_system in #( + Emscripten) : if test -n "$ac_tool_prefix"; then # Extract the first word of "${ac_tool_prefix}node", so it can be a program name with args. @@ -7788,8 +7744,8 @@ then : as_fn_append HOSTRUNNER " --experimental-wasm-memory64" fi ;; #( - WASI/*) : - HOSTRUNNER='wasmtime run --wasm max-wasm-stack=16777216 --wasi preview2 --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/' ;; #( + WASI) : + HOSTRUNNER='wasmtime run --wasm max-wasm-stack=16777216 --wasi preview2=n --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/' ;; #( *) : HOSTRUNNER='' ;; @@ -7804,13 +7760,8 @@ if test -n "$HOSTRUNNER"; then fi # LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable -case $ac_sys_system/$ac_sys_emscripten_target in #( - Emscripten/browser*) : - LIBRARY_DEPS='$(PY3LIBRARY) $(WASM_STDLIB) python.html python.worker.js' ;; #( - *) : - LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)' - ;; -esac +LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)' + LINK_PYTHON_DEPS='$(LIBRARY_DEPS)' if test "$PY_ENABLE_SHARED" = 1 || test "$enable_framework" ; then LIBRARY_DEPS="\$(LDLIBRARY) $LIBRARY_DEPS" @@ -8338,8 +8289,9 @@ if test "$Py_OPT" = 'true' ; then DEF_MAKE_ALL_RULE="profile-opt" REQUIRE_PGO="yes" DEF_MAKE_RULE="build_all" - case $CC in - *gcc*) + if test "x$ac_cv_gcc_compat" = xyes +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fno-semantic-interposition" >&5 printf %s "checking whether C compiler accepts -fno-semantic-interposition... " >&6; } if test ${ax_cv_check_cflags__Werror__fno_semantic_interposition+y} @@ -8381,8 +8333,8 @@ else $as_nop : fi - ;; - esac + +fi elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then DEF_MAKE_ALL_RULE="build_wasm" REQUIRE_PGO="no" @@ -8409,7 +8361,7 @@ printf "%s\n" "$PROFILE_TASK" >&6; } llvm_bin_dir='' llvm_path="${PATH}" -if test "${CC}" = "clang" +if test "${ac_cv_cc_name}" = "clang" then clang_bin=`which clang` # Some systems install clang elsewhere as a symlink to the real path @@ -8467,8 +8419,8 @@ printf "%s\n" "no" >&6; } fi if test "$Py_LTO" = 'true' ; then - case $CC in - *clang*) + case $ac_cv_cc_name in + clang) LDFLAGS_NOLTO="-fno-lto" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -flto=thin" >&5 printf %s "checking whether C compiler accepts -flto=thin... " >&6; } @@ -8748,14 +8700,14 @@ fi ;; esac ;; - *emcc*) + emcc) if test "$Py_LTO_POLICY" != "default"; then as_fn_error $? "emcc supports only default lto." "$LINENO" 5 fi LTOFLAGS="-flto" LTOCFLAGS="-flto" ;; - *gcc*) + gcc) if test $Py_LTO_POLICY = thin then as_fn_error $? "thin lto is not supported under gcc compiler." "$LINENO" 5 @@ -8921,10 +8873,8 @@ printf "%s\n" "$as_me: llvm-profdata found via xcrun: ${LLVM_PROFDATA}" >&6;} fi LLVM_PROF_ERR=no -# GNU Autoconf recommends the use of expr instead of basename. -CC_BASENAME=$(expr "//$CC" : '.*/\(.*\)') -case "$CC_BASENAME" in - *clang*) +case "$ac_cv_cc_name" in + clang) # Any changes made here should be reflected in the GCC+Darwin case below PGO_PROF_GEN_FLAG="-fprofile-instr-generate" PGO_PROF_USE_FLAG="-fprofile-instr-use=\"\$(shell pwd)/code.profclangd\"" @@ -8939,31 +8889,13 @@ case "$CC_BASENAME" in fi fi ;; - *gcc*) - case $ac_sys_system in - Darwin*) - PGO_PROF_GEN_FLAG="-fprofile-instr-generate" - PGO_PROF_USE_FLAG="-fprofile-instr-use=\"\$(shell pwd)/code.profclangd\"" - LLVM_PROF_MERGER=" ${LLVM_PROFDATA} merge -output=\"\$(shell pwd)/code.profclangd\" \"\$(shell pwd)\"/*.profclangr " - LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"\$(shell pwd)/code-%p.profclangr\"" - if test "${LLVM_PROF_FOUND}" = "not-found" - then - LLVM_PROF_ERR=yes - if test "${REQUIRE_PGO}" = "yes" - then - as_fn_error $? "llvm-profdata is required for a --enable-optimizations build but could not be found." "$LINENO" 5 - fi - fi - ;; - *) - PGO_PROF_GEN_FLAG="-fprofile-generate" - PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" - LLVM_PROF_MERGER="true" - LLVM_PROF_FILE="" - ;; - esac + gcc) + PGO_PROF_GEN_FLAG="-fprofile-generate" + PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" ;; - *icc*) + icc) PGO_PROF_GEN_FLAG="-prof-gen" PGO_PROF_USE_FLAG="-prof-use" LLVM_PROF_MERGER="true" @@ -9329,19 +9261,6 @@ printf "%s\n" "$BOLT_APPLY_FLAGS" >&6; } # compiler and platform. BASECFLAGS tweaks need to be made even if the # user set OPT. -case $CC in - *clang*) - cc_is_clang=1 - ;; - *) - if $CC --version 2>&1 | grep -q clang - then - cc_is_clang=1 - else - cc_is_clang= - fi -esac - save_CFLAGS=$CFLAGS CFLAGS="-fstrict-overflow -fno-strict-overflow" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if $CC supports -fstrict-overflow and -fno-strict-overflow" >&5 @@ -9465,7 +9384,7 @@ if test "${OPT-unset}" = "unset" then case $GCC in yes) - if test -n "${cc_is_clang}" + if test "${ac_cv_cc_name}" != "clang" then # bpo-30104: disable strict aliasing to compile correctly dtoa.c, # see Makefile.pre.in for more information @@ -9529,34 +9448,10 @@ then : as_fn_append LINKFORSHARED " -sPROXY_TO_PTHREAD" fi - - case $ac_sys_emscripten_target in #( - browser*) : - - if test "x$ac_sys_emscripten_target" = xbrowser-debug -then : - wasm_debug=yes -fi - as_fn_append LINKFORSHARED " --preload-file=\$(WASM_ASSETS_DIR)" - WASM_ASSETS_DIR=".\$(prefix)" - WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py" - WASM_LINKFORSHARED_DEBUG="-gsource-map --emit-symbol-map" - ;; #( - node*) : - - if test "x$ac_sys_emscripten_target" = xnode-debug -then : - wasm_debug=yes -fi - as_fn_append LDFLAGS_NODIST " --pre-js=\$(srcdir)/Tools/wasm/emscripten/node_pre.js" - as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH -sNODERAWFS" - as_fn_append LINKFORSHARED " -sEXIT_RUNTIME" - WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" - - ;; #( - *) : - ;; -esac + as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH" + as_fn_append LDFLAGS_NODIST " -sEXIT_RUNTIME" + as_fn_append LDFLAGS_NODIST " --pre-js=\$(srcdir)/Tools/wasm/emscripten/node_pre.js" + WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" if test "x$wasm_debug" = xyes then : @@ -9964,8 +9859,9 @@ fi fi -case $GCC in -yes) +if test "x$ac_cv_gcc_compat" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" @@ -10083,8 +9979,8 @@ fi # ICC doesn't recognize the option, but only emits a warning ## XXX does it emit an unused result warning and can it be disabled? - case "$CC_BASENAME" in #( - *icc*) : + case "$ac_cv_cc_name" in #( + icc) : ac_cv_disable_unused_result_warning=no @@ -10489,22 +10385,19 @@ fi Darwin*) # -Wno-long-double, -no-cpp-precomp, and -mno-fused-madd # used to be here, but non-Apple gcc doesn't accept them. - if test "${CC}" = gcc - then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking which compiler should be used" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking which compiler should be used" >&5 printf %s "checking which compiler should be used... " >&6; } - case "${UNIVERSALSDK}" in - */MacOSX10.4u.sdk) - # Build using 10.4 SDK, force usage of gcc when the - # compiler is gcc, otherwise the user will get very - # confusing error messages when building on OSX 10.6 - CC=gcc-4.0 - CPP=cpp-4.0 - ;; - esac - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 + case "${UNIVERSALSDK}" in + */MacOSX10.4u.sdk) + # Build using 10.4 SDK, force usage of gcc when the + # compiler is gcc, otherwise the user will get very + # confusing error messages when building on OSX 10.6 + CC=gcc-4.0 + CPP=cpp-4.0 + ;; + esac + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 printf "%s\n" "$CC" >&6; } - fi LIPO_INTEL64_FLAGS="" if test "${enable_universalsdk}" @@ -10650,9 +10543,9 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ # end of Darwin* tests ;; esac - ;; -*) +else $as_nop + case $ac_sys_system in OpenUNIX*|UnixWare*) BASECFLAGS="$BASECFLAGS -K pentium,host,inline,loop_unroll,alloca " @@ -10661,18 +10554,18 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ BASECFLAGS="$BASECFLAGS -belf -Ki486 -DSCO5" ;; esac - ;; -esac -case "$CC_BASENAME" in -*mpicc*) +fi + +case "$ac_cv_cc_name" in +mpicc) CFLAGS_NODIST="$CFLAGS_NODIST" ;; -*icc*) +icc) # ICC needs -fp-model strict or floats behave badly CFLAGS_NODIST="$CFLAGS_NODIST -fp-model strict" ;; -*xlc*) +xlc) CFLAGS_NODIST="$CFLAGS_NODIST -qalias=noansi -qmaxmem=-1" ;; esac @@ -13195,7 +13088,7 @@ then LDSHARED="\$(LIBPL)/ld_so_aix \$(CC) -bI:\$(LIBPL)/python.exp" ;; SunOS/5*) - if test "$GCC" = "yes" ; then + if test "$ac_cv_gcc_compat" = "yes" ; then LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared' else @@ -13203,7 +13096,7 @@ then LDCXXSHARED='$(CXX) -G' fi ;; hp*|HP*) - if test "$GCC" = "yes" ; then + if test "$ac_cv_gcc_compat" = "yes" ; then LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared' else @@ -13296,7 +13189,7 @@ then LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; OpenUNIX*|UnixWare*) - if test "$GCC" = "yes" ; then + if test "$ac_cv_gcc_compat" = "yes" ; then LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared' else @@ -13340,13 +13233,13 @@ printf %s "checking CCSHARED... " >&6; } if test -z "$CCSHARED" then case $ac_sys_system/$ac_sys_release in - SunOS*) if test "$GCC" = yes; + SunOS*) if test "$ac_cv_gcc_compat" = "yes"; then CCSHARED="-fPIC"; elif test `uname -p` = sparc; then CCSHARED="-xcode=pic32"; else CCSHARED="-Kpic"; fi;; - hp*|HP*) if test "$GCC" = yes; + hp*|HP*) if test "$ac_cv_gcc_compat" = "yes"; then CCSHARED="-fPIC"; else CCSHARED="+z"; fi;; @@ -13361,12 +13254,12 @@ fi;; FreeBSD*|NetBSD*|OpenBSD*|DragonFly*) CCSHARED="-fPIC";; Haiku*) CCSHARED="-fPIC";; OpenUNIX*|UnixWare*) - if test "$GCC" = "yes" + if test "$ac_cv_gcc_compat" = "yes" then CCSHARED="-fPIC" else CCSHARED="-KPIC" fi;; SCO_SV*) - if test "$GCC" = "yes" + if test "$ac_cv_gcc_compat" = "yes" then CCSHARED="-fPIC" else CCSHARED="-Kpic -belf" fi;; @@ -13426,13 +13319,13 @@ printf "%s\n" "#define THREAD_STACK_SIZE 0x$stack_size" >>confdefs.h then LINKFORSHARED="-Wl,--export-dynamic" fi;; - SunOS/5*) case $CC in - *gcc*) + SunOS/5*) if test "$ac_cv_gcc_compat" = "yes"; then if $CC -Xlinker --help 2>&1 | grep export-dynamic >/dev/null then LINKFORSHARED="-Xlinker --export-dynamic" - fi;; - esac;; + fi + fi + ;; CYGWIN*) if test $enable_shared = "no" then @@ -13748,12 +13641,12 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - LIBUUID_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "uuid >= 2.20" 2>&1` + LIBUUID_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "uuid >= 2.20" 2>&1` else - LIBUUID_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "uuid >= 2.20" 2>&1` + LIBUUID_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "uuid >= 2.20" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$LIBUUID_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$LIBUUID_PKG_ERRORS" >&5 save_CFLAGS=$CFLAGS @@ -14005,11 +13898,11 @@ LIBS=$save_LIBS else - LIBUUID_CFLAGS=$pkg_cv_LIBUUID_CFLAGS - LIBUUID_LIBS=$pkg_cv_LIBUUID_LIBS + LIBUUID_CFLAGS=$pkg_cv_LIBUUID_CFLAGS + LIBUUID_LIBS=$pkg_cv_LIBUUID_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - have_uuid=yes + have_uuid=yes printf "%s\n" "#define HAVE_UUID_H 1" >>confdefs.h printf "%s\n" "#define HAVE_UUID_GENERATE_TIME_SAFE 1" >>confdefs.h @@ -14697,12 +14590,12 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - LIBFFI_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libffi" 2>&1` + LIBFFI_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libffi" 2>&1` else - LIBFFI_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libffi" 2>&1` + LIBFFI_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libffi" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$LIBFFI_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$LIBFFI_PKG_ERRORS" >&5 save_CFLAGS=$CFLAGS @@ -14848,11 +14741,11 @@ LIBS=$save_LIBS else - LIBFFI_CFLAGS=$pkg_cv_LIBFFI_CFLAGS - LIBFFI_LIBS=$pkg_cv_LIBFFI_LIBS + LIBFFI_CFLAGS=$pkg_cv_LIBFFI_CFLAGS + LIBFFI_LIBS=$pkg_cv_LIBFFI_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - have_libffi=yes + have_libffi=yes fi fi @@ -15174,25 +15067,25 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libmpdec >= 2.5.0" 2>&1` + LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libmpdec >= 2.5.0" 2>&1` else - LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libmpdec >= 2.5.0" 2>&1` + LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libmpdec >= 2.5.0" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$LIBMPDEC_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$LIBMPDEC_PKG_ERRORS" >&5 - LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} + LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} LIBMPDEC_INTERNAL= elif test $pkg_failed = untried; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} + LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} LIBMPDEC_INTERNAL= else - LIBMPDEC_CFLAGS=$pkg_cv_LIBMPDEC_CFLAGS - LIBMPDEC_LIBS=$pkg_cv_LIBMPDEC_LIBS + LIBMPDEC_CFLAGS=$pkg_cv_LIBMPDEC_CFLAGS + LIBMPDEC_LIBS=$pkg_cv_LIBMPDEC_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } @@ -15323,7 +15216,7 @@ esac fi elif test $ac_cv_sizeof_size_t -eq 4; then if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then - case $CC in #( + case $ac_cv_cc_name in #( *gcc*) : libmpdec_machine=ppro ;; #( *clang*) : @@ -15443,12 +15336,12 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - LIBSQLITE3_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "sqlite3 >= 3.15.2" 2>&1` + LIBSQLITE3_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "sqlite3 >= 3.15.2" 2>&1` else - LIBSQLITE3_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "sqlite3 >= 3.15.2" 2>&1` + LIBSQLITE3_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "sqlite3 >= 3.15.2" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$LIBSQLITE3_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$LIBSQLITE3_PKG_ERRORS" >&5 LIBSQLITE3_CFLAGS=${LIBSQLITE3_CFLAGS-""} @@ -15464,8 +15357,8 @@ printf "%s\n" "no" >&6; } else - LIBSQLITE3_CFLAGS=$pkg_cv_LIBSQLITE3_CFLAGS - LIBSQLITE3_LIBS=$pkg_cv_LIBSQLITE3_LIBS + LIBSQLITE3_CFLAGS=$pkg_cv_LIBSQLITE3_CFLAGS + LIBSQLITE3_LIBS=$pkg_cv_LIBSQLITE3_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } @@ -16207,24 +16100,24 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - TCLTK_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$_QUERY" 2>&1` + TCLTK_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$_QUERY" 2>&1` else - TCLTK_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$_QUERY" 2>&1` + TCLTK_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$_QUERY" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$TCLTK_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$TCLTK_PKG_ERRORS" >&5 - found_tcltk=no + found_tcltk=no elif test $pkg_failed = untried; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - found_tcltk=no + found_tcltk=no else - TCLTK_CFLAGS=$pkg_cv_TCLTK_CFLAGS - TCLTK_LIBS=$pkg_cv_TCLTK_LIBS + TCLTK_CFLAGS=$pkg_cv_TCLTK_CFLAGS + TCLTK_LIBS=$pkg_cv_TCLTK_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - found_tcltk=yes + found_tcltk=yes fi fi @@ -16304,14 +16197,14 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - X11_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "x11" 2>&1` + X11_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "x11" 2>&1` else - X11_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "x11" 2>&1` + X11_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "x11" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$X11_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$X11_PKG_ERRORS" >&5 - as_fn_error $? "Package requirements (x11) were not met: + as_fn_error $? "Package requirements (x11) were not met: $X11_PKG_ERRORS @@ -16324,7 +16217,7 @@ See the pkg-config man page for more details." "$LINENO" 5 elif test $pkg_failed = untried; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it is in your PATH or set the PKG_CONFIG environment variable to the full @@ -16337,8 +16230,8 @@ See the pkg-config man page for more details. To get pkg-config, see . See \`config.log' for more details" "$LINENO" 5; } else - X11_CFLAGS=$pkg_cv_X11_CFLAGS - X11_LIBS=$pkg_cv_X11_LIBS + X11_CFLAGS=$pkg_cv_X11_CFLAGS + X11_LIBS=$pkg_cv_X11_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } @@ -20743,12 +20636,12 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - ZLIB_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "zlib >= 1.2.0" 2>&1` + ZLIB_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "zlib >= 1.2.0" 2>&1` else - ZLIB_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "zlib >= 1.2.0" 2>&1` + ZLIB_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "zlib >= 1.2.0" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$ZLIB_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$ZLIB_PKG_ERRORS" >&5 save_CFLAGS=$CFLAGS @@ -21006,8 +20899,8 @@ LIBS=$save_LIBS else - ZLIB_CFLAGS=$pkg_cv_ZLIB_CFLAGS - ZLIB_LIBS=$pkg_cv_ZLIB_LIBS + ZLIB_CFLAGS=$pkg_cv_ZLIB_CFLAGS + ZLIB_LIBS=$pkg_cv_ZLIB_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } @@ -21091,12 +20984,12 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - BZIP2_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "bzip2" 2>&1` + BZIP2_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "bzip2" 2>&1` else - BZIP2_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "bzip2" 2>&1` + BZIP2_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "bzip2" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$BZIP2_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$BZIP2_PKG_ERRORS" >&5 save_CFLAGS=$CFLAGS @@ -21260,11 +21153,11 @@ LIBS=$save_LIBS else - BZIP2_CFLAGS=$pkg_cv_BZIP2_CFLAGS - BZIP2_LIBS=$pkg_cv_BZIP2_LIBS + BZIP2_CFLAGS=$pkg_cv_BZIP2_CFLAGS + BZIP2_LIBS=$pkg_cv_BZIP2_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - have_bzip2=yes + have_bzip2=yes fi @@ -21319,12 +21212,12 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - LIBLZMA_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblzma" 2>&1` + LIBLZMA_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblzma" 2>&1` else - LIBLZMA_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblzma" 2>&1` + LIBLZMA_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblzma" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$LIBLZMA_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$LIBLZMA_PKG_ERRORS" >&5 save_CFLAGS=$CFLAGS @@ -21488,11 +21381,11 @@ LIBS=$save_LIBS else - LIBLZMA_CFLAGS=$pkg_cv_LIBLZMA_CFLAGS - LIBLZMA_LIBS=$pkg_cv_LIBLZMA_LIBS + LIBLZMA_CFLAGS=$pkg_cv_LIBLZMA_CFLAGS + LIBLZMA_LIBS=$pkg_cv_LIBLZMA_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - have_liblzma=yes + have_liblzma=yes fi @@ -24205,10 +24098,10 @@ if ac_fn_c_try_link "$LINENO" then : -if grep noonsees conftest$EXEEXT >/dev/null ; then +if grep noonsees conftest* > /dev/null ; then ax_cv_c_float_words_bigendian=yes fi -if grep seesnoon conftest$EXEEXT >/dev/null ; then +if grep seesnoon conftest* >/dev/null ; then if test "$ax_cv_c_float_words_bigendian" = unknown; then ax_cv_c_float_words_bigendian=no else @@ -24227,41 +24120,30 @@ printf "%s\n" "$ax_cv_c_float_words_bigendian" >&6; } case $ax_cv_c_float_words_bigendian in yes) -printf "%s\n" "#define FLOAT_WORDS_BIGENDIAN 1" >>confdefs.h +printf "%s\n" "#define DOUBLE_IS_BIG_ENDIAN_IEEE754 1" >>confdefs.h ;; no) - ;; - *) - as_fn_error $? " - -Unknown float word ordering. You need to manually preset -ax_cv_c_float_words_bigendian=no (or yes) according to your system. - - " "$LINENO" 5 ;; -esac - - -if test "$ax_cv_c_float_words_bigendian" = "yes" -then - -printf "%s\n" "#define DOUBLE_IS_BIG_ENDIAN_IEEE754 1" >>confdefs.h - -elif test "$ax_cv_c_float_words_bigendian" = "no" -then printf "%s\n" "#define DOUBLE_IS_LITTLE_ENDIAN_IEEE754 1" >>confdefs.h - -else - # Some ARM platforms use a mixed-endian representation for doubles. - # While Python doesn't currently have full support for these platforms - # (see e.g., issue 1762561), we can at least make sure that float <-> string - # conversions work. - # FLOAT_WORDS_BIGENDIAN doesn't actually detect this case, but if it's not big - # or little, then it must be this? + ;; + *) + case $host_cpu in #( + *arm*) : + # Some ARM platforms use a mixed-endian representation for + # doubles. While Python doesn't currently have full support + # for these platforms (see e.g., issue 1762561), we can at + # least make sure that float <-> string conversions work. + # FLOAT_WORDS_BIGENDIAN doesn't actually detect this case, + # but if it's not big or little, then it must be this? printf "%s\n" "#define DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754 1" >>confdefs.h + ;; #( + *) : + as_fn_error $? "Unknown float word ordering. You need to manually preset ax_cv_c_float_words_bigendian=no (or yes) according to your system." "$LINENO" 5 ;; +esac ;; +esac + -fi # The short float repr introduced in Python 3.1 requires the # correctly-rounded string <-> double conversion functions from @@ -25334,12 +25216,12 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - LIBREADLINE_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "readline" 2>&1` + LIBREADLINE_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "readline" 2>&1` else - LIBREADLINE_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "readline" 2>&1` + LIBREADLINE_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "readline" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$LIBREADLINE_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$LIBREADLINE_PKG_ERRORS" >&5 save_CFLAGS=$CFLAGS @@ -25497,8 +25379,8 @@ LIBS=$save_LIBS else - LIBREADLINE_CFLAGS=$pkg_cv_LIBREADLINE_CFLAGS - LIBREADLINE_LIBS=$pkg_cv_LIBREADLINE_LIBS + LIBREADLINE_CFLAGS=$pkg_cv_LIBREADLINE_CFLAGS + LIBREADLINE_LIBS=$pkg_cv_LIBREADLINE_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } @@ -25565,12 +25447,12 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - LIBEDIT_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libedit" 2>&1` + LIBEDIT_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libedit" 2>&1` else - LIBEDIT_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libedit" 2>&1` + LIBEDIT_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libedit" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$LIBEDIT_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$LIBEDIT_PKG_ERRORS" >&5 save_CFLAGS=$CFLAGS @@ -25732,8 +25614,8 @@ LIBS=$save_LIBS else - LIBEDIT_CFLAGS=$pkg_cv_LIBEDIT_CFLAGS - LIBEDIT_LIBS=$pkg_cv_LIBEDIT_LIBS + LIBEDIT_CFLAGS=$pkg_cv_LIBEDIT_CFLAGS + LIBEDIT_LIBS=$pkg_cv_LIBEDIT_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } @@ -26594,21 +26476,21 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncursesw" 2>&1` + CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncursesw" 2>&1` else - CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncursesw" 2>&1` + CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncursesw" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$CURSES_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$CURSES_PKG_ERRORS" >&5 - have_curses=no + have_curses=no elif test $pkg_failed = untried; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - have_curses=no + have_curses=no else - CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS - CURSES_LIBS=$pkg_cv_CURSES_LIBS + CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS + CURSES_LIBS=$pkg_cv_CURSES_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } @@ -26667,21 +26549,21 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "panelw" 2>&1` + PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "panelw" 2>&1` else - PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "panelw" 2>&1` + PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "panelw" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$PANEL_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$PANEL_PKG_ERRORS" >&5 - have_panel=no + have_panel=no elif test $pkg_failed = untried; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - have_panel=no + have_panel=no else - PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS - PANEL_LIBS=$pkg_cv_PANEL_LIBS + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } @@ -26748,21 +26630,21 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncurses" 2>&1` + CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncurses" 2>&1` else - CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncurses" 2>&1` + CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncurses" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$CURSES_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$CURSES_PKG_ERRORS" >&5 - have_curses=no + have_curses=no elif test $pkg_failed = untried; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - have_curses=no + have_curses=no else - CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS - CURSES_LIBS=$pkg_cv_CURSES_LIBS + CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS + CURSES_LIBS=$pkg_cv_CURSES_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } @@ -26821,21 +26703,21 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "panel" 2>&1` + PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "panel" 2>&1` else - PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "panel" 2>&1` + PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "panel" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$PANEL_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$PANEL_PKG_ERRORS" >&5 - have_panel=no + have_panel=no elif test $pkg_failed = untried; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - have_panel=no + have_panel=no else - PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS - PANEL_LIBS=$pkg_cv_PANEL_LIBS + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } @@ -28091,7 +27973,6 @@ done SRCDIRS="\ Modules \ - Modules/_blake2 \ Modules/_ctypes \ Modules/_decimal \ Modules/_decimal/libmpdec \ @@ -28214,8 +28095,8 @@ if test "$ac_cv_gcc_asm_for_x87" = yes; then # Some versions of gcc miscompile inline asm: # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=46491 # http://gcc.gnu.org/ml/gcc/2010-11/msg00366.html - case $CC in - *gcc*) + case $ac_cv_cc_name in + gcc) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gcc ipa-pure-const bug" >&5 printf %s "checking for gcc ipa-pure-const bug... " >&6; } saved_cflags="$CFLAGS" @@ -29101,15 +28982,7 @@ else $as_nop fi else $as_nop - - case $ac_sys_system/$ac_sys_emscripten_target in #( - Emscripten/browser*) : - TEST_MODULES=no ;; #( - *) : - TEST_MODULES=yes - ;; -esac - + TEST_MODULES=yes fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $TEST_MODULES" >&5 @@ -30778,7 +30651,8 @@ esac # The SIMD files use aligned_alloc, which is not available on older versions of # Android. -if test "$ac_sys_system" != "Linux-android" || test "$ANDROID_API_LEVEL" -ge 28; then +# The *mmintrin.h headers are x86-family-specific, so can't be used on WASI. +if test "$ac_sys_system" != "Linux-android" -a "$ac_sys_system" != "WASI" || test "$ANDROID_API_LEVEL" -ge 28; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -msse -msse2 -msse3 -msse4.1 -msse4.2" >&5 printf %s "checking whether C compiler accepts -msse -msse2 -msse3 -msse4.1 -msse4.2... " >&6; } if test ${ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2+y} @@ -30845,11 +30719,12 @@ fi # The SIMD files use aligned_alloc, which is not available on older versions of # Android. +# The *mmintrin.h headers are x86-family-specific, so can't be used on WASI. # # Although AVX support is not guaranteed on Android # (https://developer.android.com/ndk/guides/abis#86-64), this is safe because we do a # runtime CPUID check. -if test "$ac_sys_system" != "Linux-android" || test "$ANDROID_API_LEVEL" -ge 28; then +if test "$ac_sys_system" != "Linux-android" -a "$ac_sys_system" != "WASI" || test "$ANDROID_API_LEVEL" -ge 28; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -mavx2" >&5 printf %s "checking whether C compiler accepts -mavx2... " >&6; } if test ${ax_cv_check_cflags__Werror__mavx2+y} diff --git a/configure.ac b/configure.ac index 6a98cc5e2a0835..7904f8990c48ee 100644 --- a/configure.ac +++ b/configure.ac @@ -336,7 +336,7 @@ then *-*-emscripten) ac_sys_system=Emscripten ;; - *-*-wasi) + *-*-wasi*) ac_sys_system=WASI ;; *) @@ -1048,6 +1048,9 @@ AC_PROG_GREP AC_PROG_SED AC_PROG_EGREP +dnl GNU Autoconf recommends the use of expr instead of basename. +AS_VAR_SET([CC_BASENAME], [$(expr "//$CC" : '.*/\(.*\)')]) + dnl detect compiler name dnl check for xlc before clang, newer xlc's can use clang as frontend. dnl check for GCC last, other compilers set __GNUC__, too. @@ -1073,9 +1076,7 @@ EOF if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then ac_cv_cc_name=`grep -v '^#' conftest.out | grep -v '^ *$' | tr -d ' '` - if test $(expr "//$CC" : '.*/\(.*\)') = "mpicc"; then - ac_cv_cc_name="mpicc" - fi + AS_VAR_IF([CC_BASENAME], [mpicc], [ac_cv_cc_name=mpicc]) else ac_cv_cc_name="unknown" fi @@ -1104,11 +1105,11 @@ AC_SUBST([CXX]) preset_cxx="$CXX" if test -z "$CXX" then - case "$CC" in + case "$ac_cv_cc_name" in gcc) AC_PATH_TOOL([CXX], [g++], [g++], [notfound]) ;; cc) AC_PATH_TOOL([CXX], [c++], [c++], [notfound]) ;; - clang|*/clang) AC_PATH_TOOL([CXX], [clang++], [clang++], [notfound]) ;; - icc|*/icc) AC_PATH_TOOL([CXX], [icpc], [icpc], [notfound]) ;; + clang) AC_PATH_TOOL([CXX], [clang++], [clang++], [notfound]) ;; + icc) AC_PATH_TOOL([CXX], [icpc], [icpc], [notfound]) ;; esac if test "$CXX" = "notfound" then @@ -1200,7 +1201,7 @@ AS_CASE([$host/$ac_cv_cc_name], [aarch64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux ARM64, glibc, gcc+clang [aarch64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], [powerpc64le-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux on PPC64 little endian, glibc, gcc - [wasm32-unknown-wasi/clang], [PY_SUPPORT_TIER=2], dnl WebAssembly System Interface, clang + [wasm32-unknown-wasip1/clang], [PY_SUPPORT_TIER=2], dnl WebAssembly System Interface preview1, clang [x86_64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], dnl Linux on AMD64, any vendor, glibc, clang [aarch64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=3], dnl Windows ARM64, MSVC @@ -1281,30 +1282,6 @@ AS_CASE([$ac_sys_system/$ac_sys_release], ] ) -AC_MSG_CHECKING([for --with-emscripten-target]) -AC_ARG_WITH([emscripten-target], - [AS_HELP_STRING([--with-emscripten-target=@<:@browser|node@:>@], [Emscripten platform])], -[ - AS_VAR_IF([ac_sys_system], [Emscripten], [ - AS_CASE([$with_emscripten_target], - [browser], [ac_sys_emscripten_target=browser], - [node], [ac_sys_emscripten_target=node], -dnl Debug builds with source map / dwarf symbols. Py_DEBUG builds easily -dnl run out of stack space. Detached sybmols and map prohibit some -dnl optimizations and increase file size. Options are undocumented so we -dnl are free to remove them in the future. - [browser-debug], [ac_sys_emscripten_target=browser-debug], - [node-debug], [ac_sys_emscripten_target=node-debug], - [AC_MSG_ERROR([Invalid argument: --with-emscripten-target=browser|node])] - ) - ], [ - AC_MSG_ERROR([--with-emscripten-target only applies to Emscripten]) - ]) -], [ - AS_VAR_IF([ac_sys_system], [Emscripten], [ac_sys_emscripten_target=browser]) -]) -AC_MSG_RESULT([$ac_sys_emscripten_target]) - dnl On Emscripten dlopen() requires -s MAIN_MODULE and -fPIC. The flags dnl disables dead code elimination and increases the size of the WASM module dnl by about 1.5 to 2MB. MAIN_MODULE defines __wasm_mutable_globals__. @@ -1349,10 +1326,9 @@ AC_ARG_WITH([suffix], [EXEEXT=$with_suffix] ) ], [ - AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/browser*], [EXEEXT=.js], - [Emscripten/node*], [EXEEXT=.js], - [WASI/*], [EXEEXT=.wasm], + AS_CASE([$ac_sys_system], + [Emscripten], [EXEEXT=.js], + [WASI], [EXEEXT=.wasm], [EXEEXT=] ) ]) @@ -1381,7 +1357,7 @@ rmdir CaseSensitiveTestDir case $ac_sys_system in hp*|HP*) - case $CC in + case $ac_cv_cc_name in cc|*/cc) CC="$CC -Ae";; esac;; esac @@ -1467,7 +1443,7 @@ AC_MSG_RESULT([$EXPORTSYMS]) AC_SUBST([GNULD]) AC_MSG_CHECKING([for GNU ld]) ac_prog=ld -if test "$GCC" = yes; then +if test "$ac_cv_cc_name" = "gcc"; then ac_prog=`$CC -print-prog-name=ld` fi case `"$ac_prog" -V 2>&1 < /dev/null` in @@ -1637,8 +1613,8 @@ AC_MSG_CHECKING([HOSTRUNNER]) AC_ARG_VAR([HOSTRUNNER], [Program to run CPython for the host platform]) if test -z "$HOSTRUNNER" then - AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/node*], [ + AS_CASE([$ac_sys_system], + [Emscripten], [ AC_PATH_TOOL([NODE], [node], [node]) HOSTRUNNER="$NODE" AS_VAR_IF([host_cpu], [wasm64], [AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-memory64"])]) @@ -1646,7 +1622,7 @@ then dnl TODO: support other WASI runtimes dnl wasmtime starts the process with "/" as CWD. For OOT builds add the dnl directory containing _sysconfigdata to PYTHONPATH. - [WASI/*], [HOSTRUNNER='wasmtime run --wasm max-wasm-stack=16777216 --wasi preview2 --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/'], + [WASI], [HOSTRUNNER='wasmtime run --wasm max-wasm-stack=16777216 --wasi preview2=n --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/'], [HOSTRUNNER=''] ) fi @@ -1659,10 +1635,8 @@ if test -n "$HOSTRUNNER"; then fi # LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable -AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/browser*], [LIBRARY_DEPS='$(PY3LIBRARY) $(WASM_STDLIB) python.html python.worker.js'], - [LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)'] -) +LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)' + LINK_PYTHON_DEPS='$(LIBRARY_DEPS)' if test "$PY_ENABLE_SHARED" = 1 || test "$enable_framework" ; then LIBRARY_DEPS="\$(LDLIBRARY) $LIBRARY_DEPS" @@ -1874,14 +1848,12 @@ if test "$Py_OPT" = 'true' ; then DEF_MAKE_ALL_RULE="profile-opt" REQUIRE_PGO="yes" DEF_MAKE_RULE="build_all" - case $CC in - *gcc*) + AS_VAR_IF([ac_cv_gcc_compat], [yes], [ AX_CHECK_COMPILE_FLAG([-fno-semantic-interposition],[ CFLAGS_NODIST="$CFLAGS_NODIST -fno-semantic-interposition" LDFLAGS_NODIST="$LDFLAGS_NODIST -fno-semantic-interposition" ], [], [-Werror]) - ;; - esac + ]) elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then dnl Emscripten does not support shared extensions yet. Build dnl "python.[js,wasm]", "pybuilddir.txt", and "platform" files. @@ -1908,7 +1880,7 @@ AC_MSG_RESULT([$PROFILE_TASK]) llvm_bin_dir='' llvm_path="${PATH}" -if test "${CC}" = "clang" +if test "${ac_cv_cc_name}" = "clang" then clang_bin=`which clang` # Some systems install clang elsewhere as a symlink to the real path @@ -1955,8 +1927,8 @@ esac ], [AC_MSG_RESULT([no])]) if test "$Py_LTO" = 'true' ; then - case $CC in - *clang*) + case $ac_cv_cc_name in + clang) LDFLAGS_NOLTO="-fno-lto" dnl Clang linker requires -flto in order to link objects with LTO information. dnl Thin LTO is faster and works for object files with full LTO information, too. @@ -2019,14 +1991,14 @@ if test "$Py_LTO" = 'true' ; then ;; esac ;; - *emcc*) + emcc) if test "$Py_LTO_POLICY" != "default"; then AC_MSG_ERROR([emcc supports only default lto.]) fi LTOFLAGS="-flto" LTOCFLAGS="-flto" ;; - *gcc*) + gcc) if test $Py_LTO_POLICY = thin then AC_MSG_ERROR([thin lto is not supported under gcc compiler.]) @@ -2085,10 +2057,8 @@ then fi LLVM_PROF_ERR=no -# GNU Autoconf recommends the use of expr instead of basename. -AS_VAR_SET([CC_BASENAME], [$(expr "//$CC" : '.*/\(.*\)')]) -case "$CC_BASENAME" in - *clang*) +case "$ac_cv_cc_name" in + clang) # Any changes made here should be reflected in the GCC+Darwin case below PGO_PROF_GEN_FLAG="-fprofile-instr-generate" PGO_PROF_USE_FLAG="-fprofile-instr-use=\"\$(shell pwd)/code.profclangd\"" @@ -2107,35 +2077,13 @@ case "$CC_BASENAME" in fi fi ;; - *gcc*) - case $ac_sys_system in - Darwin*) - PGO_PROF_GEN_FLAG="-fprofile-instr-generate" - PGO_PROF_USE_FLAG="-fprofile-instr-use=\"\$(shell pwd)/code.profclangd\"" - LLVM_PROF_MERGER=m4_normalize(" - ${LLVM_PROFDATA} merge - -output=\"\$(shell pwd)/code.profclangd\" - \"\$(shell pwd)\"/*.profclangr - ") - LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"\$(shell pwd)/code-%p.profclangr\"" - if test "${LLVM_PROF_FOUND}" = "not-found" - then - LLVM_PROF_ERR=yes - if test "${REQUIRE_PGO}" = "yes" - then - AC_MSG_ERROR([llvm-profdata is required for a --enable-optimizations build but could not be found.]) - fi - fi - ;; - *) - PGO_PROF_GEN_FLAG="-fprofile-generate" - PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" - LLVM_PROF_MERGER="true" - LLVM_PROF_FILE="" - ;; - esac + gcc) + PGO_PROF_GEN_FLAG="-fprofile-generate" + PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" ;; - *icc*) + icc) PGO_PROF_GEN_FLAG="-prof-gen" PGO_PROF_USE_FLAG="-prof-use" LLVM_PROF_MERGER="true" @@ -2259,19 +2207,6 @@ AC_MSG_RESULT([$BOLT_APPLY_FLAGS]) # compiler and platform. BASECFLAGS tweaks need to be made even if the # user set OPT. -case $CC in - *clang*) - cc_is_clang=1 - ;; - *) - if $CC --version 2>&1 | grep -q clang - then - cc_is_clang=1 - else - cc_is_clang= - fi -esac - dnl Historically, some of our code assumed that signed integer overflow dnl is defined behaviour via twos-complement. dnl Set STRICT_OVERFLOW_CFLAGS and NO_STRICT_OVERFLOW_CFLAGS depending on compiler support. @@ -2346,7 +2281,7 @@ if test "${OPT-unset}" = "unset" then case $GCC in yes) - if test -n "${cc_is_clang}" + if test "${ac_cv_cc_name}" != "clang" then # bpo-30104: disable strict aliasing to compile correctly dtoa.c, # see Makefile.pre.in for more information @@ -2403,24 +2338,11 @@ AS_CASE([$ac_sys_system], AS_VAR_APPEND([LDFLAGS_NODIST], [" -sUSE_PTHREADS"]) AS_VAR_APPEND([LINKFORSHARED], [" -sPROXY_TO_PTHREAD"]) ]) - - AS_CASE([$ac_sys_emscripten_target], - [browser*], [ - AS_VAR_IF([ac_sys_emscripten_target], [browser-debug], [wasm_debug=yes]) - AS_VAR_APPEND([LINKFORSHARED], [" --preload-file=\$(WASM_ASSETS_DIR)"]) - WASM_ASSETS_DIR=".\$(prefix)" - WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py" - dnl separate-dwarf does not seem to work in Chrome DevTools Support. - WASM_LINKFORSHARED_DEBUG="-gsource-map --emit-symbol-map" - ], - [node*], [ - AS_VAR_IF([ac_sys_emscripten_target], [node-debug], [wasm_debug=yes]) - AS_VAR_APPEND([LDFLAGS_NODIST], [" --pre-js=\$(srcdir)/Tools/wasm/emscripten/node_pre.js"]) - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sNODERAWFS"]) - AS_VAR_APPEND([LINKFORSHARED], [" -sEXIT_RUNTIME"]) - WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" - ] - ) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH"]) + dnl not completely sure whether or not we want -sEXIT_RUNTIME, keeping it for now. + AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXIT_RUNTIME"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" --pre-js=\$(srcdir)/Tools/wasm/emscripten/node_pre.js"]) + WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" AS_VAR_IF([wasm_debug], [yes], [ AS_VAR_APPEND([LDFLAGS_NODIST], [" -sASSERTIONS"]) @@ -2526,8 +2448,7 @@ then AX_CHECK_COMPILE_FLAG([-D_FORTIFY_SOURCE=3], [CFLAGS_NODIST="$CFLAGS_NODIST -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3"], [AC_MSG_WARN([-D_FORTIFY_SOURCE=3 not supported])], [-Werror]) fi -case $GCC in -yes) +AS_VAR_IF([ac_cv_gcc_compat], [yes], [ CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" PY_CHECK_CC_WARNING([enable], [extra], [if we can add -Wextra]) @@ -2568,8 +2489,8 @@ yes) # ICC doesn't recognize the option, but only emits a warning ## XXX does it emit an unused result warning and can it be disabled? - AS_CASE(["$CC_BASENAME"], - [*icc*], [ac_cv_disable_unused_result_warning=no] + AS_CASE(["$ac_cv_cc_name"], + [icc], [ac_cv_disable_unused_result_warning=no] [PY_CHECK_CC_WARNING([disable], [unused-result])]) AS_VAR_IF([ac_cv_disable_unused_result_warning], [yes], [BASECFLAGS="$BASECFLAGS -Wno-unused-result" @@ -2662,20 +2583,17 @@ yes) Darwin*) # -Wno-long-double, -no-cpp-precomp, and -mno-fused-madd # used to be here, but non-Apple gcc doesn't accept them. - if test "${CC}" = gcc - then - AC_MSG_CHECKING([which compiler should be used]) - case "${UNIVERSALSDK}" in - */MacOSX10.4u.sdk) - # Build using 10.4 SDK, force usage of gcc when the - # compiler is gcc, otherwise the user will get very - # confusing error messages when building on OSX 10.6 - CC=gcc-4.0 - CPP=cpp-4.0 - ;; - esac - AC_MSG_RESULT([$CC]) - fi + AC_MSG_CHECKING([which compiler should be used]) + case "${UNIVERSALSDK}" in + */MacOSX10.4u.sdk) + # Build using 10.4 SDK, force usage of gcc when the + # compiler is gcc, otherwise the user will get very + # confusing error messages when building on OSX 10.6 + CC=gcc-4.0 + CPP=cpp-4.0 + ;; + esac + AC_MSG_RESULT([$CC]) LIPO_INTEL64_FLAGS="" if test "${enable_universalsdk}" @@ -2800,9 +2718,7 @@ yes) # end of Darwin* tests ;; esac - ;; - -*) +], [ case $ac_sys_system in OpenUNIX*|UnixWare*) BASECFLAGS="$BASECFLAGS -K pentium,host,inline,loop_unroll,alloca " @@ -2811,18 +2727,17 @@ yes) BASECFLAGS="$BASECFLAGS -belf -Ki486 -DSCO5" ;; esac - ;; -esac +]) -case "$CC_BASENAME" in -*mpicc*) +case "$ac_cv_cc_name" in +mpicc) CFLAGS_NODIST="$CFLAGS_NODIST" ;; -*icc*) +icc) # ICC needs -fp-model strict or floats behave badly CFLAGS_NODIST="$CFLAGS_NODIST -fp-model strict" ;; -*xlc*) +xlc) CFLAGS_NODIST="$CFLAGS_NODIST -qalias=noansi -qmaxmem=-1" ;; esac @@ -3430,7 +3345,7 @@ then LDSHARED="\$(LIBPL)/ld_so_aix \$(CC) -bI:\$(LIBPL)/python.exp" ;; SunOS/5*) - if test "$GCC" = "yes" ; then + if test "$ac_cv_gcc_compat" = "yes" ; then LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared' else @@ -3438,7 +3353,7 @@ then LDCXXSHARED='$(CXX) -G' fi ;; hp*|HP*) - if test "$GCC" = "yes" ; then + if test "$ac_cv_gcc_compat" = "yes" ; then LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared' else @@ -3531,7 +3446,7 @@ then LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; OpenUNIX*|UnixWare*) - if test "$GCC" = "yes" ; then + if test "$ac_cv_gcc_compat" = "yes" ; then LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared' else @@ -3571,13 +3486,13 @@ AC_MSG_CHECKING([CCSHARED]) if test -z "$CCSHARED" then case $ac_sys_system/$ac_sys_release in - SunOS*) if test "$GCC" = yes; + SunOS*) if test "$ac_cv_gcc_compat" = "yes"; then CCSHARED="-fPIC"; elif test `uname -p` = sparc; then CCSHARED="-xcode=pic32"; else CCSHARED="-Kpic"; fi;; - hp*|HP*) if test "$GCC" = yes; + hp*|HP*) if test "$ac_cv_gcc_compat" = "yes"; then CCSHARED="-fPIC"; else CCSHARED="+z"; fi;; @@ -3589,12 +3504,12 @@ then FreeBSD*|NetBSD*|OpenBSD*|DragonFly*) CCSHARED="-fPIC";; Haiku*) CCSHARED="-fPIC";; OpenUNIX*|UnixWare*) - if test "$GCC" = "yes" + if test "$ac_cv_gcc_compat" = "yes" then CCSHARED="-fPIC" else CCSHARED="-KPIC" fi;; SCO_SV*) - if test "$GCC" = "yes" + if test "$ac_cv_gcc_compat" = "yes" then CCSHARED="-fPIC" else CCSHARED="-Kpic -belf" fi;; @@ -3652,13 +3567,13 @@ then then LINKFORSHARED="-Wl,--export-dynamic" fi;; - SunOS/5*) case $CC in - *gcc*) + SunOS/5*) if test "$ac_cv_gcc_compat" = "yes"; then if $CC -Xlinker --help 2>&1 | grep export-dynamic >/dev/null then LINKFORSHARED="-Xlinker --export-dynamic" - fi;; - esac;; + fi + fi + ;; CYGWIN*) if test $enable_shared = "no" then @@ -4228,7 +4143,7 @@ AS_VAR_IF( fi elif test $ac_cv_sizeof_size_t -eq 4; then if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then - AS_CASE([$CC], + AS_CASE([$ac_cv_cc_name], [*gcc*], [libmpdec_machine=ppro], [*clang*], [libmpdec_machine=ppro], [libmpdec_machine=ansi32] @@ -5946,28 +5861,28 @@ AS_VAR_IF([ac_cv_gcc_asm_for_x64], [yes], [ # * Check for various properties of floating point * # ************************************************** -AX_C_FLOAT_WORDS_BIGENDIAN -if test "$ax_cv_c_float_words_bigendian" = "yes" -then - AC_DEFINE([DOUBLE_IS_BIG_ENDIAN_IEEE754], [1], - [Define if C doubles are 64-bit IEEE 754 binary format, stored - with the most significant byte first]) -elif test "$ax_cv_c_float_words_bigendian" = "no" -then - AC_DEFINE([DOUBLE_IS_LITTLE_ENDIAN_IEEE754], [1], - [Define if C doubles are 64-bit IEEE 754 binary format, stored - with the least significant byte first]) -else - # Some ARM platforms use a mixed-endian representation for doubles. - # While Python doesn't currently have full support for these platforms - # (see e.g., issue 1762561), we can at least make sure that float <-> string - # conversions work. - # FLOAT_WORDS_BIGENDIAN doesn't actually detect this case, but if it's not big - # or little, then it must be this? - AC_DEFINE([DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754], [1], - [Define if C doubles are 64-bit IEEE 754 binary format, stored - in ARM mixed-endian order (byte order 45670123)]) -fi +AX_C_FLOAT_WORDS_BIGENDIAN( + [AC_DEFINE([DOUBLE_IS_BIG_ENDIAN_IEEE754], [1], + [Define if C doubles are 64-bit IEEE 754 binary format, + stored with the most significant byte first])], + [AC_DEFINE([DOUBLE_IS_LITTLE_ENDIAN_IEEE754], [1], + [Define if C doubles are 64-bit IEEE 754 binary format, + stored with the least significant byte first])], + [AS_CASE([$host_cpu], + [*arm*], [# Some ARM platforms use a mixed-endian representation for + # doubles. While Python doesn't currently have full support + # for these platforms (see e.g., issue 1762561), we can at + # least make sure that float <-> string conversions work. + # FLOAT_WORDS_BIGENDIAN doesn't actually detect this case, + # but if it's not big or little, then it must be this? + AC_DEFINE([DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754], [1], + [Define if C doubles are 64-bit IEEE 754 binary format, + stored in ARM mixed-endian order (byte order 45670123)])], + [AC_MSG_ERROR([m4_normalize([ + Unknown float word ordering. You need to manually + preset ax_cv_c_float_words_bigendian=no (or yes) + according to your system. + ])])])]) # The short float repr introduced in Python 3.1 requires the # correctly-rounded string <-> double conversion functions from @@ -7074,7 +6989,6 @@ done AC_SUBST([SRCDIRS]) SRCDIRS="\ Modules \ - Modules/_blake2 \ Modules/_ctypes \ Modules/_decimal \ Modules/_decimal/libmpdec \ @@ -7153,8 +7067,8 @@ if test "$ac_cv_gcc_asm_for_x87" = yes; then # Some versions of gcc miscompile inline asm: # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=46491 # http://gcc.gnu.org/ml/gcc/2010-11/msg00366.html - case $CC in - *gcc*) + case $ac_cv_cc_name in + gcc) AC_MSG_CHECKING([for gcc ipa-pure-const bug]) saved_cflags="$CFLAGS" CFLAGS="-O2" @@ -7509,12 +7423,7 @@ AC_MSG_CHECKING([for --disable-test-modules]) AC_ARG_ENABLE([test-modules], [AS_HELP_STRING([--disable-test-modules], [don't build nor install test modules])], [ AS_VAR_IF([enable_test_modules], [yes], [TEST_MODULES=yes], [TEST_MODULES=no]) -], [ - AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/browser*], [TEST_MODULES=no], - [TEST_MODULES=yes] - ) -]) +], [TEST_MODULES=yes]) AC_MSG_RESULT([$TEST_MODULES]) AC_SUBST([TEST_MODULES]) @@ -7856,7 +7765,8 @@ AC_SUBST([LIBHACL_CFLAGS]) # The SIMD files use aligned_alloc, which is not available on older versions of # Android. -if test "$ac_sys_system" != "Linux-android" || test "$ANDROID_API_LEVEL" -ge 28; then +# The *mmintrin.h headers are x86-family-specific, so can't be used on WASI. +if test "$ac_sys_system" != "Linux-android" -a "$ac_sys_system" != "WASI" || test "$ANDROID_API_LEVEL" -ge 28; then dnl This can be extended here to detect e.g. Power8, which HACL* should also support. AX_CHECK_COMPILE_FLAG([-msse -msse2 -msse3 -msse4.1 -msse4.2],[ [LIBHACL_SIMD128_FLAGS="-msse -msse2 -msse3 -msse4.1 -msse4.2"] @@ -7882,11 +7792,12 @@ AC_SUBST([LIBHACL_SIMD128_OBJS]) # The SIMD files use aligned_alloc, which is not available on older versions of # Android. +# The *mmintrin.h headers are x86-family-specific, so can't be used on WASI. # # Although AVX support is not guaranteed on Android # (https://developer.android.com/ndk/guides/abis#86-64), this is safe because we do a # runtime CPUID check. -if test "$ac_sys_system" != "Linux-android" || test "$ANDROID_API_LEVEL" -ge 28; then +if test "$ac_sys_system" != "Linux-android" -a "$ac_sys_system" != "WASI" || test "$ANDROID_API_LEVEL" -ge 28; then AX_CHECK_COMPILE_FLAG([-mavx2],[ [LIBHACL_SIMD256_FLAGS="-mavx2"] AC_DEFINE([HACL_CAN_COMPILE_SIMD256], [1], [HACL* library can compile SIMD256 implementations]) diff --git a/iOS/README.rst b/iOS/README.rst index 4d7c344d5e9e17..e33455eef8f44a 100644 --- a/iOS/README.rst +++ b/iOS/README.rst @@ -351,13 +351,13 @@ Running specific tests ^^^^^^^^^^^^^^^^^^^^^^ As the test suite is being executed on an iOS simulator, it is not possible to -pass in command line arguments to configure test suite operation. To work around -this limitation, the arguments that would normally be passed as command line -arguments are configured as a static string at the start of the XCTest method -``- (void)testPython`` in ``iOSTestbedTests.m``. To pass an argument to the test -suite, add a a string to the ``argv`` definition. These arguments will be passed -to the test suite as if they had been passed to ``python -m test`` at the -command line. +pass in command line arguments to configure test suite operation. To work +around this limitation, the arguments that would normally be passed as command +line arguments are configured as part of the ``iOSTestbed-Info.plist`` file +that is used to configure the iOS testbed app. In this file, the ``TestArgs`` +key is an array containing the arguments that would be passed to ``python -m`` +on the command line (including ``test`` in position 0, the name of the test +module to be executed). Disabling automated breakpoints ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj index d57cfc3dbe0304..6819ac0eeed95f 100644 --- a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj +++ b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj @@ -17,6 +17,8 @@ 607A66502B0EFFE00010BFC8 /* Python.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; }; 607A66512B0EFFE00010BFC8 /* Python.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; 607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */ = {isa = PBXBuildFile; fileRef = 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */; }; + 608619542CB77BA900F46182 /* app_packages in Resources */ = {isa = PBXBuildFile; fileRef = 608619532CB77BA900F46182 /* app_packages */; }; + 608619562CB7819B00F46182 /* app in Resources */ = {isa = PBXBuildFile; fileRef = 608619552CB7819B00F46182 /* app */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -66,6 +68,8 @@ 607A664A2B0EFB310010BFC8 /* Python.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = Python.xcframework; sourceTree = ""; }; 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "dylib-Info-template.plist"; sourceTree = ""; }; 607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "iOSTestbed-Info.plist"; sourceTree = ""; }; + 608619532CB77BA900F46182 /* app_packages */ = {isa = PBXFileReference; lastKnownFileType = folder; path = app_packages; sourceTree = ""; }; + 608619552CB7819B00F46182 /* app */ = {isa = PBXFileReference; lastKnownFileType = folder; path = app; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -111,6 +115,8 @@ 607A66142B0EFA380010BFC8 /* iOSTestbed */ = { isa = PBXGroup; children = ( + 608619552CB7819B00F46182 /* app */, + 608619532CB77BA900F46182 /* app_packages */, 607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */, 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */, 607A66152B0EFA380010BFC8 /* AppDelegate.h */, @@ -223,7 +229,9 @@ files = ( 607A66252B0EFA390010BFC8 /* LaunchScreen.storyboard in Resources */, 607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */, + 608619562CB7819B00F46182 /* app in Resources */, 607A66222B0EFA390010BFC8 /* Assets.xcassets in Resources */, + 608619542CB77BA900F46182 /* app_packages in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -273,7 +281,7 @@ ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The location of the extension file, relative to the bundle\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n # The path to the extension file, relative to the install base\n PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n # The full dotted name of the extension module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n # Create a back reference to the .so file location in the framework\n echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\" \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; + shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The location of the extension file, relative to the bundle\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n # The path to the extension file, relative to the install base\n PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n # The full dotted name of the extension module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n # Create a back reference to the .so file location in the framework\n echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\" \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\necho \"Install app package extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app_packages\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app_packages/ \"$FULL_EXT\"\ndone\necho \"Install app extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; }; /* End PBXShellScriptBuildPhase section */ diff --git a/iOS/testbed/iOSTestbed/app/README b/iOS/testbed/iOSTestbed/app/README new file mode 100644 index 00000000000000..af22c685f87976 --- /dev/null +++ b/iOS/testbed/iOSTestbed/app/README @@ -0,0 +1,7 @@ +This folder can contain any Python application code. + +During the build, any binary modules found in this folder will be processed into +iOS Framework form. + +When the test suite runs, this folder will be on the PYTHONPATH, and will be the +working directory for the test suite. diff --git a/iOS/testbed/iOSTestbed/app_packages/README b/iOS/testbed/iOSTestbed/app_packages/README new file mode 100644 index 00000000000000..42d7fdeb813250 --- /dev/null +++ b/iOS/testbed/iOSTestbed/app_packages/README @@ -0,0 +1,7 @@ +This folder can be a target for installing any Python dependencies needed by the +test suite. + +During the build, any binary modules found in this folder will be processed into +iOS Framework form. + +When the test suite runs, this folder will be on the PYTHONPATH. diff --git a/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist b/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist index e2aa460b6fd5ee..a582f42a212783 100644 --- a/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist +++ b/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist @@ -41,8 +41,18 @@ UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight - MainModule - ios + TestArgs + + test + -uall + --single-process + --rerun + -W + + UIApplicationSceneManifest UIApplicationSupportsMultipleScenes diff --git a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m index 9bf502a808eb88..db00d43da85cbc 100644 --- a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m +++ b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m @@ -9,30 +9,38 @@ @implementation iOSTestbedTests - (void)testPython { - // Arguments to pass into the test suite runner. - // argv[0] must identify the process; any subsequent arg - // will be handled as if it were an argument to `python -m test` - const char *argv[] = { - "iOSTestbed", // argv[0] is the process that is running. - "-uall", // Enable all resources - "--single-process", // always run all tests sequentially in a single process - "--rerun", // Re-run failed tests in verbose mode - "-W", // Display test output on failure - // To run a subset of tests, add the test names below; e.g., - // "test_os", - // "test_sys", - }; - - // Start a Python interpreter. + const char **argv; int exit_code; + int failed; PyStatus status; PyPreConfig preconfig; PyConfig config; + PyObject *sys_module; + PyObject *sys_path_attr; + NSArray *test_args; NSString *python_home; + NSString *path; wchar_t *wtmp_str; NSString *resourcePath = [[NSBundle mainBundle] resourcePath]; + // Disable all color, as the Xcode log can't display color + setenv("NO_COLOR", "1", true); + + // Arguments to pass into the test suite runner. + // argv[0] must identify the process; any subsequent arg + // will be handled as if it were an argument to `python -m test` + test_args = [[NSBundle mainBundle] objectForInfoDictionaryKey:@"TestArgs"]; + if (test_args == NULL) { + NSLog(@"Unable to identify test arguments."); + } + argv = malloc(sizeof(char *) * ([test_args count] + 1)); + argv[0] = "iOSTestbed"; + for (int i = 1; i < [test_args count]; i++) { + argv[i] = [[test_args objectAtIndex:i] UTF8String]; + } + NSLog(@"Test command: %@", test_args); + // Generate an isolated Python configuration. NSLog(@"Configuring isolated Python..."); PyPreConfig_InitIsolatedConfig(&preconfig); @@ -50,7 +58,7 @@ - (void)testPython { // Ensure that signal handlers are installed config.install_signal_handlers = 1; // Run the test module. - config.run_module = Py_DecodeLocale("test", NULL); + config.run_module = Py_DecodeLocale([[test_args objectAtIndex:0] UTF8String], NULL); // For debugging - enable verbose mode. // config.verbose = 1; @@ -83,7 +91,7 @@ - (void)testPython { } NSLog(@"Configure argc/argv..."); - status = PyConfig_SetBytesArgv(&config, sizeof(argv) / sizeof(char *), (char**) argv); + status = PyConfig_SetBytesArgv(&config, [test_args count], (char**) argv); if (PyStatus_Exception(status)) { XCTFail(@"Unable to configure argc/argv: %s", status.err_msg); PyConfig_Clear(&config); @@ -98,11 +106,47 @@ - (void)testPython { return; } + sys_module = PyImport_ImportModule("sys"); + if (sys_module == NULL) { + XCTFail(@"Could not import sys module"); + return; + } + + sys_path_attr = PyObject_GetAttrString(sys_module, "path"); + if (sys_path_attr == NULL) { + XCTFail(@"Could not access sys.path"); + return; + } + + // Add the app packages path + path = [NSString stringWithFormat:@"%@/app_packages", resourcePath, nil]; + NSLog(@"App packages path: %@", path); + wtmp_str = Py_DecodeLocale([path UTF8String], NULL); + failed = PyList_Insert(sys_path_attr, 0, PyUnicode_FromString([path UTF8String])); + if (failed) { + XCTFail(@"Unable to add app packages to sys.path"); + return; + } + PyMem_RawFree(wtmp_str); + + path = [NSString stringWithFormat:@"%@/app", resourcePath, nil]; + NSLog(@"App path: %@", path); + wtmp_str = Py_DecodeLocale([path UTF8String], NULL); + failed = PyList_Insert(sys_path_attr, 0, PyUnicode_FromString([path UTF8String])); + if (failed) { + XCTFail(@"Unable to add app to sys.path"); + return; + } + PyMem_RawFree(wtmp_str); + + // Ensure the working directory is the app folder. + chdir([path UTF8String]); + // Start the test suite. Print a separator to differentiate Python startup logs from app logs NSLog(@"---------------------------------------------------------------------------"); exit_code = Py_RunMain(); - XCTAssertEqual(exit_code, 0, @"Python test suite did not pass"); + XCTAssertEqual(exit_code, 0, @"Test suite did not pass"); NSLog(@"---------------------------------------------------------------------------"); diff --git a/pyconfig.h.in b/pyconfig.h.in index fcb8a965b1e476..924d86627b0e9b 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -47,10 +47,6 @@ /* Define if --enable-ipv6 is specified */ #undef ENABLE_IPV6 -/* Define to 1 if your system stores words within floats with the most - significant word first */ -#undef FLOAT_WORDS_BIGENDIAN - /* Define if getpgrp() must be called as getpgrp(0). */ #undef GETPGRP_HAVE_ARG