diff --git a/.editorconfig b/.editorconfig index 0169eed951cd3f1..a6187d64f3ce46e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,6 +1,6 @@ root = true -[*.{py,c,cpp,h,rst,md,yml}] +[*.{py,c,cpp,h,js,rst,md,yml}] trim_trailing_whitespace = true insert_final_newline = true indent_style = space @@ -11,5 +11,5 @@ indent_size = 4 [*.rst] indent_size = 3 -[*.yml] +[*.{js,yml}] indent_size = 2 diff --git a/.gitattributes b/.gitattributes index 2a48df079e1aeb0..4f82cea7480cfe8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -77,6 +77,7 @@ Include/internal/pycore_opcode.h generated Include/internal/pycore_opcode_metadata.h generated Include/internal/pycore_*_generated.h generated Include/internal/pycore_uop_ids.h generated +Include/internal/pycore_uop_metadata.h generated Include/opcode.h generated Include/opcode_ids.h generated Include/token.h generated @@ -94,6 +95,7 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated +Python/optimizer_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ae915423ece9551..1d0bce111320ff6 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -21,6 +21,7 @@ configure* @erlend-aasland @corona10 **/*context* @1st1 **/*genobject* @markshannon **/*hamt* @1st1 +**/*jit* @brandtbucher Objects/set* @rhettinger Objects/dict* @methane @markshannon Objects/typevarobject.c @JelleZijlstra @@ -36,8 +37,9 @@ Python/flowgraph.c @markshannon @iritkatriel Python/ast_opt.c @isidentical Python/bytecodes.c @markshannon @gvanrossum Python/optimizer*.c @markshannon @gvanrossum +Python/optimizer_analysis.c @Fidget-Spinner +Python/optimizer_bytecodes.c @Fidget-Spinner Lib/test/test_patma.py @brandtbucher -Lib/test/test_peepholer.py @brandtbucher Lib/test/test_type_*.py @JelleZijlstra Lib/test/test_capi/test_misc.py @markshannon @gvanrossum Tools/c-analyzer/ @ericsnowcurrently @@ -230,8 +232,8 @@ Doc/c-api/stable.rst @encukou **/*zipfile/_path/* @jaraco # Argument Clinic -/Tools/clinic/** @erlend-aasland @AlexWaygood -/Lib/test/test_clinic.py @erlend-aasland @AlexWaygood +/Tools/clinic/** @erlend-aasland +/Lib/test/test_clinic.py @erlend-aasland Doc/howto/clinic.rst @erlend-aasland # Subinterpreters @@ -247,3 +249,7 @@ Lib/test/test_interpreters/ @ericsnowcurrently # SBOM /Misc/sbom.spdx.json @sethmlarson /Tools/build/generate_sbom.py @sethmlarson + +# Config Parser +Lib/configparser.py @jaraco +Lib/test/test_configparser.py @jaraco diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cc5ecc09fbc5923..20d1fad40ecafee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -131,11 +131,13 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.x' + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }} - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Add ccache to PATH @@ -160,7 +162,7 @@ jobs: - name: Build CPython run: | make -j4 regen-all - make regen-stdlib-module-names + make regen-stdlib-module-names regen-sbom - name: Check for changes run: | git add -u @@ -250,7 +252,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1w, 3.0.11, 3.1.3] + openssl_ver: [1.1.1w, 3.0.13, 3.1.5, 3.2.1] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl @@ -258,11 +260,13 @@ jobs: LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib steps: - uses: actions/checkout@v4 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -274,7 +278,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -304,7 +308,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' env: - OPENSSL_VER: 3.0.11 + OPENSSL_VER: 3.0.13 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 @@ -319,7 +323,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -341,11 +345,13 @@ jobs: run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR - name: Bind mount sources read-only run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -375,7 +381,7 @@ jobs: ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -r ${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt - name: 'Restore Hypothesis database' id: cache-hypothesis-database - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./hypothesis key: hypothesis-database-${{ github.head_ref || github.run_id }} @@ -415,16 +421,18 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 3.0.11 + OPENSSL_VER: 3.0.13 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - uses: actions/checkout@v4 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -440,7 +448,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml new file mode 100644 index 000000000000000..43d0b2c1b4016ca --- /dev/null +++ b/.github/workflows/jit.yml @@ -0,0 +1,131 @@ +name: JIT +on: + pull_request: + paths: + - '**jit**' + - 'Python/bytecodes.c' + - 'Python/optimizer*.c' + - 'Python/optimizer_bytecodes.c' + push: + paths: + - '**jit**' + - 'Python/bytecodes.c' + - 'Python/optimizer*.c' + - 'Python/optimizer_bytecodes.c' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + jit: + name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + target: + - i686-pc-windows-msvc/msvc + - x86_64-pc-windows-msvc/msvc + - x86_64-apple-darwin/clang + - aarch64-apple-darwin/clang + - x86_64-unknown-linux-gnu/gcc + - x86_64-unknown-linux-gnu/clang + - aarch64-unknown-linux-gnu/gcc + - aarch64-unknown-linux-gnu/clang + debug: + - true + - false + llvm: + - 16 + include: + - target: i686-pc-windows-msvc/msvc + architecture: Win32 + runner: windows-latest + compiler: msvc + - target: x86_64-pc-windows-msvc/msvc + architecture: x64 + runner: windows-latest + compiler: msvc + - target: x86_64-apple-darwin/clang + architecture: x86_64 + runner: macos-13 + compiler: clang + - target: aarch64-apple-darwin/clang + architecture: aarch64 + runner: macos-14 + compiler: clang + - target: x86_64-unknown-linux-gnu/gcc + architecture: x86_64 + runner: ubuntu-latest + compiler: gcc + - target: x86_64-unknown-linux-gnu/clang + architecture: x86_64 + runner: ubuntu-latest + compiler: clang + - target: aarch64-unknown-linux-gnu/gcc + architecture: aarch64 + runner: ubuntu-latest + compiler: gcc + # These fail because of emulation, not because of the JIT: + exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection + - target: aarch64-unknown-linux-gnu/clang + architecture: aarch64 + runner: ubuntu-latest + compiler: clang + # These fail because of emulation, not because of the JIT: + exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection + env: + CC: ${{ matrix.compiler }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Windows + if: runner.os == 'Windows' + run: | + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} + ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} + ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + + - name: macOS + if: runner.os == 'macOS' + run: | + brew install llvm@${{ matrix.llvm }} + export SDKROOT="$(xcrun --show-sdk-path)" + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + make all --jobs 4 + ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + + - name: Native Linux + if: runner.os == 'Linux' && matrix.architecture == 'x86_64' + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + make all --jobs 4 + ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + + - name: Emulated Linux + if: runner.os == 'Linux' && matrix.architecture != 'x86_64' + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + ./configure --prefix="$(pwd)/../build" + make install --jobs 4 + make clean --jobs 4 + export HOST=${{ matrix.architecture }}-linux-gnu + sudo apt install --yes "gcc-$HOST" qemu-user + ${{ !matrix.debug && matrix.compiler == 'clang' && './configure --enable-optimizations' || '' }} + ${{ !matrix.debug && matrix.compiler == 'clang' && 'make profile-run-stamp --jobs 4' || '' }} + export CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" + export CPP="$CC --preprocess" + export HOSTRUNNER=qemu-${{ matrix.architecture }} + export QEMU_LD_PREFIX="/usr/$HOST" + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes + make all --jobs 4 + ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 11928e72b9b43a1..b766785de405d2e 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -12,6 +12,7 @@ on: - "Tools/build/generate_sbom.py" - "Tools/cases_generator/**" - "Tools/clinic/**" + - "Tools/jit/**" - "Tools/peg_generator/**" - "Tools/requirements-dev.txt" - "Tools/wasm/**" @@ -38,6 +39,7 @@ jobs: "Tools/build/", "Tools/cases_generator", "Tools/clinic", + "Tools/jit", "Tools/peg_generator", "Tools/wasm", ] diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index 080204bcfd3b94f..ff5cbdf3eda749a 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -11,6 +11,7 @@ permissions: jobs: label: name: DO-NOT-MERGE / unresolved review + if: github.repository_owner == 'python' runs-on: ubuntu-latest timeout-minutes: 10 diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index e534751ee1011da..cea8f93d67b29c1 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -89,7 +89,7 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v4 - - uses: actions/cache@v3 + - uses: actions/cache@v4 with: path: ~/.cache/pip key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }} diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index c24b6e963ddfd67..65b73cd791c75c7 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -12,20 +12,30 @@ on: jobs: build_macos: name: 'build and test' - runs-on: macos-latest timeout-minutes: 60 env: HOMEBREW_NO_ANALYTICS: 1 HOMEBREW_NO_AUTO_UPDATE: 1 HOMEBREW_NO_INSTALL_CLEANUP: 1 + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 PYTHONSTRICTEXTENSIONBUILD: 1 + strategy: + fail-fast: false + matrix: + os: [ + "macos-14", # M1 + "macos-13", # Intel + ] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }} + key: ${{ github.job }}-${{ matrix.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk - name: Configure CPython diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index c2194280c0a50f4..ee64fe62a0bd0a2 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -14,7 +14,7 @@ jobs: timeout-minutes: 60 runs-on: ubuntu-20.04 env: - OPENSSL_VER: 3.0.11 + OPENSSL_VER: 3.0.13 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 @@ -29,7 +29,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -52,11 +52,13 @@ jobs: run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR - name: Bind mount sources read-only run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: ${{ inputs.options }} diff --git a/.gitignore b/.gitignore index c424a894c2a6e09..2194b393aa48211 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,7 @@ Lib/test/data/* /_bootstrap_python /Makefile /Makefile.pre +iOS/Resources/Info.plist Mac/Makefile Mac/PythonLauncher/Info.plist Mac/PythonLauncher/Makefile @@ -126,6 +127,7 @@ Tools/unicode/data/ # hendrikmuhs/ccache-action@v1 /.ccache /cross-build/ +/jit_stencils.h /platform /profile-clean-stamp /profile-run-stamp @@ -158,5 +160,5 @@ Python/frozen_modules/MANIFEST /python !/Python/ -# main branch only: ABI files are not checked/maintained +# main branch only: ABI files are not checked/maintained. Doc/data/python*.abi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 19033ce243d9d35..69d85238985150c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.7 + rev: v0.2.0 hooks: - id: ruff name: Run Ruff on Lib/test/ diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index e572815ffd6259f..1e1cabdf242bd13 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -29,7 +29,7 @@ without intermediate copying. Python provides such a facility at the C level in the form of the :ref:`buffer protocol `. This protocol has two sides: -.. index:: single: PyBufferProcs +.. index:: single: PyBufferProcs (C type) - on the producer side, a type can export a "buffer interface" which allows objects of that type to expose information about their underlying buffer. diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 5082b0cb6ad3f3e..f6fdd7574323c7e 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -22,16 +22,21 @@ bound into a function. .. c:var:: PyTypeObject PyCode_Type This is an instance of :c:type:`PyTypeObject` representing the Python - :class:`code` type. + :ref:`code object `. .. c:function:: int PyCode_Check(PyObject *co) - Return true if *co* is a :class:`code` object. This function always succeeds. + Return true if *co* is a :ref:`code object `. + This function always succeeds. -.. c:function:: int PyCode_GetNumFree(PyCodeObject *co) +.. c:function:: Py_ssize_t PyCode_GetNumFree(PyCodeObject *co) - Return the number of free variables in *co*. + Return the number of free variables in a code object. + +.. c:function:: int PyCode_GetFirstFree(PyCodeObject *co) + + Return the position of the first free variable in a code object. .. c:function:: PyCodeObject* PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable) @@ -48,7 +53,7 @@ bound into a function. .. versionchanged:: 3.11 Added ``qualname`` and ``exceptiontable`` parameters. - .. index:: single: PyCode_New + .. index:: single: PyCode_New (C function) .. versionchanged:: 3.12 @@ -61,7 +66,7 @@ bound into a function. Similar to :c:func:`PyUnstable_Code_New`, but with an extra "posonlyargcount" for positional-only arguments. The same caveats that apply to ``PyUnstable_Code_New`` also apply to this function. - .. index:: single: PyCode_NewWithPosOnlyArgs + .. index:: single: PyCode_NewWithPosOnlyArgs (C function) .. versionadded:: 3.8 as ``PyCode_NewWithPosOnlyArgs`` @@ -220,7 +225,7 @@ may change without deprecation warnings. *free* will be called on non-``NULL`` data stored under the new index. Use :c:func:`Py_DecRef` when storing :c:type:`PyObject`. - .. index:: single: _PyEval_RequestCodeExtraIndex + .. index:: single: _PyEval_RequestCodeExtraIndex (C function) .. versionadded:: 3.6 as ``_PyEval_RequestCodeExtraIndex`` @@ -238,7 +243,7 @@ may change without deprecation warnings. If no data was set under the index, set *extra* to ``NULL`` and return 0 without setting an exception. - .. index:: single: _PyCode_GetExtra + .. index:: single: _PyCode_GetExtra (C function) .. versionadded:: 3.6 as ``_PyCode_GetExtra`` @@ -253,7 +258,7 @@ may change without deprecation warnings. Set the extra data stored under the given index to *extra*. Return 0 on success. Set an exception and return -1 on failure. - .. index:: single: _PyCode_SetExtra + .. index:: single: _PyCode_SetExtra (C function) .. versionadded:: 3.6 as ``_PyCode_SetExtra`` diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 8471c98d0448721..03f3d28187bfe9a 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -174,6 +174,26 @@ Dictionary Objects .. versionadded:: 3.4 +.. c:function:: int PyDict_SetDefaultRef(PyObject *p, PyObject *key, PyObject *default_value, PyObject **result) + + Inserts *default_value* into the dictionary *p* with a key of *key* if the + key is not already present in the dictionary. If *result* is not ``NULL``, + then *\*result* is set to a :term:`strong reference` to either + *default_value*, if the key was not present, or the existing value, if *key* + was already present in the dictionary. + Returns ``1`` if the key was present and *default_value* was not inserted, + or ``0`` if the key was not present and *default_value* was inserted. + On failure, returns ``-1``, sets an exception, and sets ``*result`` + to ``NULL``. + + For clarity: if you have a strong reference to *default_value* before + calling this function, then after it returns, you hold a strong reference + to both *default_value* and *\*result* (if it's not ``NULL``). + These may refer to the same object: in that case you hold two separate + references to it. + .. versionadded:: 3.13 + + .. c:function:: int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result) Remove *key* from dictionary *p* and optionally return the removed value. diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index c7e3cd9463e5d77..ba13fd1b9973e03 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -105,7 +105,7 @@ Printing and clearing parameters help format the warning message; they have the same meaning and values as in :c:func:`PyUnicode_FromFormat`. ``PyErr_WriteUnraisable(obj)`` is roughtly equivalent to - ``PyErr_FormatUnraisable("Exception ignored in: %R, obj)``. + ``PyErr_FormatUnraisable("Exception ignored in: %R", obj)``. If *format* is ``NULL``, only the traceback is printed. .. versionadded:: 3.13 @@ -180,7 +180,7 @@ For convenience, some of these functions will always return a .. c:function:: PyObject* PyErr_SetFromErrno(PyObject *type) - .. index:: single: strerror() + .. index:: single: strerror (C function) This is a convenience function to raise an exception when a C library function has returned an error and set the C variable :c:data:`errno`. It constructs a @@ -396,7 +396,7 @@ an error value). .. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) Function similar to :c:func:`PyErr_WarnFormat`, but *category* is - :exc:`ResourceWarning` and it passes *source* to :func:`warnings.WarningMessage`. + :exc:`ResourceWarning` and it passes *source* to :class:`!warnings.WarningMessage`. .. versionadded:: 3.6 @@ -635,7 +635,7 @@ Signal Handling .. index:: pair: module; signal - single: SIGINT + single: SIGINT (C macro) single: KeyboardInterrupt (built-in exception) This function interacts with Python's signal handling. @@ -666,7 +666,7 @@ Signal Handling .. index:: pair: module; signal - single: SIGINT + single: SIGINT (C macro) single: KeyboardInterrupt (built-in exception) Simulate the effect of a :c:macro:`!SIGINT` signal arriving. @@ -732,7 +732,7 @@ Exception Classes This creates a class object derived from :exc:`Exception` (accessible in C as :c:data:`PyExc_Exception`). - The :attr:`__module__` attribute of the new class is set to the first part (up + The :attr:`!__module__` attribute of the new class is set to the first part (up to the last dot) of the *name* argument, and the class name is set to the last part (after the last dot). The *base* argument can be used to specify alternate base classes; it can either be only one class or a tuple of classes. The *dict* @@ -904,8 +904,8 @@ because the :ref:`call protocol ` takes care of recursion handling. Marks a point where a recursive C-level call is about to be performed. - If :c:macro:`USE_STACKCHECK` is defined, this function checks if the OS - stack overflowed using :c:func:`PyOS_CheckStack`. In this is the case, it + If :c:macro:`!USE_STACKCHECK` is defined, this function checks if the OS + stack overflowed using :c:func:`PyOS_CheckStack`. If this is the case, it sets a :exc:`MemoryError` and returns a nonzero value. The function then checks if the recursion limit is reached. If this is the @@ -968,59 +968,59 @@ All standard Python exceptions are available as global variables whose names are the variables: .. index:: - single: PyExc_BaseException - single: PyExc_Exception - single: PyExc_ArithmeticError - single: PyExc_AssertionError - single: PyExc_AttributeError - single: PyExc_BlockingIOError - single: PyExc_BrokenPipeError - single: PyExc_BufferError - single: PyExc_ChildProcessError - single: PyExc_ConnectionAbortedError - single: PyExc_ConnectionError - single: PyExc_ConnectionRefusedError - single: PyExc_ConnectionResetError - single: PyExc_EOFError - single: PyExc_FileExistsError - single: PyExc_FileNotFoundError - single: PyExc_FloatingPointError - single: PyExc_GeneratorExit - single: PyExc_ImportError - single: PyExc_IndentationError - single: PyExc_IndexError - single: PyExc_InterruptedError - single: PyExc_IsADirectoryError - single: PyExc_KeyError - single: PyExc_KeyboardInterrupt - single: PyExc_LookupError - single: PyExc_MemoryError - single: PyExc_ModuleNotFoundError - single: PyExc_NameError - single: PyExc_NotADirectoryError - single: PyExc_NotImplementedError - single: PyExc_OSError - single: PyExc_OverflowError - single: PyExc_PermissionError - single: PyExc_ProcessLookupError - single: PyExc_RecursionError - single: PyExc_ReferenceError - single: PyExc_RuntimeError - single: PyExc_StopAsyncIteration - single: PyExc_StopIteration - single: PyExc_SyntaxError - single: PyExc_SystemError - single: PyExc_SystemExit - single: PyExc_TabError - single: PyExc_TimeoutError - single: PyExc_TypeError - single: PyExc_UnboundLocalError - single: PyExc_UnicodeDecodeError - single: PyExc_UnicodeEncodeError - single: PyExc_UnicodeError - single: PyExc_UnicodeTranslateError - single: PyExc_ValueError - single: PyExc_ZeroDivisionError + single: PyExc_BaseException (C var) + single: PyExc_Exception (C var) + single: PyExc_ArithmeticError (C var) + single: PyExc_AssertionError (C var) + single: PyExc_AttributeError (C var) + single: PyExc_BlockingIOError (C var) + single: PyExc_BrokenPipeError (C var) + single: PyExc_BufferError (C var) + single: PyExc_ChildProcessError (C var) + single: PyExc_ConnectionAbortedError (C var) + single: PyExc_ConnectionError (C var) + single: PyExc_ConnectionRefusedError (C var) + single: PyExc_ConnectionResetError (C var) + single: PyExc_EOFError (C var) + single: PyExc_FileExistsError (C var) + single: PyExc_FileNotFoundError (C var) + single: PyExc_FloatingPointError (C var) + single: PyExc_GeneratorExit (C var) + single: PyExc_ImportError (C var) + single: PyExc_IndentationError (C var) + single: PyExc_IndexError (C var) + single: PyExc_InterruptedError (C var) + single: PyExc_IsADirectoryError (C var) + single: PyExc_KeyError (C var) + single: PyExc_KeyboardInterrupt (C var) + single: PyExc_LookupError (C var) + single: PyExc_MemoryError (C var) + single: PyExc_ModuleNotFoundError (C var) + single: PyExc_NameError (C var) + single: PyExc_NotADirectoryError (C var) + single: PyExc_NotImplementedError (C var) + single: PyExc_OSError (C var) + single: PyExc_OverflowError (C var) + single: PyExc_PermissionError (C var) + single: PyExc_ProcessLookupError (C var) + single: PyExc_RecursionError (C var) + single: PyExc_ReferenceError (C var) + single: PyExc_RuntimeError (C var) + single: PyExc_StopAsyncIteration (C var) + single: PyExc_StopIteration (C var) + single: PyExc_SyntaxError (C var) + single: PyExc_SystemError (C var) + single: PyExc_SystemExit (C var) + single: PyExc_TabError (C var) + single: PyExc_TimeoutError (C var) + single: PyExc_TypeError (C var) + single: PyExc_UnboundLocalError (C var) + single: PyExc_UnicodeDecodeError (C var) + single: PyExc_UnicodeEncodeError (C var) + single: PyExc_UnicodeError (C var) + single: PyExc_UnicodeTranslateError (C var) + single: PyExc_ValueError (C var) + single: PyExc_ZeroDivisionError (C var) +-----------------------------------------+---------------------------------+----------+ | C Name | Python Name | Notes | @@ -1151,18 +1151,18 @@ the variables: These are compatibility aliases to :c:data:`PyExc_OSError`: .. index:: - single: PyExc_EnvironmentError - single: PyExc_IOError - single: PyExc_WindowsError + single: PyExc_EnvironmentError (C var) + single: PyExc_IOError (C var) + single: PyExc_WindowsError (C var) +-------------------------------------+----------+ | C Name | Notes | +=====================================+==========+ -| :c:data:`PyExc_EnvironmentError` | | +| :c:data:`!PyExc_EnvironmentError` | | +-------------------------------------+----------+ -| :c:data:`PyExc_IOError` | | +| :c:data:`!PyExc_IOError` | | +-------------------------------------+----------+ -| :c:data:`PyExc_WindowsError` | [2]_ | +| :c:data:`!PyExc_WindowsError` | [2]_ | +-------------------------------------+----------+ .. versionchanged:: 3.3 @@ -1188,17 +1188,17 @@ names are ``PyExc_`` followed by the Python exception name. These have the type the variables: .. index:: - single: PyExc_Warning - single: PyExc_BytesWarning - single: PyExc_DeprecationWarning - single: PyExc_FutureWarning - single: PyExc_ImportWarning - single: PyExc_PendingDeprecationWarning - single: PyExc_ResourceWarning - single: PyExc_RuntimeWarning - single: PyExc_SyntaxWarning - single: PyExc_UnicodeWarning - single: PyExc_UserWarning + single: PyExc_Warning (C var) + single: PyExc_BytesWarning (C var) + single: PyExc_DeprecationWarning (C var) + single: PyExc_FutureWarning (C var) + single: PyExc_ImportWarning (C var) + single: PyExc_PendingDeprecationWarning (C var) + single: PyExc_ResourceWarning (C var) + single: PyExc_RuntimeWarning (C var) + single: PyExc_SyntaxWarning (C var) + single: PyExc_UnicodeWarning (C var) + single: PyExc_UserWarning (C var) +------------------------------------------+---------------------------------+----------+ | C Name | Python Name | Notes | diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index d3a78c588454e8e..e9019a0d500f7e5 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -96,7 +96,7 @@ the :mod:`io` APIs instead. .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) - .. index:: single: Py_PRINT_RAW + .. index:: single: Py_PRINT_RAW (C macro) Write object *obj* to file object *p*. The only supported flag for *flags* is :c:macro:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index 6b2494ee4f0ed4c..621da3eb069949c 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -83,10 +83,15 @@ rules: .. versionadded:: 3.12 -.. c:function:: TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize) +.. c:macro:: PyObject_GC_Resize(TYPE, op, newsize) - Resize an object allocated by :c:macro:`PyObject_NewVar`. Returns the - resized object or ``NULL`` on failure. *op* must not be tracked by the collector yet. + Resize an object allocated by :c:macro:`PyObject_NewVar`. + Returns the resized object of type ``TYPE*`` (refers to any C type) + or ``NULL`` on failure. + + *op* must be of type :c:expr:`PyVarObject *` + and must not be tracked by the collector yet. + *newsize* must be of type :c:type:`Py_ssize_t`. .. c:function:: void PyObject_GC_Track(PyObject *op) diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 51c20b202f091c5..7c74e9e88678dc0 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -13,20 +13,8 @@ Importing Modules single: __all__ (package variable) single: modules (in module sys) - This is a simplified interface to :c:func:`PyImport_ImportModuleEx` below, - leaving the *globals* and *locals* arguments set to ``NULL`` and *level* set - to 0. When the *name* - argument contains a dot (when it specifies a submodule of a package), the - *fromlist* argument is set to the list ``['*']`` so that the return value is the - named module rather than the top-level package containing it as would otherwise - be the case. (Unfortunately, this has an additional side effect when *name* in - fact specifies a subpackage instead of a submodule: the submodules specified in - the package's ``__all__`` variable are loaded.) Return a new reference to the - imported module, or ``NULL`` with an exception set on failure. A failing - import of a module doesn't leave the module in :data:`sys.modules`. - - This function always uses absolute imports. - + This is a wrapper around :c:func:`PyImport_Import()` which takes a + :c:expr:`const char *` as an argument instead of a :c:expr:`PyObject *`. .. c:function:: PyObject* PyImport_ImportModuleNoBlock(const char *name) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index f8fd48e781d6da1..e7199ad5e0c1b15 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -332,7 +332,7 @@ Initializing and finalizing the interpreter pair: module; __main__ pair: module; sys triple: module; search; path - single: Py_FinalizeEx() + single: Py_FinalizeEx (C function) Initialize the Python interpreter. In an application embedding Python, this should be called before using any other Python/C API functions; see @@ -661,7 +661,7 @@ operations could cause problems in a multi-threaded program: for example, when two threads simultaneously increment the reference count of the same object, the reference count could end up being incremented only once instead of twice. -.. index:: single: setswitchinterval() (in module sys) +.. index:: single: setswitchinterval (in module sys) Therefore, the rule exists that only the thread that has acquired the :term:`GIL` may operate on Python objects or call Python/C API functions. @@ -671,8 +671,7 @@ released around potentially blocking I/O operations like reading or writing a file, so that other Python threads can run in the meantime. .. index:: - single: PyThreadState - single: PyThreadState + single: PyThreadState (C type) The Python interpreter keeps some thread-specific bookkeeping information inside a data structure called :c:type:`PyThreadState`. There's also one @@ -698,8 +697,8 @@ This is so common that a pair of macros exists to simplify it:: Py_END_ALLOW_THREADS .. index:: - single: Py_BEGIN_ALLOW_THREADS - single: Py_END_ALLOW_THREADS + single: Py_BEGIN_ALLOW_THREADS (C macro) + single: Py_END_ALLOW_THREADS (C macro) The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the @@ -714,8 +713,8 @@ The block above expands to the following code:: PyEval_RestoreThread(_save); .. index:: - single: PyEval_RestoreThread() - single: PyEval_SaveThread() + single: PyEval_RestoreThread (C function) + single: PyEval_SaveThread (C function) Here is how these functions work: the global interpreter lock is used to protect the pointer to the current thread state. When releasing the lock and saving the thread state, @@ -1399,8 +1398,8 @@ function. You can create and destroy them using the following functions: may be stored internally on the :c:type:`PyInterpreterState`. .. index:: - single: Py_FinalizeEx() - single: Py_Initialize() + single: Py_FinalizeEx (C function) + single: Py_Initialize (C function) Extension modules are shared between (sub-)interpreters as follows: @@ -1428,7 +1427,7 @@ function. You can create and destroy them using the following functions: As with multi-phase initialization, this means that only C-level static and global variables are shared between these modules. - .. index:: single: close() (in module os) + .. index:: single: close (in module os) .. c:function:: PyThreadState* Py_NewInterpreter(void) @@ -1451,7 +1450,7 @@ function. You can create and destroy them using the following functions: .. c:function:: void Py_EndInterpreter(PyThreadState *tstate) - .. index:: single: Py_FinalizeEx() + .. index:: single: Py_FinalizeEx (C function) Destroy the (sub-)interpreter represented by the given thread state. The given thread state must be the current thread state. See the @@ -1543,8 +1542,6 @@ pointer and a void pointer argument. .. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) - .. index:: single: Py_AddPendingCall() - Schedule a function to be called from the main interpreter thread. On success, ``0`` is returned and *func* is queued for being called in the main thread. On failure, ``-1`` is returned without setting any exception. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 4dbca92b18b5cd2..8ef463e3f88ca8f 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -148,7 +148,7 @@ complete listing. worse performances (due to increased code size for example). The compiler is usually smarter than the developer for the cost/benefit analysis. - If Python is :ref:`built in debug mode ` (if the ``Py_DEBUG`` + If Python is :ref:`built in debug mode ` (if the :c:macro:`Py_DEBUG` macro is defined), the :c:macro:`Py_ALWAYS_INLINE` macro does nothing. It must be specified before the function return type. Usage:: @@ -325,8 +325,8 @@ objects that reference each other here; for now, the solution is "don't do that.") .. index:: - single: Py_INCREF() - single: Py_DECREF() + single: Py_INCREF (C function) + single: Py_DECREF (C function) Reference counts are always manipulated explicitly. The normal way is to use the macro :c:func:`Py_INCREF` to take a new reference to an @@ -401,8 +401,8 @@ function, that function assumes that it now owns that reference, and you are not responsible for it any longer. .. index:: - single: PyList_SetItem() - single: PyTuple_SetItem() + single: PyList_SetItem (C function) + single: PyTuple_SetItem (C function) Few functions steal references; the two notable exceptions are :c:func:`PyList_SetItem` and :c:func:`PyTuple_SetItem`, which steal a reference @@ -491,8 +491,8 @@ using :c:func:`PySequence_GetItem` (which happens to take exactly the same arguments), you do own a reference to the returned object. .. index:: - single: PyList_GetItem() - single: PySequence_GetItem() + single: PyList_GetItem (C function) + single: PySequence_GetItem (C function) Here is an example of how you could write a function that computes the sum of the items in a list of integers; once using :c:func:`PyList_GetItem`, and once @@ -587,7 +587,7 @@ caller, then to the caller's caller, and so on, until they reach the top-level interpreter, where they are reported to the user accompanied by a stack traceback. -.. index:: single: PyErr_Occurred() +.. index:: single: PyErr_Occurred (C function) For C programmers, however, error checking always has to be explicit. All functions in the Python/C API can raise exceptions, unless an explicit claim is @@ -601,8 +601,8 @@ ambiguous return value, and require explicit testing for errors with :c:func:`PyErr_Occurred`. These exceptions are always explicitly documented. .. index:: - single: PyErr_SetString() - single: PyErr_Clear() + single: PyErr_SetString (C function) + single: PyErr_Clear (C function) Exception state is maintained in per-thread storage (this is equivalent to using global storage in an unthreaded application). A thread can be in one of @@ -624,7 +624,7 @@ an exception is being passed on between C functions until it reaches the Python bytecode interpreter's main loop, which takes care of transferring it to ``sys.exc_info()`` and friends. -.. index:: single: exc_info() (in module sys) +.. index:: single: exc_info (in module sys) Note that starting with Python 1.5, the preferred, thread-safe way to access the exception state from Python code is to call the function :func:`sys.exc_info`, @@ -709,9 +709,9 @@ Here is the corresponding C code, in all its glory:: .. index:: single: incr_item() .. index:: - single: PyErr_ExceptionMatches() - single: PyErr_Clear() - single: Py_XDECREF() + single: PyErr_ExceptionMatches (C function) + single: PyErr_Clear (C function) + single: Py_XDECREF (C function) This example represents an endorsed use of the ``goto`` statement in C! It illustrates the use of :c:func:`PyErr_ExceptionMatches` and @@ -735,7 +735,7 @@ the finalization, of the Python interpreter. Most functionality of the interpreter can only be used after the interpreter has been initialized. .. index:: - single: Py_Initialize() + single: Py_Initialize (C function) pair: module; builtins pair: module; __main__ pair: module; sys @@ -770,10 +770,10 @@ environment variable :envvar:`PYTHONHOME`, or insert additional directories in front of the standard path by setting :envvar:`PYTHONPATH`. .. index:: - single: Py_GetPath() - single: Py_GetPrefix() - single: Py_GetExecPrefix() - single: Py_GetProgramFullPath() + single: Py_GetPath (C function) + single: Py_GetPrefix (C function) + single: Py_GetExecPrefix (C function) + single: Py_GetProgramFullPath (C function) The embedding application can steer the search by setting :c:member:`PyConfig.program_name` *before* calling @@ -784,7 +784,7 @@ control has to provide its own implementation of :c:func:`Py_GetPath`, :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, and :c:func:`Py_GetProgramFullPath` (all defined in :file:`Modules/getpath.c`). -.. index:: single: Py_IsInitialized() +.. index:: single: Py_IsInitialized (C function) Sometimes, it is desirable to "uninitialize" Python. For instance, the application may want to start over (make another call to @@ -812,12 +812,14 @@ available that support tracing of reference counts, debugging the memory allocator, or low-level profiling of the main interpreter loop. Only the most frequently used builds will be described in the remainder of this section. -Compiling the interpreter with the :c:macro:`Py_DEBUG` macro defined produces +.. c:macro:: Py_DEBUG + +Compiling the interpreter with the :c:macro:`!Py_DEBUG` macro defined produces what is generally meant by :ref:`a debug build of Python `. -:c:macro:`Py_DEBUG` is enabled in the Unix build by adding +:c:macro:`!Py_DEBUG` is enabled in the Unix build by adding :option:`--with-pydebug` to the :file:`./configure` command. It is also implied by the presence of the -not-Python-specific :c:macro:`_DEBUG` macro. When :c:macro:`Py_DEBUG` is enabled +not-Python-specific :c:macro:`!_DEBUG` macro. When :c:macro:`!Py_DEBUG` is enabled in the Unix build, compiler optimization is disabled. In addition to the reference count debugging described below, extra checks are @@ -832,4 +834,3 @@ after every statement run by the interpreter.) Please refer to :file:`Misc/SpecialBuilds.txt` in the Python source distribution for more detailed information. - diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index c8b64bad702f508..53eb54d3e1021a3 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -56,13 +56,21 @@ List Objects Similar to :c:func:`PyList_Size`, but without error checking. -.. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index) +.. c:function:: PyObject* PyList_GetItemRef(PyObject *list, Py_ssize_t index) Return the object at position *index* in the list pointed to by *list*. The position must be non-negative; indexing from the end of the list is not - supported. If *index* is out of bounds (<0 or >=len(list)), + supported. If *index* is out of bounds (:code:`<0 or >=len(list)`), return ``NULL`` and set an :exc:`IndexError` exception. + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index) + + Like :c:func:`PyList_GetItemRef`, but returns a + :term:`borrowed reference` instead of a :term:`strong reference`. + .. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 045604870d3c840..582f5c7bf05471c 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -113,11 +113,33 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. retrieved from the resulting value using :c:func:`PyLong_AsVoidPtr`. +.. c:function:: PyObject* PyLong_FromNativeBytes(const void* buffer, size_t n_bytes, int endianness) + + Create a Python integer from the value contained in the first *n_bytes* of + *buffer*, interpreted as a two's-complement signed number. + + *endianness* may be passed ``-1`` for the native endian that CPython was + compiled with, or else ``0`` for big endian and ``1`` for little. + + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyLong_FromUnsignedNativeBytes(const void* buffer, size_t n_bytes, int endianness) + + Create a Python integer from the value contained in the first *n_bytes* of + *buffer*, interpreted as an unsigned number. + + *endianness* may be passed ``-1`` for the native endian that CPython was + compiled with, or else ``0`` for big endian and ``1`` for little. + + .. versionadded:: 3.13 + + .. XXX alias PyLong_AS_LONG (for now) .. c:function:: long PyLong_AsLong(PyObject *obj) .. index:: - single: LONG_MAX + single: LONG_MAX (C macro) single: OverflowError (built-in exception) Return a C :c:expr:`long` representation of *obj*. If *obj* is not an @@ -210,7 +232,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: Py_ssize_t PyLong_AsSsize_t(PyObject *pylong) .. index:: - single: PY_SSIZE_T_MAX + single: PY_SSIZE_T_MAX (C macro) single: OverflowError (built-in exception) Return a C :c:type:`Py_ssize_t` representation of *pylong*. *pylong* must @@ -225,7 +247,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: unsigned long PyLong_AsUnsignedLong(PyObject *pylong) .. index:: - single: ULONG_MAX + single: ULONG_MAX (C macro) single: OverflowError (built-in exception) Return a C :c:expr:`unsigned long` representation of *pylong*. *pylong* @@ -241,7 +263,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: size_t PyLong_AsSize_t(PyObject *pylong) .. index:: - single: SIZE_MAX + single: SIZE_MAX (C macro) single: OverflowError (built-in exception) Return a C :c:type:`size_t` representation of *pylong*. *pylong* must be @@ -332,6 +354,94 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Returns ``NULL`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. +.. c:function:: Py_ssize_t PyLong_AsNativeBytes(PyObject *pylong, void* buffer, Py_ssize_t n_bytes, int endianness) + + Copy the Python integer value to a native *buffer* of size *n_bytes*:: + + int32_t value; + Py_ssize_t bytes = PyLong_AsNativeBits(pylong, &value, sizeof(value), -1); + if (bytes < 0) { + // A Python exception was set with the reason. + return NULL; + } + else if (bytes <= (Py_ssize_t)sizeof(value)) { + // Success! + } + else { + // Overflow occurred, but 'value' contains the truncated + // lowest bits of pylong. + } + + The above example may look *similar* to + :c:func:`PyLong_As* ` + but instead fills in a specific caller defined type and never raises an + error about of the :class:`int` *pylong*'s value regardless of *n_bytes* + or the returned byte count. + + To get at the entire potentially big Python value, this can be used to + reserve enough space and copy it:: + + // Ask how much space we need. + Py_ssize_t expected = PyLong_AsNativeBits(pylong, NULL, 0, -1); + if (expected < 0) { + // A Python exception was set with the reason. + return NULL; + } + assert(expected != 0); // Impossible per the API definition. + uint8_t *bignum = malloc(expected); + if (!bignum) { + PyErr_SetString(PyExc_MemoryError, "bignum malloc failed."); + return NULL; + } + // Safely get the entire value. + Py_ssize_t bytes = PyLong_AsNativeBits(pylong, bignum, expected, -1); + if (bytes < 0) { // Exception set. + free(bignum); + return NULL; + } + else if (bytes > expected) { // Be safe, should not be possible. + PyErr_SetString(PyExc_RuntimeError, + "Unexpected bignum truncation after a size check."); + free(bignum); + return NULL; + } + // The expected success given the above pre-check. + // ... use bignum ... + free(bignum); + + *endianness* may be passed ``-1`` for the native endian that CPython was + compiled with, or ``0`` for big endian and ``1`` for little. + + Returns ``-1`` with an exception raised if *pylong* cannot be interpreted as + an integer. Otherwise, return the size of the buffer required to store the + value. If this is equal to or less than *n_bytes*, the entire value was + copied. ``0`` will never be returned. + + Unless an exception is raised, all *n_bytes* of the buffer will always be + written. In the case of truncation, as many of the lowest bits of the value + as could fit are written. This allows the caller to ignore all non-negative + results if the intent is to match the typical behavior of a C-style + downcast. No exception is set on truncation. + + Values are always copied as two's-complement and sufficient buffer will be + requested to include a sign bit. For example, this may cause an value that + fits into 8 bytes when treated as unsigned to request 9 bytes, even though + all eight bytes were copied into the buffer. What has been omitted is the + zero sign bit -- redundant if the caller's intention is to treat the value + as unsigned. + + Passing zero to *n_bytes* will return the size of a buffer that would + be large enough to hold the value. This may be larger than technically + necessary, but not unreasonably so. + + .. note:: + + Passing *n_bytes=0* to this function is not an accurate way to determine + the bit length of a value. + + .. versionadded:: 3.13 + + .. c:function:: int PyUnstable_Long_IsCompact(const PyLongObject* op) Return 1 if *op* is compact, 0 otherwise. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index c05282ffc595219..9da09a21607f617 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -41,10 +41,10 @@ buffers is performed on demand by the Python memory manager through the Python/C API functions listed in this document. .. index:: - single: malloc() - single: calloc() - single: realloc() - single: free() + single: malloc (C function) + single: calloc (C function) + single: realloc (C function) + single: free (C function) To avoid memory corruption, extension writers should never try to operate on Python objects with the functions exported by the C library: :c:func:`malloc`, diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index 2aa43318e7a455c..f6038032805259f 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -20,6 +20,17 @@ any other object. read/write, otherwise it may be either read-only or read/write at the discretion of the exporter. + +.. c:macro:: PyBUF_READ + + Flag to request a readonly buffer. + + +.. c:macro:: PyBUF_WRITE + + Flag to request a writable buffer. + + .. c:function:: PyObject *PyMemoryView_FromMemory(char *mem, Py_ssize_t size, int flags) Create a memoryview object using *mem* as the underlying buffer. @@ -41,6 +52,8 @@ any other object. original memory. Otherwise, a copy is made and the memoryview points to a new bytes object. + *buffertype* can be one of :c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE`. + .. c:function:: int PyMemoryView_Check(PyObject *obj) diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 0032da9659636c5..e2943f18ddc6010 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -187,26 +187,26 @@ Implementing functions and methods PyObject *kwargs); -.. c:type:: _PyCFunctionFast +.. c:type:: PyCFunctionFast Type of the functions used to implement Python callables in C with signature :c:macro:`METH_FASTCALL`. The function signature is:: - PyObject *_PyCFunctionFast(PyObject *self, - PyObject *const *args, - Py_ssize_t nargs); + PyObject *PyCFunctionFast(PyObject *self, + PyObject *const *args, + Py_ssize_t nargs); -.. c:type:: _PyCFunctionFastWithKeywords +.. c:type:: PyCFunctionFastWithKeywords Type of the functions used to implement Python callables in C with signature :ref:`METH_FASTCALL | METH_KEYWORDS `. The function signature is:: - PyObject *_PyCFunctionFastWithKeywords(PyObject *self, - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames); + PyObject *PyCFunctionFastWithKeywords(PyObject *self, + PyObject *const *args, + Py_ssize_t nargs, + PyObject *kwnames); .. c:type:: PyCMethod @@ -290,7 +290,7 @@ There are these calling conventions: .. c:macro:: METH_FASTCALL Fast calling convention supporting only positional arguments. - The methods have the type :c:type:`_PyCFunctionFast`. + The methods have the type :c:type:`PyCFunctionFast`. The first parameter is *self*, the second parameter is a C array of :c:expr:`PyObject*` values indicating the arguments and the third parameter is the number of arguments (the length of the array). @@ -306,7 +306,7 @@ There are these calling conventions: :c:expr:`METH_FASTCALL | METH_KEYWORDS` Extension of :c:macro:`METH_FASTCALL` supporting also keyword arguments, - with methods of type :c:type:`_PyCFunctionFastWithKeywords`. + with methods of type :c:type:`PyCFunctionFastWithKeywords`. Keyword arguments are passed the same way as in the :ref:`vectorcall protocol `: there is an additional fourth :c:expr:`PyObject*` parameter @@ -561,9 +561,9 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct. .. index:: - single: READ_RESTRICTED - single: WRITE_RESTRICTED - single: RESTRICTED + single: READ_RESTRICTED (C macro) + single: WRITE_RESTRICTED (C macro) + single: RESTRICTED (C macro) .. versionchanged:: 3.10 @@ -574,7 +574,7 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: :c:macro:`Py_AUDIT_READ`; :c:macro:`!WRITE_RESTRICTED` does nothing. .. index:: - single: READONLY + single: READONLY (C macro) .. versionchanged:: 3.12 @@ -637,24 +637,24 @@ Macro name C type Python type Reading a ``NULL`` pointer raises :py:exc:`AttributeError`. .. index:: - single: T_BYTE - single: T_SHORT - single: T_INT - single: T_LONG - single: T_LONGLONG - single: T_UBYTE - single: T_USHORT - single: T_UINT - single: T_ULONG - single: T_ULONGULONG - single: T_PYSSIZET - single: T_FLOAT - single: T_DOUBLE - single: T_BOOL - single: T_CHAR - single: T_STRING - single: T_STRING_INPLACE - single: T_OBJECT_EX + single: T_BYTE (C macro) + single: T_SHORT (C macro) + single: T_INT (C macro) + single: T_LONG (C macro) + single: T_LONGLONG (C macro) + single: T_UBYTE (C macro) + single: T_USHORT (C macro) + single: T_UINT (C macro) + single: T_ULONG (C macro) + single: T_ULONGULONG (C macro) + single: T_PYSSIZET (C macro) + single: T_FLOAT (C macro) + single: T_DOUBLE (C macro) + single: T_BOOL (C macro) + single: T_CHAR (C macro) + single: T_STRING (C macro) + single: T_STRING_INPLACE (C macro) + single: T_OBJECT_EX (C macro) single: structmember.h .. versionadded:: 3.12 diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index e3c54b075114ff7..d6fca1a0b0a219c 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -5,6 +5,7 @@ Operating System Utilities ========================== + .. c:function:: PyObject* PyOS_FSPath(PyObject *path) Return the file system representation for *path*. If the object is a @@ -97,27 +98,30 @@ Operating System Utilities .. c:function:: int PyOS_CheckStack() + .. index:: single: USE_STACKCHECK (C macro) + Return true when the interpreter runs out of stack space. This is a reliable - check, but is only available when :c:macro:`USE_STACKCHECK` is defined (currently + check, but is only available when :c:macro:`!USE_STACKCHECK` is defined (currently on certain versions of Windows using the Microsoft Visual C++ compiler). - :c:macro:`USE_STACKCHECK` will be defined automatically; you should never + :c:macro:`!USE_STACKCHECK` will be defined automatically; you should never change the definition in your own code. +.. c:type:: void (*PyOS_sighandler_t)(int) + + .. c:function:: PyOS_sighandler_t PyOS_getsig(int i) Return the current signal handler for signal *i*. This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions - directly! :c:type:`PyOS_sighandler_t` is a typedef alias for :c:expr:`void - (\*)(int)`. + directly! .. c:function:: PyOS_sighandler_t PyOS_setsig(int i, PyOS_sighandler_t h) Set the signal handler for signal *i* to be *h*; return the old signal handler. This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do - not call those functions directly! :c:type:`PyOS_sighandler_t` is a typedef - alias for :c:expr:`void (\*)(int)`. + not call those functions directly! .. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) @@ -342,10 +346,8 @@ accessible to C code. They all work with the current interpreter thread's silently abort the operation by raising an error subclassed from :class:`Exception` (other errors will not be silenced). - The hook function is of type :c:expr:`int (*)(const char *event, PyObject - *args, void *userData)`, where *args* is guaranteed to be a - :c:type:`PyTupleObject`. The hook function is always called with the GIL - held by the Python interpreter that raised the event. + The hook function is always called with the GIL held by the Python + interpreter that raised the event. See :pep:`578` for a detailed description of auditing. Functions in the runtime and standard library that raise events are listed in the @@ -354,12 +356,21 @@ accessible to C code. They all work with the current interpreter thread's .. audit-event:: sys.addaudithook "" c.PySys_AddAuditHook - If the interpreter is initialized, this function raises a auditing event + If the interpreter is initialized, this function raises an auditing event ``sys.addaudithook`` with no arguments. If any existing hooks raise an exception derived from :class:`Exception`, the new hook will not be added and the exception is cleared. As a result, callers cannot assume that their hook has been added unless they control all existing hooks. + .. c:namespace:: NULL + .. c:type:: int (*Py_AuditHookFunction) (const char *event, PyObject *args, void *userData) + + The type of the hook function. + *event* is the C string event argument passed to :c:func:`PySys_Audit` or + :c:func:`PySys_AuditTuple`. + *args* is guaranteed to be a :c:type:`PyTupleObject`. + *userData* is the argument passed to PySys_AddAuditHook(). + .. versionadded:: 3.8 @@ -371,7 +382,7 @@ Process Control .. c:function:: void Py_FatalError(const char *message) - .. index:: single: abort() + .. index:: single: abort (C function) Print a fatal error message and kill the process. No cleanup is performed. This function should only be invoked when a condition is detected that would @@ -391,8 +402,8 @@ Process Control .. c:function:: void Py_Exit(int status) .. index:: - single: Py_FinalizeEx() - single: exit() + single: Py_FinalizeEx (C function) + single: exit (C function) Exit the current process. This calls :c:func:`Py_FinalizeEx` and then calls the standard C library function ``exit(status)``. If :c:func:`Py_FinalizeEx` @@ -405,7 +416,7 @@ Process Control .. c:function:: int Py_AtExit(void (*func) ()) .. index:: - single: Py_FinalizeEx() + single: Py_FinalizeEx (C function) single: cleanup functions Register a cleanup function to be called by :c:func:`Py_FinalizeEx`. The cleanup diff --git a/Doc/c-api/time.rst b/Doc/c-api/time.rst new file mode 100644 index 000000000000000..7791cdb17810550 --- /dev/null +++ b/Doc/c-api/time.rst @@ -0,0 +1,83 @@ +.. highlight:: c + +PyTime C API +============ + +.. versionadded:: 3.13 + +The clock C API provides access to system clocks. +It is similar to the Python :mod:`time` module. + +For C API related to the :mod:`datetime` module, see :ref:`datetimeobjects`. + + +Types +----- + +.. c:type:: PyTime_t + + A timestamp or duration in nanoseconds, represented as a signed 64-bit + integer. + + The reference point for timestamps depends on the clock used. For example, + :c:func:`PyTime_Time` returns timestamps relative to the UNIX epoch. + + The supported range is around [-292.3 years; +292.3 years]. + Using the Unix epoch (January 1st, 1970) as reference, the supported date + range is around [1677-09-21; 2262-04-11]. + The exact limits are exposed as constants: + +.. c:var:: PyTime_t PyTime_MIN + + Minimum value of :c:type:`PyTime_t`. + +.. c:var:: PyTime_t PyTime_MAX + + Maximum value of :c:type:`PyTime_t`. + + +Clock Functions +--------------- + +The following functions take a pointer to a :c:expr:`PyTime_t` that they +set to the value of a particular clock. +Details of each clock are given in the documentation of the corresponding +Python function. + +The functions return ``0`` on success, or ``-1`` (with an exception set) +on failure. + +On integer overflow, they set the :c:data:`PyExc_OverflowError` exception and +set ``*result`` to the value clamped to the ``[PyTime_MIN; PyTime_MAX]`` +range. +(On current systems, integer overflows are likely caused by misconfigured +system time.) + +As any other C API (unless otherwise specified), the functions must be called +with the :term:`GIL` held. + +.. c:function:: int PyTime_Monotonic(PyTime_t *result) + + Read the monotonic clock. + See :func:`time.monotonic` for important details on this clock. + +.. c:function:: int PyTime_PerfCounter(PyTime_t *result) + + Read the performance counter. + See :func:`time.perf_counter` for important details on this clock. + +.. c:function:: int PyTime_Time(PyTime_t *result) + + Read the “wall clock” time. + See :func:`time.time` for details important on this clock. + + +Conversion functions +-------------------- + +.. c:function:: double PyTime_AsSecondsDouble(PyTime_t t) + + Convert a timestamp to a number of seconds as a C :c:expr:`double`. + + The function cannot fail, but note that :c:expr:`double` has limited + accuracy for large values. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 5541eaa521803bc..666ffe89605c56f 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -854,7 +854,12 @@ wchar_t Support Copy the Unicode object contents into the :c:type:`wchar_t` buffer *wstr*. At most *size* :c:type:`wchar_t` characters are copied (excluding a possibly trailing null termination character). Return the number of :c:type:`wchar_t` characters - copied or ``-1`` in case of an error. Note that the resulting :c:expr:`wchar_t*` + copied or ``-1`` in case of an error. + + When *wstr* is ``NULL``, instead return the *size* that would be required + to store all of *unicode* including a terminating null. + + Note that the resulting :c:expr:`wchar_t*` string may or may not be null-terminated. It is the responsibility of the caller to make sure that the :c:expr:`wchar_t*` string is null-terminated in case this is required by the application. Also, note that the :c:expr:`wchar_t*` string diff --git a/Doc/c-api/utilities.rst b/Doc/c-api/utilities.rst index 48ae54acebe8879..9d0abf440f791d4 100644 --- a/Doc/c-api/utilities.rst +++ b/Doc/c-api/utilities.rst @@ -20,4 +20,5 @@ and parsing function arguments and constructing Python values from C values. hash.rst reflection.rst codec.rst + time.rst perfmaps.rst diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 324518c035096b4..67167444d0a6852 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -322,7 +322,7 @@ the same library that the Python runtime is using. .. c:var:: int Py_eval_input - .. index:: single: Py_CompileString() + .. index:: single: Py_CompileString (C function) The start symbol from the Python grammar for isolated expressions; for use with :c:func:`Py_CompileString`. @@ -330,7 +330,7 @@ the same library that the Python runtime is using. .. c:var:: int Py_file_input - .. index:: single: Py_CompileString() + .. index:: single: Py_CompileString (C function) The start symbol from the Python grammar for sequences of statements as read from a file or other source; for use with :c:func:`Py_CompileString`. This is @@ -339,7 +339,7 @@ the same library that the Python runtime is using. .. c:var:: int Py_single_input - .. index:: single: Py_CompileString() + .. index:: single: Py_CompileString (C function) The start symbol from the Python grammar for a single statement; for use with :c:func:`Py_CompileString`. This is the symbol used for the interactive diff --git a/Doc/conf.py b/Doc/conf.py index 458954370debe28..f4c75c5758cb281 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -6,7 +6,9 @@ # The contents of this file are pickled, so don't put values in the namespace # that aren't pickleable (module imports are okay, they're removed automatically). -import sys, os, time +import os +import sys +import time sys.path.append(os.path.abspath('tools/extensions')) sys.path.append(os.path.abspath('includes')) @@ -55,13 +57,19 @@ # General substitutions. project = 'Python' -copyright = '2001-%s, Python Software Foundation' % time.strftime('%Y') +copyright = f"2001-{time.strftime('%Y')}, Python Software Foundation" # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. import patchlevel version, release = patchlevel.get_version_info() +rst_epilog = f""" +.. |python_version_literal| replace:: ``Python {version}`` +.. |python_x_dot_y_literal| replace:: ``python{version}`` +.. |usr_local_bin_python_x_dot_y_literal| replace:: ``/usr/local/bin/python{version}`` +""" + # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: today = '' @@ -89,22 +97,34 @@ nitpick_ignore = [ # Standard C functions ('c:func', 'calloc'), + ('c:func', 'ctime'), ('c:func', 'dlopen'), ('c:func', 'exec'), ('c:func', 'fcntl'), + ('c:func', 'flock'), ('c:func', 'fork'), ('c:func', 'free'), + ('c:func', 'gettimeofday'), ('c:func', 'gmtime'), + ('c:func', 'grantpt'), + ('c:func', 'ioctl'), + ('c:func', 'localeconv'), ('c:func', 'localtime'), ('c:func', 'main'), ('c:func', 'malloc'), + ('c:func', 'mktime'), + ('c:func', 'posix_openpt'), ('c:func', 'printf'), + ('c:func', 'ptsname'), + ('c:func', 'ptsname_r'), ('c:func', 'realloc'), ('c:func', 'snprintf'), ('c:func', 'sprintf'), ('c:func', 'stat'), + ('c:func', 'strftime'), ('c:func', 'system'), ('c:func', 'time'), + ('c:func', 'unlockpt'), ('c:func', 'vsnprintf'), # Standard C types ('c:type', 'FILE'), @@ -123,11 +143,14 @@ ('c:type', 'wchar_t'), ('c:type', '__int64'), ('c:type', 'unsigned __int64'), + ('c:type', 'double'), # Standard C structures ('c:struct', 'in6_addr'), ('c:struct', 'in_addr'), ('c:struct', 'stat'), ('c:struct', 'statvfs'), + ('c:struct', 'timeval'), + ('c:struct', 'timespec'), # Standard C macros ('c:macro', 'LLONG_MAX'), ('c:macro', 'LLONG_MIN'), @@ -254,15 +277,16 @@ ('py:attr', '__annotations__'), ('py:meth', '__missing__'), ('py:attr', '__wrapped__'), + ('py:attr', 'decimal.Context.clamp'), ('py:meth', 'index'), # list.index, tuple.index, etc. ] -# gh-106948: Copy standard C types declared in the "c:type" domain to the -# "c:identifier" domain, since "c:function" markup looks for types in the -# "c:identifier" domain. Use list() to not iterate on items which are being -# added +# gh-106948: Copy standard C types declared in the "c:type" domain and C +# structures declared in the "c:struct" domain to the "c:identifier" domain, +# since "c:function" markup looks for types in the "c:identifier" domain. Use +# list() to not iterate on items which are being added for role, name in list(nitpick_ignore): - if role == 'c:type': + if role in ('c:type', 'c:struct'): nitpick_ignore.append(('c:identifier', name)) del role, name @@ -292,6 +316,9 @@ 'root_include_title': False # We use the version switcher instead. } +if os.getenv("READTHEDOCS"): + html_theme_options["hosted_on"] = 'Read the Docs' + # Override stylesheet fingerprinting for Windows CHM htmlhelp to fix GH-91207 # https://github.com/python/cpython/issues/91207 if any('htmlhelp' in arg for arg in sys.argv): @@ -300,7 +327,7 @@ print("It may be removed in the future\n") # Short title used e.g. for HTML tags. -html_short_title = '%s Documentation' % release +html_short_title = f'{release} Documentation' # Deployment preview information # (See .readthedocs.yml and https://docs.readthedocs.io/en/stable/reference/environment-variables.html) @@ -349,12 +376,9 @@ latex_engine = 'xelatex' -# Get LaTeX to handle Unicode correctly latex_elements = { -} - -# Additional stuff for the LaTeX preamble. -latex_elements['preamble'] = r''' + # For the LaTeX preamble. + 'preamble': r''' \authoraddress{ \sphinxstrong{Python Software Foundation}\\ Email: \sphinxemail{docs@python.org} @@ -362,13 +386,12 @@ \let\Verbatim=\OriginalVerbatim \let\endVerbatim=\endOriginalVerbatim \setcounter{tocdepth}{2} -''' - -# The paper size ('letter' or 'a4'). -latex_elements['papersize'] = 'a4' - -# The font size ('10pt', '11pt' or '12pt'). -latex_elements['pointsize'] = '10pt' +''', + # The paper size ('letter' or 'a4'). + 'papersize': 'a4', + # The font size ('10pt', '11pt' or '12pt'). + 'pointsize': '10pt', +} # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). @@ -431,9 +454,9 @@ # Regexes to find C items in the source files. coverage_c_regexes = { - 'cfunction': (r'^PyAPI_FUNC\(.*\)\s+([^_][\w_]+)'), - 'data': (r'^PyAPI_DATA\(.*\)\s+([^_][\w_]+)'), - 'macro': (r'^#define ([^_][\w_]+)\(.*\)[\s|\\]'), + 'cfunction': r'^PyAPI_FUNC\(.*\)\s+([^_][\w_]+)', + 'data': r'^PyAPI_DATA\(.*\)\s+([^_][\w_]+)', + 'macro': r'^#define ([^_][\w_]+)\(.*\)[\s|\\]', } # The coverage checker will ignore all C items whose names match these regexes diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index f719ce153b239a9..62a96146d605ff7 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -1133,6 +1133,10 @@ PyList_GetItem:PyObject*::0: PyList_GetItem:PyObject*:list:0: PyList_GetItem:Py_ssize_t:index:: +PyList_GetItemRef:PyObject*::+1: +PyList_GetItemRef:PyObject*:list:0: +PyList_GetItemRef:Py_ssize_t:index:: + PyList_GetSlice:PyObject*::+1: PyList_GetSlice:PyObject*:list:0: PyList_GetSlice:Py_ssize_t:low:: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 811b1bd84d24174..25629b4da053da8 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -42,6 +42,8 @@ function,PyBytes_Repr,3.2,, function,PyBytes_Size,3.2,, var,PyBytes_Type,3.2,, type,PyCFunction,3.2,, +type,PyCFunctionFast,3.13,, +type,PyCFunctionFastWithKeywords,3.13,, type,PyCFunctionWithKeywords,3.2,, function,PyCFunction_GetFlags,3.2,, function,PyCFunction_GetFunction,3.2,, @@ -220,6 +222,7 @@ var,PyExc_GeneratorExit,3.2,, var,PyExc_IOError,3.2,, var,PyExc_ImportError,3.2,, var,PyExc_ImportWarning,3.2,, +var,PyExc_IncompleteInputError,3.13,, var,PyExc_IndentationError,3.2,, var,PyExc_IndexError,3.2,, var,PyExc_InterruptedError,3.7,, @@ -335,6 +338,7 @@ var,PyListRevIter_Type,3.2,, function,PyList_Append,3.2,, function,PyList_AsTuple,3.2,, function,PyList_GetItem,3.2,, +function,PyList_GetItemRef,3.13,, function,PyList_GetSlice,3.2,, function,PyList_Insert,3.2,, function,PyList_New,3.2,, diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 745fc10a22d1612..b70e1b1fe57e67c 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -547,7 +547,7 @@ reference count of an object and are safe in the presence of ``NULL`` pointers (but note that *temp* will not be ``NULL`` in this context). More info on them in section :ref:`refcounts`. -.. index:: single: PyObject_CallObject() +.. index:: single: PyObject_CallObject (C function) Later, when it is time to call the function, you call the C function :c:func:`PyObject_CallObject`. This function has two arguments, both pointers to @@ -638,7 +638,7 @@ the above example, we use :c:func:`Py_BuildValue` to construct the dictionary. : Extracting Parameters in Extension Functions ============================================ -.. index:: single: PyArg_ParseTuple() +.. index:: single: PyArg_ParseTuple (C function) The :c:func:`PyArg_ParseTuple` function is declared as follows:: @@ -730,7 +730,7 @@ Some example calls:: Keyword Parameters for Extension Functions ========================================== -.. index:: single: PyArg_ParseTupleAndKeywords() +.. index:: single: PyArg_ParseTupleAndKeywords (C function) The :c:func:`PyArg_ParseTupleAndKeywords` function is declared as follows:: diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 7a92b3257c6cd31..473a418809cff10 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -89,8 +89,8 @@ If your type supports garbage collection, the destructor should call } .. index:: - single: PyErr_Fetch() - single: PyErr_Restore() + single: PyErr_Fetch (C function) + single: PyErr_Restore (C function) One important requirement of the deallocator function is that it leaves any pending exceptions alone. This is important since deallocators are frequently diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index 476a43d9c288f18..e2f8004c7e3aea5 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -405,22 +405,37 @@ lists. When in doubt, use a mutex! Can't we get rid of the Global Interpreter Lock? ------------------------------------------------ -.. XXX link to dbeazley's talk about GIL? - The :term:`global interpreter lock` (GIL) is often seen as a hindrance to Python's deployment on high-end multiprocessor server machines, because a multi-threaded Python program effectively only uses one CPU, due to the insistence that (almost) all Python code can only run while the GIL is held. -Back in the days of Python 1.5, Greg Stein actually implemented a comprehensive +With the approval of :pep:`703` work is now underway to remove the GIL from the +CPython implementation of Python. Initially it will be implemented as an +optional compiler flag when building the interpreter, and so separate +builds will be available with and without the GIL. Long-term, the hope is +to settle on a single build, once the performance implications of removing the +GIL are fully understood. Python 3.13 is likely to be the first release +containing this work, although it may not be completely functional in this +release. + +The current work to remove the GIL is based on a +`fork of Python 3.9 with the GIL removed <https://github.com/colesbury/nogil>`_ +by Sam Gross. +Prior to that, +in the days of Python 1.5, Greg Stein actually implemented a comprehensive patch set (the "free threading" patches) that removed the GIL and replaced it -with fine-grained locking. Adam Olsen recently did a similar experiment +with fine-grained locking. Adam Olsen did a similar experiment in his `python-safethread <https://code.google.com/archive/p/python-safethread>`_ -project. Unfortunately, both experiments exhibited a sharp drop in single-thread +project. Unfortunately, both of these earlier experiments exhibited a sharp +drop in single-thread performance (at least 30% slower), due to the amount of fine-grained locking -necessary to compensate for the removal of the GIL. +necessary to compensate for the removal of the GIL. The Python 3.9 fork +is the first attempt at removing the GIL with an acceptable performance +impact. -This doesn't mean that you can't make good use of Python on multi-CPU machines! +The presence of the GIL in current Python releases +doesn't mean that you can't make good use of Python on multi-CPU machines! You just have to be creative with dividing the work up between multiple *processes* rather than multiple *threads*. The :class:`~concurrent.futures.ProcessPoolExecutor` class in the new @@ -434,22 +449,13 @@ thread of execution is in the C code and allow other threads to get some work done. Some standard library modules such as :mod:`zlib` and :mod:`hashlib` already do this. -It has been suggested that the GIL should be a per-interpreter-state lock rather -than truly global; interpreters then wouldn't be able to share objects. -Unfortunately, this isn't likely to happen either. It would be a tremendous -amount of work, because many object implementations currently have global state. -For example, small integers and short strings are cached; these caches would -have to be moved to the interpreter state. Other object types have their own -free list; these free lists would have to be moved to the interpreter state. -And so on. - -And I doubt that it can even be done in finite time, because the same problem -exists for 3rd party extensions. It is likely that 3rd party extensions are -being written at a faster rate than you can convert them to store all their -global state in the interpreter state. - -And finally, once you have multiple interpreters not sharing any state, what -have you gained over running each interpreter in a separate process? +An alternative approach to reducing the impact of the GIL is +to make the GIL a per-interpreter-state lock rather than truly global. +This was :ref:`first implemented in Python 3.12 <whatsnew312-pep684>` and is +available in the C API. A Python interface to it is expected in Python 3.13. +The main limitation to it at the moment is likely to be 3rd party extension +modules, since these must be written with multiple interpreters in mind in +order to be usable, so many older extension modules will not be usable. Input and Output diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 098bfffb104ef69..f656e32514c7178 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -341,7 +341,7 @@ Glossary docstring A string literal which appears as the first expression in a class, function or module. While ignored when the suite is executed, it is - recognized by the compiler and put into the :attr:`__doc__` attribute + recognized by the compiler and put into the :attr:`!__doc__` attribute of the enclosing class, function or module. Since it is available via introspection, it is the canonical place for documentation of the object. @@ -1104,10 +1104,12 @@ Glossary The :class:`collections.abc.Sequence` abstract base class defines a much richer interface that goes beyond just :meth:`~object.__getitem__` and :meth:`~object.__len__`, adding - :meth:`count`, :meth:`index`, :meth:`~object.__contains__`, and + :meth:`!count`, :meth:`!index`, :meth:`~object.__contains__`, and :meth:`~object.__reversed__`. Types that implement this expanded interface can be registered explicitly using - :func:`~abc.ABCMeta.register`. + :func:`~abc.ABCMeta.register`. For more documentation on sequence + methods generally, see + :ref:`Common Sequence Operations <typesseq-common>`. set comprehension A compact way to process all or part of the elements in an iterable and diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 75346f2c7618c2e..51f9f4a6556e576 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -1,8 +1,8 @@ .. _descriptorhowto: -====================== -Descriptor HowTo Guide -====================== +================ +Descriptor Guide +================ :Author: Raymond Hettinger :Contact: <python at rcn dot com> @@ -1004,31 +1004,42 @@ here is a pure Python equivalent: if doc is None and fget is not None: doc = fget.__doc__ self.__doc__ = doc - self._name = '' + self._name = None def __set_name__(self, owner, name): self._name = name + @property + def __name__(self): + return self._name if self._name is not None else self.fget.__name__ + + @__name__.setter + def __name__(self, value): + self._name = value + def __get__(self, obj, objtype=None): if obj is None: return self if self.fget is None: raise AttributeError( - f'property {self._name!r} of {type(obj).__name__!r} object has no getter' + f'property {self.__name__!r} of {type(obj).__name__!r} ' + 'object has no getter' ) return self.fget(obj) def __set__(self, obj, value): if self.fset is None: raise AttributeError( - f'property {self._name!r} of {type(obj).__name__!r} object has no setter' + f'property {self.__name__!r} of {type(obj).__name__!r} ' + 'object has no setter' ) self.fset(obj, value) def __delete__(self, obj): if self.fdel is None: raise AttributeError( - f'property {self._name!r} of {type(obj).__name__!r} object has no deleter' + f'property {self.__name__!r} of {type(obj).__name__!r} ' + 'object has no deleter' ) self.fdel(obj) @@ -1192,6 +1203,10 @@ roughly equivalent to: "Emulate method_getattro() in Objects/classobject.c" return getattr(self.__func__, name) + def __get__(self, obj, objtype=None): + "Emulate method_descr_get() in Objects/classobject.c" + return self + To support automatic creation of methods, functions include the :meth:`__get__` method for binding methods during attribute access. This means that functions are non-data descriptors that return bound methods @@ -1214,8 +1229,20 @@ descriptor works in practice: .. testcode:: class D: - def f(self, x): - return x + def f(self): + return self + + class D2: + pass + +.. doctest:: + :hide: + + >>> d = D() + >>> d2 = D2() + >>> d2.f = d.f.__get__(d2, D2) + >>> d2.f() is d + True The function has a :term:`qualified name` attribute to support introspection: diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 1e9ac9b6761b647..30be15230fc088e 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -497,13 +497,30 @@ the :meth:`~Enum.__repr__` omits the inherited class' name. For example:: >>> Creature.DOG <Creature.DOG: size='medium', legs=4> -Use the :func:`!dataclass` argument ``repr=False`` +Use the :func:`~dataclasses.dataclass` argument ``repr=False`` to use the standard :func:`repr`. .. versionchanged:: 3.12 Only the dataclass fields are shown in the value area, not the dataclass' name. +.. note:: + + Adding :func:`~dataclasses.dataclass` decorator to :class:`Enum` + and its subclasses is not supported. It will not raise any errors, + but it will produce very strange results at runtime, such as members + being equal to each other:: + + >>> @dataclass # don't do this: it does not make any sense + ... class Color(Enum): + ... RED = 1 + ... BLUE = 2 + ... + >>> Color.RED is Color.BLUE + False + >>> Color.RED == Color.BLUE # problem is here: they should not be equal + True + Pickling -------- diff --git a/Doc/howto/gdb_helpers.rst b/Doc/howto/gdb_helpers.rst new file mode 100644 index 000000000000000..53bbf7ddaa2ab96 --- /dev/null +++ b/Doc/howto/gdb_helpers.rst @@ -0,0 +1,449 @@ +.. _gdb: + +========================================================= +Debugging C API extensions and CPython Internals with GDB +========================================================= + +.. highlight:: none + +This document explains how the Python GDB extension, ``python-gdb.py``, can +be used with the GDB debugger to debug CPython extensions and the +CPython interpreter itself. + +When debugging low-level problems such as crashes or deadlocks, a low-level +debugger, such as GDB, is useful to diagnose and correct the issue. +By default, GDB (or any of its front-ends) doesn't support high-level +information specific to the CPython interpreter. + +The ``python-gdb.py`` extension adds CPython interpreter information to GDB. +The extension helps introspect the stack of currently executing Python functions. +Given a Python object represented by a :c:expr:`PyObject *` pointer, +the extension surfaces the type and value of the object. + +Developers who are working on CPython extensions or tinkering with parts +of CPython that are written in C can use this document to learn how to use the +``python-gdb.py`` extension with GDB. + +.. note:: + + This document assumes that you are familiar with the basics of GDB and the + CPython C API. It consolidates guidance from the + `devguide <https://devguide.python.org>`_ and the + `Python wiki <https://wiki.python.org/moin/DebuggingWithGdb>`_. + + +Prerequisites +============= + +You need to have: + +- GDB 7 or later. (For earlier versions of GDB, see ``Misc/gdbinit`` in the + sources of Python 3.11 or earlier.) +- GDB-compatible debugging information for Python and any extension you are + debugging. +- The ``python-gdb.py`` extension. + +The extension is built with Python, but might be distributed separately or +not at all. Below, we include tips for a few common systems as examples. +Note that even if the instructions match your system, they might be outdated. + + +Setup with Python built from source +----------------------------------- + +When you build CPython from source, debugging information should be available, +and the build should add a ``python-gdb.py`` file to the root directory of +your repository. + +To activate support, you must add the directory containing ``python-gdb.py`` +to GDB's "auto-load-safe-path". +If you haven't done this, recent versions of GDB will print out a warning +with instructions on how to do this. + +.. note:: + + If you do not see instructions for your version of GDB, put this in your + configuration file (``~/.gdbinit`` or ``~/.config/gdb/gdbinit``):: + + add-auto-load-safe-path /path/to/cpython + + You can also add multiple paths, separated by ``:``. + + +Setup for Python from a Linux distro +------------------------------------ + +Most Linux systems provide debug information for the system Python +in a package called ``python-debuginfo``, ``python-dbg`` or similar. +For example: + +- Fedora: + + .. code-block:: shell + + sudo dnf install gdb + sudo dnf debuginfo-install python3 + +- Ubuntu: + + .. code-block:: shell + + sudo apt install gdb python3-dbg + +On several recent Linux systems, GDB can download debugging symbols +automatically using *debuginfod*. +However, this will not install the ``python-gdb.py`` extension; +you generally do need to install the debug info package separately. + + +Using the Debug build and Development mode +========================================== + +For easier debugging, you might want to: + +- Use a :ref:`debug build <debug-build>` of Python. (When building from source, + use ``configure --with-pydebug``. On Linux distros, install and run a package + like ``python-debug`` or ``python-dbg``, if available.) +- Use the runtime :ref:`development mode <devmode>` (``-X dev``). + +Both enable extra assertions and disable some optimizations. +Sometimes this hides the bug you are trying to find, but in most cases they +make the process easier. + + +Using the ``python-gdb`` extension +================================== + +When the extension is loaded, it provides two main features: +pretty printers for Python values, and additional commands. + +Pretty-printers +--------------- + +This is what a GDB backtrace looks like (truncated) when this extension is +enabled:: + + #0 0x000000000041a6b1 in PyObject_Malloc (nbytes=Cannot access memory at address 0x7fffff7fefe8 + ) at Objects/obmalloc.c:748 + #1 0x000000000041b7c0 in _PyObject_DebugMallocApi (id=111 'o', nbytes=24) at Objects/obmalloc.c:1445 + #2 0x000000000041b717 in _PyObject_DebugMalloc (nbytes=24) at Objects/obmalloc.c:1412 + #3 0x000000000044060a in _PyUnicode_New (length=11) at Objects/unicodeobject.c:346 + #4 0x00000000004466aa in PyUnicodeUCS2_DecodeUTF8Stateful (s=0x5c2b8d "__lltrace__", size=11, errors=0x0, consumed= + 0x0) at Objects/unicodeobject.c:2531 + #5 0x0000000000446647 in PyUnicodeUCS2_DecodeUTF8 (s=0x5c2b8d "__lltrace__", size=11, errors=0x0) + at Objects/unicodeobject.c:2495 + #6 0x0000000000440d1b in PyUnicodeUCS2_FromStringAndSize (u=0x5c2b8d "__lltrace__", size=11) + at Objects/unicodeobject.c:551 + #7 0x0000000000440d94 in PyUnicodeUCS2_FromString (u=0x5c2b8d "__lltrace__") at Objects/unicodeobject.c:569 + #8 0x0000000000584abd in PyDict_GetItemString (v= + {'Yuck': <type at remote 0xad4730>, '__builtins__': <module at remote 0x7ffff7fd5ee8>, '__file__': 'Lib/test/crashers/nasty_eq_vs_dict.py', '__package__': None, 'y': <Yuck(i=0) at remote 0xaacd80>, 'dict': {0: 0, 1: 1, 2: 2, 3: 3}, '__cached__': None, '__name__': '__main__', 'z': <Yuck(i=0) at remote 0xaace60>, '__doc__': None}, key= + 0x5c2b8d "__lltrace__") at Objects/dictobject.c:2171 + +Notice how the dictionary argument to ``PyDict_GetItemString`` is displayed +as its ``repr()``, rather than an opaque ``PyObject *`` pointer. + +The extension works by supplying a custom printing routine for values of type +``PyObject *``. If you need to access lower-level details of an object, then +cast the value to a pointer of the appropriate type. For example:: + + (gdb) p globals + $1 = {'__builtins__': <module at remote 0x7ffff7fb1868>, '__name__': + '__main__', 'ctypes': <module at remote 0x7ffff7f14360>, '__doc__': None, + '__package__': None} + + (gdb) p *(PyDictObject*)globals + $2 = {ob_refcnt = 3, ob_type = 0x3dbdf85820, ma_fill = 5, ma_used = 5, + ma_mask = 7, ma_table = 0x63d0f8, ma_lookup = 0x3dbdc7ea70 + <lookdict_string>, ma_smalltable = {{me_hash = 7065186196740147912, + me_key = '__builtins__', me_value = <module at remote 0x7ffff7fb1868>}, + {me_hash = -368181376027291943, me_key = '__name__', + me_value ='__main__'}, {me_hash = 0, me_key = 0x0, me_value = 0x0}, + {me_hash = 0, me_key = 0x0, me_value = 0x0}, + {me_hash = -9177857982131165996, me_key = 'ctypes', + me_value = <module at remote 0x7ffff7f14360>}, + {me_hash = -8518757509529533123, me_key = '__doc__', me_value = None}, + {me_hash = 0, me_key = 0x0, me_value = 0x0}, { + me_hash = 6614918939584953775, me_key = '__package__', me_value = None}}} + +Note that the pretty-printers do not actually call ``repr()``. +For basic types, they try to match its result closely. + +An area that can be confusing is that the custom printer for some types look a +lot like GDB's built-in printer for standard types. For example, the +pretty-printer for a Python ``int`` (:c:expr:`PyLongObject *`) +gives a representation that is not distinguishable from one of a +regular machine-level integer:: + + (gdb) p some_machine_integer + $3 = 42 + + (gdb) p some_python_integer + $4 = 42 + +The internal structure can be revealed with a cast to :c:expr:`PyLongObject *`: + + (gdb) p *(PyLongObject*)some_python_integer + $5 = {ob_base = {ob_base = {ob_refcnt = 8, ob_type = 0x3dad39f5e0}, ob_size = 1}, + ob_digit = {42}} + +A similar confusion can arise with the ``str`` type, where the output looks a +lot like gdb's built-in printer for ``char *``:: + + (gdb) p ptr_to_python_str + $6 = '__builtins__' + +The pretty-printer for ``str`` instances defaults to using single-quotes (as +does Python's ``repr`` for strings) whereas the standard printer for ``char *`` +values uses double-quotes and contains a hexadecimal address:: + + (gdb) p ptr_to_char_star + $7 = 0x6d72c0 "hello world" + +Again, the implementation details can be revealed with a cast to +:c:expr:`PyUnicodeObject *`:: + + (gdb) p *(PyUnicodeObject*)$6 + $8 = {ob_base = {ob_refcnt = 33, ob_type = 0x3dad3a95a0}, length = 12, + str = 0x7ffff2128500, hash = 7065186196740147912, state = 1, defenc = 0x0} + +``py-list`` +----------- + + The extension adds a ``py-list`` command, which + lists the Python source code (if any) for the current frame in the selected + thread. The current line is marked with a ">":: + + (gdb) py-list + 901 if options.profile: + 902 options.profile = False + 903 profile_me() + 904 return + 905 + >906 u = UI() + 907 if not u.quit: + 908 try: + 909 gtk.main() + 910 except KeyboardInterrupt: + 911 # properly quit on a keyboard interrupt... + + Use ``py-list START`` to list at a different line number within the Python + source, and ``py-list START,END`` to list a specific range of lines within + the Python source. + +``py-up`` and ``py-down`` +------------------------- + + The ``py-up`` and ``py-down`` commands are analogous to GDB's regular ``up`` + and ``down`` commands, but try to move at the level of CPython frames, rather + than C frames. + + GDB is not always able to read the relevant frame information, depending on + the optimization level with which CPython was compiled. Internally, the + commands look for C frames that are executing the default frame evaluation + function (that is, the core bytecode interpreter loop within CPython) and + look up the value of the related ``PyFrameObject *``. + + They emit the frame number (at the C level) within the thread. + + For example:: + + (gdb) py-up + #37 Frame 0x9420b04, for file /usr/lib/python2.6/site-packages/ + gnome_sudoku/main.py, line 906, in start_game () + u = UI() + (gdb) py-up + #40 Frame 0x948e82c, for file /usr/lib/python2.6/site-packages/ + gnome_sudoku/gnome_sudoku.py, line 22, in start_game(main=<module at remote 0xb771b7f4>) + main.start_game() + (gdb) py-up + Unable to find an older python frame + + so we're at the top of the Python stack. + + The frame numbers correspond to those displayed by GDB's standard + ``backtrace`` command. + The command skips C frames which are not executing Python code. + + Going back down:: + + (gdb) py-down + #37 Frame 0x9420b04, for file /usr/lib/python2.6/site-packages/gnome_sudoku/main.py, line 906, in start_game () + u = UI() + (gdb) py-down + #34 (unable to read python frame information) + (gdb) py-down + #23 (unable to read python frame information) + (gdb) py-down + #19 (unable to read python frame information) + (gdb) py-down + #14 Frame 0x99262ac, for file /usr/lib/python2.6/site-packages/gnome_sudoku/game_selector.py, line 201, in run_swallowed_dialog (self=<NewOrSavedGameSelector(new_game_model=<gtk.ListStore at remote 0x98fab44>, puzzle=None, saved_games=[{'gsd.auto_fills': 0, 'tracking': {}, 'trackers': {}, 'notes': [], 'saved_at': 1270084485, 'game': '7 8 0 0 0 0 0 5 6 0 0 9 0 8 0 1 0 0 0 4 6 0 0 0 0 7 0 6 5 0 0 0 4 7 9 2 0 0 0 9 0 1 0 0 0 3 9 7 6 0 0 0 1 8 0 6 0 0 0 0 2 8 0 0 0 5 0 4 0 6 0 0 2 1 0 0 0 0 0 4 5\n7 8 0 0 0 0 0 5 6 0 0 9 0 8 0 1 0 0 0 4 6 0 0 0 0 7 0 6 5 1 8 3 4 7 9 2 0 0 0 9 0 1 0 0 0 3 9 7 6 0 0 0 1 8 0 6 0 0 0 0 2 8 0 0 0 5 0 4 0 6 0 0 2 1 0 0 0 0 0 4 5', 'gsd.impossible_hints': 0, 'timer.__absolute_start_time__': <float at remote 0x984b474>, 'gsd.hints': 0, 'timer.active_time': <float at remote 0x984b494>, 'timer.total_time': <float at remote 0x984b464>}], dialog=<gtk.Dialog at remote 0x98faaa4>, saved_game_model=<gtk.ListStore at remote 0x98fad24>, sudoku_maker=<SudokuMaker(terminated=False, played=[], batch_siz...(truncated) + swallower.run_dialog(self.dialog) + (gdb) py-down + #11 Frame 0x9aead74, for file /usr/lib/python2.6/site-packages/gnome_sudoku/dialog_swallower.py, line 48, in run_dialog (self=<SwappableArea(running=<gtk.Dialog at remote 0x98faaa4>, main_page=0) at remote 0x98fa6e4>, d=<gtk.Dialog at remote 0x98faaa4>) + gtk.main() + (gdb) py-down + #8 (unable to read python frame information) + (gdb) py-down + Unable to find a newer python frame + + and we're at the bottom of the Python stack. + + Note that in Python 3.12 and newer, the same C stack frame can be used for + multiple Python stack frames. This means that ``py-up`` and ``py-down`` + may move multiple Python frames at once. For example:: + + (gdb) py-up + #6 Frame 0x7ffff7fb62b0, for file /tmp/rec.py, line 5, in recursive_function (n=0) + time.sleep(5) + #6 Frame 0x7ffff7fb6240, for file /tmp/rec.py, line 7, in recursive_function (n=1) + recursive_function(n-1) + #6 Frame 0x7ffff7fb61d0, for file /tmp/rec.py, line 7, in recursive_function (n=2) + recursive_function(n-1) + #6 Frame 0x7ffff7fb6160, for file /tmp/rec.py, line 7, in recursive_function (n=3) + recursive_function(n-1) + #6 Frame 0x7ffff7fb60f0, for file /tmp/rec.py, line 7, in recursive_function (n=4) + recursive_function(n-1) + #6 Frame 0x7ffff7fb6080, for file /tmp/rec.py, line 7, in recursive_function (n=5) + recursive_function(n-1) + #6 Frame 0x7ffff7fb6020, for file /tmp/rec.py, line 9, in <module> () + recursive_function(5) + (gdb) py-up + Unable to find an older python frame + + +``py-bt`` +--------- + + The ``py-bt`` command attempts to display a Python-level backtrace of the + current thread. + + For example:: + + (gdb) py-bt + #8 (unable to read python frame information) + #11 Frame 0x9aead74, for file /usr/lib/python2.6/site-packages/gnome_sudoku/dialog_swallower.py, line 48, in run_dialog (self=<SwappableArea(running=<gtk.Dialog at remote 0x98faaa4>, main_page=0) at remote 0x98fa6e4>, d=<gtk.Dialog at remote 0x98faaa4>) + gtk.main() + #14 Frame 0x99262ac, for file /usr/lib/python2.6/site-packages/gnome_sudoku/game_selector.py, line 201, in run_swallowed_dialog (self=<NewOrSavedGameSelector(new_game_model=<gtk.ListStore at remote 0x98fab44>, puzzle=None, saved_games=[{'gsd.auto_fills': 0, 'tracking': {}, 'trackers': {}, 'notes': [], 'saved_at': 1270084485, 'game': '7 8 0 0 0 0 0 5 6 0 0 9 0 8 0 1 0 0 0 4 6 0 0 0 0 7 0 6 5 0 0 0 4 7 9 2 0 0 0 9 0 1 0 0 0 3 9 7 6 0 0 0 1 8 0 6 0 0 0 0 2 8 0 0 0 5 0 4 0 6 0 0 2 1 0 0 0 0 0 4 5\n7 8 0 0 0 0 0 5 6 0 0 9 0 8 0 1 0 0 0 4 6 0 0 0 0 7 0 6 5 1 8 3 4 7 9 2 0 0 0 9 0 1 0 0 0 3 9 7 6 0 0 0 1 8 0 6 0 0 0 0 2 8 0 0 0 5 0 4 0 6 0 0 2 1 0 0 0 0 0 4 5', 'gsd.impossible_hints': 0, 'timer.__absolute_start_time__': <float at remote 0x984b474>, 'gsd.hints': 0, 'timer.active_time': <float at remote 0x984b494>, 'timer.total_time': <float at remote 0x984b464>}], dialog=<gtk.Dialog at remote 0x98faaa4>, saved_game_model=<gtk.ListStore at remote 0x98fad24>, sudoku_maker=<SudokuMaker(terminated=False, played=[], batch_siz...(truncated) + swallower.run_dialog(self.dialog) + #19 (unable to read python frame information) + #23 (unable to read python frame information) + #34 (unable to read python frame information) + #37 Frame 0x9420b04, for file /usr/lib/python2.6/site-packages/gnome_sudoku/main.py, line 906, in start_game () + u = UI() + #40 Frame 0x948e82c, for file /usr/lib/python2.6/site-packages/gnome_sudoku/gnome_sudoku.py, line 22, in start_game (main=<module at remote 0xb771b7f4>) + main.start_game() + + The frame numbers correspond to those displayed by GDB's standard + ``backtrace`` command. + +``py-print`` +------------ + + The ``py-print`` command looks up a Python name and tries to print it. + It looks in locals within the current thread, then globals, then finally + builtins:: + + (gdb) py-print self + local 'self' = <SwappableArea(running=<gtk.Dialog at remote 0x98faaa4>, + main_page=0) at remote 0x98fa6e4> + (gdb) py-print __name__ + global '__name__' = 'gnome_sudoku.dialog_swallower' + (gdb) py-print len + builtin 'len' = <built-in function len> + (gdb) py-print scarlet_pimpernel + 'scarlet_pimpernel' not found + + If the current C frame corresponds to multiple Python frames, ``py-print`` + only considers the first one. + +``py-locals`` +------------- + + The ``py-locals`` command looks up all Python locals within the current + Python frame in the selected thread, and prints their representations:: + + (gdb) py-locals + self = <SwappableArea(running=<gtk.Dialog at remote 0x98faaa4>, + main_page=0) at remote 0x98fa6e4> + d = <gtk.Dialog at remote 0x98faaa4> + + If the current C frame corresponds to multiple Python frames, locals from + all of them will be shown:: + + (gdb) py-locals + Locals for recursive_function + n = 0 + Locals for recursive_function + n = 1 + Locals for recursive_function + n = 2 + Locals for recursive_function + n = 3 + Locals for recursive_function + n = 4 + Locals for recursive_function + n = 5 + Locals for <module> + + +Use with GDB commands +===================== + +The extension commands complement GDB's built-in commands. +For example, you can use a frame numbers shown by ``py-bt`` with the ``frame`` +command to go a specific frame within the selected thread, like this:: + + (gdb) py-bt + (output snipped) + #68 Frame 0xaa4560, for file Lib/test/regrtest.py, line 1548, in <module> () + main() + (gdb) frame 68 + #68 0x00000000004cd1e6 in PyEval_EvalFrameEx (f=Frame 0xaa4560, for file Lib/test/regrtest.py, line 1548, in <module> (), throwflag=0) at Python/ceval.c:2665 + 2665 x = call_function(&sp, oparg); + (gdb) py-list + 1543 # Run the tests in a context manager that temporary changes the CWD to a + 1544 # temporary and writable directory. If it's not possible to create or + 1545 # change the CWD, the original CWD will be used. The original CWD is + 1546 # available from test_support.SAVEDCWD. + 1547 with test_support.temp_cwd(TESTCWD, quiet=True): + >1548 main() + +The ``info threads`` command will give you a list of the threads within the +process, and you can use the ``thread`` command to select a different one:: + + (gdb) info threads + 105 Thread 0x7fffefa18710 (LWP 10260) sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:86 + 104 Thread 0x7fffdf5fe710 (LWP 10259) sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:86 + * 1 Thread 0x7ffff7fe2700 (LWP 10145) 0x00000038e46d73e3 in select () at ../sysdeps/unix/syscall-template.S:82 + +You can use ``thread apply all COMMAND`` or (``t a a COMMAND`` for short) to run +a command on all threads. With ``py-bt``, this lets you see what every +thread is doing at the Python level:: + + (gdb) t a a py-bt + + Thread 105 (Thread 0x7fffefa18710 (LWP 10260)): + #5 Frame 0x7fffd00019d0, for file /home/david/coding/python-svn/Lib/threading.py, line 155, in _acquire_restore (self=<_RLock(_Verbose__verbose=False, _RLock__owner=140737354016512, _RLock__block=<thread.lock at remote 0x858770>, _RLock__count=1) at remote 0xd7ff40>, count_owner=(1, 140737213728528), count=1, owner=140737213728528) + self.__block.acquire() + #8 Frame 0x7fffac001640, for file /home/david/coding/python-svn/Lib/threading.py, line 269, in wait (self=<_Condition(_Condition__lock=<_RLock(_Verbose__verbose=False, _RLock__owner=140737354016512, _RLock__block=<thread.lock at remote 0x858770>, _RLock__count=1) at remote 0xd7ff40>, acquire=<instancemethod at remote 0xd80260>, _is_owned=<instancemethod at remote 0xd80160>, _release_save=<instancemethod at remote 0xd803e0>, release=<instancemethod at remote 0xd802e0>, _acquire_restore=<instancemethod at remote 0xd7ee60>, _Verbose__verbose=False, _Condition__waiters=[]) at remote 0xd7fd10>, timeout=None, waiter=<thread.lock at remote 0x858a90>, saved_state=(1, 140737213728528)) + self._acquire_restore(saved_state) + #12 Frame 0x7fffb8001a10, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 348, in f () + cond.wait() + #16 Frame 0x7fffb8001c40, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 37, in task (tid=140737213728528) + f() + + Thread 104 (Thread 0x7fffdf5fe710 (LWP 10259)): + #5 Frame 0x7fffe4001580, for file /home/david/coding/python-svn/Lib/threading.py, line 155, in _acquire_restore (self=<_RLock(_Verbose__verbose=False, _RLock__owner=140737354016512, _RLock__block=<thread.lock at remote 0x858770>, _RLock__count=1) at remote 0xd7ff40>, count_owner=(1, 140736940992272), count=1, owner=140736940992272) + self.__block.acquire() + #8 Frame 0x7fffc8002090, for file /home/david/coding/python-svn/Lib/threading.py, line 269, in wait (self=<_Condition(_Condition__lock=<_RLock(_Verbose__verbose=False, _RLock__owner=140737354016512, _RLock__block=<thread.lock at remote 0x858770>, _RLock__count=1) at remote 0xd7ff40>, acquire=<instancemethod at remote 0xd80260>, _is_owned=<instancemethod at remote 0xd80160>, _release_save=<instancemethod at remote 0xd803e0>, release=<instancemethod at remote 0xd802e0>, _acquire_restore=<instancemethod at remote 0xd7ee60>, _Verbose__verbose=False, _Condition__waiters=[]) at remote 0xd7fd10>, timeout=None, waiter=<thread.lock at remote 0x858860>, saved_state=(1, 140736940992272)) + self._acquire_restore(saved_state) + #12 Frame 0x7fffac001c90, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 348, in f () + cond.wait() + #16 Frame 0x7fffac0011c0, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 37, in task (tid=140736940992272) + f() + + Thread 1 (Thread 0x7ffff7fe2700 (LWP 10145)): + #5 Frame 0xcb5380, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 16, in _wait () + time.sleep(0.01) + #8 Frame 0x7fffd00024a0, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 378, in _check_notify (self=<ConditionTests(_testMethodName='test_notify', _resultForDoCleanups=<TestResult(_original_stdout=<cStringIO.StringO at remote 0xc191e0>, skipped=[], _mirrorOutput=False, testsRun=39, buffer=False, _original_stderr=<file at remote 0x7ffff7fc6340>, _stdout_buffer=<cStringIO.StringO at remote 0xc9c7f8>, _stderr_buffer=<cStringIO.StringO at remote 0xc9c790>, _moduleSetUpFailed=False, expectedFailures=[], errors=[], _previousTestClass=<type at remote 0x928310>, unexpectedSuccesses=[], failures=[], shouldStop=False, failfast=False) at remote 0xc185a0>, _threads=(0,), _cleanups=[], _type_equality_funcs={<type at remote 0x7eba00>: <instancemethod at remote 0xd750e0>, <type at remote 0x7e7820>: <instancemethod at remote 0xd75160>, <type at remote 0x7e30e0>: <instancemethod at remote 0xd75060>, <type at remote 0x7e7d20>: <instancemethod at remote 0xd751e0>, <type at remote 0x7f19e0...(truncated) + _wait() diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index a835bb5f13bd1c9..8b334555ab64633 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -13,10 +13,10 @@ Currently, the HOWTOs are: .. toctree:: :maxdepth: 1 - pyporting.rst cporting.rst curses.rst descriptor.rst + gdb_helpers.rst enum.rst functional.rst logging.rst diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index ea494f2fdbbce42..f7d885ec88483d7 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -1744,13 +1744,11 @@ to the above, as in the following example:: return self.fmt.format(*self.args) class StyleAdapter(logging.LoggerAdapter): - def __init__(self, logger, extra=None): - super().__init__(logger, extra or {}) - - def log(self, level, msg, /, *args, **kwargs): + def log(self, level, msg, /, *args, stacklevel=1, **kwargs): if self.isEnabledFor(level): msg, kwargs = self.process(msg, kwargs) - self.logger._log(level, Message(msg, args), (), **kwargs) + self.logger.log(level, Message(msg, args), **kwargs, + stacklevel=stacklevel+1) logger = StyleAdapter(logging.getLogger(__name__)) @@ -1762,7 +1760,7 @@ to the above, as in the following example:: main() The above script should log the message ``Hello, world!`` when run with -Python 3.2 or later. +Python 3.8 or later. .. currentmodule:: logging @@ -1933,30 +1931,28 @@ This dictionary is passed to :func:`~config.dictConfig` to put the configuration LOGGING = { 'version': 1, - 'disable_existing_loggers': True, + 'disable_existing_loggers': False, 'formatters': { 'verbose': { - 'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s' + 'format': '{levelname} {asctime} {module} {process:d} {thread:d} {message}', + 'style': '{', }, 'simple': { - 'format': '%(levelname)s %(message)s' + 'format': '{levelname} {message}', + 'style': '{', }, }, 'filters': { 'special': { '()': 'project.logging.SpecialFilter', 'foo': 'bar', - } + }, }, 'handlers': { - 'null': { - 'level':'DEBUG', - 'class':'django.utils.log.NullHandler', - }, - 'console':{ - 'level':'DEBUG', - 'class':'logging.StreamHandler', - 'formatter': 'simple' + 'console': { + 'level': 'INFO', + 'class': 'logging.StreamHandler', + 'formatter': 'simple', }, 'mail_admins': { 'level': 'ERROR', @@ -1966,9 +1962,8 @@ This dictionary is passed to :func:`~config.dictConfig` to put the configuration }, 'loggers': { 'django': { - 'handlers':['null'], + 'handlers': ['console'], 'propagate': True, - 'level':'INFO', }, 'django.request': { 'handlers': ['mail_admins'], diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index f164b461c93b9cd..347330e98dd00c9 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -520,7 +520,7 @@ custom handlers) are the following configuration methods: * The :meth:`~Handler.setLevel` method, just as in logger objects, specifies the lowest severity that will be dispatched to the appropriate destination. Why - are there two :func:`setLevel` methods? The level set in the logger + are there two :meth:`~Handler.setLevel` methods? The level set in the logger determines which severity of messages it will pass to its handlers. The level set in each handler determines which messages that handler will send on. @@ -774,29 +774,29 @@ What happens if no configuration is provided If no logging configuration is provided, it is possible to have a situation where a logging event needs to be output, but no handlers can be found to -output the event. The behaviour of the logging package in these -circumstances is dependent on the Python version. +output the event. -For versions of Python prior to 3.2, the behaviour is as follows: +The event is output using a 'handler of last resort', stored in +:data:`lastResort`. This internal handler is not associated with any +logger, and acts like a :class:`~logging.StreamHandler` which writes the +event description message to the current value of ``sys.stderr`` (therefore +respecting any redirections which may be in effect). No formatting is +done on the message - just the bare event description message is printed. +The handler's level is set to ``WARNING``, so all events at this and +greater severities will be output. -* If *logging.raiseExceptions* is ``False`` (production mode), the event is - silently dropped. +.. versionchanged:: 3.2 -* If *logging.raiseExceptions* is ``True`` (development mode), a message - 'No handlers could be found for logger X.Y.Z' is printed once. + For versions of Python prior to 3.2, the behaviour is as follows: -In Python 3.2 and later, the behaviour is as follows: + * If :data:`raiseExceptions` is ``False`` (production mode), the event is + silently dropped. -* The event is output using a 'handler of last resort', stored in - ``logging.lastResort``. This internal handler is not associated with any - logger, and acts like a :class:`~logging.StreamHandler` which writes the - event description message to the current value of ``sys.stderr`` (therefore - respecting any redirections which may be in effect). No formatting is - done on the message - just the bare event description message is printed. - The handler's level is set to ``WARNING``, so all events at this and - greater severities will be output. + * If :data:`raiseExceptions` is ``True`` (development mode), a message + 'No handlers could be found for logger X.Y.Z' is printed once. -To obtain the pre-3.2 behaviour, ``logging.lastResort`` can be set to ``None``. + To obtain the pre-3.2 behaviour, + :data:`lastResort` can be set to ``None``. .. _library-config: @@ -998,7 +998,7 @@ Logged messages are formatted for presentation through instances of the use with the % operator and a dictionary. For formatting multiple messages in a batch, instances of -:class:`~handlers.BufferingFormatter` can be used. In addition to the format +:class:`BufferingFormatter` can be used. In addition to the format string (which is applied to each message in the batch), there is provision for header and trailer format strings. @@ -1034,7 +1034,8 @@ checks to see if a module-level variable, :data:`raiseExceptions`, is set. If set, a traceback is printed to :data:`sys.stderr`. If not set, the exception is swallowed. -.. note:: The default value of :data:`raiseExceptions` is ``True``. This is +.. note:: + The default value of :data:`raiseExceptions` is ``True``. This is because during development, you typically want to be notified of any exceptions that occur. It's advised that you set :data:`raiseExceptions` to ``False`` for production usage. @@ -1072,7 +1073,7 @@ You can write code like this:: expensive_func2()) so that if the logger's threshold is set above ``DEBUG``, the calls to -:func:`expensive_func1` and :func:`expensive_func2` are never made. +``expensive_func1`` and ``expensive_func2`` are never made. .. note:: In some cases, :meth:`~Logger.isEnabledFor` can itself be more expensive than you'd like (e.g. for deeply nested loggers where an explicit diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index 501b16d82d4d6fc..d560364107bd125 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -1,3 +1,5 @@ +:orphan: + .. _pyporting-howto: ************************************* @@ -6,423 +8,30 @@ How to port Python 2 Code to Python 3 :author: Brett Cannon -.. topic:: Abstract - - Python 2 reached its official end-of-life at the start of 2020. This means - that no new bug reports, fixes, or changes will be made to Python 2 - it's - no longer supported. - - This guide is intended to provide you with a path to Python 3 for your - code, that includes compatibility with Python 2 as a first step. - - If you are looking to port an extension module instead of pure Python code, - please see :ref:`cporting-howto`. - - The archived python-porting_ mailing list may contain some useful guidance. - - -The Short Explanation -===================== - -To achieve Python 2/3 compatibility in a single code base, the basic steps -are: - -#. Only worry about supporting Python 2.7 -#. Make sure you have good test coverage (coverage.py_ can help; - ``python -m pip install coverage``) -#. Learn the differences between Python 2 and 3 -#. Use Futurize_ (or Modernize_) to update your code (e.g. ``python -m pip install future``) -#. Use Pylint_ to help make sure you don't regress on your Python 3 support - (``python -m pip install pylint``) -#. Use caniusepython3_ to find out which of your dependencies are blocking your - use of Python 3 (``python -m pip install caniusepython3``) -#. Once your dependencies are no longer blocking you, use continuous integration - to make sure you stay compatible with Python 2 and 3 (tox_ can help test - against multiple versions of Python; ``python -m pip install tox``) -#. Consider using optional :term:`static type checking <static type checker>` - to make sure your type usage - works in both Python 2 and 3 (e.g. use mypy_ to check your typing under both - Python 2 and Python 3; ``python -m pip install mypy``). - -.. note:: - - Note: Using ``python -m pip install`` guarantees that the ``pip`` you invoke - is the one installed for the Python currently in use, whether it be - a system-wide ``pip`` or one installed within a - :ref:`virtual environment <tut-venv>`. - -Details -======= - -Even if other factors - say, dependencies over which you have no control - -still require you to support Python 2, that does not prevent you taking the -step of including Python 3 support. - -Most changes required to support Python 3 lead to cleaner code using newer -practices even in Python 2 code. - - -Different versions of Python 2 ------------------------------- - -Ideally, your code should be compatible with Python 2.7, which was the -last supported version of Python 2. - -Some of the tools mentioned in this guide will not work with Python 2.6. - -If absolutely necessary, the six_ project can help you support Python 2.5 and -3 simultaneously. Do realize, though, that nearly all the projects listed in -this guide will not be available to you. - -If you are able to skip Python 2.5 and older, the required changes to your -code will be minimal. At worst you will have to use a function instead of a -method in some instances or have to import a function instead of using a -built-in one. - - -Make sure you specify the proper version support in your ``setup.py`` file --------------------------------------------------------------------------- - -In your ``setup.py`` file you should have the proper `trove classifier`_ -specifying what versions of Python you support. As your project does not support -Python 3 yet you should at least have -``Programming Language :: Python :: 2 :: Only`` specified. Ideally you should -also specify each major/minor version of Python that you do support, e.g. -``Programming Language :: Python :: 2.7``. - - -Have good test coverage ------------------------ - -Once you have your code supporting the oldest version of Python 2 you want it -to, you will want to make sure your test suite has good coverage. A good rule of -thumb is that if you want to be confident enough in your test suite that any -failures that appear after having tools rewrite your code are actual bugs in the -tools and not in your code. If you want a number to aim for, try to get over 80% -coverage (and don't feel bad if you find it hard to get better than 90% -coverage). If you don't already have a tool to measure test coverage then -coverage.py_ is recommended. - - -Be aware of the differences between Python 2 and 3 --------------------------------------------------- - -Once you have your code well-tested you are ready to begin porting your code to -Python 3! But to fully understand how your code is going to change and what -you want to look out for while you code, you will want to learn what changes -Python 3 makes in terms of Python 2. - -Some resources for understanding the differences and their implications for you -code: - -* the :ref:`"What's New" <whatsnew-index>` doc for each release of Python 3 -* the `Porting to Python 3`_ book (which is free online) -* the handy `cheat sheet`_ from the Python-Future project. - - -Update your code ----------------- - -There are tools available that can port your code automatically. - -Futurize_ does its best to make Python 3 idioms and practices exist in Python -2, e.g. backporting the ``bytes`` type from Python 3 so that you have -semantic parity between the major versions of Python. This is the better -approach for most cases. - -Modernize_, on the other hand, is more conservative and targets a Python 2/3 -subset of Python, directly relying on six_ to help provide compatibility. - -A good approach is to run the tool over your test suite first and visually -inspect the diff to make sure the transformation is accurate. After you have -transformed your test suite and verified that all the tests still pass as -expected, then you can transform your application code knowing that any tests -which fail is a translation failure. - -Unfortunately the tools can't automate everything to make your code work under -Python 3, and you will also need to read the tools' documentation in case some -options you need are turned off by default. - -Key issues to be aware of and check for: - -Division -++++++++ - -In Python 3, ``5 / 2 == 2.5`` and not ``2`` as it was in Python 2; all -division between ``int`` values result in a ``float``. This change has -actually been planned since Python 2.2 which was released in 2002. Since then -users have been encouraged to add ``from __future__ import division`` to any -and all files which use the ``/`` and ``//`` operators or to be running the -interpreter with the ``-Q`` flag. If you have not been doing this then you -will need to go through your code and do two things: - -#. Add ``from __future__ import division`` to your files -#. Update any division operator as necessary to either use ``//`` to use floor - division or continue using ``/`` and expect a float - -The reason that ``/`` isn't simply translated to ``//`` automatically is that if -an object defines a ``__truediv__`` method but not ``__floordiv__`` then your -code would begin to fail (e.g. a user-defined class that uses ``/`` to -signify some operation but not ``//`` for the same thing or at all). +Python 2 reached its official end-of-life at the start of 2020. This means +that no new bug reports, fixes, or changes will be made to Python 2 - it's +no longer supported: see :pep:`373` and +`status of Python versions <https://devguide.python.org/versions>`_. +If you are looking to port an extension module instead of pure Python code, +please see :ref:`cporting-howto`. -Text versus binary data -+++++++++++++++++++++++ +The archived python-porting_ mailing list may contain some useful guidance. -In Python 2 you could use the ``str`` type for both text and binary data. -Unfortunately this confluence of two different concepts could lead to brittle -code which sometimes worked for either kind of data, sometimes not. It also -could lead to confusing APIs if people didn't explicitly state that something -that accepted ``str`` accepted either text or binary data instead of one -specific type. This complicated the situation especially for anyone supporting -multiple languages as APIs wouldn't bother explicitly supporting ``unicode`` -when they claimed text data support. +Since Python 3.13 the original porting guide was discontinued. +You can find the old guide in the +`archive <https://docs.python.org/3.12/howto/pyporting.html>`_. -Python 3 made text and binary data distinct types that cannot simply be mixed -together. For any code that deals only with text or only binary data, this -separation doesn't pose an issue. But for code that has to deal with both, it -does mean you might have to now care about when you are using text compared -to binary data, which is why this cannot be entirely automated. -Decide which APIs take text and which take binary (it is **highly** recommended -you don't design APIs that can take both due to the difficulty of keeping the -code working; as stated earlier it is difficult to do well). In Python 2 this -means making sure the APIs that take text can work with ``unicode`` and those -that work with binary data work with the ``bytes`` type from Python 3 -(which is a subset of ``str`` in Python 2 and acts as an alias for ``bytes`` -type in Python 2). Usually the biggest issue is realizing which methods exist -on which types in Python 2 and 3 simultaneously (for text that's ``unicode`` -in Python 2 and ``str`` in Python 3, for binary that's ``str``/``bytes`` in -Python 2 and ``bytes`` in Python 3). +Third-party guides +================== -The following table lists the **unique** methods of each data type across -Python 2 and 3 (e.g., the ``decode()`` method is usable on the equivalent binary -data type in either Python 2 or 3, but it can't be used by the textual data -type consistently between Python 2 and 3 because ``str`` in Python 3 doesn't -have the method). Do note that as of Python 3.5 the ``__mod__`` method was -added to the bytes type. +There are also multiple third-party guides that might be useful: -======================== ===================== -**Text data** **Binary data** ------------------------- --------------------- -\ decode ------------------------- --------------------- -encode ------------------------- --------------------- -format ------------------------- --------------------- -isdecimal ------------------------- --------------------- -isnumeric -======================== ===================== +- `Guide by Fedora <https://portingguide.readthedocs.io>`_ +- `PyCon 2020 tutorial <https://www.youtube.com/watch?v=JgIgEjASOlk>`_ +- `Guide by DigitalOcean <https://www.digitalocean.com/community/tutorials/how-to-port-python-2-code-to-python-3>`_ +- `Guide by ActiveState <https://www.activestate.com/blog/how-to-migrate-python-2-applications-to-python-3>`_ -Making the distinction easier to handle can be accomplished by encoding and -decoding between binary data and text at the edge of your code. This means that -when you receive text in binary data, you should immediately decode it. And if -your code needs to send text as binary data then encode it as late as possible. -This allows your code to work with only text internally and thus eliminates -having to keep track of what type of data you are working with. -The next issue is making sure you know whether the string literals in your code -represent text or binary data. You should add a ``b`` prefix to any -literal that presents binary data. For text you should add a ``u`` prefix to -the text literal. (There is a :mod:`__future__` import to force all unspecified -literals to be Unicode, but usage has shown it isn't as effective as adding a -``b`` or ``u`` prefix to all literals explicitly) - -You also need to be careful about opening files. Possibly you have not always -bothered to add the ``b`` mode when opening a binary file (e.g., ``rb`` for -binary reading). Under Python 3, binary files and text files are clearly -distinct and mutually incompatible; see the :mod:`io` module for details. -Therefore, you **must** make a decision of whether a file will be used for -binary access (allowing binary data to be read and/or written) or textual access -(allowing text data to be read and/or written). You should also use :func:`io.open` -for opening files instead of the built-in :func:`open` function as the :mod:`io` -module is consistent from Python 2 to 3 while the built-in :func:`open` function -is not (in Python 3 it's actually :func:`io.open`). Do not bother with the -outdated practice of using :func:`codecs.open` as that's only necessary for -keeping compatibility with Python 2.5. - -The constructors of both ``str`` and ``bytes`` have different semantics for the -same arguments between Python 2 and 3. Passing an integer to ``bytes`` in Python 2 -will give you the string representation of the integer: ``bytes(3) == '3'``. -But in Python 3, an integer argument to ``bytes`` will give you a bytes object -as long as the integer specified, filled with null bytes: -``bytes(3) == b'\x00\x00\x00'``. A similar worry is necessary when passing a -bytes object to ``str``. In Python 2 you just get the bytes object back: -``str(b'3') == b'3'``. But in Python 3 you get the string representation of the -bytes object: ``str(b'3') == "b'3'"``. - -Finally, the indexing of binary data requires careful handling (slicing does -**not** require any special handling). In Python 2, -``b'123'[1] == b'2'`` while in Python 3 ``b'123'[1] == 50``. Because binary data -is simply a collection of binary numbers, Python 3 returns the integer value for -the byte you index on. But in Python 2 because ``bytes == str``, indexing -returns a one-item slice of bytes. The six_ project has a function -named ``six.indexbytes()`` which will return an integer like in Python 3: -``six.indexbytes(b'123', 1)``. - -To summarize: - -#. Decide which of your APIs take text and which take binary data -#. Make sure that your code that works with text also works with ``unicode`` and - code for binary data works with ``bytes`` in Python 2 (see the table above - for what methods you cannot use for each type) -#. Mark all binary literals with a ``b`` prefix, textual literals with a ``u`` - prefix -#. Decode binary data to text as soon as possible, encode text as binary data as - late as possible -#. Open files using :func:`io.open` and make sure to specify the ``b`` mode when - appropriate -#. Be careful when indexing into binary data - - -Use feature detection instead of version detection -++++++++++++++++++++++++++++++++++++++++++++++++++ - -Inevitably you will have code that has to choose what to do based on what -version of Python is running. The best way to do this is with feature detection -of whether the version of Python you're running under supports what you need. -If for some reason that doesn't work then you should make the version check be -against Python 2 and not Python 3. To help explain this, let's look at an -example. - -Let's pretend that you need access to a feature of :mod:`importlib` that -is available in Python's standard library since Python 3.3 and available for -Python 2 through importlib2_ on PyPI. You might be tempted to write code to -access e.g. the :mod:`importlib.abc` module by doing the following:: - - import sys - - if sys.version_info[0] == 3: - from importlib import abc - else: - from importlib2 import abc - -The problem with this code is what happens when Python 4 comes out? It would -be better to treat Python 2 as the exceptional case instead of Python 3 and -assume that future Python versions will be more compatible with Python 3 than -Python 2:: - - import sys - - if sys.version_info[0] > 2: - from importlib import abc - else: - from importlib2 import abc - -The best solution, though, is to do no version detection at all and instead rely -on feature detection. That avoids any potential issues of getting the version -detection wrong and helps keep you future-compatible:: - - try: - from importlib import abc - except ImportError: - from importlib2 import abc - - -Prevent compatibility regressions ---------------------------------- - -Once you have fully translated your code to be compatible with Python 3, you -will want to make sure your code doesn't regress and stop working under -Python 3. This is especially true if you have a dependency which is blocking you -from actually running under Python 3 at the moment. - -To help with staying compatible, any new modules you create should have -at least the following block of code at the top of it:: - - from __future__ import absolute_import - from __future__ import division - from __future__ import print_function - -You can also run Python 2 with the ``-3`` flag to be warned about various -compatibility issues your code triggers during execution. If you turn warnings -into errors with ``-Werror`` then you can make sure that you don't accidentally -miss a warning. - -You can also use the Pylint_ project and its ``--py3k`` flag to lint your code -to receive warnings when your code begins to deviate from Python 3 -compatibility. This also prevents you from having to run Modernize_ or Futurize_ -over your code regularly to catch compatibility regressions. This does require -you only support Python 2.7 and Python 3.4 or newer as that is Pylint's -minimum Python version support. - - -Check which dependencies block your transition ----------------------------------------------- - -**After** you have made your code compatible with Python 3 you should begin to -care about whether your dependencies have also been ported. The caniusepython3_ -project was created to help you determine which projects --- directly or indirectly -- are blocking you from supporting Python 3. There -is both a command-line tool as well as a web interface at -https://caniusepython3.com. - -The project also provides code which you can integrate into your test suite so -that you will have a failing test when you no longer have dependencies blocking -you from using Python 3. This allows you to avoid having to manually check your -dependencies and to be notified quickly when you can start running on Python 3. - - -Update your ``setup.py`` file to denote Python 3 compatibility --------------------------------------------------------------- - -Once your code works under Python 3, you should update the classifiers in -your ``setup.py`` to contain ``Programming Language :: Python :: 3`` and to not -specify sole Python 2 support. This will tell anyone using your code that you -support Python 2 **and** 3. Ideally you will also want to add classifiers for -each major/minor version of Python you now support. - - -Use continuous integration to stay compatible ---------------------------------------------- - -Once you are able to fully run under Python 3 you will want to make sure your -code always works under both Python 2 and 3. Probably the best tool for running -your tests under multiple Python interpreters is tox_. You can then integrate -tox with your continuous integration system so that you never accidentally break -Python 2 or 3 support. - -You may also want to use the ``-bb`` flag with the Python 3 interpreter to -trigger an exception when you are comparing bytes to strings or bytes to an int -(the latter is available starting in Python 3.5). By default type-differing -comparisons simply return ``False``, but if you made a mistake in your -separation of text/binary data handling or indexing on bytes you wouldn't easily -find the mistake. This flag will raise an exception when these kinds of -comparisons occur, making the mistake much easier to track down. - - -Consider using optional static type checking --------------------------------------------- - -Another way to help port your code is to use a :term:`static type checker` like -mypy_ or pytype_ on your code. These tools can be used to analyze your code as -if it's being run under Python 2, then you can run the tool a second time as if -your code is running under Python 3. By running a static type checker twice like -this you can discover if you're e.g. misusing binary data type in one version -of Python compared to another. If you add optional type hints to your code you -can also explicitly state whether your APIs use textual or binary data, helping -to make sure everything functions as expected in both versions of Python. - - -.. _caniusepython3: https://pypi.org/project/caniusepython3 -.. _cheat sheet: https://python-future.org/compatible_idioms.html -.. _coverage.py: https://pypi.org/project/coverage -.. _Futurize: https://python-future.org/automatic_conversion.html -.. _importlib2: https://pypi.org/project/importlib2 -.. _Modernize: https://python-modernize.readthedocs.io/ -.. _mypy: https://mypy-lang.org/ -.. _Porting to Python 3: http://python3porting.com/ -.. _Pylint: https://pypi.org/project/pylint - -.. _Python 3 Q & A: https://ncoghlan-devs-python-notes.readthedocs.io/en/latest/python3/questions_and_answers.html - -.. _pytype: https://github.com/google/pytype -.. _python-future: https://python-future.org/ .. _python-porting: https://mail.python.org/pipermail/python-porting/ -.. _six: https://pypi.org/project/six -.. _tox: https://pypi.org/project/tox -.. _trove classifier: https://pypi.org/classifiers - -.. _Why Python 3 exists: https://snarky.ca/why-python-3-exists diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index 38dd09f0a721d2f..b98f91e023bdfc7 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -1,10 +1,9 @@ .. _sortinghowto: -Sorting HOW TO -************** +Sorting Techniques +****************** :Author: Andrew Dalke and Raymond Hettinger -:Release: 0.1 Python lists have a built-in :meth:`list.sort` method that modifies the list @@ -56,7 +55,7 @@ For example, here's a case-insensitive string comparison: .. doctest:: - >>> sorted("This is a test string from Andrew".split(), key=str.lower) + >>> sorted("This is a test string from Andrew".split(), key=str.casefold) ['a', 'Andrew', 'from', 'is', 'string', 'test', 'This'] The value of the *key* parameter should be a function (or other callable) that @@ -97,10 +96,14 @@ The same technique works for objects with named attributes. For example: >>> sorted(student_objects, key=lambda student: student.age) # sort by age [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] -Operator Module Functions -========================= +Objects with named attributes can be made by a regular class as shown +above, or they can be instances of :class:`~dataclasses.dataclass` or +a :term:`named tuple`. -The key-function patterns shown above are very common, so Python provides +Operator Module Functions and Partial Function Evaluation +========================================================= + +The :term:`key function` patterns shown above are very common, so Python provides convenience functions to make accessor functions easier and faster. The :mod:`operator` module has :func:`~operator.itemgetter`, :func:`~operator.attrgetter`, and a :func:`~operator.methodcaller` function. @@ -128,6 +131,24 @@ sort by *grade* then by *age*: >>> sorted(student_objects, key=attrgetter('grade', 'age')) [('john', 'A', 15), ('dave', 'B', 10), ('jane', 'B', 12)] +The :mod:`functools` module provides another helpful tool for making +key-functions. The :func:`~functools.partial` function can reduce the +`arity <https://en.wikipedia.org/wiki/Arity>`_ of a multi-argument +function making it suitable for use as a key-function. + +.. doctest:: + + >>> from functools import partial + >>> from unicodedata import normalize + + >>> names = 'Zoë Åbjørn Núñez Élana Zeke Abe Nubia Eloise'.split() + + >>> sorted(names, key=partial(normalize, 'NFD')) + ['Abe', 'Åbjørn', 'Eloise', 'Élana', 'Nubia', 'Núñez', 'Zeke', 'Zoë'] + + >>> sorted(names, key=partial(normalize, 'NFC')) + ['Abe', 'Eloise', 'Nubia', 'Núñez', 'Zeke', 'Zoë', 'Åbjørn', 'Élana'] + Ascending and Descending ======================== @@ -200,6 +221,8 @@ This idiom is called Decorate-Sort-Undecorate after its three steps: For example, to sort the student data by *grade* using the DSU approach: +.. doctest:: + >>> decorated = [(student.grade, i, student) for i, student in enumerate(student_objects)] >>> decorated.sort() >>> [student for grade, i, student in decorated] # undecorate @@ -282,7 +305,11 @@ Odds and Ends [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] However, note that ``<`` can fall back to using :meth:`~object.__gt__` if - :meth:`~object.__lt__` is not implemented (see :func:`object.__lt__`). + :meth:`~object.__lt__` is not implemented (see :func:`object.__lt__` + for details on the mechanics). To avoid surprises, :pep:`8` + recommends that all six comparison methods be implemented. + The :func:`~functools.total_ordering` decorator is provided to make that + task easier. * Key functions need not depend directly on the objects being sorted. A key function can also access external resources. For instance, if the student grades @@ -295,3 +322,24 @@ Odds and Ends >>> newgrades = {'john': 'F', 'jane':'A', 'dave': 'C'} >>> sorted(students, key=newgrades.__getitem__) ['jane', 'dave', 'john'] + +Partial Sorts +============= + +Some applications require only some of the data to be ordered. The standard +library provides several tools that do less work than a full sort: + +* :func:`min` and :func:`max` return the smallest and largest values, + respectively. These functions make a single pass over the input data and + require almost no auxiliary memory. + +* :func:`heapq.nsmallest` and :func:`heapq.nlargest` return + the *n* smallest and largest values, respectively. These functions + make a single pass over the data keeping only *n* elements in memory + at a time. For values of *n* that are small relative to the number of + inputs, these functions make far fewer comparisons than a full sort. + +* :func:`heapq.heappush` and :func:`heapq.heappop` create and maintain a + partially sorted arrangement of data that keeps the smallest element + at position ``0``. These functions are suitable for implementing + priority queues which are commonly used for task scheduling. diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index d261e4a4f338a56..762f8b4695b3dd9 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -8,20 +8,68 @@ -------------- -:mod:`__future__` is a real module, and serves three purposes: +Imports of the form ``from __future__ import feature`` are called +:ref:`future statements <future>`. These are special-cased by the Python compiler +to allow the use of new Python features in modules containing the future statement +before the release in which the feature becomes standard. + +While these future statements are given additional special meaning by the +Python compiler, they are still executed like any other import statement and +the :mod:`__future__` exists and is handled by the import system the same way +any other Python module would be. This design serves three purposes: * To avoid confusing existing tools that analyze import statements and expect to find the modules they're importing. -* To ensure that :ref:`future statements <future>` run under releases prior to - 2.1 at least yield runtime exceptions (the import of :mod:`__future__` will - fail, because there was no module of that name prior to 2.1). - * To document when incompatible changes were introduced, and when they will be --- or were --- made mandatory. This is a form of executable documentation, and can be inspected programmatically via importing :mod:`__future__` and examining its contents. +* To ensure that :ref:`future statements <future>` run under releases prior to + Python 2.1 at least yield runtime exceptions (the import of :mod:`__future__` + will fail, because there was no module of that name prior to 2.1). + +Module Contents +--------------- + +No feature description will ever be deleted from :mod:`__future__`. Since its +introduction in Python 2.1 the following features have found their way into the +language using this mechanism: + ++------------------+-------------+--------------+---------------------------------------------+ +| feature | optional in | mandatory in | effect | ++==================+=============+==============+=============================================+ +| nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: | +| | | | *Statically Nested Scopes* | ++------------------+-------------+--------------+---------------------------------------------+ +| generators | 2.2.0a1 | 2.3 | :pep:`255`: | +| | | | *Simple Generators* | ++------------------+-------------+--------------+---------------------------------------------+ +| division | 2.2.0a2 | 3.0 | :pep:`238`: | +| | | | *Changing the Division Operator* | ++------------------+-------------+--------------+---------------------------------------------+ +| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: | +| | | | *Imports: Multi-Line and Absolute/Relative* | ++------------------+-------------+--------------+---------------------------------------------+ +| with_statement | 2.5.0a1 | 2.6 | :pep:`343`: | +| | | | *The "with" Statement* | ++------------------+-------------+--------------+---------------------------------------------+ +| print_function | 2.6.0a2 | 3.0 | :pep:`3105`: | +| | | | *Make print a function* | ++------------------+-------------+--------------+---------------------------------------------+ +| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | +| | | | *Bytes literals in Python 3000* | ++------------------+-------------+--------------+---------------------------------------------+ +| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | +| | | | *StopIteration handling inside generators* | ++------------------+-------------+--------------+---------------------------------------------+ +| annotations | 3.7.0b1 | TBD [1]_ | :pep:`563`: | +| | | | *Postponed evaluation of annotations* | ++------------------+-------------+--------------+---------------------------------------------+ + +.. XXX Adding a new entry? Remember to update simple_stmts.rst, too. + .. _future-classes: .. class:: _Feature @@ -65,43 +113,6 @@ dynamically compiled code. This flag is stored in the :attr:`_Feature.compiler_flag` attribute on :class:`_Feature` instances. -No feature description will ever be deleted from :mod:`__future__`. Since its -introduction in Python 2.1 the following features have found their way into the -language using this mechanism: - -+------------------+-------------+--------------+---------------------------------------------+ -| feature | optional in | mandatory in | effect | -+==================+=============+==============+=============================================+ -| nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: | -| | | | *Statically Nested Scopes* | -+------------------+-------------+--------------+---------------------------------------------+ -| generators | 2.2.0a1 | 2.3 | :pep:`255`: | -| | | | *Simple Generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| division | 2.2.0a2 | 3.0 | :pep:`238`: | -| | | | *Changing the Division Operator* | -+------------------+-------------+--------------+---------------------------------------------+ -| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: | -| | | | *Imports: Multi-Line and Absolute/Relative* | -+------------------+-------------+--------------+---------------------------------------------+ -| with_statement | 2.5.0a1 | 2.6 | :pep:`343`: | -| | | | *The "with" Statement* | -+------------------+-------------+--------------+---------------------------------------------+ -| print_function | 2.6.0a2 | 3.0 | :pep:`3105`: | -| | | | *Make print a function* | -+------------------+-------------+--------------+---------------------------------------------+ -| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | -| | | | *Bytes literals in Python 3000* | -+------------------+-------------+--------------+---------------------------------------------+ -| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | -| | | | *StopIteration handling inside generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | TBD [1]_ | :pep:`563`: | -| | | | *Postponed evaluation of annotations* | -+------------------+-------------+--------------+---------------------------------------------+ - -.. XXX Adding a new entry? Remember to update simple_stmts.rst, too. - .. [1] ``from __future__ import annotations`` was previously scheduled to become mandatory in Python 3.10, but the Python Steering Council @@ -115,3 +126,6 @@ language using this mechanism: :ref:`future` How the compiler treats future imports. + + :pep:`236` - Back to the __future__ + The original proposal for the __future__ mechanism. diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 1395d457f874b0c..eaddd44e2defd71 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -745,7 +745,7 @@ The add_argument() method .. method:: ArgumentParser.add_argument(name or flags..., [action], [nargs], \ [const], [default], [type], [choices], [required], \ - [help], [metavar], [dest]) + [help], [metavar], [dest], [deprecated]) Define how a single command-line argument should be parsed. Each parameter has its own more detailed description below, but in short they are: @@ -777,6 +777,8 @@ The add_argument() method * dest_ - The name of the attribute to be added to the object returned by :meth:`parse_args`. + * deprecated_ - Whether or not use of the argument is deprecated. + The following sections describe how each of these are used. @@ -1439,6 +1441,34 @@ behavior:: >>> parser.parse_args('--foo XXX'.split()) Namespace(bar='XXX') + +.. _deprecated: + +deprecated +^^^^^^^^^^ + +During a project's lifetime, some arguments may need to be removed from the +command line. Before removing them, you should inform +your users that the arguments are deprecated and will be removed. +The ``deprecated`` keyword argument of +:meth:`~ArgumentParser.add_argument`, which defaults to ``False``, +specifies if the argument is deprecated and will be removed +in the future. +For arguments, if ``deprecated`` is ``True``, then a warning will be +printed to standard error when the argument is used:: + + >>> import argparse + >>> parser = argparse.ArgumentParser(prog='snake.py') + >>> parser.add_argument('--legs', default=0, type=int, deprecated=True) + >>> parser.parse_args([]) + Namespace(legs=0) + >>> parser.parse_args(['--legs', '4']) # doctest: +SKIP + snake.py: warning: option '--legs' is deprecated + Namespace(legs=4) + +.. versionchanged:: 3.13 + + Action classes ^^^^^^^^^^^^^^ @@ -1842,7 +1872,8 @@ Sub-commands {foo,bar} additional help - Furthermore, ``add_parser`` supports an additional ``aliases`` argument, + Furthermore, :meth:`~_SubParsersAction.add_parser` supports an additional + *aliases* argument, which allows multiple strings to refer to the same subparser. This example, like ``svn``, aliases ``co`` as a shorthand for ``checkout``:: @@ -1853,6 +1884,20 @@ Sub-commands >>> parser.parse_args(['co', 'bar']) Namespace(foo='bar') + :meth:`~_SubParsersAction.add_parser` supports also an additional + *deprecated* argument, which allows to deprecate the subparser. + + >>> import argparse + >>> parser = argparse.ArgumentParser(prog='chicken.py') + >>> subparsers = parser.add_subparsers() + >>> run = subparsers.add_parser('run') + >>> fly = subparsers.add_parser('fly', deprecated=True) + >>> parser.parse_args(['fly']) # doctest: +SKIP + chicken.py: warning: command 'fly' is deprecated + Namespace() + + .. versionadded:: 3.13 + One particularly effective way of handling sub-commands is to combine the use of the :meth:`add_subparsers` method with calls to :meth:`set_defaults` so that each subparser knows which Python function it should execute. For diff --git a/Doc/library/array.rst b/Doc/library/array.rst index a0e8bb20a098fdc..cdf21db8779fe8f 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -215,6 +215,13 @@ The module defines the following type: Remove the first occurrence of *x* from the array. + .. method:: clear() + + Remove all elements from the array. + + .. versionadded:: 3.13 + + .. method:: reverse() Reverse the order of the items in the array. @@ -272,4 +279,3 @@ Examples:: `NumPy <https://numpy.org/>`_ The NumPy package defines another array type. - diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index c943c2f498173e8..d629393f023c8a0 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -103,20 +103,15 @@ Node classes For example, to create and populate an :class:`ast.UnaryOp` node, you could use :: - node = ast.UnaryOp() - node.op = ast.USub() - node.operand = ast.Constant() - node.operand.value = 5 - node.operand.lineno = 0 - node.operand.col_offset = 0 - node.lineno = 0 - node.col_offset = 0 - - or the more compact :: - node = ast.UnaryOp(ast.USub(), ast.Constant(5, lineno=0, col_offset=0), lineno=0, col_offset=0) + If a field that is optional in the grammar is omitted from the constructor, + it defaults to ``None``. If a list field is omitted, it defaults to the empty + list. If any other field is omitted, a :exc:`DeprecationWarning` is raised + and the AST node will not have this field. In Python 3.15, this condition will + raise an error. + .. versionchanged:: 3.8 Class :class:`ast.Constant` is now used for all constants. @@ -140,6 +135,14 @@ Node classes In the meantime, instantiating them will return an instance of a different class. +.. deprecated-removed:: 3.13 3.15 + + Previous versions of Python allowed the creation of AST nodes that were missing + required fields. Similarly, AST node constructors allowed arbitrary keyword + arguments that were set as attributes of the AST node, even if they did not + match any of the fields of the AST node. This behavior is deprecated and will + be removed in Python 3.15. + .. note:: The descriptions of the specific node classes displayed here were initially adapted from the fantastic `Green Tree diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 3f734f544afe216..7c08d65f26bc27e 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -362,6 +362,11 @@ Datagram Transports This method does not block; it buffers the data and arranges for it to be sent out asynchronously. + .. versionchanged:: 3.13 + This method can be called with an empty bytes object to send a + zero-length datagram. The buffer size calculation used for flow + control is also updated to account for the datagram header. + .. method:: DatagramTransport.abort() Close the transport immediately, without waiting for pending @@ -417,8 +422,8 @@ Subprocess Transports Stop the subprocess. - On POSIX systems, this method sends SIGTERM to the subprocess. - On Windows, the Windows API function TerminateProcess() is called to + On POSIX systems, this method sends :py:const:`~signal.SIGTERM` to the subprocess. + On Windows, the Windows API function :c:func:`!TerminateProcess` is called to stop the subprocess. See also :meth:`subprocess.Popen.terminate`. diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index bf35b1cb798aee1..817a6ff3052f4aa 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -240,7 +240,7 @@ their completion. .. note:: - On Windows, :py:data:`SIGTERM` is an alias for :meth:`terminate`. + On Windows, :py:const:`~signal.SIGTERM` is an alias for :meth:`terminate`. ``CTRL_C_EVENT`` and ``CTRL_BREAK_EVENT`` can be sent to processes started with a *creationflags* parameter which includes ``CREATE_NEW_PROCESS_GROUP``. @@ -249,10 +249,10 @@ their completion. Stop the child process. - On POSIX systems this method sends :py:const:`signal.SIGTERM` to the + On POSIX systems this method sends :py:const:`~signal.SIGTERM` to the child process. - On Windows the Win32 API function :c:func:`TerminateProcess` is + On Windows the Win32 API function :c:func:`!TerminateProcess` is called to stop the child process. .. method:: kill() diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 05bdf5488af143c..3cf8e2737e85dcd 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -216,8 +216,8 @@ Condition .. method:: notify(n=1) - Wake up at most *n* tasks (1 by default) waiting on this - condition. The method is no-op if no tasks are waiting. + Wake up *n* tasks (1 by default) waiting on this + condition. If fewer than *n* tasks are waiting they are all awakened. The lock must be acquired before this method is called and released shortly after. If called with an *unlocked* lock @@ -257,12 +257,18 @@ Condition Once awakened, the Condition re-acquires its lock and this method returns ``True``. + Note that a task *may* return from this call spuriously, + which is why the caller should always re-check the state + and be prepared to :meth:`wait` again. For this reason, you may + prefer to use :meth:`wait_for` instead. + .. coroutinemethod:: wait_for(predicate) Wait until a predicate becomes *true*. The predicate must be a callable which result will be - interpreted as a boolean value. The final value is the + interpreted as a boolean value. The method will repeatedly + :meth:`wait` until the predicate evaluates to *true*. The final value is the return value. diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index 3dbef69580d9b30..43a8bd2d7cd1339 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -4,8 +4,8 @@ .. module:: atexit :synopsis: Register and execute cleanup functions. -.. moduleauthor:: Skip Montanaro <skip@pobox.com> -.. sectionauthor:: Skip Montanaro <skip@pobox.com> +.. moduleauthor:: Skip Montanaro <skip.montanaro@gmail.com> +.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com> -------------- diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index d5b6af8c1928eff..e596893358f3fb3 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -244,6 +244,24 @@ The modern interface provides: .. versionadded:: 3.4 +.. function:: z85encode(s) + + Encode the :term:`bytes-like object` *s* using Z85 (as used in ZeroMQ) + and return the encoded :class:`bytes`. See `Z85 specification + <https://rfc.zeromq.org/spec/32/>`_ for more information. + + .. versionadded:: 3.13 + + +.. function:: z85decode(s) + + Decode the Z85-encoded :term:`bytes-like object` or ASCII string *s* and + return the decoded :class:`bytes`. See `Z85 specification + <https://rfc.zeromq.org/spec/32/>`_ for more information. + + .. versionadded:: 3.13 + + The legacy interface: .. function:: decode(input, output) diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index 52f0ca7c013482b..7bf4308a96d0f55 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -148,8 +148,8 @@ The :mod:`bdb` module also defines two classes: .. method:: reset() - Set the :attr:`botframe`, :attr:`stopframe`, :attr:`returnframe` and - :attr:`quitting` attributes with values ready to start debugging. + Set the :attr:`!botframe`, :attr:`!stopframe`, :attr:`!returnframe` and + :attr:`quitting <Bdb.set_quit>` attributes with values ready to start debugging. .. method:: trace_dispatch(frame, event, arg) @@ -182,7 +182,7 @@ The :mod:`bdb` module also defines two classes: If the debugger should stop on the current line, invoke the :meth:`user_line` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set + Raise a :exc:`BdbQuit` exception if the :attr:`quitting <Bdb.set_quit>` flag is set (which can be set from :meth:`user_line`). Return a reference to the :meth:`trace_dispatch` method for further tracing in that scope. @@ -190,7 +190,7 @@ The :mod:`bdb` module also defines two classes: If the debugger should stop on this function call, invoke the :meth:`user_call` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set + Raise a :exc:`BdbQuit` exception if the :attr:`quitting <Bdb.set_quit>` flag is set (which can be set from :meth:`user_call`). Return a reference to the :meth:`trace_dispatch` method for further tracing in that scope. @@ -198,7 +198,7 @@ The :mod:`bdb` module also defines two classes: If the debugger should stop on this function return, invoke the :meth:`user_return` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set + Raise a :exc:`BdbQuit` exception if the :attr:`quitting <Bdb.set_quit>` flag is set (which can be set from :meth:`user_return`). Return a reference to the :meth:`trace_dispatch` method for further tracing in that scope. @@ -206,7 +206,7 @@ The :mod:`bdb` module also defines two classes: If the debugger should stop at this exception, invokes the :meth:`user_exception` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set + Raise a :exc:`BdbQuit` exception if the :attr:`quitting <Bdb.set_quit>` flag is set (which can be set from :meth:`user_exception`). Return a reference to the :meth:`trace_dispatch` method for further tracing in that scope. @@ -293,7 +293,9 @@ The :mod:`bdb` module also defines two classes: .. method:: set_quit() - Set the :attr:`quitting` attribute to ``True``. This raises :exc:`BdbQuit` in + .. index:: single: quitting (bdb.Bdb attribute) + + Set the :attr:`!quitting` attribute to ``True``. This raises :exc:`BdbQuit` in the next call to one of the :meth:`!dispatch_\*` methods. @@ -383,7 +385,7 @@ The :mod:`bdb` module also defines two classes: .. method:: run(cmd, globals=None, locals=None) Debug a statement executed via the :func:`exec` function. *globals* - defaults to :attr:`__main__.__dict__`, *locals* defaults to *globals*. + defaults to :attr:`!__main__.__dict__`, *locals* defaults to *globals*. .. method:: runeval(expr, globals=None, locals=None) diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 6586f539a8da4fc..e699a7284ac8023 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -512,7 +512,7 @@ to interactively print a calendar. python -m calendar [-h] [-L LOCALE] [-e ENCODING] [-t {text,html}] [-w WIDTH] [-l LINES] [-s SPACING] [-m MONTHS] [-c CSS] - [year] [month] + [-f FIRST_WEEKDAY] [year] [month] For example, to print a calendar for the year 2000: @@ -586,10 +586,17 @@ The following options are accepted: or as an HTML document. +.. option:: --first-weekday FIRST_WEEKDAY, -f FIRST_WEEKDAY + + The weekday to start each week. + Must be a number between 0 (Monday) and 6 (Sunday). + Defaults to 0. + + .. versionadded:: 3.13 + .. option:: year The year to print the calendar for. - Must be a number between 1 and 9999. Defaults to the current year. diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 4617624686b1f3b..a757f19b99448c1 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1132,7 +1132,8 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp875 | | Greek | +-----------------+--------------------------------+--------------------------------+ -| cp932 | 932, ms932, mskanji, ms-kanji | Japanese | +| cp932 | 932, ms932, mskanji, ms-kanji, | Japanese | +| | windows-31j | | +-----------------+--------------------------------+--------------------------------+ | cp949 | 949, ms949, uhc | Korean | +-----------------+--------------------------------+--------------------------------+ @@ -1540,13 +1541,13 @@ This module implements the ANSI codepage (CP_ACP). .. availability:: Windows. -.. versionchanged:: 3.3 - Support any error handler. - .. versionchanged:: 3.2 Before 3.2, the *errors* argument was ignored; ``'replace'`` was always used to encode, and ``'ignore'`` to decode. +.. versionchanged:: 3.3 + Support any error handler. + :mod:`encodings.utf_8_sig` --- UTF-8 codec with BOM signature ------------------------------------------------------------- diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 582bb18f752bd59..7bcaba60c6ddbd9 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -136,8 +136,8 @@ ABC Inherits from Abstract Methods Mi :class:`Collection` ``__len__`` ``index``, and ``count`` :class:`MutableSequence` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods and - ``__setitem__``, ``append``, ``reverse``, ``extend``, ``pop``, - ``__delitem__``, ``remove``, and ``__iadd__`` + ``__setitem__``, ``append``, ``clear``, ``reverse``, ``extend``, + ``__delitem__``, ``pop``, ``remove``, and ``__iadd__`` ``__len__``, ``insert`` diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 7a5589e68b30525..4ee7820585d3a21 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -4,7 +4,7 @@ .. module:: csv :synopsis: Write and read tabular data to and from delimited files. -.. sectionauthor:: Skip Montanaro <skip@pobox.com> +.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com> **Source code:** :source:`Lib/csv.py` @@ -88,7 +88,7 @@ The :mod:`csv` module defines the following functions: Return a writer object responsible for converting the user's data into delimited strings on the given file-like object. *csvfile* can be any object with a - :func:`write` method. If *csvfile* is a file object, it should be opened with + :meth:`~io.TextIOBase.write` method. If *csvfile* is a file object, it should be opened with ``newline=''`` [1]_. An optional *dialect* parameter can be given which is used to define a set of parameters specific to a particular CSV dialect. It may be an instance of a subclass of the @@ -197,10 +197,10 @@ The :mod:`csv` module defines the following classes: Create an object which operates like a regular writer but maps dictionaries onto output rows. The *fieldnames* parameter is a :mod:`sequence <collections.abc>` of keys that identify the order in which values in the - dictionary passed to the :meth:`writerow` method are written to file + dictionary passed to the :meth:`~csvwriter.writerow` method are written to file *f*. The optional *restval* parameter specifies the value to be written if the dictionary is missing a key in *fieldnames*. If the - dictionary passed to the :meth:`writerow` method contains a key not found in + dictionary passed to the :meth:`~csvwriter.writerow` method contains a key not found in *fieldnames*, the optional *extrasaction* parameter indicates what action to take. If it is set to ``'raise'``, the default value, a :exc:`ValueError` @@ -244,7 +244,6 @@ The :mod:`csv` module defines the following classes: with open('students.csv', 'w', newline='') as csvfile: writer = csv.writer(csvfile, dialect='unix') - ^^^^^^^^^^^^^^ .. class:: excel() @@ -351,6 +350,8 @@ The :mod:`csv` module defines the following constants: Instructs :class:`reader` objects to interpret an empty (unquoted) field as None and to otherwise behave as :data:`QUOTE_ALL`. + .. versionadded:: 3.12 + .. data:: QUOTE_STRINGS Instructs :class:`writer` objects to always place quotes around fields @@ -360,6 +361,8 @@ The :mod:`csv` module defines the following constants: Instructs :class:`reader` objects to interpret an empty (unquoted) string as ``None`` and to otherwise behave as :data:`QUOTE_NONNUMERIC`. + .. versionadded:: 3.12 + The :mod:`csv` module defines the following exception: @@ -374,8 +377,8 @@ Dialects and Formatting Parameters To make it easier to specify the format of input and output records, specific formatting parameters are grouped together into dialects. A dialect is a -subclass of the :class:`Dialect` class having a set of specific methods and a -single :meth:`validate` method. When creating :class:`reader` or +subclass of the :class:`Dialect` class containing various attributes +describing the format of the CSV file. When creating :class:`reader` or :class:`writer` objects, the programmer can specify a string or a subclass of the :class:`Dialect` class as the dialect parameter. In addition to, or instead of, the *dialect* parameter, the programmer can also specify individual @@ -492,9 +495,9 @@ DictReader objects have the following public attribute: Writer Objects -------------- -:class:`Writer` objects (:class:`DictWriter` instances and objects returned by +:class:`writer` objects (:class:`DictWriter` instances and objects returned by the :func:`writer` function) have the following public methods. A *row* must be -an iterable of strings or numbers for :class:`Writer` objects and a dictionary +an iterable of strings or numbers for :class:`writer` objects and a dictionary mapping fieldnames to strings or numbers (by passing them through :func:`str` first) for :class:`DictWriter` objects. Note that complex numbers are written out surrounded by parens. This may cause some problems for other programs which diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index ef3a9a0f5898af4..eed18201e3ede05 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -200,7 +200,7 @@ calls). Python objects that can directly be used as parameters in these function calls. ``None`` is passed as a C ``NULL`` pointer, bytes objects and strings are passed as pointer to the memory block that contains their data (:c:expr:`char *` or -:c:expr:`wchar_t *`). Python integers are passed as the platforms default C +:c:expr:`wchar_t *`). Python integers are passed as the platform's default C :c:expr:`int` type, their value is masked to fit into the C type. Before we move on calling functions with other parameter types, we have to learn @@ -670,6 +670,10 @@ compiler does it. It is possible to override this behavior by specifying a :attr:`~Structure._pack_` class attribute in the subclass definition. This must be set to a positive integer and specifies the maximum alignment for the fields. This is what ``#pragma pack(n)`` also does in MSVC. +It is also possible to set a minimum alignment for how the subclass itself is packed in the +same way ``#pragma align(n)`` works in MSVC. +This can be achieved by specifying a ::attr:`~Structure._align_` class attribute +in the subclass definition. :mod:`ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the @@ -1113,10 +1117,6 @@ api:: >>> print(hex(version.value)) 0x30c00a0 -If the interpreter would have been started with :option:`-O`, the sample would -have printed ``c_long(1)``, or ``c_long(2)`` if :option:`-OO` would have been -specified. - An extended example which also demonstrates the use of pointers accesses the :c:data:`PyImport_FrozenModules` pointer exported by Python. @@ -1441,7 +1441,7 @@ function exported by these libraries, and reacquired afterwards. All these classes can be instantiated by calling them with at least one argument, the pathname of the shared library. If you have an existing handle to an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platforms :c:func:`!dlopen` or +parameter, otherwise the underlying platform's :c:func:`!dlopen` or :c:func:`!LoadLibrary` function is used to load the library into the process, and to get a handle to it. @@ -1452,7 +1452,7 @@ configurable. The *use_errno* parameter, when set to true, enables a ctypes mechanism that allows accessing the system :data:`errno` error number in a safe way. -:mod:`ctypes` maintains a thread-local copy of the systems :data:`errno` +:mod:`ctypes` maintains a thread-local copy of the system's :data:`errno` variable; if you call foreign functions created with ``use_errno=True`` then the :data:`errno` value before the function call is swapped with the ctypes private copy, the same happens immediately after the function call. @@ -2534,6 +2534,12 @@ fields, or any other data types containing pointer type fields. Setting this attribute to 0 is the same as not setting it at all. + .. attribute:: _align_ + + An optional small integer that allows overriding the alignment of + the structure when being packed or unpacked to/from memory. + Setting this attribute to 0 is the same as not setting it at all. + .. attribute:: _anonymous_ An optional sequence that lists the names of unnamed (anonymous) fields. diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 88f2e0251b1e519..4ada69d63abada1 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -538,8 +538,8 @@ that has to be called, it is common to call this method in a class Rectangle: def __init__(self, height, width): - self.height = height - self.width = width + self.height = height + self.width = width @dataclass class Square(Rectangle): diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index b36f8c19cd6040b..4602132f37f7331 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -14,7 +14,7 @@ .. XXX what order should the types be discussed in? -The :mod:`datetime` module supplies classes for manipulating dates and times. +The :mod:`!datetime` module supplies classes for manipulating dates and times. While date and time arithmetic is supported, the focus of the implementation is on efficient attribute extraction for output formatting and manipulation. @@ -70,7 +70,7 @@ These :class:`tzinfo` objects capture information about the offset from UTC time, the time zone name, and whether daylight saving time is in effect. Only one concrete :class:`tzinfo` class, the :class:`timezone` class, is -supplied by the :mod:`datetime` module. The :class:`timezone` class can +supplied by the :mod:`!datetime` module. The :class:`timezone` class can represent simple timezones with fixed offsets from UTC, such as UTC itself or North American EST and EDT timezones. Supporting timezones at deeper levels of detail is up to the application. The rules for time adjustment across the @@ -80,7 +80,7 @@ standard suitable for every application aside from UTC. Constants --------- -The :mod:`datetime` module exports the following constants: +The :mod:`!datetime` module exports the following constants: .. data:: MINYEAR @@ -130,8 +130,8 @@ Available Types .. class:: timedelta :noindex: - A duration expressing the difference between two :class:`date`, :class:`.time`, - or :class:`.datetime` instances to microsecond resolution. + A duration expressing the difference between two :class:`.datetime` + or :class:`date` instances to microsecond resolution. .. class:: tzinfo @@ -203,7 +203,7 @@ objects. -------------------------- A :class:`timedelta` object represents a duration, the difference between two -dates or times. +:class:`.datetime` or :class:`date` instances. .. class:: timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0) @@ -400,30 +400,7 @@ objects (see below). the :func:`divmod` function. True division and multiplication of a :class:`timedelta` object by a :class:`float` object are now supported. - -Comparisons of :class:`timedelta` objects are supported, with some caveats. - -The comparisons ``==`` or ``!=`` *always* return a :class:`bool`, no matter -the type of the compared object:: - - >>> from datetime import timedelta - >>> delta1 = timedelta(seconds=57) - >>> delta2 = timedelta(hours=25, seconds=2) - >>> delta2 != delta1 - True - >>> delta2 == 5 - False - -For all other comparisons (such as ``<`` and ``>``), when a :class:`timedelta` -object is compared to an object of a different type, :exc:`TypeError` -is raised:: - - >>> delta2 > delta1 - True - >>> delta2 > 5 - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - TypeError: '>' not supported between instances of 'datetime.timedelta' and 'int' +:class:`timedelta` objects support equality and order comparisons. In Boolean contexts, a :class:`timedelta` object is considered to be true if and only if it isn't equal to ``timedelta(0)``. @@ -536,7 +513,15 @@ Other constructors, all class methods: .. classmethod:: date.fromisoformat(date_string) Return a :class:`date` corresponding to a *date_string* given in any valid - ISO 8601 format, except ordinal dates (e.g. ``YYYY-DDD``):: + ISO 8601 format, with the following exceptions: + + 1. Reduced precision dates are not currently supported (``YYYY-MM``, + ``YYYY``). + 2. Extended date representations are not currently supported + (``±YYYYYY-MM-DD``). + 3. Ordinal dates are not currently supported (``YYYY-OOO``). + + Examples:: >>> from datetime import date >>> date.fromisoformat('2019-12-04') @@ -606,8 +591,13 @@ Supported operations: +-------------------------------+----------------------------------------------+ | ``timedelta = date1 - date2`` | \(3) | +-------------------------------+----------------------------------------------+ -| ``date1 < date2`` | *date1* is considered less than *date2* when | -| | *date1* precedes *date2* in time. (4) | +| | ``date1 == date2`` | Equality comparison. (4) | +| | ``date1 != date2`` | | ++-------------------------------+----------------------------------------------+ +| | ``date1 < date2`` | Order comparison. (5) | +| | ``date1 > date2`` | | +| | ``date1 <= date2`` | | +| | ``date1 >= date2`` | | +-------------------------------+----------------------------------------------+ Notes: @@ -627,15 +617,28 @@ Notes: timedelta.microseconds are 0, and date2 + timedelta == date1 after. (4) + :class:`date` objects are equal if they represent the same date. + + :class:`!date` objects that are not also :class:`.datetime` instances + are never equal to :class:`!datetime` objects, even if they represent + the same date. + +(5) + *date1* is considered less than *date2* when *date1* precedes *date2* in time. In other words, ``date1 < date2`` if and only if ``date1.toordinal() < - date2.toordinal()``. Date comparison raises :exc:`TypeError` if - the other comparand isn't also a :class:`date` object. However, - ``NotImplemented`` is returned instead if the other comparand has a - :meth:`timetuple` attribute. This hook gives other kinds of date objects a - chance at implementing mixed-type comparison. If not, when a :class:`date` - object is compared to an object of a different type, :exc:`TypeError` is raised - unless the comparison is ``==`` or ``!=``. The latter cases return - :const:`False` or :const:`True`, respectively. + date2.toordinal()``. + + Order comparison between a :class:`!date` object that is not also a + :class:`.datetime` instance and a :class:`!datetime` object raises + :exc:`TypeError`. + +.. versionchanged:: 3.13 + Comparison between :class:`.datetime` object and an instance of + the :class:`date` subclass that is not a :class:`!datetime` subclass + no longer coverts the latter to :class:`!date`, ignoring the time part + and the time zone. + The default behavior can be changed by overriding the special comparison + methods in subclasses. In Boolean contexts, all :class:`date` objects are considered to be true. @@ -1017,8 +1020,12 @@ Other constructors, all class methods: 1. Time zone offsets may have fractional seconds. 2. The ``T`` separator may be replaced by any single unicode character. - 3. Ordinal dates are not currently supported. - 4. Fractional hours and minutes are not supported. + 3. Fractional hours and minutes are not supported. + 4. Reduced precision dates are not currently supported (``YYYY-MM``, + ``YYYY``). + 5. Extended date representations are not currently supported + (``±YYYYYY-MM-DD``). + 6. Ordinal dates are not currently supported (``YYYY-OOO``). Examples:: @@ -1158,8 +1165,13 @@ Supported operations: +---------------------------------------+--------------------------------+ | ``timedelta = datetime1 - datetime2`` | \(3) | +---------------------------------------+--------------------------------+ -| ``datetime1 < datetime2`` | Compares :class:`.datetime` to | -| | :class:`.datetime`. (4) | +| | ``datetime1 == datetime2`` | Equality comparison. (4) | +| | ``datetime1 != datetime2`` | | ++---------------------------------------+--------------------------------+ +| | ``datetime1 < datetime2`` | Order comparison. (5) | +| | ``datetime1 > datetime2`` | | +| | ``datetime1 <= datetime2`` | | +| | ``datetime1 >= datetime2`` | | +---------------------------------------+--------------------------------+ (1) @@ -1187,39 +1199,44 @@ Supported operations: are done in this case. If both are aware and have different :attr:`~.datetime.tzinfo` attributes, ``a-b`` acts - as if *a* and *b* were first converted to naive UTC datetimes first. The + as if *a* and *b* were first converted to naive UTC datetimes. The result is ``(a.replace(tzinfo=None) - a.utcoffset()) - (b.replace(tzinfo=None) - b.utcoffset())`` except that the implementation never overflows. (4) - *datetime1* is considered less than *datetime2* when *datetime1* precedes - *datetime2* in time. + :class:`.datetime` objects are equal if they represent the same date + and time, taking into account the time zone. - If one comparand is naive and the other is aware, :exc:`TypeError` - is raised if an order comparison is attempted. For equality - comparisons, naive instances are never equal to aware instances. + Naive and aware :class:`!datetime` objects are never equal. - If both comparands are aware, and have the same :attr:`~.datetime.tzinfo` attribute, the - common :attr:`~.datetime.tzinfo` attribute is ignored and the base datetimes are - compared. If both comparands are aware and have different :attr:`~.datetime.tzinfo` - attributes, the comparands are first adjusted by subtracting their UTC - offsets (obtained from ``self.utcoffset()``). + If both comparands are aware and have different :attr:`~.datetime.tzinfo` + attributes, the comparison acts as comparands were first converted to UTC + datetimes except that the implementation never overflows. + :class:`!datetime` instances in a repeated interval are never equal to + :class:`!datetime` instances in other time zone. - .. versionchanged:: 3.3 - Equality comparisons between aware and naive :class:`.datetime` - instances don't raise :exc:`TypeError`. +(5) + *datetime1* is considered less than *datetime2* when *datetime1* precedes + *datetime2* in time, taking into account the time zone. - .. note:: + Order comparison between naive and aware :class:`.datetime` objects + raises :exc:`TypeError`. + + If both comparands are aware and have different :attr:`~.datetime.tzinfo` + attributes, the comparison acts as comparands were first converted to UTC + datetimes except that the implementation never overflows. + +.. versionchanged:: 3.3 + Equality comparisons between aware and naive :class:`.datetime` + instances don't raise :exc:`TypeError`. - In order to stop comparison from falling back to the default scheme of comparing - object addresses, datetime comparison normally raises :exc:`TypeError` if the - other comparand isn't also a :class:`.datetime` object. However, - ``NotImplemented`` is returned instead if the other comparand has a - :meth:`timetuple` attribute. This hook gives other kinds of date objects a - chance at implementing mixed-type comparison. If not, when a :class:`.datetime` - object is compared to an object of a different type, :exc:`TypeError` is raised - unless the comparison is ``==`` or ``!=``. The latter cases return - :const:`False` or :const:`True`, respectively. +.. versionchanged:: 3.13 + Comparison between :class:`.datetime` object and an instance of + the :class:`date` subclass that is not a :class:`!datetime` subclass + no longer coverts the latter to :class:`!date`, ignoring the time part + and the time zone. + The default behavior can be changed by overriding the special comparison + methods in subclasses. Instance methods: @@ -1255,7 +1272,7 @@ Instance methods: ``tzinfo=None`` can be specified to create a naive datetime from an aware datetime with no conversion of date and time data. - :class:`datetime` objects are also supported by generic function + :class:`.datetime` objects are also supported by generic function :func:`copy.replace`. .. versionchanged:: 3.6 @@ -1347,22 +1364,22 @@ Instance methods: where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` is the day number within the current year starting with ``1`` for January - 1st. The :attr:`tm_isdst` flag of the result is set according to the + 1st. The :attr:`~time.struct_time.tm_isdst` flag of the result is set according to the :meth:`dst` method: :attr:`.tzinfo` is ``None`` or :meth:`dst` returns - ``None``, :attr:`tm_isdst` is set to ``-1``; else if :meth:`dst` returns a - non-zero value, :attr:`tm_isdst` is set to ``1``; else :attr:`tm_isdst` is + ``None``, :attr:`!tm_isdst` is set to ``-1``; else if :meth:`dst` returns a + non-zero value, :attr:`!tm_isdst` is set to ``1``; else :attr:`!tm_isdst` is set to ``0``. .. method:: datetime.utctimetuple() If :class:`.datetime` instance *d* is naive, this is the same as - ``d.timetuple()`` except that :attr:`tm_isdst` is forced to 0 regardless of what + ``d.timetuple()`` except that :attr:`~.time.struct_time.tm_isdst` is forced to 0 regardless of what ``d.dst()`` returns. DST is never in effect for a UTC time. If *d* is aware, *d* is normalized to UTC time, by subtracting ``d.utcoffset()``, and a :class:`time.struct_time` for the - normalized time is returned. :attr:`tm_isdst` is forced to 0. Note + normalized time is returned. :attr:`!tm_isdst` is forced to 0. Note that an :exc:`OverflowError` may be raised if *d*.year was ``MINYEAR`` or ``MAXYEAR`` and UTC adjustment spills over a year boundary. @@ -1550,7 +1567,7 @@ Instance methods: Examples of Usage: :class:`.datetime` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Examples of working with :class:`~datetime.datetime` objects: +Examples of working with :class:`.datetime` objects: .. doctest:: @@ -1678,7 +1695,7 @@ Usage of ``KabulTz`` from above:: :class:`.time` Objects ---------------------- -A :class:`time` object represents a (local) time of day, independent of any particular +A :class:`.time` object represents a (local) time of day, independent of any particular day, and subject to adjustment via a :class:`tzinfo` object. .. class:: time(hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0) @@ -1754,24 +1771,21 @@ Instance attributes (read-only): .. versionadded:: 3.6 -:class:`.time` objects support comparison of :class:`.time` to :class:`.time`, -where *a* is considered less -than *b* when *a* precedes *b* in time. If one comparand is naive and the other -is aware, :exc:`TypeError` is raised if an order comparison is attempted. For equality -comparisons, naive instances are never equal to aware instances. +:class:`.time` objects support equality and order comparisons, +where *a* is considered less than *b* when *a* precedes *b* in time. + +Naive and aware :class:`!time` objects are never equal. +Order comparison between naive and aware :class:`!time` objects raises +:exc:`TypeError`. If both comparands are aware, and have -the same :attr:`~time.tzinfo` attribute, the common :attr:`~time.tzinfo` attribute is +the same :attr:`~.time.tzinfo` attribute, the common :attr:`!tzinfo` attribute is ignored and the base times are compared. If both comparands are aware and -have different :attr:`~time.tzinfo` attributes, the comparands are first adjusted by -subtracting their UTC offsets (obtained from ``self.utcoffset()``). In order -to stop mixed-type comparisons from falling back to the default comparison by -object address, when a :class:`.time` object is compared to an object of a -different type, :exc:`TypeError` is raised unless the comparison is ``==`` or -``!=``. The latter cases return :const:`False` or :const:`True`, respectively. +have different :attr:`!tzinfo` attributes, the comparands are first adjusted by +subtracting their UTC offsets (obtained from ``self.utcoffset()``). .. versionchanged:: 3.3 - Equality comparisons between aware and naive :class:`~datetime.time` instances + Equality comparisons between aware and naive :class:`.time` instances don't raise :exc:`TypeError`. In Boolean contexts, a :class:`.time` object is always considered to be true. @@ -1797,7 +1811,7 @@ Other constructor: be truncated). 4. Fractional hours and minutes are not supported. - Examples:: + Examples: .. doctest:: @@ -1836,7 +1850,7 @@ Instance methods: ``tzinfo=None`` can be specified to create a naive :class:`.time` from an aware :class:`.time`, without conversion of the time data. - :class:`time` objects are also supported by generic function + :class:`.time` objects are also supported by generic function :func:`copy.replace`. .. versionchanged:: 3.6 @@ -1981,7 +1995,7 @@ Examples of working with a :class:`.time` object:: You need to derive a concrete subclass, and (at least) supply implementations of the standard :class:`tzinfo` methods needed by the - :class:`.datetime` methods you use. The :mod:`datetime` module provides + :class:`.datetime` methods you use. The :mod:`!datetime` module provides :class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can represent timezones with fixed offset from UTC such as UTC itself or North American EST and EDT. @@ -1994,7 +2008,7 @@ Examples of working with a :class:`.time` object:: A concrete subclass of :class:`tzinfo` may need to implement the following methods. Exactly which methods are needed depends on the uses made of aware - :mod:`datetime` objects. If in doubt, simply implement all of them. + :mod:`!datetime` objects. If in doubt, simply implement all of them. .. method:: tzinfo.utcoffset(dt) @@ -2035,7 +2049,7 @@ Examples of working with a :class:`.time` object:: already been added to the UTC offset returned by :meth:`utcoffset`, so there's no need to consult :meth:`dst` unless you're interested in obtaining DST info separately. For example, :meth:`datetime.timetuple` calls its :attr:`~.datetime.tzinfo` - attribute's :meth:`dst` method to determine how the :attr:`tm_isdst` flag + attribute's :meth:`dst` method to determine how the :attr:`~time.struct_time.tm_isdst` flag should be set, and :meth:`tzinfo.fromutc` calls :meth:`dst` to account for DST changes when crossing time zones. @@ -2051,7 +2065,7 @@ Examples of working with a :class:`.time` object:: relies on this, but cannot detect violations; it's the programmer's responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee this, it may be able to override the default implementation of - :meth:`tzinfo.fromutc` to work correctly with :meth:`astimezone` regardless. + :meth:`tzinfo.fromutc` to work correctly with :meth:`~.datetime.astimezone` regardless. Most implementations of :meth:`dst` will probably look like one of these two:: @@ -2080,7 +2094,7 @@ Examples of working with a :class:`.time` object:: .. method:: tzinfo.tzname(dt) Return the time zone name corresponding to the :class:`.datetime` object *dt*, as - a string. Nothing about string names is defined by the :mod:`datetime` module, + a string. Nothing about string names is defined by the :mod:`!datetime` module, and there's no requirement that it mean anything in particular. For example, "GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all valid replies. Return ``None`` if a string name isn't known. Note that this is @@ -2128,7 +2142,7 @@ There is one more :class:`tzinfo` method that a subclass may wish to override: different years. An example of a time zone the default :meth:`fromutc` implementation may not handle correctly in all cases is one where the standard offset (from UTC) depends on the specific date and time passed, which can happen - for political reasons. The default implementations of :meth:`astimezone` and + for political reasons. The default implementations of :meth:`~.datetime.astimezone` and :meth:`fromutc` may not produce the result you want if the result is one of the hours straddling the moment the standard offset changes. @@ -2194,10 +2208,10 @@ hour that can't be spelled unambiguously in local wall time: the last hour of daylight time. In Eastern, that's times of the form 5:MM UTC on the day daylight time ends. The local wall clock leaps from 1:59 (daylight time) back to 1:00 (standard time) again. Local times of the form 1:MM are ambiguous. -:meth:`astimezone` mimics the local clock's behavior by mapping two adjacent UTC +:meth:`~.datetime.astimezone` mimics the local clock's behavior by mapping two adjacent UTC hours into the same local hour then. In the Eastern example, UTC times of the form 5:MM and 6:MM both map to 1:MM when converted to Eastern, but earlier times -have the :attr:`~datetime.fold` attribute set to 0 and the later times have it set to 1. +have the :attr:`~.datetime.fold` attribute set to 0 and the later times have it set to 1. For example, at the Fall back transition of 2016, we get:: >>> u0 = datetime(2016, 11, 6, 4, tzinfo=timezone.utc) @@ -2212,10 +2226,10 @@ For example, at the Fall back transition of 2016, we get:: 07:00:00 UTC = 02:00:00 EST 0 Note that the :class:`.datetime` instances that differ only by the value of the -:attr:`~datetime.fold` attribute are considered equal in comparisons. +:attr:`~.datetime.fold` attribute are considered equal in comparisons. Applications that can't bear wall-time ambiguities should explicitly check the -value of the :attr:`~datetime.fold` attribute or avoid using hybrid +value of the :attr:`~.datetime.fold` attribute or avoid using hybrid :class:`tzinfo` subclasses; there are no ambiguities when using :class:`timezone`, or any other fixed-offset :class:`tzinfo` subclass (such as a class representing only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). @@ -2223,7 +2237,7 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. seealso:: :mod:`zoneinfo` - The :mod:`datetime` module has a basic :class:`timezone` class (for + The :mod:`!datetime` module has a basic :class:`timezone` class (for handling arbitrary fixed offsets from UTC) and its :attr:`timezone.utc` attribute (a UTC timezone instance). @@ -2241,7 +2255,7 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. _datetime-timezone: :class:`timezone` Objects --------------------------- +------------------------- The :class:`timezone` class is a subclass of :class:`tzinfo`, each instance of which represents a timezone defined by a fixed offset from @@ -2316,8 +2330,8 @@ Class attributes: .. _strftime-strptime-behavior: -:meth:`strftime` and :meth:`strptime` Behavior ----------------------------------------------- +:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Behavior +-------------------------------------------------------------------- :class:`date`, :class:`.datetime`, and :class:`.time` objects all support a ``strftime(format)`` method, to create a string representing the time under the @@ -2327,8 +2341,8 @@ Conversely, the :meth:`datetime.strptime` class method creates a :class:`.datetime` object from a string representing a date and time and a corresponding format string. -The table below provides a high-level comparison of :meth:`strftime` -versus :meth:`strptime`: +The table below provides a high-level comparison of :meth:`~.datetime.strftime` +versus :meth:`~.datetime.strptime`: +----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ | | ``strftime`` | ``strptime`` | @@ -2345,8 +2359,8 @@ versus :meth:`strptime`: .. _format-codes: -:meth:`strftime` and :meth:`strptime` Format Codes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Format Codes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ These methods accept format codes that can be used to parse and format dates:: @@ -2485,13 +2499,13 @@ convenience. These parameters all correspond to ISO 8601 date values. | | naive). | -03:07:12.345216 | | +-----------+--------------------------------+------------------------+-------+ -These may not be available on all platforms when used with the :meth:`strftime` +These may not be available on all platforms when used with the :meth:`~.datetime.strftime` method. The ISO 8601 year and ISO 8601 week directives are not interchangeable -with the year and week number directives above. Calling :meth:`strptime` with +with the year and week number directives above. Calling :meth:`~.datetime.strptime` with incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. The full set of format codes supported varies across platforms, because Python -calls the platform C library's :func:`strftime` function, and platform +calls the platform C library's :c:func:`strftime` function, and platform variations are common. To see the full set of format codes supported on your platform, consult the :manpage:`strftime(3)` documentation. There are also differences between platforms in handling of unsupported format specifiers. @@ -2507,9 +2521,9 @@ Technical Detail Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's ``time.strftime(fmt, d.timetuple())`` although not all objects support a -:meth:`timetuple` method. +:meth:`~date.timetuple` method. -For the :meth:`datetime.strptime` class method, the default value is +For the :meth:`.datetime.strptime` class method, the default value is ``1900-01-01T00:00:00.000``: any components not specified in the format string will be pulled from the default value. [#]_ @@ -2522,7 +2536,7 @@ information, which are supported in ``datetime.strptime`` but are discarded by ``time.strptime``. For :class:`.time` objects, the format codes for year, month, and day should not -be used, as :class:`time` objects have no such values. If they're used anyway, +be used, as :class:`!time` objects have no such values. If they're used anyway, ``1900`` is substituted for the year, and ``1`` for the month and day. For :class:`date` objects, the format codes for hours, minutes, seconds, and @@ -2544,27 +2558,27 @@ Notes: contain non-ASCII characters. (2) - The :meth:`strptime` method can parse years in the full [1, 9999] range, but + The :meth:`~.datetime.strptime` method can parse years in the full [1, 9999] range, but years < 1000 must be zero-filled to 4-digit width. .. versionchanged:: 3.2 - In previous versions, :meth:`strftime` method was restricted to + In previous versions, :meth:`~.datetime.strftime` method was restricted to years >= 1900. .. versionchanged:: 3.3 - In version 3.2, :meth:`strftime` method was restricted to + In version 3.2, :meth:`~.datetime.strftime` method was restricted to years >= 1000. (3) - When used with the :meth:`strptime` method, the ``%p`` directive only affects + When used with the :meth:`~.datetime.strptime` method, the ``%p`` directive only affects the output hour field if the ``%I`` directive is used to parse the hour. (4) - Unlike the :mod:`time` module, the :mod:`datetime` module does not support + Unlike the :mod:`time` module, the :mod:`!datetime` module does not support leap seconds. (5) - When used with the :meth:`strptime` method, the ``%f`` directive + When used with the :meth:`~.datetime.strptime` method, the ``%f`` directive accepts from one to six digits and zero pads on the right. ``%f`` is an extension to the set of format characters in the C standard (but implemented separately in datetime objects, and therefore always @@ -2577,7 +2591,7 @@ Notes: For an aware object: ``%z`` - :meth:`utcoffset` is transformed into a string of the form + :meth:`~.datetime.utcoffset` is transformed into a string of the form ``±HHMM[SS[.ffffff]]``, where ``HH`` is a 2-digit string giving the number of UTC offset hours, ``MM`` is a 2-digit string giving the number of UTC offset minutes, ``SS`` is a 2-digit string giving the number of UTC offset @@ -2585,14 +2599,14 @@ Notes: offset microseconds. The ``ffffff`` part is omitted when the offset is a whole number of seconds and both the ``ffffff`` and the ``SS`` part is omitted when the offset is a whole number of minutes. For example, if - :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is + :meth:`~.datetime.utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is replaced with the string ``'-0330'``. .. versionchanged:: 3.7 The UTC offset is not restricted to a whole number of minutes. .. versionchanged:: 3.7 - When the ``%z`` directive is provided to the :meth:`strptime` method, + When the ``%z`` directive is provided to the :meth:`~.datetime.strptime` method, the UTC offsets can have a colon as a separator between hours, minutes and seconds. For example, ``'+01:00:00'`` will be parsed as an offset of one hour. @@ -2603,11 +2617,11 @@ Notes: hours, minutes and seconds. ``%Z`` - In :meth:`strftime`, ``%Z`` is replaced by an empty string if - :meth:`tzname` returns ``None``; otherwise ``%Z`` is replaced by the + In :meth:`~.datetime.strftime`, ``%Z`` is replaced by an empty string if + :meth:`~.datetime.tzname` returns ``None``; otherwise ``%Z`` is replaced by the returned value, which must be a string. - :meth:`strptime` only accepts certain values for ``%Z``: + :meth:`~.datetime.strptime` only accepts certain values for ``%Z``: 1. any value in ``time.tzname`` for your machine's locale 2. the hard-coded values ``UTC`` and ``GMT`` @@ -2617,23 +2631,23 @@ Notes: invalid values. .. versionchanged:: 3.2 - When the ``%z`` directive is provided to the :meth:`strptime` method, an + When the ``%z`` directive is provided to the :meth:`~.datetime.strptime` method, an aware :class:`.datetime` object will be produced. The ``tzinfo`` of the result will be set to a :class:`timezone` instance. (7) - When used with the :meth:`strptime` method, ``%U`` and ``%W`` are only used + When used with the :meth:`~.datetime.strptime` method, ``%U`` and ``%W`` are only used in calculations when the day of the week and the calendar year (``%Y``) are specified. (8) Similar to ``%U`` and ``%W``, ``%V`` is only used in calculations when the day of the week and the ISO year (``%G``) are specified in a - :meth:`strptime` format string. Also note that ``%G`` and ``%Y`` are not + :meth:`~.datetime.strptime` format string. Also note that ``%G`` and ``%Y`` are not interchangeable. (9) - When used with the :meth:`strptime` method, the leading zero is optional + When used with the :meth:`~.datetime.strptime` method, the leading zero is optional for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%j``, ``%U``, ``%W``, and ``%V``. Format ``%y`` does require a leading zero. diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 076e86143d06a6f..227b55c4315419e 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -8,8 +8,13 @@ -------------- -:mod:`dbm` is a generic interface to variants of the DBM database --- -:mod:`dbm.gnu` or :mod:`dbm.ndbm`. If none of these modules is installed, the +:mod:`dbm` is a generic interface to variants of the DBM database: + +* :mod:`dbm.sqlite3` +* :mod:`dbm.gnu` +* :mod:`dbm.ndbm` + +If none of these modules are installed, the slow-but-simple implementation in module :mod:`dbm.dumb` will be used. There is a `third party interface <https://www.jcea.es/programacion/pybsddb.htm>`_ to the Oracle Berkeley DB. @@ -25,8 +30,8 @@ the Oracle Berkeley DB. .. function:: whichdb(filename) This function attempts to guess which of the several simple database modules - available --- :mod:`dbm.gnu`, :mod:`dbm.ndbm` or :mod:`dbm.dumb` --- should - be used to open a given file. + available --- :mod:`dbm.sqlite3`, :mod:`dbm.gnu`, :mod:`dbm.ndbm`, + or :mod:`dbm.dumb` --- should be used to open a given file. Return one of the following values: @@ -52,9 +57,9 @@ the Oracle Berkeley DB. .. |flag_n| replace:: Always create a new, empty database, open for reading and writing. -.. |incompat_note| replace:: - The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible - and can not be used interchangeably. +.. |mode_param_doc| replace:: + The Unix file access mode of the file (default: octal ``0o666``), + used only when the database has to be created. .. function:: open(file, flag='r', mode=0o666) @@ -69,14 +74,13 @@ the Oracle Berkeley DB. :type file: :term:`path-like object` :param str flag: - * ``'r'`` (default), |flag_r| - * ``'w'``, |flag_w| - * ``'c'``, |flag_c| - * ``'n'``, |flag_n| + * ``'r'`` (default): |flag_r| + * ``'w'``: |flag_w| + * ``'c'``: |flag_c| + * ``'n'``: |flag_n| :param int mode: - The Unix file access mode of the file (default: octal ``0o666``), - used only when the database has to be created. + |mode_param_doc| .. versionchanged:: 3.11 *file* accepts a :term:`path-like object`. @@ -141,6 +145,46 @@ then prints out the contents of the database:: The individual submodules are described in the following sections. +:mod:`dbm.sqlite3` --- SQLite backend for dbm +--------------------------------------------- + +.. module:: dbm.sqlite3 + :platform: All + :synopsis: SQLite backend for dbm + +.. versionadded:: 3.13 + +**Source code:** :source:`Lib/dbm/sqlite3.py` + +-------------- + +This module uses the standard library :mod:`sqlite3` module to provide an +SQLite backend for the :mod:`dbm` module. +The files created by :mod:`dbm.sqlite3` can thus be opened by :mod:`sqlite3`, +or any other SQLite browser, including the SQLite CLI. + +.. function:: open(filename, /, flag="r", mode=0o666) + + Open an SQLite database. + The returned object behaves like a :term:`mapping`, + implements a :meth:`!close` method, + and supports a "closing" context manager via the :keyword:`with` keyword. + + :param filename: + The path to the database to be opened. + :type filename: :term:`path-like object` + + :param str flag: + + * ``'r'`` (default): |flag_r| + * ``'w'``: |flag_w| + * ``'c'``: |flag_c| + * ``'n'``: |flag_n| + + :param mode: + The Unix file access mode of the file (default: octal ``0o666``), + used only when the database has to be created. + :mod:`dbm.gnu` --- GNU database manager --------------------------------------- @@ -157,11 +201,10 @@ The :mod:`dbm.gnu` module provides an interface to the :abbr:`GDBM (GNU dbm)` library, similar to the :mod:`dbm.ndbm` module, but with additional functionality like crash tolerance. -:class:`!gdbm` objects behave similar to :term:`mappings <mapping>`, -except that keys and values are always converted to :class:`bytes` before storing, -and the :meth:`!items` and :meth:`!values` methods are not supported. +.. note:: -.. note:: |incompat_note| + The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + and can not be used interchangeably. .. exception:: error @@ -171,47 +214,46 @@ and the :meth:`!items` and :meth:`!values` methods are not supported. .. function:: open(filename, flag="r", mode=0o666, /) - Open a GDBM database and return a :class:`!gdbm` object. The *filename* - argument is the name of the database file. - - The optional *flag* argument can be: + Open a GDBM database and return a :class:`!gdbm` object. - .. csv-table:: - :header: "Value", "Meaning" + :param filename: + The database file to open. + :type filename: :term:`path-like object` - ``'r'`` (default), |flag_r| - ``'w'``, |flag_w| - ``'c'``, |flag_c| - ``'n'``, |flag_n| + :param str flag: + * ``'r'`` (default): |flag_r| + * ``'w'``: |flag_w| + * ``'c'``: |flag_c| + * ``'n'``: |flag_n| - The following additional characters may be appended to the flag to control - how the database is opened: + The following additional characters may be appended + to control how the database is opened: - +---------+--------------------------------------------+ - | Value | Meaning | - +=========+============================================+ - | ``'f'`` | Open the database in fast mode. Writes | - | | to the database will not be synchronized. | - +---------+--------------------------------------------+ - | ``'s'`` | Synchronized mode. This will cause changes | - | | to the database to be immediately written | - | | to the file. | - +---------+--------------------------------------------+ - | ``'u'`` | Do not lock database. | - +---------+--------------------------------------------+ + * ``'f'``: Open the database in fast mode. + Writes to the database will not be synchronized. + * ``'s'``: Synchronized mode. + Changes to the database will be written immediately to the file. + * ``'u'``: Do not lock database. - Not all flags are valid for all versions of GDBM. The module constant - :const:`open_flags` is a string of supported flag characters. The exception - :exc:`error` is raised if an invalid flag is specified. + Not all flags are valid for all versions of GDBM. + See the :data:`open_flags` member for a list of supported flag characters. - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666``. + :param int mode: + |mode_param_doc| - In addition to the dictionary-like methods, :class:`gdbm` objects have the - following methods: + :raises error: + If an invalid *flag* argument is passed. .. versionchanged:: 3.11 - Accepts :term:`path-like object` for filename. + *filename* accepts a :term:`path-like object`. + + .. data:: open_flags + + A string of characters the *flag* parameter of :meth:`~dbm.gnu.open` supports. + + :class:`!gdbm` objects behave similar to :term:`mappings <mapping>`, + but :meth:`!items` and :meth:`!values` methods are not supported. + The following methods are also provided: .. method:: gdbm.firstkey() @@ -268,14 +310,13 @@ and the :meth:`!items` and :meth:`!values` methods are not supported. The :mod:`dbm.ndbm` module provides an interface to the :abbr:`NDBM (New Database Manager)` library. -:class:`!ndbm` objects behave similar to :term:`mappings <mapping>`, -except that keys and values are always stored as :class:`bytes`, -and the :meth:`!items` and :meth:`!values` methods are not supported. - This module can be used with the "classic" NDBM interface or the :abbr:`GDBM (GNU dbm)` compatibility interface. -.. note:: |incompat_note| +.. note:: + + The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + and can not be used interchangeably. .. warning:: @@ -298,25 +339,24 @@ This module can be used with the "classic" NDBM interface or the .. function:: open(filename, flag="r", mode=0o666, /) Open an NDBM database and return an :class:`!ndbm` object. - The *filename* argument is the name of the database file - (without the :file:`.dir` or :file:`.pag` extensions). - - The optional *flag* argument must be one of these values: - .. csv-table:: - :header: "Value", "Meaning" + :param filename: + The basename of the database file + (without the :file:`.dir` or :file:`.pag` extensions). + :type filename: :term:`path-like object` - ``'r'`` (default), |flag_r| - ``'w'``, |flag_w| - ``'c'``, |flag_c| - ``'n'``, |flag_n| + :param str flag: + * ``'r'`` (default): |flag_r| + * ``'w'``: |flag_w| + * ``'c'``: |flag_c| + * ``'n'``: |flag_n| - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666`` (and will be - modified by the prevailing umask). + :param int mode: + |mode_param_doc| - In addition to the dictionary-like methods, :class:`!ndbm` objects - provide the following method: + :class:`!ndbm` objects behave similar to :term:`mappings <mapping>`, + but :meth:`!items` and :meth:`!values` methods are not supported. + The following methods are also provided: .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. @@ -355,8 +395,6 @@ The :mod:`dbm.dumb` module provides a persistent :class:`dict`-like interface which is written entirely in Python. Unlike other :mod:`dbm` backends, such as :mod:`dbm.gnu`, no external library is required. -As with other :mod:`dbm` backends, -the keys and values are always stored as :class:`bytes`. The :mod:`!dbm.dumb` module defines the following: @@ -382,17 +420,13 @@ The :mod:`!dbm.dumb` module defines the following: :type database: :term:`path-like object` :param str flag: - .. csv-table:: - :header: "Value", "Meaning" - - ``'r'``, |flag_r| - ``'w'``, |flag_w| - ``'c'`` (default), |flag_c| - ``'n'``, |flag_n| + * ``'r'``: |flag_r| + * ``'w'``: |flag_w| + * ``'c'`` (default): |flag_c| + * ``'n'``: |flag_n| :param int mode: - The Unix file access mode of the file (default: ``0o666``), - used only when the database has to be created. + |mode_param_doc| .. warning:: It is possible to crash the Python interpreter when loading a database @@ -400,7 +434,7 @@ The :mod:`!dbm.dumb` module defines the following: Python's AST compiler. .. versionchanged:: 3.5 - :func:`open` always creates a new database when *flag* is ``'n'``. + :func:`~dbm.dumb.open` always creates a new database when *flag* is ``'n'``. .. versionchanged:: 3.8 A database opened read-only if *flag* is ``'r'``. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index e654760fb91c650..190e994a12cc713 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1606,7 +1606,7 @@ iterations of the loop. value = STACK.pop() result = func(value) - STACK.push(result) + STACK.append(result) * ``oparg == 1``: call :func:`str` on *value* * ``oparg == 2``: call :func:`repr` on *value* @@ -1623,7 +1623,7 @@ iterations of the loop. value = STACK.pop() result = value.__format__("") - STACK.push(result) + STACK.append(result) Used for implementing formatted literal strings (f-strings). @@ -1636,7 +1636,7 @@ iterations of the loop. spec = STACK.pop() value = STACK.pop() result = value.__format__(spec) - STACK.push(result) + STACK.append(result) Used for implementing formatted literal strings (f-strings). @@ -1783,7 +1783,7 @@ iterations of the loop. arg2 = STACK.pop() arg1 = STACK.pop() result = intrinsic2(arg1, arg2) - STACK.push(result) + STACK.append(result) The operand determines which intrinsic function is called: diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 07b15e23b2c10a4..6e7de004cd52a1a 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -286,6 +286,19 @@ Data Types appropriate value will be chosen for you. See :class:`auto` for the details. + .. attribute:: Enum._name_ + + Name of the member. + + .. attribute:: Enum._value_ + + Value of the member, can be set in :meth:`~object.__new__`. + + .. attribute:: Enum._order_ + + No longer used, kept for backward compatibility. + (class attribute, removed during class creation). + .. attribute:: Enum._ignore_ ``_ignore_`` is only used during creation and is removed from the @@ -337,6 +350,17 @@ Data Types >>> PowersOfThree.SECOND.value 9 + .. method:: Enum.__init__(self, \*args, \**kwds) + + By default, does nothing. If multiple values are given in the member + assignment, those values become separate arguments to ``__init__``; e.g. + + >>> from enum import Enum + >>> class Weekday(Enum): + ... MONDAY = 1, 'Mon' + + ``Weekday.__init__()`` would be called as ``Weekday.__init__(self, 1, 'Mon')`` + .. method:: Enum.__init_subclass__(cls, \**kwds) A *classmethod* that is used to further configure subsequent subclasses. @@ -364,6 +388,21 @@ Data Types >>> Build('deBUG') <Build.DEBUG: 'debug'> + .. method:: Enum.__new__(cls, \*args, \**kwds) + + By default, doesn't exist. If specified, either in the enum class + definition or in a mixin class (such as ``int``), all values given + in the member assignment will be passed; e.g. + + >>> from enum import Enum + >>> class MyIntEnum(Enum): + ... SEVENTEEN = '1a', 16 + + results in the call ``int('1a', 16)`` and a value of ``17`` for the member. + + ..note:: When writing a custom ``__new__``, do not use ``super().__new__`` -- + call the appropriate ``__new__`` instead. + .. method:: Enum.__repr__(self) Returns the string used for *repr()* calls. By default, returns the @@ -477,9 +516,9 @@ Data Types .. class:: Flag - *Flag* members support the bitwise operators ``&`` (*AND*), ``|`` (*OR*), - ``^`` (*XOR*), and ``~`` (*INVERT*); the results of those operators are members - of the enumeration. + ``Flag`` is the same as :class:`Enum`, but its members support the bitwise + operators ``&`` (*AND*), ``|`` (*OR*), ``^`` (*XOR*), and ``~`` (*INVERT*); + the results of those operators are members of the enumeration. .. method:: __contains__(self, value) @@ -511,9 +550,7 @@ Data Types >>> list(purple) [<Color.RED: 1>, <Color.BLUE: 4>] - .. versionchanged:: 3.11 - - Aliases are no longer returned during iteration. + .. versionadded:: 3.11 .. method:: __len__(self): @@ -802,8 +839,8 @@ Supported ``_sunder_`` names - :attr:`~Enum._ignore_` -- a list of names, either as a :class:`list` or a :class:`str`, that will not be transformed into members, and will be removed from the final class -- :attr:`~Enum._order_` -- used in Python 2/3 code to ensure member order is - consistent (class attribute, removed during class creation) +- :attr:`~Enum._order_` -- no longer used, kept for backward + compatibility (class attribute, removed during class creation) - :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for an enum member; may be overridden diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index f821776c2861331..88417b40e4aa7f9 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -16,7 +16,7 @@ equivalent, even if they have the same name. .. index:: pair: statement; raise -The built-in exceptions listed below can be generated by the interpreter or +The built-in exceptions listed in this chapter can be generated by the interpreter or built-in functions. Except where mentioned, they have an "associated value" indicating the detailed cause of the error. This may be a string or a tuple of several items of information (e.g., an error code and a string explaining the @@ -416,6 +416,24 @@ The following exceptions are the exceptions that are usually raised. handling in C, most floating point operations are not checked. +.. exception:: PythonFinalizationError + + This exception is derived from :exc:`RuntimeError`. It is raised when + an operation is blocked during interpreter shutdown also known as + :term:`Python finalization <interpreter shutdown>`. + + Examples of operations which can be blocked with a + :exc:`PythonFinalizationError` during the Python finalization: + + * Creating a new Python thread. + * :func:`os.fork`. + + See also the :func:`sys.is_finalizing` function. + + .. versionadded:: 3.13 + Previously, a plain :exc:`RuntimeError` was raised. + + .. exception:: RecursionError This exception is derived from :exc:`RuntimeError`. It is raised when the diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index 13ad2dd7da5090a..b93d6ac7aab9565 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -13,10 +13,10 @@ ---------------- -This module performs file control and I/O control on file descriptors. It is an -interface to the :c:func:`fcntl` and :c:func:`ioctl` Unix routines. For a -complete description of these calls, see :manpage:`fcntl(2)` and -:manpage:`ioctl(2)` Unix manual pages. +This module performs file and I/O control on file descriptors. It is an +interface to the :c:func:`fcntl` and :c:func:`ioctl` Unix routines. +See the :manpage:`fcntl(2)` and :manpage:`ioctl(2)` Unix manual pages +for full details. .. availability:: Unix, not Emscripten, not WASI. @@ -101,7 +101,7 @@ The module defines the following functions: most likely to result in a segmentation violation or a more subtle data corruption. - If the :c:func:`fcntl` fails, an :exc:`OSError` is raised. + If the :c:func:`fcntl` call fails, an :exc:`OSError` is raised. .. audit-event:: fcntl.fcntl fd,cmd,arg fcntl.fcntl @@ -139,7 +139,7 @@ The module defines the following functions: buffer 1024 bytes long which is then passed to :func:`ioctl` and copied back into the supplied buffer. - If the :c:func:`ioctl` fails, an :exc:`OSError` exception is raised. + If the :c:func:`ioctl` call fails, an :exc:`OSError` exception is raised. An example:: @@ -164,7 +164,7 @@ The module defines the following functions: :manpage:`flock(2)` for details. (On some systems, this function is emulated using :c:func:`fcntl`.) - If the :c:func:`flock` fails, an :exc:`OSError` exception is raised. + If the :c:func:`flock` call fails, an :exc:`OSError` exception is raised. .. audit-event:: fcntl.flock fd,operation fcntl.flock @@ -176,17 +176,28 @@ The module defines the following functions: method are accepted as well) of the file to lock or unlock, and *cmd* is one of the following values: - * :const:`LOCK_UN` -- unlock - * :const:`LOCK_SH` -- acquire a shared lock - * :const:`LOCK_EX` -- acquire an exclusive lock + .. data:: LOCK_UN - When *cmd* is :const:`LOCK_SH` or :const:`LOCK_EX`, it can also be - bitwise ORed with :const:`LOCK_NB` to avoid blocking on lock acquisition. - If :const:`LOCK_NB` is used and the lock cannot be acquired, an + Release an existing lock. + + .. data:: LOCK_SH + + Acquire a shared lock. + + .. data:: LOCK_EX + + Acquire an exclusive lock. + + .. data:: LOCK_NB + + Bitwise OR with any of the other three ``LOCK_*`` constants to make + the request non-blocking. + + If :const:`!LOCK_NB` is used and the lock cannot be acquired, an :exc:`OSError` will be raised and the exception will have an *errno* - attribute set to :const:`EACCES` or :const:`EAGAIN` (depending on the + attribute set to :const:`~errno.EACCES` or :const:`~errno.EAGAIN` (depending on the operating system; for portability, check for both values). On at least some - systems, :const:`LOCK_EX` can only be used if the file descriptor refers to a + systems, :const:`!LOCK_EX` can only be used if the file descriptor refers to a file opened for writing. *len* is the number of bytes to lock, *start* is the byte offset at diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index 2f98a272c297ae3..8d1aae018ada121 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -104,7 +104,7 @@ FTP objects :param timeout: A timeout in seconds for blocking operations like :meth:`connect` (default: the global default timeout setting). - :type timeout: int | None + :type timeout: float | None :param source_address: |param_doc_source_address| @@ -178,7 +178,7 @@ FTP objects :param timeout: A timeout in seconds for the connection attempt (default: the global default timeout setting). - :type timeout: int | None + :type timeout: float | None :param source_address: |param_doc_source_address| @@ -232,8 +232,8 @@ FTP objects .. method:: FTP.voidcmd(cmd) Send a simple command string to the server and handle the response. Return - nothing if a response code corresponding to success (codes in the range - 200--299) is received. Raise :exc:`error_reply` otherwise. + the response string if the response code corresponds to success (codes in + the range 200--299). Raise :exc:`error_reply` otherwise. .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.voidcmd @@ -483,7 +483,7 @@ FTP_TLS objects :param timeout: A timeout in seconds for blocking operations like :meth:`~FTP.connect` (default: the global default timeout setting). - :type timeout: int | None + :type timeout: float | None :param source_address: |param_doc_source_address| diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 348788a1cb13c34..171d85fe7d4880c 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -526,9 +526,20 @@ are always available. They are listed here in alphabetical order. .. function:: eval(expression, globals=None, locals=None) - The arguments are a string and optional globals and locals. If provided, - *globals* must be a dictionary. If provided, *locals* can be any mapping - object. + :param expression: + A Python expression. + :type expression: :class:`str` | :ref:`code object <code-objects>` + + :param globals: + The global namespace (default: ``None``). + :type globals: :class:`dict` | ``None`` + + :param locals: + The local namespace (default: ``None``). + :type locals: :term:`mapping` | ``None`` + + :returns: The result of the evaluated expression. + :raises: Syntax errors are reported as exceptions. The *expression* argument is parsed and evaluated as a Python expression (technically speaking, a condition list) using the *globals* and *locals* @@ -545,8 +556,7 @@ are always available. They are listed here in alphabetical order. :term:`nested scopes <nested scope>` (non-locals) in the enclosing environment. - The return value is the result of - the evaluated expression. Syntax errors are reported as exceptions. Example: + Example: >>> x = 1 >>> eval('x+1') @@ -1577,6 +1587,16 @@ are always available. They are listed here in alphabetical order. If :func:`sys.displayhook` is not accessible, this function will raise :exc:`RuntimeError`. + This class has a custom representation that can be evaluated:: + + class Person: + def __init__(self, name, age): + self.name = name + self.age = age + + def __repr__(self): + return f"Person('{self.name}', {self.age})" + .. function:: reversed(seq) diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 50cde09fa10a9d9..79be215a7660454 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -61,7 +61,7 @@ The module defines the following items: .. exception:: BadGzipFile - An exception raised for invalid gzip files. It inherits :exc:`OSError`. + An exception raised for invalid gzip files. It inherits from :exc:`OSError`. :exc:`EOFError` and :exc:`zlib.error` can also be raised for invalid gzip files. @@ -287,4 +287,3 @@ Command line options .. option:: -h, --help Show the help message. - diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 12a6d768437ea54..2fe188be641c2d1 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -649,6 +649,11 @@ internal consistency, so you should know what you're doing if you do that. :const:`None`. +.. attribute:: Cookie.domain + + Cookie domain (a string). + + .. attribute:: Cookie.path Cookie path (a string, eg. ``'/acme/rocket_launchers'``). diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index f8b3e39c4f54f09..8a74cadb98a0dbd 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -340,6 +340,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Functions wrapped in :func:`functools.partial` now return ``True`` if the wrapped function is a Python generator function. + .. versionchanged:: 3.13 + Functions wrapped in :func:`functools.partialmethod` now return ``True`` + if the wrapped function is a Python generator function. .. function:: isgenerator(object) @@ -363,6 +366,10 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Sync functions marked with :func:`markcoroutinefunction` now return ``True``. + .. versionchanged:: 3.13 + Functions wrapped in :func:`functools.partialmethod` now return ``True`` + if the wrapped function is a :term:`coroutine function`. + .. function:: markcoroutinefunction(func) @@ -429,6 +436,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Functions wrapped in :func:`functools.partial` now return ``True`` if the wrapped function is a :term:`asynchronous generator` function. + .. versionchanged:: 3.13 + Functions wrapped in :func:`functools.partialmethod` now return ``True`` + if the wrapped function is a :term:`coroutine function`. .. function:: isasyncgen(object) diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 6736aa9ee2b0efc..8eb531aa4ea2487 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -466,7 +466,7 @@ I/O Base Classes .. class:: RawIOBase - Base class for raw binary streams. It inherits :class:`IOBase`. + Base class for raw binary streams. It inherits from :class:`IOBase`. Raw binary streams typically provide low-level access to an underlying OS device or API, and do not try to encapsulate it in high-level primitives @@ -519,7 +519,7 @@ I/O Base Classes .. class:: BufferedIOBase Base class for binary streams that support some kind of buffering. - It inherits :class:`IOBase`. + It inherits from :class:`IOBase`. The main difference with :class:`RawIOBase` is that methods :meth:`read`, :meth:`readinto` and :meth:`write` will try (respectively) to read as much @@ -633,7 +633,7 @@ Raw File I/O .. class:: FileIO(name, mode='r', closefd=True, opener=None) A raw binary stream representing an OS-level file containing bytes data. It - inherits :class:`RawIOBase`. + inherits from :class:`RawIOBase`. The *name* can be one of two things: @@ -696,7 +696,7 @@ than raw I/O does. .. class:: BytesIO(initial_bytes=b'') - A binary stream using an in-memory bytes buffer. It inherits + A binary stream using an in-memory bytes buffer. It inherits from :class:`BufferedIOBase`. The buffer is discarded when the :meth:`~IOBase.close` method is called. @@ -745,7 +745,7 @@ than raw I/O does. .. class:: BufferedReader(raw, buffer_size=DEFAULT_BUFFER_SIZE) A buffered binary stream providing higher-level access to a readable, non - seekable :class:`RawIOBase` raw binary stream. It inherits + seekable :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedIOBase`. When reading data from this object, a larger amount of data may be @@ -783,7 +783,7 @@ than raw I/O does. .. class:: BufferedWriter(raw, buffer_size=DEFAULT_BUFFER_SIZE) A buffered binary stream providing higher-level access to a writeable, non - seekable :class:`RawIOBase` raw binary stream. It inherits + seekable :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedIOBase`. When writing to this object, data is normally placed into an internal @@ -818,7 +818,7 @@ than raw I/O does. .. class:: BufferedRandom(raw, buffer_size=DEFAULT_BUFFER_SIZE) A buffered binary stream providing higher-level access to a seekable - :class:`RawIOBase` raw binary stream. It inherits :class:`BufferedReader` + :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedReader` and :class:`BufferedWriter`. The constructor creates a reader and writer for a seekable raw stream, given @@ -834,7 +834,7 @@ than raw I/O does. A buffered binary stream providing higher-level access to two non seekable :class:`RawIOBase` raw binary streams---one readable, the other writeable. - It inherits :class:`BufferedIOBase`. + It inherits from :class:`BufferedIOBase`. *reader* and *writer* are :class:`RawIOBase` objects that are readable and writeable respectively. If the *buffer_size* is omitted it defaults to @@ -857,7 +857,7 @@ Text I/O .. class:: TextIOBase Base class for text streams. This class provides a character and line based - interface to stream I/O. It inherits :class:`IOBase`. + interface to stream I/O. It inherits from :class:`IOBase`. :class:`TextIOBase` provides or overrides these data attributes and methods in addition to those from :class:`IOBase`: @@ -946,7 +946,7 @@ Text I/O line_buffering=False, write_through=False) A buffered text stream providing higher-level access to a - :class:`BufferedIOBase` buffered binary stream. It inherits + :class:`BufferedIOBase` buffered binary stream. It inherits from :class:`TextIOBase`. *encoding* gives the name of the encoding that the stream will be decoded or @@ -1073,7 +1073,7 @@ Text I/O .. class:: StringIO(initial_value='', newline='\n') - A text stream using an in-memory text buffer. It inherits + A text stream using an in-memory text buffer. It inherits from :class:`TextIOBase`. The text buffer is discarded when the :meth:`~IOBase.close` method is @@ -1124,7 +1124,7 @@ Text I/O .. class:: IncrementalNewlineDecoder A helper codec that decodes newlines for :term:`universal newlines` mode. - It inherits :class:`codecs.IncrementalDecoder`. + It inherits from :class:`codecs.IncrementalDecoder`. Performance diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 338a5f9615aae31..c26f6c89b4920a1 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -688,6 +688,14 @@ loops that truncate the stream. else: break + Note, the element that first fails the predicate condition is + consumed from the input iterator and there is no way to access it. + This could be an issue if an application wants to further consume the + input iterator after takewhile has been run to exhaustion. To work + around this problem, consider using `more-iterools before_and_after() + <https://more-itertools.readthedocs.io/en/stable/api.html#more_itertools.before_and_after>`_ + instead. + .. function:: tee(iterable, n=2) @@ -770,7 +778,7 @@ The primary purpose of the itertools recipes is educational. The recipes show various ways of thinking about individual tools — for example, that ``chain.from_iterable`` is related to the concept of flattening. The recipes also give ideas about ways that the tools can be combined — for example, how -``compress()`` and ``range()`` can work together. The recipes also show patterns +``starmap()`` and ``repeat()`` can work together. The recipes also show patterns for using itertools with the :mod:`operator` and :mod:`collections` modules as well as with the built-in itertools such as ``map()``, ``filter()``, ``reversed()``, and ``enumerate()``. @@ -855,10 +863,9 @@ which incur interpreter overhead. "Given a predicate that returns True or False, count the True results." return sum(map(pred, iterable)) - def all_equal(iterable): + def all_equal(iterable, key=None): "Returns True if all the elements are equal to each other." - g = groupby(iterable) - return next(g, True) and not next(g, False) + return len(take(2, groupby(iterable, key))) <= 1 def first_true(iterable, default=False, pred=None): """Returns the first true value in the iterable. @@ -1004,32 +1011,6 @@ which incur interpreter overhead. except exception: pass - def before_and_after(predicate, it): - """ Variant of takewhile() that allows complete - access to the remainder of the iterator. - - >>> it = iter('ABCdEfGhI') - >>> all_upper, remainder = before_and_after(str.isupper, it) - >>> ''.join(all_upper) - 'ABC' - >>> ''.join(remainder) # takewhile() would lose the 'd' - 'dEfGhI' - - Note that the true iterator must be fully consumed - before the remainder iterator can generate valid results. - """ - it = iter(it) - transition = [] - - def true_iterator(): - for elem in it: - if predicate(elem): - yield elem - else: - transition.append(elem) - return - - return true_iterator(), chain(transition, it) The following recipes have a more mathematical flavor: @@ -1243,6 +1224,8 @@ The following recipes have a more mathematical flavor: >>> [all_equal(s) for s in ('', 'A', 'AAAA', 'AAAB', 'AAABA')] [True, True, True, False, False] + >>> [all_equal(s, key=str.casefold) for s in ('', 'A', 'AaAa', 'AAAB', 'AAABA')] + [True, True, True, False, False] >>> quantify(range(99), lambda x: x%2==0) 50 @@ -1543,13 +1526,6 @@ The following recipes have a more mathematical flavor: >>> list(odds) [1, 3, 5, 7, 9] - >>> it = iter('ABCdEfGhI') - >>> all_upper, remainder = before_and_after(str.isupper, it) - >>> ''.join(all_upper) - 'ABC' - >>> ''.join(remainder) - 'dEfGhI' - >>> list(subslices('ABCD')) ['A', 'AB', 'ABC', 'ABCD', 'B', 'BC', 'BCD', 'C', 'CD', 'D'] @@ -1640,6 +1616,32 @@ The following recipes have a more mathematical flavor: result.append(pool[-1-n]) return tuple(result) + def before_and_after(predicate, it): + """ Variant of takewhile() that allows complete + access to the remainder of the iterator. + + >>> it = iter('ABCdEfGhI') + >>> all_upper, remainder = before_and_after(str.isupper, it) + >>> ''.join(all_upper) + 'ABC' + >>> ''.join(remainder) # takewhile() would lose the 'd' + 'dEfGhI' + + Note that the true iterator must be fully consumed + before the remainder iterator can generate valid results. + """ + it = iter(it) + transition = [] + + def true_iterator(): + for elem in it: + if predicate(elem): + yield elem + else: + transition.append(elem) + return + + return true_iterator(), chain(transition, it) .. doctest:: :hide: @@ -1669,3 +1671,10 @@ The following recipes have a more mathematical flavor: >>> combos = list(combinations(iterable, r)) >>> all(nth_combination(iterable, r, i) == comb for i, comb in enumerate(combos)) True + + >>> it = iter('ABCdEfGhI') + >>> all_upper, remainder = before_and_after(str.isupper, it) + >>> ''.join(all_upper) + 'ABC' + >>> ''.join(remainder) + 'dEfGhI' diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 0ce4b697145cb3e..c82ff9dc325b4c8 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -106,7 +106,7 @@ Extending :class:`JSONEncoder`:: ... if isinstance(obj, complex): ... return [obj.real, obj.imag] ... # Let the base class default method raise the TypeError - ... return json.JSONEncoder.default(self, obj) + ... return super().default(obj) ... >>> json.dumps(2 + 1j, cls=ComplexEncoder) '[2.0, 1.0]' @@ -504,7 +504,7 @@ Encoders and Decoders else: return list(iterable) # Let the base class default method raise the TypeError - return json.JSONEncoder.default(self, o) + return super().default(o) .. method:: encode(o) diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 4b756d10b4c586c..39eb41ce1f16708 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -531,12 +531,12 @@ subclasses. However, the :meth:`!__init__` method in subclasses needs to call This method should be called from handlers when an exception is encountered during an :meth:`emit` call. If the module-level attribute - ``raiseExceptions`` is ``False``, exceptions get silently ignored. This is + :data:`raiseExceptions` is ``False``, exceptions get silently ignored. This is what is mostly wanted for a logging system - most users will not care about errors in the logging system, they are more interested in application errors. You could, however, replace this with a custom handler if you wish. The specified record is the one which was being processed when the exception - occurred. (The default value of ``raiseExceptions`` is ``True``, as that is + occurred. (The default value of :data:`raiseExceptions` is ``True``, as that is more useful during development). @@ -1494,6 +1494,18 @@ Module-Level Attributes .. versionadded:: 3.2 +.. attribute:: raiseExceptions + + Used to see if exceptions during handling should be propagated. + + Default: ``True``. + + If :data:`raiseExceptions` is ``False``, + exceptions get silently ignored. This is what is mostly wanted + for a logging system - most users will not care about errors in + the logging system, they are more interested in application errors. + + Integration with the warnings module ------------------------------------ diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index fa5b273093f583f..a613548c9e518e0 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -1136,8 +1136,8 @@ When a :class:`!MaildirMessage` instance is created based upon a leading "From " or trailing newline. For convenience, *time_* may be specified and will be formatted appropriately and appended to *from_*. If *time_* is specified, it should be a :class:`time.struct_time` instance, a - tuple suitable for passing to :meth:`time.strftime`, or ``True`` (to use - :meth:`time.gmtime`). + tuple suitable for passing to :func:`time.strftime`, or ``True`` (to use + :func:`time.gmtime`). .. method:: get_flags() @@ -1508,8 +1508,8 @@ When a :class:`!BabylMessage` instance is created based upon an leading "From " or trailing newline. For convenience, *time_* may be specified and will be formatted appropriately and appended to *from_*. If *time_* is specified, it should be a :class:`time.struct_time` instance, a - tuple suitable for passing to :meth:`time.strftime`, or ``True`` (to use - :meth:`time.gmtime`). + tuple suitable for passing to :func:`time.strftime`, or ``True`` (to use + :func:`time.gmtime`). .. method:: get_flags() diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 9caf7230eed0aa9..3c850317f608582 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -239,11 +239,11 @@ Number-theoretic and representation functions See also :func:`math.ulp`. + .. versionadded:: 3.9 + .. versionchanged:: 3.12 Added the *steps* argument. - .. versionadded:: 3.9 - .. function:: perm(n, k=None) Return the number of ways to choose *k* items from *n* items @@ -680,11 +680,11 @@ Constants >>> math.isnan(float('nan')) True + .. versionadded:: 3.5 + .. versionchanged:: 3.11 It is now always available. - .. versionadded:: 3.5 - .. impl-detail:: diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index 2a6d980ab78a609..ac3458c86fd4c4f 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -252,3 +252,18 @@ Other Functions .. data:: CRTDBG_REPORT_MODE Returns current *mode* for the specified *type*. + + +.. data:: CRT_ASSEMBLY_VERSION + + The CRT Assembly version, from the :file:`crtassem.h` header file. + + +.. data:: VC_ASSEMBLY_PUBLICKEYTOKEN + + The VC Assembly public key token, from the :file:`crtassem.h` header file. + + +.. data:: LIBRARIES_ASSEMBLY_NAME_PREFIX + + The Libraries Assembly name prefix, from the :file:`crtassem.h` header file. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 789a84b02d59d23..0b87de4c61e6aa2 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -56,7 +56,7 @@ will print to standard output :: The :class:`Process` class -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ In :mod:`multiprocessing`, processes are spawned by creating a :class:`Process` object and then calling its :meth:`~Process.start` method. :class:`Process` @@ -102,7 +102,7 @@ necessary, see :ref:`multiprocessing-programming`. .. _multiprocessing-start-methods: Contexts and start methods -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ Depending on the platform, :mod:`multiprocessing` supports three ways to start a process. These *start methods* are @@ -150,18 +150,18 @@ to start a process. These *start methods* are over Unix pipes such as Linux. -.. versionchanged:: 3.8 - - On macOS, the *spawn* start method is now the default. The *fork* start - method should be considered unsafe as it can lead to crashes of the - subprocess as macOS system libraries may start threads. See :issue:`33725`. - .. versionchanged:: 3.4 *spawn* added on all POSIX platforms, and *forkserver* added for some POSIX platforms. Child processes no longer inherit all of the parents inheritable handles on Windows. +.. versionchanged:: 3.8 + + On macOS, the *spawn* start method is now the default. The *fork* start + method should be considered unsafe as it can lead to crashes of the + subprocess as macOS system libraries may start threads. See :issue:`33725`. + On POSIX using the *spawn* or *forkserver* start methods will also start a *resource tracker* process which tracks the unlinked named system resources (such as named semaphores or @@ -231,7 +231,7 @@ library user. Exchanging objects between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :mod:`multiprocessing` supports two types of communication channel between processes: @@ -283,7 +283,7 @@ processes: Synchronization between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :mod:`multiprocessing` contains equivalents of all the synchronization primitives from :mod:`threading`. For instance one can use a lock to ensure @@ -309,7 +309,7 @@ mixed up. Sharing state between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ As mentioned above, when doing concurrent programming it is usually best to avoid using shared state as far as possible. This is particularly true when @@ -399,7 +399,7 @@ However, if you really do need to use some shared data then Using a pool of workers -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ The :class:`~multiprocessing.pool.Pool` class represents a pool of worker processes. It has methods which allows tasks to be offloaded to the worker @@ -490,7 +490,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the :class:`Process` and exceptions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: Process(group=None, target=None, name=None, args=(), kwargs={}, \ *, daemon=None) @@ -519,7 +519,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the to the process. .. versionchanged:: 3.3 - Added the *daemon* argument. + Added the *daemon* parameter. .. method:: run() @@ -649,8 +649,8 @@ The :mod:`multiprocessing` package mostly replicates the API of the .. method:: terminate() - Terminate the process. On POSIX this is done using the ``SIGTERM`` signal; - on Windows :c:func:`TerminateProcess` is used. Note that exit handlers and + Terminate the process. On POSIX this is done using the :py:const:`~signal.SIGTERM` signal; + on Windows :c:func:`!TerminateProcess` is used. Note that exit handlers and finally clauses, etc., will not be executed. Note that descendant processes of the process will *not* be terminated -- @@ -724,7 +724,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the Raised by methods with a timeout when the timeout expires. Pipes and Queues -~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ When using multiple processes, one generally uses message passing for communication between processes and avoids having to use any synchronization @@ -981,7 +981,7 @@ For an example of the usage of queues for interprocess communication see Miscellaneous -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ .. function:: active_children() @@ -1084,13 +1084,13 @@ Miscellaneous The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'`` or ``None``. See :ref:`multiprocessing-start-methods`. -.. versionchanged:: 3.8 + .. versionadded:: 3.4 - On macOS, the *spawn* start method is now the default. The *fork* start - method should be considered unsafe as it can lead to crashes of the - subprocess. See :issue:`33725`. + .. versionchanged:: 3.8 - .. versionadded:: 3.4 + On macOS, the *spawn* start method is now the default. The *fork* start + method should be considered unsafe as it can lead to crashes of the + subprocess. See :issue:`33725`. .. function:: set_executable(executable) @@ -1150,7 +1150,7 @@ Miscellaneous Connection Objects -~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^ .. currentmodule:: multiprocessing.connection @@ -1245,8 +1245,7 @@ Connection objects are usually created using Connection objects themselves can now be transferred between processes using :meth:`Connection.send` and :meth:`Connection.recv`. - .. versionadded:: 3.3 - Connection objects now support the context management protocol -- see + Connection objects also now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the connection object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. @@ -1292,7 +1291,7 @@ For example: Synchronization primitives -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ .. currentmodule:: multiprocessing @@ -1481,7 +1480,7 @@ object -- see :ref:`multiprocessing-managers`. Shared :mod:`ctypes` Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It is possible to create shared objects using shared memory which can be inherited by child processes. @@ -1543,7 +1542,7 @@ inherited by child processes. The :mod:`multiprocessing.sharedctypes` module ->>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +"""""""""""""""""""""""""""""""""""""""""""""" .. module:: multiprocessing.sharedctypes :synopsis: Allocate ctypes objects from shared memory. @@ -1709,7 +1708,7 @@ The results printed are :: .. _multiprocessing-managers: Managers -~~~~~~~~ +^^^^^^^^ Managers provide a way to create data which can be shared between different processes, including sharing over a network between processes running on @@ -1954,7 +1953,7 @@ their parent process exits. The manager classes are defined in the Customized managers ->>>>>>>>>>>>>>>>>>> +""""""""""""""""""" To create one's own manager, one creates a subclass of :class:`BaseManager` and uses the :meth:`~BaseManager.register` classmethod to register new types or @@ -1981,7 +1980,7 @@ callables with the manager class. For example:: Using a remote manager ->>>>>>>>>>>>>>>>>>>>>> +"""""""""""""""""""""" It is possible to run a manager server on one machine and have clients use it from other machines (assuming that the firewalls involved allow it). @@ -2044,7 +2043,7 @@ client to access it remotely:: .. _multiprocessing-proxy_objects: Proxy Objects -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ A proxy is an object which *refers* to a shared object which lives (presumably) in a different process. The shared object is said to be the *referent* of the @@ -2196,7 +2195,7 @@ demonstrates a level of control over the synchronization. Cleanup ->>>>>>> +""""""" A proxy object uses a weakref callback so that when it gets garbage collected it deregisters itself from the manager which owns its referent. @@ -2206,7 +2205,7 @@ any proxies referring to it. Process Pools -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ .. module:: multiprocessing.pool :synopsis: Create pools of processes. @@ -2250,11 +2249,11 @@ with the :class:`Pool` class. as CPython does not assure that the finalizer of the pool will be called (see :meth:`object.__del__` for more information). - .. versionadded:: 3.2 - *maxtasksperchild* + .. versionchanged:: 3.2 + Added the *maxtasksperchild* parameter. - .. versionadded:: 3.4 - *context* + .. versionchanged:: 3.4 + Added the *context* parameter. .. versionchanged:: 3.13 *processes* uses :func:`os.process_cpu_count` by default, instead of @@ -2380,7 +2379,7 @@ with the :class:`Pool` class. Wait for the worker processes to exit. One must call :meth:`close` or :meth:`terminate` before using :meth:`join`. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Pool objects now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the pool object, and :meth:`~contextmanager.__exit__` calls :meth:`terminate`. @@ -2442,7 +2441,7 @@ The following example demonstrates the use of a pool:: .. _multiprocessing-listeners-clients: Listeners and Clients -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^ .. module:: multiprocessing.connection :synopsis: API for dealing with sockets. @@ -2549,7 +2548,7 @@ multiple connections at the same time. The address from which the last accepted connection came. If this is unavailable then it is ``None``. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Listener objects now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the listener object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. @@ -2665,7 +2664,7 @@ wait for messages from multiple processes at once:: .. _multiprocessing-address-formats: Address Formats ->>>>>>>>>>>>>>> +""""""""""""""" * An ``'AF_INET'`` address is a tuple of the form ``(hostname, port)`` where *hostname* is a string and *port* is an integer. @@ -2685,7 +2684,7 @@ an ``'AF_PIPE'`` address rather than an ``'AF_UNIX'`` address. .. _multiprocessing-auth-keys: Authentication keys -~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ When one uses :meth:`Connection.recv <Connection.recv>`, the data received is automatically @@ -2711,7 +2710,7 @@ Suitable authentication keys can also be generated by using :func:`os.urandom`. Logging -~~~~~~~ +^^^^^^^ Some support for logging is available. Note, however, that the :mod:`logging` package does not use process shared locks so it is possible (depending on the @@ -2759,7 +2758,7 @@ For a full table of logging levels, see the :mod:`logging` module. The :mod:`multiprocessing.dummy` module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: multiprocessing.dummy :synopsis: Dumb wrapper around threading. @@ -2818,7 +2817,7 @@ There are certain guidelines and idioms which should be adhered to when using All start methods -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ The following applies to all start methods. @@ -2977,9 +2976,9 @@ Beware of replacing :data:`sys.stdin` with a "file like object" For more information, see :issue:`5155`, :issue:`5313` and :issue:`5331` The *spawn* and *forkserver* start methods -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -There are a few extra restriction which don't apply to the *fork* +There are a few extra restrictions which don't apply to the *fork* start method. More picklability diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 10d7f061fb759b7..933fd07d62418a8 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -23,7 +23,7 @@ processes, a :class:`~multiprocessing.managers.BaseManager` subclass, :class:`~multiprocessing.managers.SharedMemoryManager`, is also provided in the :mod:`multiprocessing.managers` module. -In this module, shared memory refers to "System V style" shared memory blocks +In this module, shared memory refers to "POSIX style" shared memory blocks (though is not necessarily implemented explicitly as such) and does not refer to "distributed shared memory". This style of shared memory permits distinct processes to potentially read and write to a common (or shared) region of diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 34bc76b231de921..3ee2b7db1e511b9 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -79,7 +79,7 @@ the :mod:`glob` module.) .. function:: commonpath(paths) - Return the longest common sub-path of each pathname in the sequence + Return the longest common sub-path of each pathname in the iterable *paths*. Raise :exc:`ValueError` if *paths* contain both absolute and relative pathnames, the *paths* are on the different drives or if *paths* is empty. Unlike :func:`commonprefix`, this returns a @@ -92,6 +92,9 @@ the :mod:`glob` module.) .. versionchanged:: 3.6 Accepts a sequence of :term:`path-like objects <path-like object>`. + .. versionchanged:: 3.13 + Any iterable can now be passed, rather than just sequences. + .. function:: commonprefix(list) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 0008ec6a40c76ff..cc9f3e75a80c517 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1122,6 +1122,20 @@ as internal buffering of data. .. versionchanged:: 3.12 Added support for pipes on Windows. + +.. function:: grantpt(fd, /) + + Grant access to the slave pseudo-terminal device associated with the + master pseudo-terminal device to which the file descriptor *fd* refers. + The file descriptor *fd* is not closed upon failure. + + Calls the C standard library function :c:func:`grantpt`. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.13 + + .. function:: isatty(fd, /) Return ``True`` if the file descriptor *fd* is open and connected to a @@ -1429,6 +1443,23 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo .. versionadded:: 3.3 +.. function:: posix_openpt(oflag, /) + + Open and return a file descriptor for a master pseudo-terminal device. + + Calls the C standard library function :c:func:`posix_openpt`. The *oflag* + argument is used to set file status flags and file access modes as + specified in the manual page of :c:func:`posix_openpt` of your system. + + The returned file descriptor is :ref:`non-inheritable <fd_inheritance>`. + If the value :data:`O_CLOEXEC` is available on the system, it is added to + *oflag*. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.13 + + .. function:: preadv(fd, buffers, offset, flags=0, /) Read from a file descriptor *fd* at a position of *offset* into mutable @@ -1486,6 +1517,21 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo .. versionadded:: 3.7 +.. function:: ptsname(fd, /) + + Return the name of the slave pseudo-terminal device associated with the + master pseudo-terminal device to which the file descriptor *fd* refers. + The file descriptor *fd* is not closed upon failure. + + Calls the reentrant C standard library function :c:func:`ptsname_r` if + it is available; otherwise, the C standard library function + :c:func:`ptsname`, which is not guaranteed to be thread-safe, is called. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.13 + + .. function:: pwrite(fd, str, offset, /) Write the bytestring in *str* to file descriptor *fd* at position of @@ -1738,6 +1784,19 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo .. availability:: Unix. +.. function:: unlockpt(fd, /) + + Unlock the slave pseudo-terminal device associated with the master + pseudo-terminal device to which the file descriptor *fd* refers. + The file descriptor *fd* is not closed upon failure. + + Calls the C standard library function :c:func:`unlockpt`. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.13 + + .. function:: write(fd, str, /) Write the bytestring in *str* to file descriptor *fd*. diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index f1aba793fda03e7..4b461a5d4a2949c 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -572,6 +572,9 @@ Pure paths provide the following methods and properties: >>> PurePath('/a/b/c.py').full_match('**/*.py') True + .. seealso:: + :ref:`pathlib-pattern-language` documentation. + As with other methods, case-sensitivity follows platform defaults:: >>> PurePosixPath('b.py').full_match('*.PY') @@ -991,11 +994,6 @@ call fails (for example because the path doesn't exist). [PosixPath('pathlib.py'), PosixPath('setup.py'), PosixPath('test_pathlib.py')] >>> sorted(Path('.').glob('*/*.py')) [PosixPath('docs/conf.py')] - - Patterns are the same as for :mod:`fnmatch`, with the addition of "``**``" - which means "this directory and all subdirectories, recursively". In other - words, it enables recursive globbing:: - >>> sorted(Path('.').glob('**/*.py')) [PosixPath('build/lib/pathlib.py'), PosixPath('docs/conf.py'), @@ -1003,13 +1001,8 @@ call fails (for example because the path doesn't exist). PosixPath('setup.py'), PosixPath('test_pathlib.py')] - .. note:: - Using the "``**``" pattern in large directory trees may consume - an inordinate amount of time. - - .. tip:: - Set *follow_symlinks* to ``True`` or ``False`` to improve performance - of recursive globbing. + .. seealso:: + :ref:`pathlib-pattern-language` documentation. This method calls :meth:`Path.is_dir` on the top-level directory and propagates any :exc:`OSError` exception that is raised. Subsequent @@ -1025,11 +1018,11 @@ call fails (for example because the path doesn't exist). wildcards. Set *follow_symlinks* to ``True`` to always follow symlinks, or ``False`` to treat all symlinks as files. - .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob + .. tip:: + Set *follow_symlinks* to ``True`` or ``False`` to improve performance + of recursive globbing. - .. versionchanged:: 3.11 - Return only directories if *pattern* ends with a pathname components - separator (:data:`~os.sep` or :data:`~os.altsep`). + .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob .. versionchanged:: 3.12 The *case_sensitive* parameter was added. @@ -1038,13 +1031,29 @@ call fails (for example because the path doesn't exist). The *follow_symlinks* parameter was added. .. versionchanged:: 3.13 - Emits :exc:`FutureWarning` if the pattern ends with "``**``". In a - future Python release, patterns with this ending will match both files - and directories. Add a trailing slash to match only directories. + The *pattern* parameter accepts a :term:`path-like object`. + + +.. method:: Path.rglob(pattern, *, case_sensitive=None, follow_symlinks=None) + + Glob the given relative *pattern* recursively. This is like calling + :func:`Path.glob` with "``**/``" added in front of the *pattern*. + + .. seealso:: + :ref:`pathlib-pattern-language` and :meth:`Path.glob` documentation. + + .. audit-event:: pathlib.Path.rglob self,pattern pathlib.Path.rglob + + .. versionchanged:: 3.12 + The *case_sensitive* parameter was added. + + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. .. versionchanged:: 3.13 The *pattern* parameter accepts a :term:`path-like object`. + .. method:: Path.group(*, follow_symlinks=True) Return the name of the group owning the file. :exc:`KeyError` is raised @@ -1472,44 +1481,6 @@ call fails (for example because the path doesn't exist). strict mode, and no exception is raised in non-strict mode. In previous versions, :exc:`RuntimeError` is raised no matter the value of *strict*. -.. method:: Path.rglob(pattern, *, case_sensitive=None, follow_symlinks=None) - - Glob the given relative *pattern* recursively. This is like calling - :func:`Path.glob` with "``**/``" added in front of the *pattern*, where - *patterns* are the same as for :mod:`fnmatch`:: - - >>> sorted(Path().rglob("*.py")) - [PosixPath('build/lib/pathlib.py'), - PosixPath('docs/conf.py'), - PosixPath('pathlib.py'), - PosixPath('setup.py'), - PosixPath('test_pathlib.py')] - - By default, or when the *case_sensitive* keyword-only argument is set to - ``None``, this method matches paths using platform-specific casing rules: - typically, case-sensitive on POSIX, and case-insensitive on Windows. - Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. - - By default, or when the *follow_symlinks* keyword-only argument is set to - ``None``, this method follows symlinks except when expanding "``**``" - wildcards. Set *follow_symlinks* to ``True`` to always follow symlinks, or - ``False`` to treat all symlinks as files. - - .. audit-event:: pathlib.Path.rglob self,pattern pathlib.Path.rglob - - .. versionchanged:: 3.11 - Return only directories if *pattern* ends with a pathname components - separator (:data:`~os.sep` or :data:`~os.altsep`). - - .. versionchanged:: 3.12 - The *case_sensitive* parameter was added. - - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. - - .. versionchanged:: 3.13 - The *pattern* parameter accepts a :term:`path-like object`. - .. method:: Path.rmdir() Remove this directory. The directory must be empty. @@ -1640,6 +1611,81 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.10 The *newline* parameter was added. + +.. _pathlib-pattern-language: + +Pattern language +---------------- + +The following wildcards are supported in patterns for +:meth:`~PurePath.full_match`, :meth:`~Path.glob` and :meth:`~Path.rglob`: + +``**`` (entire segment) + Matches any number of file or directory segments, including zero. +``*`` (entire segment) + Matches one file or directory segment. +``*`` (part of a segment) + Matches any number of non-separator characters, including zero. +``?`` + Matches one non-separator character. +``[seq]`` + Matches one character in *seq*. +``[!seq]`` + Matches one character not in *seq*. + +For a literal match, wrap the meta-characters in brackets. +For example, ``"[?]"`` matches the character ``"?"``. + +The "``**``" wildcard enables recursive globbing. A few examples: + +========================= =========================================== +Pattern Meaning +========================= =========================================== +"``**/*``" Any path with at least one segment. +"``**/*.py``" Any path with a final segment ending "``.py``". +"``assets/**``" Any path starting with "``assets/``". +"``assets/**/*``" Any path starting with "``assets/``", excluding "``assets/``" itself. +========================= =========================================== + +.. note:: + Globbing with the "``**``" wildcard visits every directory in the tree. + Large directory trees may take a long time to search. + +.. versionchanged:: 3.13 + Globbing with a pattern that ends with "``**``" returns both files and + directories. In previous versions, only directories were returned. + +In :meth:`Path.glob` and :meth:`~Path.rglob`, a trailing slash may be added to +the pattern to match only directories. + +.. versionchanged:: 3.11 + Globbing with a pattern that ends with a pathname components separator + (:data:`~os.sep` or :data:`~os.altsep`) returns only directories. + + +Comparison to the :mod:`glob` module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The patterns accepted and results generated by :meth:`Path.glob` and +:meth:`Path.rglob` differ slightly from those by the :mod:`glob` module: + +1. Files beginning with a dot are not special in pathlib. This is + like passing ``include_hidden=True`` to :func:`glob.glob`. +2. "``**``" pattern components are always recursive in pathlib. This is like + passing ``recursive=True`` to :func:`glob.glob`. +3. "``**``" pattern components do not follow symlinks by default in pathlib. + This behaviour has no equivalent in :func:`glob.glob`, but you can pass + ``follow_symlinks=True`` to :meth:`Path.glob` for compatible behaviour. +4. Like all :class:`PurePath` and :class:`Path` objects, the values returned + from :meth:`Path.glob` and :meth:`Path.rglob` don't include trailing + slashes. +5. The values returned from pathlib's ``path.glob()`` and ``path.rglob()`` + include the *path* as a prefix, unlike the results of + ``glob.glob(root_dir=path)``. +6. ``bytes``-based paths and :ref:`paths relative to directory descriptors + <dir_fd>` are not supported by pathlib. + + Correspondence to tools in the :mod:`os` module ----------------------------------------------- diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index cfb251fca5c7cd9..cb517681fa81b96 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -272,13 +272,13 @@ The :mod:`pickle` module defines three exceptions: .. exception:: PickleError - Common base class for the other pickling exceptions. It inherits + Common base class for the other pickling exceptions. It inherits from :exc:`Exception`. .. exception:: PicklingError Error raised when an unpicklable object is encountered by :class:`Pickler`. - It inherits :exc:`PickleError`. + It inherits from :exc:`PickleError`. Refer to :ref:`pickle-picklable` to learn what kinds of objects can be pickled. @@ -286,7 +286,7 @@ The :mod:`pickle` module defines three exceptions: .. exception:: UnpicklingError Error raised when there is a problem unpickling an object, such as a data - corruption or a security violation. It inherits :exc:`PickleError`. + corruption or a security violation. It inherits from :exc:`PickleError`. Note that other exceptions may also be raised during unpickling, including (but not necessarily limited to) AttributeError, EOFError, ImportError, and @@ -356,7 +356,7 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, :func:`copyreg.pickle`. It is a mapping whose keys are classes and whose values are reduction functions. A reduction function takes a single argument of the associated class and should - conform to the same interface as a :meth:`__reduce__` + conform to the same interface as a :meth:`~object.__reduce__` method. By default, a pickler object will not have a @@ -376,7 +376,7 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, Special reducer that can be defined in :class:`Pickler` subclasses. This method has priority over any reducer in the :attr:`dispatch_table`. It - should conform to the same interface as a :meth:`__reduce__` method, and + should conform to the same interface as a :meth:`~object.__reduce__` method, and can optionally return ``NotImplemented`` to fallback on :attr:`dispatch_table`-registered reducers to pickle ``obj``. @@ -516,7 +516,7 @@ The following types can be pickled: * classes accessible from the top level of a module; -* instances of such classes whose the result of calling :meth:`__getstate__` +* instances of such classes whose the result of calling :meth:`~object.__getstate__` is picklable (see section :ref:`pickle-inst` for details). Attempts to pickle unpicklable objects will raise the :exc:`PicklingError` @@ -552,7 +552,7 @@ purpose, so you can fix bugs in a class or add methods to the class and still load objects that were created with an earlier version of the class. If you plan to have long-lived objects that will see many versions of a class, it may be worthwhile to put a version number in the objects so that suitable -conversions can be made by the class's :meth:`__setstate__` method. +conversions can be made by the class's :meth:`~object.__setstate__` method. .. _pickle-inst: @@ -567,7 +567,7 @@ customize, and control how class instances are pickled and unpickled. In most cases, no additional code is needed to make instances picklable. By default, pickle will retrieve the class and the attributes of an instance via -introspection. When a class instance is unpickled, its :meth:`__init__` method +introspection. When a class instance is unpickled, its :meth:`~object.__init__` method is usually *not* invoked. The default behaviour first creates an uninitialized instance and then restores the saved attributes. The following code shows an implementation of this behaviour:: @@ -653,35 +653,35 @@ methods: .. note:: - If :meth:`__getstate__` returns a false value, the :meth:`__setstate__` - method will not be called upon unpickling. + If :meth:`__reduce__` returns a state with value ``None`` at pickling, + the :meth:`__setstate__` method will not be called upon unpickling. Refer to the section :ref:`pickle-state` for more information about how to use -the methods :meth:`__getstate__` and :meth:`__setstate__`. +the methods :meth:`~object.__getstate__` and :meth:`~object.__setstate__`. .. note:: - At unpickling time, some methods like :meth:`__getattr__`, - :meth:`__getattribute__`, or :meth:`__setattr__` may be called upon the + At unpickling time, some methods like :meth:`~object.__getattr__`, + :meth:`~object.__getattribute__`, or :meth:`~object.__setattr__` may be called upon the instance. In case those methods rely on some internal invariant being - true, the type should implement :meth:`__new__` to establish such an - invariant, as :meth:`__init__` is not called when unpickling an + true, the type should implement :meth:`~object.__new__` to establish such an + invariant, as :meth:`~object.__init__` is not called when unpickling an instance. .. index:: pair: copy; protocol As we shall see, pickle does not use directly the methods described above. In fact, these methods are part of the copy protocol which implements the -:meth:`__reduce__` special method. The copy protocol provides a unified +:meth:`~object.__reduce__` special method. The copy protocol provides a unified interface for retrieving the data necessary for pickling and copying objects. [#]_ -Although powerful, implementing :meth:`__reduce__` directly in your classes is +Although powerful, implementing :meth:`~object.__reduce__` directly in your classes is error prone. For this reason, class designers should use the high-level -interface (i.e., :meth:`__getnewargs_ex__`, :meth:`__getstate__` and -:meth:`__setstate__`) whenever possible. We will show, however, cases where -using :meth:`__reduce__` is the only option or leads to more efficient pickling +interface (i.e., :meth:`~object.__getnewargs_ex__`, :meth:`~object.__getstate__` and +:meth:`~object.__setstate__`) whenever possible. We will show, however, cases where +using :meth:`!__reduce__` is the only option or leads to more efficient pickling or both. .. method:: object.__reduce__() @@ -716,8 +716,9 @@ or both. These items will be appended to the object either using ``obj.append(item)`` or, in batch, using ``obj.extend(list_of_items)``. This is primarily used for list subclasses, but may be used by other - classes as long as they have :meth:`append` and :meth:`extend` methods with - the appropriate signature. (Whether :meth:`append` or :meth:`extend` is + classes as long as they have + :ref:`append and extend methods <typesseq-common>` with + the appropriate signature. (Whether :meth:`!append` or :meth:`!extend` is used depends on which pickle protocol version is used as well as the number of items to append, so both must be supported.) @@ -793,8 +794,8 @@ any other code which depends on pickling, then one can create a pickler with a private dispatch table. The global dispatch table managed by the :mod:`copyreg` module is -available as :data:`copyreg.dispatch_table`. Therefore, one may -choose to use a modified copy of :data:`copyreg.dispatch_table` as a +available as :data:`!copyreg.dispatch_table`. Therefore, one may +choose to use a modified copy of :data:`!copyreg.dispatch_table` as a private dispatch table. For example :: @@ -833,12 +834,12 @@ Handling Stateful Objects single: __setstate__() (copy protocol) Here's an example that shows how to modify pickling behavior for a class. -The :class:`TextReader` class opens a text file, and returns the line number and +The :class:`!TextReader` class below opens a text file, and returns the line number and line contents each time its :meth:`!readline` method is called. If a -:class:`TextReader` instance is pickled, all attributes *except* the file object +:class:`!TextReader` instance is pickled, all attributes *except* the file object member are saved. When the instance is unpickled, the file is reopened, and -reading resumes from the last location. The :meth:`__setstate__` and -:meth:`__getstate__` methods are used to implement this behavior. :: +reading resumes from the last location. The :meth:`!__setstate__` and +:meth:`!__getstate__` methods are used to implement this behavior. :: class TextReader: """Print and number lines in a text file.""" @@ -903,7 +904,7 @@ functions and classes. For those cases, it is possible to subclass from the :class:`Pickler` class and implement a :meth:`~Pickler.reducer_override` method. This method can return an -arbitrary reduction tuple (see :meth:`__reduce__`). It can alternatively return +arbitrary reduction tuple (see :meth:`~object.__reduce__`). It can alternatively return ``NotImplemented`` to fallback to the traditional behavior. If both the :attr:`~Pickler.dispatch_table` and @@ -971,7 +972,7 @@ provided by pickle protocol 5 and higher. Provider API ^^^^^^^^^^^^ -The large data objects to be pickled must implement a :meth:`__reduce_ex__` +The large data objects to be pickled must implement a :meth:`~object.__reduce_ex__` method specialized for protocol 5 and higher, which returns a :class:`PickleBuffer` instance (instead of e.g. a :class:`bytes` object) for any large data. diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index e883acd67d6c72f..2a2eb098646364d 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -31,7 +31,93 @@ Dictionaries are sorted by key before the display is computed. .. versionchanged:: 3.10 Added support for pretty-printing :class:`dataclasses.dataclass`. -The :mod:`pprint` module defines one class: +.. _pprint-functions: + +Functions +--------- + +.. function:: pp(object, *args, sort_dicts=False, **kwargs) + + Prints the formatted representation of *object* followed by a newline. + If *sort_dicts* is false (the default), dictionaries will be displayed with + their keys in insertion order, otherwise the dict keys will be sorted. + *args* and *kwargs* will be passed to :func:`~pprint.pprint` as formatting + parameters. + + .. versionadded:: 3.8 + + +.. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ + compact=False, sort_dicts=True, underscore_numbers=False) + + Prints the formatted representation of *object* on *stream*, followed by a + newline. If *stream* is ``None``, :data:`sys.stdout` is used. This may be used + in the interactive interpreter instead of the :func:`print` function for + inspecting values (you can even reassign ``print = pprint.pprint`` for use + within a scope). + + The configuration parameters *stream*, *indent*, *width*, *depth*, + *compact*, *sort_dicts* and *underscore_numbers* are passed to the + :class:`PrettyPrinter` constructor and their meanings are as + described in its documentation above. + + >>> import pprint + >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] + >>> stuff.insert(0, stuff) + >>> pprint.pprint(stuff) + [<Recursion on list with id=...>, + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni'] + +.. function:: pformat(object, indent=1, width=80, depth=None, *, \ + compact=False, sort_dicts=True, underscore_numbers=False) + + Return the formatted representation of *object* as a string. *indent*, + *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* are + passed to the :class:`PrettyPrinter` constructor as formatting parameters + and their meanings are as described in its documentation above. + + +.. function:: isreadable(object) + + .. index:: pair: built-in function; eval + + Determine if the formatted representation of *object* is "readable", or can be + used to reconstruct the value using :func:`eval`. This always returns ``False`` + for recursive objects. + + >>> pprint.isreadable(stuff) + False + + +.. function:: isrecursive(object) + + Determine if *object* requires a recursive representation. This function is + subject to the same limitations as noted in :func:`saferepr` below and may raise an + :exc:`RecursionError` if it fails to detect a recursive object. + + +.. function:: saferepr(object) + + Return a string representation of *object*, protected against recursion in + some common data structures, namely instances of :class:`dict`, :class:`list` + and :class:`tuple` or subclasses whose ``__repr__`` has not been overridden. If the + representation of object exposes a recursive entry, the recursive reference + will be represented as ``<Recursion on typename with id=number>``. The + representation is not otherwise formatted. + + >>> pprint.saferepr(stuff) + "[<Recursion on list with id=...>, 'spam', 'eggs', 'lumberjack', 'knights', 'ni']" + +.. _prettyprinter-objects: + +PrettyPrinter Objects +--------------------- + +This module defines one class: .. First the implementation class: @@ -44,9 +130,9 @@ The :mod:`pprint` module defines one class: Construct a :class:`PrettyPrinter` instance. This constructor understands several keyword parameters. - *stream* (default ``sys.stdout``) is a :term:`file-like object` to + *stream* (default :data:`!sys.stdout`) is a :term:`file-like object` to which the output will be written by calling its :meth:`!write` method. - If both *stream* and ``sys.stdout`` are ``None``, then + If both *stream* and :data:`!sys.stdout` are ``None``, then :meth:`~PrettyPrinter.pprint` silently returns. Other values configure the manner in which nesting of complex data @@ -87,7 +173,7 @@ The :mod:`pprint` module defines one class: Added the *underscore_numbers* parameter. .. versionchanged:: 3.11 - No longer attempts to write to ``sys.stdout`` if it is ``None``. + No longer attempts to write to :data:`!sys.stdout` if it is ``None``. >>> import pprint >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] @@ -112,89 +198,6 @@ The :mod:`pprint` module defines one class: >>> pp.pprint(tup) ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...))))))) -.. function:: pformat(object, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) - - Return the formatted representation of *object* as a string. *indent*, - *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* are - passed to the :class:`PrettyPrinter` constructor as formatting parameters - and their meanings are as described in its documentation above. - - -.. function:: pp(object, *args, sort_dicts=False, **kwargs) - - Prints the formatted representation of *object* followed by a newline. - If *sort_dicts* is false (the default), dictionaries will be displayed with - their keys in insertion order, otherwise the dict keys will be sorted. - *args* and *kwargs* will be passed to :func:`pprint` as formatting - parameters. - - .. versionadded:: 3.8 - - -.. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) - - Prints the formatted representation of *object* on *stream*, followed by a - newline. If *stream* is ``None``, ``sys.stdout`` is used. This may be used - in the interactive interpreter instead of the :func:`print` function for - inspecting values (you can even reassign ``print = pprint.pprint`` for use - within a scope). - - The configuration parameters *stream*, *indent*, *width*, *depth*, - *compact*, *sort_dicts* and *underscore_numbers* are passed to the - :class:`PrettyPrinter` constructor and their meanings are as - described in its documentation above. - - >>> import pprint - >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> stuff.insert(0, stuff) - >>> pprint.pprint(stuff) - [<Recursion on list with id=...>, - 'spam', - 'eggs', - 'lumberjack', - 'knights', - 'ni'] - -.. function:: isreadable(object) - - .. index:: pair: built-in function; eval - - Determine if the formatted representation of *object* is "readable", or can be - used to reconstruct the value using :func:`eval`. This always returns ``False`` - for recursive objects. - - >>> pprint.isreadable(stuff) - False - - -.. function:: isrecursive(object) - - Determine if *object* requires a recursive representation. This function is - subject to the same limitations as noted in :func:`saferepr` below and may raise an - :exc:`RecursionError` if it fails to detect a recursive object. - - -One more support function is also defined: - -.. function:: saferepr(object) - - Return a string representation of *object*, protected against recursion in - some common data structures, namely instances of :class:`dict`, :class:`list` - and :class:`tuple` or subclasses whose ``__repr__`` has not been overridden. If the - representation of object exposes a recursive entry, the recursive reference - will be represented as ``<Recursion on typename with id=number>``. The - representation is not otherwise formatted. - - >>> pprint.saferepr(stuff) - "[<Recursion on list with id=...>, 'spam', 'eggs', 'lumberjack', 'knights', 'ni']" - - -.. _prettyprinter-objects: - -PrettyPrinter Objects ---------------------- :class:`PrettyPrinter` instances have the following methods: @@ -258,7 +261,7 @@ are converted to strings. The default implementation uses the internals of the Example ------- -To demonstrate several uses of the :func:`pprint` function and its parameters, +To demonstrate several uses of the :func:`~pprint.pprint` function and its parameters, let's fetch information about a project from `PyPI <https://pypi.org>`_:: >>> import json @@ -267,7 +270,7 @@ let's fetch information about a project from `PyPI <https://pypi.org>`_:: >>> with urlopen('https://pypi.org/pypi/sampleproject/json') as resp: ... project_info = json.load(resp)['info'] -In its basic form, :func:`pprint` shows the whole object:: +In its basic form, :func:`~pprint.pprint` shows the whole object:: >>> pprint.pprint(project_info) {'author': 'The Python Packaging Authority', diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index cc059b66fcb84b5..3ca802e024bc27f 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -299,6 +299,13 @@ functions: Create a :class:`~pstats.Stats` object based on the current profile and print the results to stdout. + The *sort* parameter specifies the sorting order of the displayed + statistics. It accepts a single key or a tuple of keys to enable + multi-level sorting, as in :func:`Stats.sort_stats <pstats.Stats.sort_stats>`. + + .. versionadded:: 3.13 + :meth:`~Profile.print_stats` now accepts a tuple of keys. + .. method:: dump_stats(filename) Write the results of the current profile to *filename*. diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 935e872480efda7..a6ae8fdaa4991ce 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -214,7 +214,8 @@ XMLParser Objects :meth:`CharacterDataHandler` callback whenever possible. This can improve performance substantially since Expat normally breaks character data into chunks at every line ending. This attribute is false by default, and may be changed at - any time. + any time. Note that when it is false, data that does not contain newlines + may be chunked too. .. attribute:: xmlparser.buffer_used @@ -372,7 +373,10 @@ otherwise stated. marked content, and ignorable whitespace. Applications which must distinguish these cases can use the :attr:`StartCdataSectionHandler`, :attr:`EndCdataSectionHandler`, and :attr:`ElementDeclHandler` callbacks to - collect the required information. + collect the required information. Note that the character data may be + chunked even if it is short and so you may receive more than one call to + :meth:`CharacterDataHandler`. Set the :attr:`buffer_text` instance attribute + to ``True`` to avoid that. .. method:: xmlparser.UnparsedEntityDeclHandler(entityName, base, systemId, publicId, notationName) diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index b2b787c5a8260cf..f2a6dbf589fd872 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -93,6 +93,14 @@ The :mod:`queue` module defines the following classes and exceptions: on a :class:`Queue` object which is full. +.. exception:: ShutDown + + Exception raised when :meth:`~Queue.put` or :meth:`~Queue.get` is called on + a :class:`Queue` object which has been shut down. + + .. versionadded:: 3.13 + + .. _queueobjects: Queue Objects @@ -135,6 +143,8 @@ provide the public methods described below. immediately available, else raise the :exc:`Full` exception (*timeout* is ignored in that case). + Raises :exc:`ShutDown` if the queue has been shut down. + .. method:: Queue.put_nowait(item) @@ -155,6 +165,9 @@ provide the public methods described below. an uninterruptible wait on an underlying lock. This means that no exceptions can occur, and in particular a SIGINT will not trigger a :exc:`KeyboardInterrupt`. + Raises :exc:`ShutDown` if the queue has been shut down and is empty, or if + the queue has been shut down immediately. + .. method:: Queue.get_nowait() @@ -174,6 +187,9 @@ fully processed by daemon consumer threads. processed (meaning that a :meth:`task_done` call was received for every item that had been :meth:`put` into the queue). + ``shutdown(immediate=True)`` calls :meth:`task_done` for each remaining item + in the queue. + Raises a :exc:`ValueError` if called more times than there were items placed in the queue. @@ -214,6 +230,27 @@ Example of how to wait for enqueued tasks to be completed:: print('All work completed') +Terminating queues +^^^^^^^^^^^^^^^^^^ + +:class:`Queue` objects can be made to prevent further interaction by shutting +them down. + +.. method:: Queue.shutdown(immediate=False) + + Shut down the queue, making :meth:`~Queue.get` and :meth:`~Queue.put` raise + :exc:`ShutDown`. + + By default, :meth:`~Queue.get` on a shut down queue will only raise once the + queue is empty. Set *immediate* to true to make :meth:`~Queue.get` raise + immediately instead. + + All blocked callers of :meth:`~Queue.put` will be unblocked. If *immediate* + is true, also unblock callers of :meth:`~Queue.get` and :meth:`~Queue.join`. + + .. versionadded:: 3.13 + + SimpleQueue Objects ------------------- diff --git a/Doc/library/random.rst b/Doc/library/random.rst index d0ced2416c9578d..8fbce18c56f17c9 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -301,7 +301,8 @@ be found in any statistics text. ``a <= b`` and ``b <= N <= a`` for ``b < a``. The end-point value ``b`` may or may not be included in the range - depending on floating-point rounding in the equation ``a + (b-a) * random()``. + depending on floating-point rounding in the expression + ``a + (b-a) * random()``. .. function:: triangular(low, high, mode) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 0a8c88b50cdeec5..0336121c2bc631d 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -1344,7 +1344,8 @@ when there is no match, you can test whether there was a match with a simple Escapes such as ``\n`` are converted to the appropriate characters, and numeric backreferences (``\1``, ``\2``) and named backreferences (``\g<1>``, ``\g<name>``) are replaced by the contents of the - corresponding group. + corresponding group. The backreference ``\g<0>`` will be + replaced by the entire match. .. versionchanged:: 3.5 Unmatched groups are replaced with an empty string. @@ -1597,7 +1598,7 @@ To find out what card the pair consists of, one could use the Simulating scanf() ^^^^^^^^^^^^^^^^^^ -.. index:: single: scanf() +.. index:: single: scanf (C function) Python does not currently have an equivalent to :c:func:`!scanf`. Regular expressions are generally more powerful, though also more verbose, than diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 1adafcaa02eab97..54c6d9f3b32b1aa 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -5,7 +5,7 @@ :platform: Unix :synopsis: GNU readline support for Python. -.. sectionauthor:: Skip Montanaro <skip@pobox.com> +.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com> -------------- diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 01bac5afd0b9b38..4c980dd97f9394f 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -36,7 +36,7 @@ scheduler: Example:: >>> import sched, time - >>> s = sched.scheduler(time.monotonic, time.sleep) + >>> s = sched.scheduler(time.time, time.sleep) >>> def print_time(a='default'): ... print("From print_time", time.time(), a) ... diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index d9ec2cbc47e6114..4f07b9f6040d240 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -39,7 +39,7 @@ Directory and files operations .. function:: copyfileobj(fsrc, fdst[, length]) - Copy the contents of the file-like object *fsrc* to the file-like object *fdst*. + Copy the contents of the :term:`file-like object <file object>` *fsrc* to the file-like object *fdst*. The integer *length*, if given, is the buffer size. In particular, a negative *length* value means to copy the data without looping over the source data in chunks; by default the data is read in chunks to avoid uncontrolled memory @@ -52,7 +52,7 @@ Directory and files operations Copy the contents (no metadata) of the file named *src* to a file named *dst* and return *dst* in the most efficient way possible. - *src* and *dst* are path-like objects or path names given as strings. + *src* and *dst* are :term:`path-like objects <path-like object>` or path names given as strings. *dst* must be the complete target file name; look at :func:`~shutil.copy` for a copy that accepts a target directory path. If *src* and *dst* @@ -94,7 +94,7 @@ Directory and files operations .. function:: copymode(src, dst, *, follow_symlinks=True) Copy the permission bits from *src* to *dst*. The file contents, owner, and - group are unaffected. *src* and *dst* are path-like objects or path names + group are unaffected. *src* and *dst* are :term:`path-like objects <path-like object>` or path names given as strings. If *follow_symlinks* is false, and both *src* and *dst* are symbolic links, :func:`copymode` will attempt to modify the mode of *dst* itself (rather @@ -113,7 +113,7 @@ Directory and files operations Copy the permission bits, last access time, last modification time, and flags from *src* to *dst*. On Linux, :func:`copystat` also copies the "extended attributes" where possible. The file contents, owner, and - group are unaffected. *src* and *dst* are path-like objects or path + group are unaffected. *src* and *dst* are :term:`path-like objects <path-like object>` or path names given as strings. If *follow_symlinks* is false, and *src* and *dst* both @@ -274,16 +274,16 @@ Directory and files operations .. audit-event:: shutil.copytree src,dst shutil.copytree - .. versionchanged:: 3.3 - Copy metadata when *symlinks* is false. - Now returns *dst*. - .. versionchanged:: 3.2 Added the *copy_function* argument to be able to provide a custom copy function. Added the *ignore_dangling_symlinks* argument to silence dangling symlinks errors when *symlinks* is false. + .. versionchanged:: 3.3 + Copy metadata when *symlinks* is false. + Now returns *dst*. + .. versionchanged:: 3.8 Platform-specific fast-copy syscalls may be used internally in order to copy the file more efficiently. See @@ -360,21 +360,24 @@ Directory and files operations .. function:: move(src, dst, copy_function=copy2) - Recursively move a file or directory (*src*) to another location (*dst*) - and return the destination. + Recursively move a file or directory (*src*) to another location and return + the destination. - If the destination is an existing directory, then *src* is moved inside that - directory. If the destination already exists but is not a directory, it may - be overwritten depending on :func:`os.rename` semantics. + If *dst* is an existing directory or a symlink to a directory, then *src* + is moved inside that directory. The destination path in that directory must + not already exist. + + If *dst* already exists but is not a directory, it may be overwritten + depending on :func:`os.rename` semantics. If the destination is on the current filesystem, then :func:`os.rename` is - used. Otherwise, *src* is copied to *dst* using *copy_function* and then - removed. In case of symlinks, a new symlink pointing to the target of *src* - will be created in or as *dst* and *src* will be removed. + used. Otherwise, *src* is copied to the destination using *copy_function* + and then removed. In case of symlinks, a new symlink pointing to the target + of *src* will be created as the destination and *src* will be removed. - If *copy_function* is given, it must be a callable that takes two arguments - *src* and *dst*, and will be used to copy *src* to *dst* if - :func:`os.rename` cannot be used. If the source is a directory, + If *copy_function* is given, it must be a callable that takes two arguments, + *src* and the destination, and will be used to copy *src* to the destination + if :func:`os.rename` cannot be used. If the source is a directory, :func:`copytree` is called, passing it the *copy_function*. The default *copy_function* is :func:`copy2`. Using :func:`~shutil.copy` as the *copy_function* allows the move to succeed when it is not possible to also @@ -586,7 +589,9 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Create an archive file (such as zip or tar) and return its name. *base_name* is the name of the file to create, including the path, minus - any format-specific extension. *format* is the archive format: one of + any format-specific extension. + + *format* is the archive format: one of "zip" (if the :mod:`zlib` module is available), "tar", "gztar" (if the :mod:`zlib` module is available), "bztar" (if the :mod:`bz2` module is available), or "xztar" (if the :mod:`lzma` module is available). diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 4bfb0d8c2cfeacd..3a931e25de91e5f 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -445,6 +445,11 @@ Constants Added ``IP_PKTINFO``, ``IP_UNBLOCK_SOURCE``, ``IP_BLOCK_SOURCE``, ``IP_ADD_SOURCE_MEMBERSHIP``, ``IP_DROP_SOURCE_MEMBERSHIP``. + .. versionchanged:: 3.13 + Added ``SO_BINDTOIFINDEX``. On Linux this constant can be used in the + same way that ``SO_BINDTODEVICE`` is used, but with the index of a + network interface instead of its name. + .. data:: AF_CAN PF_CAN SOL_CAN_* @@ -1605,8 +1610,9 @@ to sockets. Receive data from the socket. The return value is a bytes object representing the data received. The maximum amount of data to be received at once is specified - by *bufsize*. See the Unix manual page :manpage:`recv(2)` for the meaning of - the optional argument *flags*; it defaults to zero. + by *bufsize*. A returned empty bytes object indicates that the client has disconnected. + See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument + *flags*; it defaults to zero. .. note:: diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 5fd213fa613c8d4..864b1dadb785623 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -494,7 +494,7 @@ This is the server side:: def handle(self): # self.request is the TCP socket connected to the client self.data = self.request.recv(1024).strip() - print("{} wrote:".format(self.client_address[0])) + print("Received from {}:".format(self.client_address[0])) print(self.data) # just send back the same data, but upper-cased self.request.sendall(self.data.upper()) @@ -525,8 +525,9 @@ objects that simplify communication by providing the standard file interface):: The difference is that the ``readline()`` call in the second handler will call ``recv()`` multiple times until it encounters a newline character, while the -single ``recv()`` call in the first handler will just return what has been sent -from the client in one ``sendall()`` call. +single ``recv()`` call in the first handler will just return what has been +received so far from the client's ``sendall()`` call (typically all of it, but +this is not guaranteed by the TCP protocol). This is the client side:: diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index c3406b166c3d89d..87d5ef1e42ca3ac 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1137,12 +1137,19 @@ Connection objects .. _Loading an Extension: https://www.sqlite.org/loadext.html#loading_an_extension_ - .. method:: iterdump + .. method:: iterdump(*, filter=None) Return an :term:`iterator` to dump the database as SQL source code. Useful when saving an in-memory database for later restoration. Similar to the ``.dump`` command in the :program:`sqlite3` shell. + :param filter: + + An optional ``LIKE`` pattern for database objects to dump, e.g. ``prefix_%``. + If ``None`` (the default), all database objects will be included. + + :type filter: str | None + Example: .. testcode:: @@ -1158,6 +1165,8 @@ Connection objects :ref:`sqlite3-howto-encoding` + .. versionchanged:: 3.13 + Added the *filter* parameter. .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index f9648fa6744bdc0..fa81c3f208cff7e 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1975,7 +1975,7 @@ to speed up repeated connections from the same clients. .. versionchanged:: 3.10 - The flag had no effect with OpenSSL before version 1.1.1k. Python 3.8.9, + The flag had no effect with OpenSSL before version 1.1.1l. Python 3.8.9, 3.9.3, and 3.10 include workarounds for previous versions. .. attribute:: SSLContext.security_level diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 0417b3f38a98079..1785c6bcc212b74 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -76,6 +76,7 @@ or sample. :func:`fmean` Fast, floating point arithmetic mean, with optional weighting. :func:`geometric_mean` Geometric mean of data. :func:`harmonic_mean` Harmonic mean of data. +:func:`kde` Estimate the probability density distribution of the data. :func:`median` Median (middle value) of data. :func:`median_low` Low median of data. :func:`median_high` High median of data. @@ -259,6 +260,54 @@ However, for reading convenience, most of the examples show sorted sequences. .. versionchanged:: 3.10 Added support for *weights*. + +.. function:: kde(data, h, kernel='normal') + + `Kernel Density Estimation (KDE) + <https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf>`_: + Create a continuous probability density function from discrete samples. + + The basic idea is to smooth the data using `a kernel function + <https://en.wikipedia.org/wiki/Kernel_(statistics)>`_. + to help draw inferences about a population from a sample. + + The degree of smoothing is controlled by the scaling parameter *h* + which is called the bandwidth. Smaller values emphasize local + features while larger values give smoother results. + + The *kernel* determines the relative weights of the sample data + points. Generally, the choice of kernel shape does not matter + as much as the more influential bandwidth smoothing parameter. + + Kernels that give some weight to every sample point include + *normal* or *gauss*, *logistic*, and *sigmoid*. + + Kernels that only give weight to sample points within the bandwidth + include *rectangular* or *uniform*, *triangular*, *parabolic* or + *epanechnikov*, *quartic* or *biweight*, *triweight*, and *cosine*. + + A :exc:`StatisticsError` will be raised if the *data* sequence is empty. + + `Wikipedia has an example + <https://en.wikipedia.org/wiki/Kernel_density_estimation#Example>`_ + where we can use :func:`kde` to generate and plot a probability + density function estimated from a small sample: + + .. doctest:: + + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde(sample, h=1.5) + >>> xarr = [i/100 for i in range(-750, 1100)] + >>> yarr = [f_hat(x) for x in xarr] + + The points in ``xarr`` and ``yarr`` can be used to make a PDF plot: + + .. image:: kde_example.png + :alt: Scatter plot of the estimated probability density function. + + .. versionadded:: 3.13 + + .. function:: median(data) Return the median (middle value) of numeric data, using the common "mean of @@ -1095,46 +1144,6 @@ The final prediction goes to the largest posterior. This is known as the 'female' -Kernel density estimation -************************* - -It is possible to estimate a continuous probability density function -from a fixed number of discrete samples. - -The basic idea is to smooth the data using `a kernel function such as a -normal distribution, triangular distribution, or uniform distribution -<https://en.wikipedia.org/wiki/Kernel_(statistics)#Kernel_functions_in_common_use>`_. -The degree of smoothing is controlled by a scaling parameter, ``h``, -which is called the *bandwidth*. - -.. testcode:: - - def kde_normal(sample, h): - "Create a continuous probability density function from a sample." - # Smooth the sample with a normal distribution kernel scaled by h. - kernel_h = NormalDist(0.0, h).pdf - n = len(sample) - def pdf(x): - return sum(kernel_h(x - x_i) for x_i in sample) / n - return pdf - -`Wikipedia has an example -<https://en.wikipedia.org/wiki/Kernel_density_estimation#Example>`_ -where we can use the ``kde_normal()`` recipe to generate and plot -a probability density function estimated from a small sample: - -.. doctest:: - - >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] - >>> f_hat = kde_normal(sample, h=1.5) - >>> xarr = [i/100 for i in range(-750, 1100)] - >>> yarr = [f_hat(x) for x in xarr] - -The points in ``xarr`` and ``yarr`` can be used to make a PDF plot: - -.. image:: kde_example.png - :alt: Scatter plot of the estimated probability density function. - .. # This modelines must appear within the last ten lines of the file. kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 9028ff5c134fa92..1a4c12590c1018d 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1528,7 +1528,7 @@ between them will be implicitly converted to a single string literal. That is, ``("spam " "eggs") == "spam eggs"``. See :ref:`strings` for more about the various forms of string literal, -including supported escape sequences, and the ``r`` ("raw") prefix that +including supported :ref:`escape sequences <escape-sequences>`, and the ``r`` ("raw") prefix that disables most escape sequence processing. Strings may also be created from other objects using the :class:`str` diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index e2e6fc542e3e671..3e507c1c7e7c85c 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -160,6 +160,21 @@ following table: If the first character is not one of these, ``'@'`` is assumed. +.. note:: + + The number 1023 (``0x3ff`` in hexadecimal) has the following byte representations: + + * ``03 ff`` in big-endian (``>``) + * ``ff 03`` in little-endian (``<``) + + Python example: + + >>> import struct + >>> struct.pack('>h', 1023) + b'\x03\xff' + >>> struct.pack('<h', 1023) + b'\xff\x03' + Native byte order is big-endian or little-endian, depending on the host system. For example, Intel x86, AMD64 (x86-64), and Apple M1 are little-endian; IBM z and many legacy architectures are big-endian. diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index c437ce770b37d0d..1dcfea58a8e89f6 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -664,7 +664,8 @@ functions. If given, *startupinfo* will be a :class:`STARTUPINFO` object, which is passed to the underlying ``CreateProcess`` function. - *creationflags*, if given, can be one or more of the following flags: + + If given, *creationflags*, can be one or more of the following flags: * :data:`CREATE_NEW_CONSOLE` * :data:`CREATE_NEW_PROCESS_GROUP` @@ -856,8 +857,8 @@ Instances of the :class:`Popen` class have the following methods: .. method:: Popen.terminate() - Stop the child. On POSIX OSs the method sends SIGTERM to the - child. On Windows the Win32 API function :c:func:`TerminateProcess` is called + Stop the child. On POSIX OSs the method sends :py:const:`~signal.SIGTERM` to the + child. On Windows the Win32 API function :c:func:`!TerminateProcess` is called to stop the child. diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 46159dcef940e7c..47568387f9a7ce0 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -97,7 +97,7 @@ Examining Symbol Tables .. class:: Function - A namespace for a function or method. This class inherits + A namespace for a function or method. This class inherits from :class:`SymbolTable`. .. method:: get_parameters() @@ -123,7 +123,7 @@ Examining Symbol Tables .. class:: Class - A namespace of a class. This class inherits :class:`SymbolTable`. + A namespace of a class. This class inherits from :class:`SymbolTable`. .. method:: get_methods() diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index abf2c393a44928d..380ba1090b39b3b 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -16,12 +16,12 @@ always available. On POSIX systems where Python was built with the standard ``configure`` script, this contains the ABI flags as specified by :pep:`3149`. + .. versionadded:: 3.2 + .. versionchanged:: 3.8 Default flags became an empty string (``m`` flag for pymalloc has been removed). - .. versionadded:: 3.2 - .. availability:: Unix. @@ -195,6 +195,17 @@ always available. This function should be used for internal and specialized purposes only. + .. deprecated:: 3.13 + Use the more general :func:`_clear_internal_caches` function instead. + + +.. function:: _clear_internal_caches() + + Clear all internal performance-related caches. Use this function *only* to + release unnecessary references and memory blocks when hunting for leaks. + + .. versionadded:: 3.13 + .. function:: _current_frames() @@ -724,7 +735,7 @@ always available. regardless of their size. This function is mainly useful for tracking and debugging memory leaks. Because of the interpreter's internal caches, the result can vary from call to call; you may have to call - :func:`_clear_type_cache()` and :func:`gc.collect()` to get more + :func:`_clear_internal_caches()` and :func:`gc.collect()` to get more predictable results. If a Python build or implementation cannot reasonably compute this @@ -1191,6 +1202,8 @@ always available. Return :const:`True` if the main Python interpreter is :term:`shutting down <interpreter shutdown>`. Return :const:`False` otherwise. + See also the :exc:`PythonFinalizationError` exception. + .. versionadded:: 3.5 .. data:: last_exc @@ -1293,7 +1306,10 @@ always available. The list of the original command line arguments passed to the Python executable. - See also :data:`sys.argv`. + The elements of :data:`sys.orig_argv` are the arguments to the Python interpreter, + while the elements of :data:`sys.argv` are the arguments to the user's program. + Arguments consumed by the interpreter itself will be present in :data:`sys.orig_argv` + and missing from :data:`sys.argv`. .. versionadded:: 3.10 diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 34a738a7f1c41f0..2134293a0bb0de1 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -673,6 +673,7 @@ be finalized; only the internally used file object will be closed. See the .. attribute:: TarFile.pax_headers + :type: dict A dictionary containing key-value pairs of pax global headers. @@ -838,26 +839,31 @@ A ``TarInfo`` object has the following public data attributes: attribute. .. attribute:: TarInfo.chksum + :type: int Header checksum. .. attribute:: TarInfo.devmajor + :type: int Device major number. .. attribute:: TarInfo.devminor + :type: int Device minor number. .. attribute:: TarInfo.offset + :type: int The tar header starts here. .. attribute:: TarInfo.offset_data + :type: int The file's data starts here. diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 9173db07fd00718..7d28f6253457263 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -143,7 +143,7 @@ guidelines to be followed: arg = (1, 2, 3) When using this pattern, remember that all classes that inherit from - :class:`unittest.TestCase` are run as tests. The :class:`Mixin` class in the example above + :class:`unittest.TestCase` are run as tests. The :class:`!TestFuncAcceptsSequencesMixin` class in the example above does not have any data and so can't be run by itself, thus it does not inherit from :class:`unittest.TestCase`. @@ -324,9 +324,9 @@ The :mod:`test.support` module defines the following constants: .. data:: Py_DEBUG - True if Python is built with the :c:macro:`Py_DEBUG` macro defined: if - Python is :ref:`built in debug mode <debug-build>` - (:option:`./configure --with-pydebug <--with-pydebug>`). + True if Python was built with the :c:macro:`Py_DEBUG` macro + defined, that is, if + Python was :ref:`built in debug mode <debug-build>`. .. versionadded:: 3.12 diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 5fbf9379b8202c0..cb511471579b694 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -987,18 +987,15 @@ method. The :meth:`~Event.wait` method blocks until the flag is true. .. method:: wait(timeout=None) - Block until the internal flag is true. If the internal flag is true on - entry, return immediately. Otherwise, block until another thread calls - :meth:`.set` to set the flag to true, or until the optional timeout occurs. + Block as long as the internal flag is false and the timeout, if given, + has not expired. The return value represents the + reason that this blocking method returned; ``True`` if returning because + the internal flag is set to true, or ``False`` if a timeout is given and + the the internal flag did not become true within the given wait time. When the timeout argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds - (or fractions thereof). - - This method returns ``True`` if and only if the internal flag has been set to - true, either before the wait call or after the wait starts, so it will - always return ``True`` except if a timeout is given and the operation - times out. + floating point number specifying a timeout for the operation in seconds, + or fractions thereof. .. versionchanged:: 3.1 Previously, the method always returned ``None``. diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 2782a961363666b..88c15f19bedf8c8 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -433,6 +433,10 @@ Functions | ``%d`` | Day of the month as a decimal number [01,31]. | | | | | | +-----------+------------------------------------------------+-------+ + | ``%f`` | Microseconds as a decimal number | \(1) | + | | [000000,999999]. | | + | | | | + +-----------+------------------------------------------------+-------+ | ``%H`` | Hour (24-hour clock) as a decimal number | | | | [00,23]. | | +-----------+------------------------------------------------+-------+ @@ -448,13 +452,13 @@ Functions | ``%M`` | Minute as a decimal number [00,59]. | | | | | | +-----------+------------------------------------------------+-------+ - | ``%p`` | Locale's equivalent of either AM or PM. | \(1) | + | ``%p`` | Locale's equivalent of either AM or PM. | \(2) | | | | | +-----------+------------------------------------------------+-------+ - | ``%S`` | Second as a decimal number [00,61]. | \(2) | + | ``%S`` | Second as a decimal number [00,61]. | \(3) | | | | | +-----------+------------------------------------------------+-------+ - | ``%U`` | Week number of the year (Sunday as the first | \(3) | + | ``%U`` | Week number of the year (Sunday as the first | \(4) | | | day of the week) as a decimal number [00,53]. | | | | All days in a new year preceding the first | | | | Sunday are considered to be in week 0. | | @@ -465,7 +469,7 @@ Functions | ``%w`` | Weekday as a decimal number [0(Sunday),6]. | | | | | | +-----------+------------------------------------------------+-------+ - | ``%W`` | Week number of the year (Monday as the first | \(3) | + | ``%W`` | Week number of the year (Monday as the first | \(4) | | | day of the week) as a decimal number [00,53]. | | | | All days in a new year preceding the first | | | | Monday are considered to be in week 0. | | @@ -500,17 +504,23 @@ Functions Notes: (1) + The ``%f`` format directive only applies to :func:`strptime`, + not to :func:`strftime`. However, see also :meth:`datetime.datetime.strptime` and + :meth:`datetime.datetime.strftime` where the ``%f`` format directive + :ref:`applies to microseconds <format-codes>`. + + (2) When used with the :func:`strptime` function, the ``%p`` directive only affects the output hour field if the ``%I`` directive is used to parse the hour. .. _leap-second: - (2) + (3) The range really is ``0`` to ``61``; value ``60`` is valid in timestamps representing `leap seconds`_ and value ``61`` is supported for historical reasons. - (3) + (4) When used with the :func:`strptime` function, ``%U`` and ``%W`` are only used in calculations when the day of the week and the year are specified. diff --git a/Doc/library/types.rst b/Doc/library/types.rst index c8c981024c1aeb1..b856544e44207c5 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -188,7 +188,7 @@ Standard names are defined for the following types: .. index:: pair: built-in function; compile - The type for code objects such as returned by :func:`compile`. + The type of :ref:`code objects <code-objects>` such as returned by :func:`compile`. .. audit-event:: code.__new__ code,filename,name,argcount,posonlyargcount,kwonlyargcount,nlocals,stacksize,flags types.CodeType @@ -196,14 +196,6 @@ Standard names are defined for the following types: required by the initializer. The audit event only occurs for direct instantiation of code objects, and is not raised for normal compilation. - .. method:: CodeType.replace(**kwargs) - - Return a copy of the code object with new values for the specified fields. - - Code objects are also supported by generic function :func:`copy.replace`. - - .. versionadded:: 3.8 - .. data:: CellType The type for cell objects: such objects are used as containers for diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index eca20b94ec8e740..b0a5d96c38d375b 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -2426,6 +2426,14 @@ passed in. >>> m.mock_calls == [call(1), call(1, 2), ANY] True +:data:`ANY` is not limited to comparisons with call objects and so +can also be used in test assertions:: + + class TestStringMethods(unittest.TestCase): + + def test_split(self): + s = 'hello world' + self.assertEqual(s.split(), ['hello', ANY]) FILTER_DIR diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index f063e463753e0b8..b5a49d9c5923876 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -5,7 +5,7 @@ :synopsis: Load a robots.txt file and answer questions about fetchability of other URLs. -.. sectionauthor:: Skip Montanaro <skip@pobox.com> +.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com> **Source code:** :source:`Lib/urllib/robotparser.py` diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index aa18873f223a6bc..2e7ff345a062342 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -286,15 +286,15 @@ creation according to their needs, the :class:`EnvBuilder` class. the virtual environment. - .. versionchanged:: 3.12 - The attribute ``lib_path`` was added to the context, and the context - object was documented. - .. versionchanged:: 3.11 The *venv* :ref:`sysconfig installation scheme <installation_paths>` is used to construct the paths of the created directories. + .. versionchanged:: 3.12 + The attribute ``lib_path`` was added to the context, and the context + object was documented. + .. method:: create_configuration(context) Creates the ``pyvenv.cfg`` configuration file in the environment. diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index a9c469707e82273..500398636e11ae8 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -396,7 +396,7 @@ Available Functions ------------------- -.. function:: warn(message, category=None, stacklevel=1, source=None, \*, skip_file_prefixes=None) +.. function:: warn(message, category=None, stacklevel=1, source=None, *, skip_file_prefixes=None) Issue a warning, or maybe ignore it or raise an exception. The *category* argument, if given, must be a :ref:`warning category class <warning-categories>`; it diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index fe92400fb08dfdc..75a7915c15240d2 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -625,6 +625,8 @@ Functions target. Returns an :term:`iterator` providing ``(event, elem)`` pairs; it has a ``root`` attribute that references the root element of the resulting XML tree once *source* is fully read. + The iterator has the :meth:`!close` method that closes the internal + file object if *source* is a filename. Note that while :func:`iterparse` builds the tree incrementally, it issues blocking reads on *source* (or the file it names). As such, it's unsuitable @@ -647,6 +649,9 @@ Functions .. versionchanged:: 3.8 The ``comment`` and ``pi`` events were added. + .. versionchanged:: 3.13 + Added the :meth:`!close` method. + .. function:: parse(source, parser=None) @@ -664,7 +669,7 @@ Functions given. Returns an element instance, representing a processing instruction. Note that :class:`XMLParser` skips over processing instructions - in the input instead of creating comment objects for them. An + in the input instead of creating PI objects for them. An :class:`ElementTree` will only contain processing instruction nodes if they have been inserted into to the tree using one of the :class:`Element` methods. @@ -1302,8 +1307,8 @@ TreeBuilder Objects .. method:: pi(target, text) - Creates a comment with the given *target* name and *text*. If - ``insert_pis`` is true, this will also add it to the tree. + Creates a process instruction with the given *target* name and *text*. + If ``insert_pis`` is true, this will also add it to the tree. .. versionadded:: 3.8 diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index 909022ea4ba6a4b..662cc459197e2ca 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -68,6 +68,7 @@ quadratic blowup **Vulnerable** (1) **Vulnerable** (1) **Vulnerable* external entity expansion Safe (5) Safe (2) Safe (3) Safe (5) Safe (4) `DTD`_ retrieval Safe (5) Safe Safe Safe (5) Safe decompression bomb Safe Safe Safe Safe **Vulnerable** +large tokens **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) ========================= ================== ================== ================== ================== ================== 1. Expat 2.4.1 and newer is not vulnerable to the "billion laughs" and @@ -81,6 +82,11 @@ decompression bomb Safe Safe Safe 4. :mod:`xmlrpc.client` doesn't expand external entities and omits them. 5. Since Python 3.7.1, external general entities are no longer processed by default. +6. Expat 2.6.0 and newer is not vulnerable to denial of service + through quadratic runtime caused by parsing large tokens. + Items still listed as vulnerable due to + potential reliance on system-provided libraries. Check + :const:`!pyexpat.EXPAT_VERSION`. billion laughs / exponential entity expansion @@ -114,6 +120,13 @@ decompression bomb files. For an attacker it can reduce the amount of transmitted data by three magnitudes or more. +large tokens + Expat needs to re-parse unfinished tokens; without the protection + introduced in Expat 2.6.0, this can lead to quadratic runtime that can + be used to cause denial of service in the application parsing XML. + The issue is known as + `CVE-2023-52425 <https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-52425>`_. + The documentation for `defusedxml`_ on PyPI has further information about all known attack vectors with examples and references. diff --git a/Doc/license.rst b/Doc/license.rst index 9fc0ff7161a5914..cbe918bd1acfe30 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -1095,3 +1095,35 @@ which is distributed under the MIT license:: LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +Global Unbounded Sequences (GUS) +-------------------------------- + +The file :file:`Python/qsbr.c` is adapted from FreeBSD's "Global Unbounded +Sequences" safe memory reclamation scheme in +`subr_smr.c <https://github.com/freebsd/freebsd-src/blob/main/sys/kern/subr_smr.c>`_. +The file is distributed under the 2-Clause BSD License:: + + Copyright (c) 2019,2020 Jeffrey Roberson <jeff@FreeBSD.org> + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice unmodified, this list of conditions, and the following + disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 0a1c1d58558e94f..6438abbba10e378 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -34,7 +34,7 @@ represented by objects.) Every object has an identity, a type and a value. An object's *identity* never changes once it has been created; you may think of it as the object's address in -memory. The ':keyword:`is`' operator compares the identity of two objects; the +memory. The :keyword:`is` operator compares the identity of two objects; the :func:`id` function returns an integer representing its identity. .. impl-detail:: @@ -81,7 +81,7 @@ are still reachable. Note that the use of the implementation's tracing or debugging facilities may keep objects alive that would normally be collectable. Also note that catching -an exception with a ':keyword:`try`...\ :keyword:`except`' statement may keep +an exception with a :keyword:`try`...\ :keyword:`except` statement may keep objects alive. Some objects contain references to "external" resources such as open files or @@ -89,8 +89,8 @@ windows. It is understood that these resources are freed when the object is garbage-collected, but since garbage collection is not guaranteed to happen, such objects also provide an explicit way to release the external resource, usually a :meth:`!close` method. Programs are strongly recommended to explicitly -close such objects. The ':keyword:`try`...\ :keyword:`finally`' statement -and the ':keyword:`with`' statement provide convenient ways to do this. +close such objects. The :keyword:`try`...\ :keyword:`finally` statement +and the :keyword:`with` statement provide convenient ways to do this. .. index:: single: container @@ -1134,6 +1134,8 @@ Special read-only attributes * - .. attribute:: codeobject.co_qualname - The fully qualified function name + .. versionadded:: 3.11 + * - .. attribute:: codeobject.co_argcount - The total number of positional :term:`parameters <parameter>` (including positional-only parameters and parameters with default values) @@ -1290,6 +1292,14 @@ Methods on code objects :pep:`626` - Precise line numbers for debugging and other tools. The PEP that introduced the :meth:`!co_lines` method. +.. method:: codeobject.replace(**kwargs) + + Return a copy of the code object with new values for the specified fields. + + Code objects are also supported by the generic function :func:`copy.replace`. + + .. versionadded:: 3.8 + .. _frame-objects: @@ -1988,8 +1998,8 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances. .. method:: object.__dir__(self) - Called when :func:`dir` is called on the object. A sequence must be - returned. :func:`dir` converts the returned sequence to a list and sorts it. + Called when :func:`dir` is called on the object. An iterable must be + returned. :func:`dir` converts the returned iterable to a list and sorts it. Customizing module attribute access @@ -2009,7 +2019,7 @@ not found on a module object through the normal lookup, i.e. the module ``__dict__`` before raising an :exc:`AttributeError`. If found, it is called with the attribute name and the result is returned. -The ``__dir__`` function should accept no arguments, and return a sequence of +The ``__dir__`` function should accept no arguments, and return an iterable of strings that represents the names accessible on module. If present, this function overrides the standard :func:`dir` search on a module. diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 87ebdc1ca1c9c6a..50e0f97a6534af2 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1890,8 +1890,9 @@ the unpacking. .. index:: pair: trailing; comma -The trailing comma is required only to create a single tuple (a.k.a. a -*singleton*); it is optional in all other cases. A single expression without a +A trailing comma is required only to create a one-item tuple, +such as ``1,``; it is optional in all other cases. +A single expression without a trailing comma doesn't create a tuple, but rather yields the value of that expression. (To create an empty tuple, use an empty pair of parentheses: ``()``.) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 8b6847ef2a7d76a..5fbc24c6ee65a51 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -3,31 +3,23 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. Doc/c-api/descriptor.rst -Doc/c-api/exceptions.rst Doc/c-api/float.rst -Doc/c-api/gcsupport.rst Doc/c-api/init.rst Doc/c-api/init_config.rst Doc/c-api/intro.rst -Doc/c-api/memoryview.rst Doc/c-api/module.rst Doc/c-api/stable.rst -Doc/c-api/sys.rst Doc/c-api/type.rst Doc/c-api/typeobj.rst Doc/extending/extending.rst Doc/glossary.rst Doc/howto/descriptor.rst Doc/howto/enum.rst -Doc/howto/logging.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst Doc/library/asyncio-policy.rst Doc/library/asyncio-subprocess.rst -Doc/library/bdb.rst Doc/library/collections.rst -Doc/library/csv.rst -Doc/library/datetime.rst Doc/library/dbm.rst Doc/library/decimal.rst Doc/library/email.charset.rst @@ -35,10 +27,8 @@ Doc/library/email.compat32-message.rst Doc/library/email.errors.rst Doc/library/email.parser.rst Doc/library/email.policy.rst -Doc/library/enum.rst Doc/library/exceptions.rst Doc/library/faulthandler.rst -Doc/library/fcntl.rst Doc/library/functools.rst Doc/library/http.cookiejar.rst Doc/library/http.server.rst @@ -50,7 +40,6 @@ Doc/library/mmap.rst Doc/library/multiprocessing.rst Doc/library/optparse.rst Doc/library/os.rst -Doc/library/pickle.rst Doc/library/pickletools.rst Doc/library/platform.rst Doc/library/plistlib.rst @@ -89,16 +78,11 @@ Doc/reference/compound_stmts.rst Doc/reference/datamodel.rst Doc/tutorial/datastructures.rst Doc/using/windows.rst -Doc/whatsnew/2.0.rst -Doc/whatsnew/2.1.rst -Doc/whatsnew/2.2.rst Doc/whatsnew/2.4.rst Doc/whatsnew/2.5.rst Doc/whatsnew/2.6.rst Doc/whatsnew/2.7.rst Doc/whatsnew/3.0.rst -Doc/whatsnew/3.1.rst -Doc/whatsnew/3.2.rst Doc/whatsnew/3.3.rst Doc/whatsnew/3.4.rst Doc/whatsnew/3.5.rst diff --git a/Doc/tools/static/changelog_search.js b/Doc/tools/static/changelog_search.js index c881a9bd4c84a76..0a77c0d71ae9375 100644 --- a/Doc/tools/static/changelog_search.js +++ b/Doc/tools/static/changelog_search.js @@ -1,53 +1,59 @@ -$(document).ready(function() { - // add the search form and bind the events - $('h1').after([ - '<p>Filter entries by content:', - '<input type="text" value="" id="searchbox" style="width: 50%">', - '<input type="submit" id="searchbox-submit" value="Filter"></p>' - ].join('\n')); +document.addEventListener("DOMContentLoaded", function () { + // add the search form and bind the events + document + .querySelector("h1") + .insertAdjacentHTML( + "afterend", + [ + "<p>Filter entries by content:", + '<input type="text" value="" id="searchbox" style="width: 50%">', + '<input type="submit" id="searchbox-submit" value="Filter"></p>', + ].join("\n"), + ); - function dofilter() { - try { - var query = new RegExp($('#searchbox').val(), 'i'); + function doFilter() { + let query; + try { + query = new RegExp(document.querySelector("#searchbox").value, "i"); + } catch (e) { + return; // not a valid regex (yet) + } + // find headers for the versions (What's new in Python X.Y.Z?) + const h2s = document.querySelectorAll("#changelog h2"); + for (const h2 of h2s) { + let sections_found = 0; + // find headers for the sections (Core, Library, etc.) + const h3s = h2.parentNode.querySelectorAll("h3"); + for (const h3 of h3s) { + let entries_found = 0; + // find all the entries + const lis = h3.parentNode.querySelectorAll("li"); + for (let li of lis) { + // check if the query matches the entry + if (query.test(li.textContent)) { + li.style.display = "block"; + entries_found++; + } else { + li.style.display = "none"; + } } - catch (e) { - return; // not a valid regex (yet) + // if there are entries, show the section, otherwise hide it + if (entries_found > 0) { + h3.parentNode.style.display = "block"; + sections_found++; + } else { + h3.parentNode.style.display = "none"; } - // find headers for the versions (What's new in Python X.Y.Z?) - $('#changelog h2').each(function(index1, h2) { - var h2_parent = $(h2).parent(); - var sections_found = 0; - // find headers for the sections (Core, Library, etc.) - h2_parent.find('h3').each(function(index2, h3) { - var h3_parent = $(h3).parent(); - var entries_found = 0; - // find all the entries - h3_parent.find('li').each(function(index3, li) { - var li = $(li); - // check if the query matches the entry - if (query.test(li.text())) { - li.show(); - entries_found++; - } - else { - li.hide(); - } - }); - // if there are entries, show the section, otherwise hide it - if (entries_found > 0) { - h3_parent.show(); - sections_found++; - } - else { - h3_parent.hide(); - } - }); - if (sections_found > 0) - h2_parent.show(); - else - h2_parent.hide(); - }); + } + if (sections_found > 0) { + h2.parentNode.style.display = "block"; + } else { + h2.parentNode.style.display = "none"; + } } - $('#searchbox').keyup(dofilter); - $('#searchbox-submit').click(dofilter); + } + document.querySelector("#searchbox").addEventListener("keyup", doFilter); + document + .querySelector("#searchbox-submit") + .addEventListener("click", doFilter); }); diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 87614d082a1d4e0..de2827461e2f241 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -48,10 +48,9 @@ objects: :noindex: Remove the item at the given position in the list, and return it. If no index - is specified, ``a.pop()`` removes and returns the last item in the list. (The - square brackets around the *i* in the method signature denote that the parameter - is optional, not that you should type square brackets at that position. You - will see this notation frequently in the Python Library Reference.) + is specified, ``a.pop()`` removes and returns the last item in the list. + It raises an :exc:`IndexError` if the list is empty or the index is + outside the list range. .. method:: list.clear() diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index 42ebf2b3d294a8b..299b6c2777adc0e 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -10,7 +10,7 @@ Using the Python Interpreter Invoking the Interpreter ======================== -The Python interpreter is usually installed as :file:`/usr/local/bin/python3.13` +The Python interpreter is usually installed as |usr_local_bin_python_x_dot_y_literal| on those machines where it is available; putting :file:`/usr/local/bin` in your Unix shell's search path makes it possible to start it by typing the command: @@ -24,7 +24,7 @@ Python guru or system administrator. (E.g., :file:`/usr/local/python` is a popular alternative location.) On Windows machines where you have installed Python from the :ref:`Microsoft Store -<windows-store>`, the :file:`python3.13` command will be available. If you have +<windows-store>`, the |python_x_dot_y_literal| command will be available. If you have the :ref:`py.exe launcher <launcher>` installed, you can use the :file:`py` command. See :ref:`setting-envvars` for other ways to launch Python. diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 63f4b5e1ce02073..9def2a5714950bc 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -15,7 +15,7 @@ operating system:: >>> import os >>> os.getcwd() # Return the current working directory - 'C:\\Python312' + 'C:\\Python313' >>> os.chdir('/server/accesslogs') # Change current working directory >>> os.system('mkdir today') # Run the command mkdir in the system shell 0 diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 33f311db3a24d23..09b6f3d91bcfed7 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -279,7 +279,7 @@ applications include caching objects that are expensive to create:: Traceback (most recent call last): File "<stdin>", line 1, in <module> d['primary'] # entry was automatically removed - File "C:/python312/lib/weakref.py", line 46, in __getitem__ + File "C:/python313/lib/weakref.py", line 46, in __getitem__ o = self.data[key]() KeyError: 'primary' diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index eb1413af2cbc3df..e99993238895f91 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -10,41 +10,46 @@ Using Python on a Mac Python on a Mac running macOS is in principle very similar to Python on any other Unix platform, but there are a number of additional features such as -the IDE and the Package Manager that are worth pointing out. +the integrated development environment (IDE) and the Package Manager that are +worth pointing out. + .. _getting-osx: +.. _getting-and-installing-macpython: -Getting and Installing MacPython -================================ +Getting and Installing Python +============================= macOS used to come with Python 2.7 pre-installed between versions 10.8 and `12.3 <https://developer.apple.com/documentation/macos-release-notes/macos-12_3-release-notes#Python>`_. -You are invited to install the most recent version of Python 3 from the Python -website (https://www.python.org). A current "universal binary" build of Python, -which runs natively on the Mac's new Intel and legacy PPC CPU's, is available -there. +You are invited to install the most recent version of Python 3 from the `Python +website <https://www.python.org/downloads/macos/>`__. +A current "universal2 binary" build of Python, which runs natively on the Mac's +new Apple Silicon and legacy Intel processors, is available there. What you get after installing is a number of things: -* A :file:`Python 3.12` folder in your :file:`Applications` folder. In here +* A |python_version_literal| folder in your :file:`Applications` folder. In here you find IDLE, the development environment that is a standard part of official - Python distributions; and PythonLauncher, which handles double-clicking Python + Python distributions; and :program:`Python Launcher`, which handles double-clicking Python scripts from the Finder. * A framework :file:`/Library/Frameworks/Python.framework`, which includes the Python executable and libraries. The installer adds this location to your shell - path. To uninstall MacPython, you can simply remove these three things. A - symlink to the Python executable is placed in /usr/local/bin/. - -The Apple-provided build of Python is installed in -:file:`/System/Library/Frameworks/Python.framework` and :file:`/usr/bin/python`, -respectively. You should never modify or delete these, as they are -Apple-controlled and are used by Apple- or third-party software. Remember that -if you choose to install a newer Python version from python.org, you will have -two different but functional Python installations on your computer, so it will -be important that your paths and usages are consistent with what you want to do. - -IDLE includes a help menu that allows you to access Python documentation. If you + path. To uninstall Python, you can remove these three things. A + symlink to the Python executable is placed in :file:`/usr/local/bin/`. + +.. note:: + + On macOS 10.8-12.3, the Apple-provided build of Python is installed in + :file:`/System/Library/Frameworks/Python.framework` and :file:`/usr/bin/python`, + respectively. You should never modify or delete these, as they are + Apple-controlled and are used by Apple- or third-party software. Remember that + if you choose to install a newer Python version from python.org, you will have + two different but functional Python installations on your computer, so it will + be important that your paths and usages are consistent with what you want to do. + +IDLE includes a Help menu that allows you to access Python documentation. If you are completely new to Python you should start reading the tutorial introduction in that document. @@ -56,29 +61,29 @@ How to run a Python script -------------------------- Your best way to get started with Python on macOS is through the IDLE -integrated development environment, see section :ref:`ide` and use the Help menu +integrated development environment; see section :ref:`ide` and use the Help menu when the IDE is running. If you want to run Python scripts from the Terminal window command line or from the Finder you first need an editor to create your script. macOS comes with a -number of standard Unix command line editors, :program:`vim` and -:program:`emacs` among them. If you want a more Mac-like editor, -:program:`BBEdit` or :program:`TextWrangler` from Bare Bones Software (see -http://www.barebones.com/products/bbedit/index.html) are good choices, as is -:program:`TextMate` (see https://macromates.com/). Other editors include -:program:`Gvim` (https://macvim.org/macvim/) and :program:`Aquamacs` -(http://aquamacs.org/). +number of standard Unix command line editors, :program:`vim` +:program:`nano` among them. If you want a more Mac-like editor, +:program:`BBEdit` from Bare Bones Software (see +https://www.barebones.com/products/bbedit/index.html) are good choices, as is +:program:`TextMate` (see https://macromates.com). Other editors include +:program:`MacVim` (https://macvim.org) and :program:`Aquamacs` +(https://aquamacs.org). To run your script from the Terminal window you must make sure that :file:`/usr/local/bin` is in your shell search path. To run your script from the Finder you have two options: -* Drag it to :program:`PythonLauncher` +* Drag it to :program:`Python Launcher`. -* Select :program:`PythonLauncher` as the default application to open your - script (or any .py script) through the finder Info window and double-click it. - :program:`PythonLauncher` has various preferences to control how your script is +* Select :program:`Python Launcher` as the default application to open your + script (or any ``.py`` script) through the finder Info window and double-click it. + :program:`Python Launcher` has various preferences to control how your script is launched. Option-dragging allows you to change these for one invocation, or use its Preferences menu to change things globally. @@ -103,10 +108,11 @@ Python on macOS honors all standard Unix environment variables such as :envvar:`PYTHONPATH`, but setting these variables for programs started from the Finder is non-standard as the Finder does not read your :file:`.profile` or :file:`.cshrc` at startup. You need to create a file -:file:`~/.MacOSX/environment.plist`. See Apple's Technical Document QA1067 for -details. +:file:`~/.MacOSX/environment.plist`. See Apple's +`Technical Q&A QA1067 <https://developer.apple.com/library/archive/qa/qa1067/_index.html>`__ +for details. -For more information on installation Python packages in MacPython, see section +For more information on installation Python packages, see section :ref:`mac-package-manager`. @@ -115,9 +121,9 @@ For more information on installation Python packages in MacPython, see section The IDE ======= -MacPython ships with the standard IDLE development environment. A good +Python ships with the standard IDLE development environment. A good introduction to using IDLE can be found at -http://www.hashcollision.org/hkn/python/idle_intro/index.html. +https://www.hashcollision.org/hkn/python/idle_intro/index.html. .. _mac-package-manager: @@ -130,8 +136,10 @@ This section has moved to the `Python Packaging User Guide`_. .. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/installing-packages/ -GUI Programming on the Mac -========================== +.. _gui-programming-on-the-mac: + +GUI Programming +=============== There are several options for building GUI applications on the Mac with Python. @@ -151,20 +159,25 @@ macOS. Packages and documentation are available from https://www.wxpython.org. macOS. More information can be found at https://riverbankcomputing.com/software/pyqt/intro. +*PySide* is another cross-platform Qt-based toolkit. More information at +https://www.qt.io/qt-for-python. + -Distributing Python Applications on the Mac -=========================================== +.. _distributing-python-applications-on-the-mac: + +Distributing Python Applications +================================ The standard tool for deploying standalone Python applications on the Mac is -:program:`py2app`. More information on installing and using py2app can be found -at https://pypi.org/project/py2app/. +:program:`py2app`. More information on installing and using :program:`py2app` +can be found at https://pypi.org/project/py2app/. Other Resources =============== -The MacPython mailing list is an excellent support resource for Python users and -developers on the Mac: +The Pythonmac-SIG mailing list is an excellent support resource for Python users +and developers on the Mac: https://www.python.org/community/sigs/current/pythonmac-sig/ diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc index 1cf438b198a9afa..354eb1541ceac28 100644 --- a/Doc/using/venv-create.inc +++ b/Doc/using/venv-create.inc @@ -14,14 +14,14 @@ used at environment creation time). It also creates an (initially empty) ``Lib\site-packages``). If an existing directory is specified, it will be re-used. +.. versionchanged:: 3.5 + The use of ``venv`` is now recommended for creating virtual environments. + .. deprecated:: 3.6 ``pyvenv`` was the recommended tool for creating virtual environments for Python 3.3 and 3.4, and is :ref:`deprecated in Python 3.6 <whatsnew36-venv>`. -.. versionchanged:: 3.5 - The use of ``venv`` is now recommended for creating virtual environments. - .. highlight:: none On Windows, invoke the ``venv`` command as follows:: diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 2a0e7b4b06f5865..cc4db34b04d900b 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -14,8 +14,8 @@ know about when using Python on Microsoft Windows. Unlike most Unix systems and services, Windows does not include a system supported installation of Python. To make Python available, the CPython team -has compiled Windows installers (MSI packages) with every `release -<https://www.python.org/download/releases/>`_ for many years. These installers +has compiled Windows installers with every `release +<https://www.python.org/downloads/>`_ for many years. These installers are primarily intended to add a per-user installation of Python, with the core interpreter and library being used by a single user. The installer is also able to install for all users of a single machine, and a separate ZIP file is diff --git a/Doc/whatsnew/2.0.rst b/Doc/whatsnew/2.0.rst index f4a9d23699de53c..1a949ec40358072 100644 --- a/Doc/whatsnew/2.0.rst +++ b/Doc/whatsnew/2.0.rst @@ -217,13 +217,13 @@ often use the ``codecs.lookup(encoding)`` function, which returns a was consumed. * *stream_reader* is a class that supports decoding input from a stream. - *stream_reader(file_obj)* returns an object that supports the :meth:`read`, - :meth:`readline`, and :meth:`readlines` methods. These methods will all + *stream_reader(file_obj)* returns an object that supports the :meth:`!read`, + :meth:`!readline`, and :meth:`!readlines` methods. These methods will all translate from the given encoding and return Unicode strings. * *stream_writer*, similarly, is a class that supports encoding output to a stream. *stream_writer(file_obj)* returns an object that supports the - :meth:`write` and :meth:`writelines` methods. These methods expect Unicode + :meth:`!write` and :meth:`!writelines` methods. These methods expect Unicode strings, translating them to the given encoding on output. For example, the following code writes a Unicode string into a file, encoding @@ -356,8 +356,8 @@ variable ``a`` by 2, equivalent to the slightly lengthier ``a = a + 2``. The full list of supported assignment operators is ``+=``, ``-=``, ``*=``, ``/=``, ``%=``, ``**=``, ``&=``, ``|=``, ``^=``, ``>>=``, and ``<<=``. Python classes can override the augmented assignment operators by defining methods -named :meth:`__iadd__`, :meth:`__isub__`, etc. For example, the following -:class:`Number` class stores a number and supports using += to create a new +named :meth:`!__iadd__`, :meth:`!__isub__`, etc. For example, the following +:class:`!Number` class stores a number and supports using += to create a new instance with an incremented value. .. The empty groups below prevent conversion to guillemets. @@ -374,7 +374,7 @@ instance with an incremented value. n += 3 print n.value -The :meth:`__iadd__` special method is called with the value of the increment, +The :meth:`!__iadd__` special method is called with the value of the increment, and should return a new instance with an appropriately modified value; this return value is bound as the new value of the variable on the left-hand side. @@ -390,10 +390,10 @@ String Methods ============== Until now string-manipulation functionality was in the :mod:`string` module, -which was usually a front-end for the :mod:`strop` module written in C. The -addition of Unicode posed a difficulty for the :mod:`strop` module, because the +which was usually a front-end for the :mod:`!strop` module written in C. The +addition of Unicode posed a difficulty for the :mod:`!strop` module, because the functions would all need to be rewritten in order to accept either 8-bit or -Unicode strings. For functions such as :func:`string.replace`, which takes 3 +Unicode strings. For functions such as :func:`!string.replace`, which takes 3 string arguments, that means eight possible permutations, and correspondingly complicated code. @@ -416,13 +416,13 @@ The old :mod:`string` module is still around for backwards compatibility, but it mostly acts as a front-end to the new string methods. Two methods which have no parallel in pre-2.0 versions, although they did exist -in JPython for quite some time, are :meth:`startswith` and :meth:`endswith`. +in JPython for quite some time, are :meth:`!startswith` and :meth:`!endswith`. ``s.startswith(t)`` is equivalent to ``s[:len(t)] == t``, while ``s.endswith(t)`` is equivalent to ``s[-len(t):] == t``. -One other method which deserves special mention is :meth:`join`. The -:meth:`join` method of a string receives one parameter, a sequence of strings, -and is equivalent to the :func:`string.join` function from the old :mod:`string` +One other method which deserves special mention is :meth:`!join`. The +:meth:`!join` method of a string receives one parameter, a sequence of strings, +and is equivalent to the :func:`!string.join` function from the old :mod:`string` module, with the arguments reversed. In other words, ``s.join(seq)`` is equivalent to the old ``string.join(seq, s)``. @@ -503,9 +503,9 @@ Minor Language Changes A new syntax makes it more convenient to call a given function with a tuple of arguments and/or a dictionary of keyword arguments. In Python 1.5 and earlier, -you'd use the :func:`apply` built-in function: ``apply(f, args, kw)`` calls the -function :func:`f` with the argument tuple *args* and the keyword arguments in -the dictionary *kw*. :func:`apply` is the same in 2.0, but thanks to a patch +you'd use the :func:`!apply` built-in function: ``apply(f, args, kw)`` calls the +function :func:`!f` with the argument tuple *args* and the keyword arguments in +the dictionary *kw*. :func:`!apply` is the same in 2.0, but thanks to a patch from Greg Ewing, ``f(*args, **kw)`` is a shorter and clearer way to achieve the same effect. This syntax is symmetrical with the syntax for defining functions:: @@ -518,7 +518,7 @@ functions:: The ``print`` statement can now have its output directed to a file-like object by following the ``print`` with ``>> file``, similar to the redirection operator in Unix shells. Previously you'd either have to use the -:meth:`write` method of the file-like object, which lacks the convenience and +:meth:`!write` method of the file-like object, which lacks the convenience and simplicity of ``print``, or you could assign a new value to ``sys.stdout`` and then restore the old value. For sending output to standard error, it's much easier to write this:: @@ -540,7 +540,7 @@ Previously there was no way to implement a class that overrode Python's built-in true if *obj* is present in the sequence *seq*; Python computes this by simply trying every index of the sequence until either *obj* is found or an :exc:`IndexError` is encountered. Moshe Zadka contributed a patch which adds a -:meth:`__contains__` magic method for providing a custom implementation for +:meth:`!__contains__` magic method for providing a custom implementation for :keyword:`!in`. Additionally, new built-in objects written in C can define what :keyword:`!in` means for them via a new slot in the sequence protocol. @@ -562,7 +562,7 @@ the python-dev mailing list for the discussion leading up to this implementation, and some useful relevant links. Note that comparisons can now also raise exceptions. In earlier versions of Python, a comparison operation such as ``cmp(a,b)`` would always produce an answer, even if a user-defined -:meth:`__cmp__` method encountered an error, since the resulting exception would +:meth:`!__cmp__` method encountered an error, since the resulting exception would simply be silently swallowed. .. Starting URL: @@ -607,7 +607,7 @@ seq1, seq2)`` is that :func:`map` pads the sequences with ``None`` if the sequences aren't all of the same length, while :func:`zip` truncates the returned list to the length of the shortest argument sequence. -The :func:`int` and :func:`long` functions now accept an optional "base" +The :func:`int` and :func:`!long` functions now accept an optional "base" parameter when the first argument is a string. ``int('123', 10)`` returns 123, while ``int('123', 16)`` returns 291. ``int(123, 16)`` raises a :exc:`TypeError` exception with the message "can't convert non-string with @@ -620,8 +620,8 @@ would be ``(2, 0, 1, 'beta', 1)``. *level* is a string such as ``"alpha"``, ``"beta"``, or ``"final"`` for a final release. Dictionaries have an odd new method, ``setdefault(key, default)``, which -behaves similarly to the existing :meth:`get` method. However, if the key is -missing, :meth:`setdefault` both returns the value of *default* as :meth:`get` +behaves similarly to the existing :meth:`!get` method. However, if the key is +missing, :meth:`!setdefault` both returns the value of *default* as :meth:`!get` would do, and also inserts it into the dictionary as the value for *key*. Thus, the following lines of code:: @@ -656,7 +656,7 @@ break. The change which will probably break the most code is tightening up the arguments accepted by some methods. Some methods would take multiple arguments and treat them as a tuple, particularly various list methods such as -:meth:`append` and :meth:`insert`. In earlier versions of Python, if ``L`` is +:meth:`!append` and :meth:`!insert`. In earlier versions of Python, if ``L`` is a list, ``L.append( 1,2 )`` appends the tuple ``(1,2)`` to the list. In Python 2.0 this causes a :exc:`TypeError` exception to be raised, with the message: 'append requires exactly 1 argument; 2 given'. The fix is to simply add an @@ -693,7 +693,7 @@ advantage of this fact will break in 2.0. Some work has been done to make integers and long integers a bit more interchangeable. In 1.5.2, large-file support was added for Solaris, to allow -reading files larger than 2 GiB; this made the :meth:`tell` method of file +reading files larger than 2 GiB; this made the :meth:`!tell` method of file objects return a long integer instead of a regular integer. Some code would subtract two file offsets and attempt to use the result to multiply a sequence or slice a string, but this raised a :exc:`TypeError`. In 2.0, long integers @@ -701,7 +701,7 @@ can be used to multiply or slice a sequence, and it'll behave as you'd intuitively expect it to; ``3L * 'abc'`` produces 'abcabcabc', and ``(0,1,2,3)[2L:4L]`` produces (2,3). Long integers can also be used in various contexts where previously only integers were accepted, such as in the -:meth:`seek` method of file objects, and in the formats supported by the ``%`` +:meth:`!seek` method of file objects, and in the formats supported by the ``%`` operator (``%d``, ``%i``, ``%x``, etc.). For example, ``"%d" % 2L**64`` will produce the string ``18446744073709551616``. @@ -715,7 +715,7 @@ digit. Taking the :func:`repr` of a float now uses a different formatting precision than :func:`str`. :func:`repr` uses ``%.17g`` format string for C's -:func:`sprintf`, while :func:`str` uses ``%.12g`` as before. The effect is that +:func:`!sprintf`, while :func:`str` uses ``%.12g`` as before. The effect is that :func:`repr` may occasionally show more decimal places than :func:`str`, for certain numbers. For example, the number 8.1 can't be represented exactly in binary, so ``repr(8.1)`` is ``'8.0999999999999996'``, while str(8.1) is @@ -723,7 +723,7 @@ binary, so ``repr(8.1)`` is ``'8.0999999999999996'``, while str(8.1) is The ``-X`` command-line option, which turned all standard exceptions into strings instead of classes, has been removed; the standard exceptions will now -always be classes. The :mod:`exceptions` module containing the standard +always be classes. The :mod:`!exceptions` module containing the standard exceptions was translated from Python to a built-in C module, written by Barry Warsaw and Fredrik Lundh. @@ -879,11 +879,11 @@ joins the basic set of Python documentation. XML Modules =========== -Python 1.5.2 included a simple XML parser in the form of the :mod:`xmllib` +Python 1.5.2 included a simple XML parser in the form of the :mod:`!xmllib` module, contributed by Sjoerd Mullender. Since 1.5.2's release, two different interfaces for processing XML have become common: SAX2 (version 2 of the Simple API for XML) provides an event-driven interface with some similarities to -:mod:`xmllib`, and the DOM (Document Object Model) provides a tree-based +:mod:`!xmllib`, and the DOM (Document Object Model) provides a tree-based interface, transforming an XML document into a tree of nodes that can be traversed and modified. Python 2.0 includes a SAX2 interface and a stripped-down DOM interface as part of the :mod:`xml` package. Here we will give a brief @@ -898,9 +898,9 @@ SAX2 Support SAX defines an event-driven interface for parsing XML. To use SAX, you must write a SAX handler class. Handler classes inherit from various classes provided by SAX, and override various methods that will then be called by the -XML parser. For example, the :meth:`startElement` and :meth:`endElement` +XML parser. For example, the :meth:`~xml.sax.handler.ContentHandler.startElement` and :meth:`~xml.sax.handler.ContentHandler.endElement` methods are called for every starting and end tag encountered by the parser, the -:meth:`characters` method is called for every chunk of character data, and so +:meth:`~xml.sax.handler.ContentHandler.characters` method is called for every chunk of character data, and so forth. The advantage of the event-driven approach is that the whole document doesn't @@ -940,8 +940,8 @@ DOM Support ----------- The Document Object Model is a tree-based representation for an XML document. A -top-level :class:`Document` instance is the root of the tree, and has a single -child which is the top-level :class:`Element` instance. This :class:`Element` +top-level :class:`!Document` instance is the root of the tree, and has a single +child which is the top-level :class:`!Element` instance. This :class:`!Element` has children nodes representing character data and any sub-elements, which may have further children of their own, and so forth. Using the DOM you can traverse the resulting tree any way you like, access element and attribute @@ -955,18 +955,18 @@ simply writing ``<tag1>``...\ ``</tag1>`` to a file. The DOM implementation included with Python lives in the :mod:`xml.dom.minidom` module. It's a lightweight implementation of the Level 1 DOM with support for -XML namespaces. The :func:`parse` and :func:`parseString` convenience +XML namespaces. The :func:`!parse` and :func:`!parseString` convenience functions are provided for generating a DOM tree:: from xml.dom import minidom doc = minidom.parse('hamlet.xml') -``doc`` is a :class:`Document` instance. :class:`Document`, like all the other -DOM classes such as :class:`Element` and :class:`Text`, is a subclass of the -:class:`Node` base class. All the nodes in a DOM tree therefore support certain -common methods, such as :meth:`toxml` which returns a string containing the XML +``doc`` is a :class:`!Document` instance. :class:`!Document`, like all the other +DOM classes such as :class:`!Element` and :class:`Text`, is a subclass of the +:class:`!Node` base class. All the nodes in a DOM tree therefore support certain +common methods, such as :meth:`!toxml` which returns a string containing the XML representation of the node and its children. Each class also has special -methods of its own; for example, :class:`Element` and :class:`Document` +methods of its own; for example, :class:`!Element` and :class:`!Document` instances have a method to find all child elements with a given tag name. Continuing from the previous 2-line example:: @@ -995,7 +995,7 @@ its children can be easily modified by deleting, adding, or removing nodes:: root.insertBefore( root.childNodes[0], root.childNodes[20] ) Again, I will refer you to the Python documentation for a complete listing of -the different :class:`Node` classes and their various methods. +the different :class:`!Node` classes and their various methods. Relationship to PyXML @@ -1020,7 +1020,7 @@ features in PyXML include: * The xmlproc validating parser, written by Lars Marius Garshol. -* The :mod:`sgmlop` parser accelerator module, written by Fredrik Lundh. +* The :mod:`!sgmlop` parser accelerator module, written by Fredrik Lundh. .. ====================================================================== @@ -1031,7 +1031,7 @@ Module changes Lots of improvements and bugfixes were made to Python's extensive standard library; some of the affected modules include :mod:`readline`, :mod:`ConfigParser <configparser>`, :mod:`!cgi`, :mod:`calendar`, :mod:`posix`, :mod:`readline`, -:mod:`xmllib`, :mod:`!aifc`, :mod:`!chunk`, :mod:`wave`, :mod:`random`, :mod:`shelve`, +:mod:`!xmllib`, :mod:`!aifc`, :mod:`!chunk`, :mod:`wave`, :mod:`random`, :mod:`shelve`, and :mod:`!nntplib`. Consult the CVS logs for the exact patch-by-patch details. Brian Gallew contributed OpenSSL support for the :mod:`socket` module. OpenSSL @@ -1039,16 +1039,17 @@ is an implementation of the Secure Socket Layer, which encrypts the data being sent over a socket. When compiling Python, you can edit :file:`Modules/Setup` to include SSL support, which adds an additional function to the :mod:`socket` module: ``socket.ssl(socket, keyfile, certfile)``, which takes a socket -object and returns an SSL socket. The :mod:`httplib` and :mod:`urllib` modules +object and returns an SSL socket. The :mod:`httplib <http>` and :mod:`urllib` modules were also changed to support ``https://`` URLs, though no one has implemented FTP or SMTP over SSL. -The :mod:`httplib` module has been rewritten by Greg Stein to support HTTP/1.1. -Backward compatibility with the 1.5 version of :mod:`httplib` is provided, +The :mod:`httplib <http>` module has been rewritten by Greg Stein to support HTTP/1.1. + +Backward compatibility with the 1.5 version of :mod:`!httplib` is provided, though using HTTP/1.1 features such as pipelining will require rewriting code to use a different set of interfaces. -The :mod:`Tkinter` module now supports Tcl/Tk version 8.1, 8.2, or 8.3, and +The :mod:`!Tkinter` module now supports Tcl/Tk version 8.1, 8.2, or 8.3, and support for the older 7.x versions has been dropped. The Tkinter module now supports displaying Unicode strings in Tk widgets. Also, Fredrik Lundh contributed an optimization which makes operations like ``create_line`` and @@ -1083,11 +1084,11 @@ module. calling :func:`atexit.register` with the function to be called on exit. (Contributed by Skip Montanaro.) -* :mod:`codecs`, :mod:`encodings`, :mod:`unicodedata`: Added as part of the new +* :mod:`codecs`, :mod:`!encodings`, :mod:`unicodedata`: Added as part of the new Unicode support. -* :mod:`filecmp`: Supersedes the old :mod:`cmp`, :mod:`cmpcache` and - :mod:`dircmp` modules, which have now become deprecated. (Contributed by Gordon +* :mod:`filecmp`: Supersedes the old :mod:`!cmp`, :mod:`!cmpcache` and + :mod:`!dircmp` modules, which have now become deprecated. (Contributed by Gordon MacMillan and Moshe Zadka.) * :mod:`gettext`: This module provides internationalization (I18N) and @@ -1105,10 +1106,10 @@ module. be passed to functions that expect ordinary strings, such as the :mod:`re` module. (Contributed by Sam Rushing, with some extensions by A.M. Kuchling.) -* :mod:`pyexpat`: An interface to the Expat XML parser. (Contributed by Paul +* :mod:`!pyexpat`: An interface to the Expat XML parser. (Contributed by Paul Prescod.) -* :mod:`robotparser`: Parse a :file:`robots.txt` file, which is used for writing +* :mod:`robotparser <urllib.robotparser>`: Parse a :file:`robots.txt` file, which is used for writing web spiders that politely avoid certain areas of a web site. The parser accepts the contents of a :file:`robots.txt` file, builds a set of rules from it, and can then answer questions about the fetchability of a given URL. (Contributed @@ -1117,7 +1118,7 @@ module. * :mod:`tabnanny`: A module/script to check Python source code for ambiguous indentation. (Contributed by Tim Peters.) -* :mod:`UserString`: A base class useful for deriving objects that behave like +* :mod:`!UserString`: A base class useful for deriving objects that behave like strings. * :mod:`webbrowser`: A module that provides a platform independent way to launch @@ -1129,10 +1130,10 @@ module. :file:`Tools/idle/BrowserControl.py`, and adapted for the standard library by Fred.) -* :mod:`_winreg`: An interface to the Windows registry. :mod:`_winreg` is an +* :mod:`_winreg <winreg>`: An interface to the Windows registry. :mod:`!_winreg` is an adaptation of functions that have been part of PythonWin since 1995, but has now been added to the core distribution, and enhanced to support Unicode. - :mod:`_winreg` was written by Bill Tutt and Mark Hammond. + :mod:`!_winreg` was written by Bill Tutt and Mark Hammond. * :mod:`zipfile`: A module for reading and writing ZIP-format archives. These are archives produced by :program:`PKZIP` on DOS/Windows or :program:`zip` on @@ -1184,13 +1185,13 @@ Deleted and Deprecated Modules ============================== A few modules have been dropped because they're obsolete, or because there are -now better ways to do the same thing. The :mod:`stdwin` module is gone; it was +now better ways to do the same thing. The :mod:`!stdwin` module is gone; it was for a platform-independent windowing toolkit that's no longer developed. A number of modules have been moved to the :file:`lib-old` subdirectory: -:mod:`cmp`, :mod:`cmpcache`, :mod:`dircmp`, :mod:`dump`, :mod:`find`, -:mod:`grep`, :mod:`packmail`, :mod:`poly`, :mod:`util`, :mod:`whatsound`, -:mod:`zmod`. If you have code which relies on a module that's been moved to +:mod:`!cmp`, :mod:`!cmpcache`, :mod:`!dircmp`, :mod:`!dump`, :mod:`!find`, +:mod:`!grep`, :mod:`!packmail`, :mod:`!poly`, :mod:`!util`, :mod:`!whatsound`, +:mod:`!zmod`. If you have code which relies on a module that's been moved to :file:`lib-old`, you can simply add that directory to ``sys.path`` to get them back, but you're encouraged to update any code that uses these modules. diff --git a/Doc/whatsnew/2.1.rst b/Doc/whatsnew/2.1.rst index 6d2d3cc02b87689..b4002f06e92adc2 100644 --- a/Doc/whatsnew/2.1.rst +++ b/Doc/whatsnew/2.1.rst @@ -48,7 +48,7 @@ nested recursive function definition doesn't work:: return g(value-1) + 1 ... -The function :func:`g` will always raise a :exc:`NameError` exception, because +The function :func:`!g` will always raise a :exc:`NameError` exception, because the binding of the name ``g`` isn't in either its local namespace or in the module-level namespace. This isn't much of a problem in practice (how often do you recursively define interior functions like this?), but this also made using @@ -104,7 +104,7 @@ To make the preceding explanation a bit clearer, here's an example:: Line 4 containing the ``exec`` statement is a syntax error, since ``exec`` would define a new local variable named ``x`` whose value should -be accessed by :func:`g`. +be accessed by :func:`!g`. This shouldn't be much of a limitation, since ``exec`` is rarely used in most Python code (and when it is used, it's often a sign of a poor design @@ -161,7 +161,7 @@ PEP 207: Rich Comparisons In earlier versions, Python's support for implementing comparisons on user-defined classes and extension types was quite simple. Classes could implement a -:meth:`__cmp__` method that was given two instances of a class, and could only +:meth:`!__cmp__` method that was given two instances of a class, and could only return 0 if they were equal or +1 or -1 if they weren't; the method couldn't raise an exception or return anything other than a Boolean value. Users of Numeric Python often found this model too weak and restrictive, because in the @@ -175,21 +175,21 @@ In Python 2.1, rich comparisons were added in order to support this need. Python classes can now individually overload each of the ``<``, ``<=``, ``>``, ``>=``, ``==``, and ``!=`` operations. The new magic method names are: -+-----------+----------------+ -| Operation | Method name | -+===========+================+ -| ``<`` | :meth:`__lt__` | -+-----------+----------------+ -| ``<=`` | :meth:`__le__` | -+-----------+----------------+ -| ``>`` | :meth:`__gt__` | -+-----------+----------------+ -| ``>=`` | :meth:`__ge__` | -+-----------+----------------+ -| ``==`` | :meth:`__eq__` | -+-----------+----------------+ -| ``!=`` | :meth:`__ne__` | -+-----------+----------------+ ++-----------+------------------------+ +| Operation | Method name | ++===========+========================+ +| ``<`` | :meth:`~object.__lt__` | ++-----------+------------------------+ +| ``<=`` | :meth:`~object.__le__` | ++-----------+------------------------+ +| ``>`` | :meth:`~object.__gt__` | ++-----------+------------------------+ +| ``>=`` | :meth:`~object.__ge__` | ++-----------+------------------------+ +| ``==`` | :meth:`~object.__eq__` | ++-----------+------------------------+ +| ``!=`` | :meth:`~object.__ne__` | ++-----------+------------------------+ (The magic methods are named after the corresponding Fortran operators ``.LT.``. ``.LE.``, &c. Numeric programmers are almost certainly quite familiar with @@ -208,7 +208,7 @@ The built-in ``cmp(A,B)`` function can use the rich comparison machinery, and now accepts an optional argument specifying which comparison operation to use; this is given as one of the strings ``"<"``, ``"<="``, ``">"``, ``">="``, ``"=="``, or ``"!="``. If called without the optional third argument, -:func:`cmp` will only return -1, 0, or +1 as in previous versions of Python; +:func:`!cmp` will only return -1, 0, or +1 as in previous versions of Python; otherwise it will call the appropriate method and can return any Python object. There are also corresponding changes of interest to C programmers; there's a new @@ -245,7 +245,7 @@ out warnings that you don't want to be displayed. Third-party modules can also use this framework to deprecate old features that they no longer wish to support. -For example, in Python 2.1 the :mod:`regex` module is deprecated, so importing +For example, in Python 2.1 the :mod:`!regex` module is deprecated, so importing it causes a warning to be printed:: >>> import regex @@ -262,7 +262,7 @@ can be used to specify a particular warning category. Filters can be added to disable certain warnings; a regular expression pattern can be applied to the message or to the module name in order to suppress a -warning. For example, you may have a program that uses the :mod:`regex` module +warning. For example, you may have a program that uses the :mod:`!regex` module and not want to spare the time to convert it to use the :mod:`re` module right now. The warning can be suppressed by calling :: @@ -274,7 +274,7 @@ now. The warning can be suppressed by calling :: This adds a filter that will apply only to warnings of the class :class:`DeprecationWarning` triggered in the :mod:`__main__` module, and applies -a regular expression to only match the message about the :mod:`regex` module +a regular expression to only match the message about the :mod:`!regex` module being deprecated, and will cause such warnings to be ignored. Warnings can also be printed only once, printed every time the offending code is executed, or turned into exceptions that will cause the program to stop (unless the @@ -368,7 +368,7 @@ dictionary:: This version works for simple things such as integers, but it has a side effect; the ``_cache`` dictionary holds a reference to the return values, so they'll never be deallocated until the Python process exits and cleans up. This isn't -very noticeable for integers, but if :func:`f` returns an object, or a data +very noticeable for integers, but if :func:`!f` returns an object, or a data structure that takes up a lot of memory, this can be a problem. Weak references provide a way to implement a cache that won't keep objects alive @@ -379,7 +379,7 @@ created by calling ``wr = weakref.ref(obj)``. The object being referred to is returned by calling the weak reference as if it were a function: ``wr()``. It will return the referenced object, or ``None`` if the object no longer exists. -This makes it possible to write a :func:`memoize` function whose cache doesn't +This makes it possible to write a :func:`!memoize` function whose cache doesn't keep objects alive, by storing weak references in the cache. :: _cache = {} @@ -402,7 +402,7 @@ weak references --- an object referenced only by proxy objects is deallocated -- but instead of requiring an explicit call to retrieve the object, the proxy transparently forwards all operations to the object as long as the object still exists. If the object is deallocated, attempting to use a proxy will cause a -:exc:`weakref.ReferenceError` exception to be raised. :: +:exc:`!weakref.ReferenceError` exception to be raised. :: proxy = weakref.proxy(obj) proxy.attr # Equivalent to obj.attr @@ -446,7 +446,7 @@ The dictionary containing attributes can be accessed as the function's :attr:`~object.__dict__`. Unlike the :attr:`~object.__dict__` attribute of class instances, in functions you can actually assign a new dictionary to :attr:`~object.__dict__`, though the new value is restricted to a regular Python dictionary; you *can't* be -tricky and set it to a :class:`UserDict` instance, or any other random object +tricky and set it to a :class:`!UserDict` instance, or any other random object that behaves like a mapping. @@ -584,11 +584,11 @@ available from the Distutils SIG at https://www.python.org/community/sigs/curren New and Improved Modules ======================== -* Ka-Ping Yee contributed two new modules: :mod:`inspect.py`, a module for - getting information about live Python code, and :mod:`pydoc.py`, a module for +* Ka-Ping Yee contributed two new modules: :mod:`!inspect.py`, a module for + getting information about live Python code, and :mod:`!pydoc.py`, a module for interactively converting docstrings to HTML or text. As a bonus, :file:`Tools/scripts/pydoc`, which is now automatically installed, uses - :mod:`pydoc.py` to display documentation given a Python module, package, or + :mod:`!pydoc.py` to display documentation given a Python module, package, or class name. For example, ``pydoc xml.dom`` displays the following:: Python Library Documentation: package xml.dom in xml @@ -617,7 +617,7 @@ New and Improved Modules Kent Beck's Smalltalk testing framework. See https://pyunit.sourceforge.net/ for more information about PyUnit. -* The :mod:`difflib` module contains a class, :class:`SequenceMatcher`, which +* The :mod:`difflib` module contains a class, :class:`~difflib.SequenceMatcher`, which compares two sequences and computes the changes required to transform one sequence into the other. For example, this module can be used to write a tool similar to the Unix :program:`diff` program, and in fact the sample program @@ -633,7 +633,7 @@ New and Improved Modules 2.1 includes an updated version of the :mod:`xml` package. Some of the noteworthy changes include support for Expat 1.2 and later versions, the ability for Expat parsers to handle files in any encoding supported by Python, and - various bugfixes for SAX, DOM, and the :mod:`minidom` module. + various bugfixes for SAX, DOM, and the :mod:`!minidom` module. * Ping also contributed another hook for handling uncaught exceptions. :func:`sys.excepthook` can be set to a callable object. When an exception isn't @@ -643,8 +643,8 @@ New and Improved Modules printing an extended traceback that not only lists the stack frames, but also lists the function arguments and the local variables for each frame. -* Various functions in the :mod:`time` module, such as :func:`asctime` and - :func:`localtime`, require a floating point argument containing the time in +* Various functions in the :mod:`time` module, such as :func:`~time.asctime` and + :func:`~time.localtime`, require a floating point argument containing the time in seconds since the epoch. The most common use of these functions is to work with the current time, so the floating point argument has been made optional; when a value isn't provided, the current time will be used. For example, log file @@ -724,10 +724,10 @@ of the more notable changes are: a discussion in comp.lang.python. A new module and method for file objects was also added, contributed by Jeff - Epler. The new method, :meth:`xreadlines`, is similar to the existing - :func:`xrange` built-in. :func:`xreadlines` returns an opaque sequence object + Epler. The new method, :meth:`!xreadlines`, is similar to the existing + :func:`!xrange` built-in. :func:`!xreadlines` returns an opaque sequence object that only supports being iterated over, reading a line on every iteration but - not reading the entire file into memory as the existing :meth:`readlines` method + not reading the entire file into memory as the existing :meth:`!readlines` method does. You'd use it like this:: for line in sys.stdin.xreadlines(): @@ -737,7 +737,7 @@ of the more notable changes are: For a fuller discussion of the line I/O changes, see the python-dev summary for January 1--15, 2001 at https://mail.python.org/pipermail/python-dev/2001-January/. -* A new method, :meth:`popitem`, was added to dictionaries to enable +* A new method, :meth:`~dict.popitem`, was added to dictionaries to enable destructively iterating through the contents of a dictionary; this can be faster for large dictionaries because there's no need to construct a list containing all the keys or values. ``D.popitem()`` removes a random ``(key, value)`` pair diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 6efc23a82de9232..e6c13f957b8d54a 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -53,9 +53,9 @@ A long time ago I wrote a web page listing flaws in Python's design. One of the most significant flaws was that it's impossible to subclass Python types implemented in C. In particular, it's not possible to subclass built-in types, so you can't just subclass, say, lists in order to add a single useful method to -them. The :mod:`UserList` module provides a class that supports all of the +them. The :mod:`!UserList` module provides a class that supports all of the methods of lists and that can be subclassed further, but there's lots of C code -that expects a regular Python list and won't accept a :class:`UserList` +that expects a regular Python list and won't accept a :class:`~collections.UserList` instance. Python 2.2 fixes this, and in the process adds some exciting new capabilities. @@ -69,7 +69,7 @@ A brief summary: * It's also possible to automatically call methods on accessing or setting an instance attribute by using a new mechanism called :dfn:`properties`. Many uses - of :meth:`__getattr__` can be rewritten to use properties instead, making the + of :meth:`~object.__getattr__` can be rewritten to use properties instead, making the resulting code simpler and faster. As a small side benefit, attributes can now have docstrings, too. @@ -120,7 +120,7 @@ added so if no built-in type is suitable, you can just subclass This means that :keyword:`class` statements that don't have any base classes are always classic classes in Python 2.2. (Actually you can also change this by -setting a module-level variable named :attr:`__metaclass__` --- see :pep:`253` +setting a module-level variable named :attr:`!__metaclass__` --- see :pep:`253` for the details --- but it's easier to just subclass :class:`object`.) The type objects for the built-in types are available as built-ins, named using @@ -134,8 +134,8 @@ type objects that behave as factories when called. :: 123 To make the set of types complete, new type objects such as :func:`dict` and -:func:`file` have been added. Here's a more interesting example, adding a -:meth:`lock` method to file objects:: +:func:`!file` have been added. Here's a more interesting example, adding a +:meth:`!lock` method to file objects:: class LockableFile(file): def lock (self, operation, length=0, start=0, whence=0): @@ -146,7 +146,7 @@ To make the set of types complete, new type objects such as :func:`dict` and The now-obsolete :mod:`!posixfile` module contained a class that emulated all of a file object's methods and also added a :meth:`!lock` method, but this class couldn't be passed to internal functions that expected a built-in file, -something which is possible with our new :class:`LockableFile`. +something which is possible with our new :class:`!LockableFile`. Descriptors @@ -154,11 +154,11 @@ Descriptors In previous versions of Python, there was no consistent way to discover what attributes and methods were supported by an object. There were some informal -conventions, such as defining :attr:`__members__` and :attr:`__methods__` +conventions, such as defining :attr:`!__members__` and :attr:`!__methods__` attributes that were lists of names, but often the author of an extension type or a class wouldn't bother to define them. You could fall back on inspecting the :attr:`~object.__dict__` of an object, but when class inheritance or an arbitrary -:meth:`__getattr__` hook were in use this could still be inaccurate. +:meth:`!__getattr__` hook were in use this could still be inaccurate. The one big idea underlying the new class model is that an API for describing the attributes of an object using :dfn:`descriptors` has been formalized. @@ -171,7 +171,7 @@ attributes of their own: * :attr:`~definition.__name__` is the attribute's name. -* :attr:`__doc__` is the attribute's docstring. +* :attr:`!__doc__` is the attribute's docstring. * ``__get__(object)`` is a method that retrieves the attribute value from *object*. @@ -186,7 +186,7 @@ are:: descriptor = obj.__class__.x descriptor.__get__(obj) -For methods, :meth:`descriptor.__get__` returns a temporary object that's +For methods, :meth:`!descriptor.__get__` returns a temporary object that's callable, and wraps up the instance and the method to be called on it. This is also why static methods and class methods are now possible; they have descriptors that wrap up just the method, or the method and the class. As a @@ -204,7 +204,7 @@ methods are defined like this:: ... g = classmethod(g) -The :func:`staticmethod` function takes the function :func:`f`, and returns it +The :func:`staticmethod` function takes the function :func:`!f`, and returns it wrapped up in a descriptor so it can be stored in the class object. You might expect there to be special syntax for creating such methods (``def static f``, ``defstatic f()``, or something like that) but no such syntax has been defined @@ -232,10 +232,10 @@ like this:: f = eiffelmethod(f, pre_f, post_f) -Note that a person using the new :func:`eiffelmethod` doesn't have to understand +Note that a person using the new :func:`!eiffelmethod` doesn't have to understand anything about descriptors. This is why I think the new features don't increase the basic complexity of the language. There will be a few wizards who need to -know about it in order to write :func:`eiffelmethod` or the ZODB or whatever, +know about it in order to write :func:`!eiffelmethod` or the ZODB or whatever, but most users will just write code on top of the resulting libraries and ignore the implementation details. @@ -263,10 +263,10 @@ from :pep:`253` by Guido van Rossum):: The lookup rule for classic classes is simple but not very smart; the base classes are searched depth-first, going from left to right. A reference to -:meth:`D.save` will search the classes :class:`D`, :class:`B`, and then -:class:`A`, where :meth:`save` would be found and returned. :meth:`C.save` -would never be found at all. This is bad, because if :class:`C`'s :meth:`save` -method is saving some internal state specific to :class:`C`, not calling it will +:meth:`!D.save` will search the classes :class:`!D`, :class:`!B`, and then +:class:`!A`, where :meth:`!save` would be found and returned. :meth:`!C.save` +would never be found at all. This is bad, because if :class:`!C`'s :meth:`!save` +method is saving some internal state specific to :class:`!C`, not calling it will result in that state never getting saved. New-style classes follow a different algorithm that's a bit more complicated to @@ -276,22 +276,22 @@ produces more useful results for really complicated inheritance graphs.) #. List all the base classes, following the classic lookup rule and include a class multiple times if it's visited repeatedly. In the above example, the list - of visited classes is [:class:`D`, :class:`B`, :class:`A`, :class:`C`, - :class:`A`]. + of visited classes is [:class:`!D`, :class:`!B`, :class:`!A`, :class:`!C`, + :class:`!A`]. #. Scan the list for duplicated classes. If any are found, remove all but one occurrence, leaving the *last* one in the list. In the above example, the list - becomes [:class:`D`, :class:`B`, :class:`C`, :class:`A`] after dropping + becomes [:class:`!D`, :class:`!B`, :class:`!C`, :class:`!A`] after dropping duplicates. -Following this rule, referring to :meth:`D.save` will return :meth:`C.save`, +Following this rule, referring to :meth:`!D.save` will return :meth:`!C.save`, which is the behaviour we're after. This lookup rule is the same as the one followed by Common Lisp. A new built-in function, :func:`super`, provides a way to get at a class's superclasses without having to reimplement Python's algorithm. The most commonly used form will be ``super(class, obj)``, which returns a bound superclass object (not the actual class object). This form will be used in methods to call a method in the superclass; for example, -:class:`D`'s :meth:`save` method would look like this:: +:class:`!D`'s :meth:`!save` method would look like this:: class D (B,C): def save (self): @@ -309,7 +309,7 @@ Attribute Access ---------------- A fair number of sophisticated Python classes define hooks for attribute access -using :meth:`__getattr__`; most commonly this is done for convenience, to make +using :meth:`~object.__getattr__`; most commonly this is done for convenience, to make code more readable by automatically mapping an attribute access such as ``obj.parent`` into a method call such as ``obj.get_parent``. Python 2.2 adds some new ways of controlling attribute access. @@ -321,22 +321,22 @@ instance's dictionary. New-style classes also support a new method, ``__getattribute__(attr_name)``. The difference between the two methods is -that :meth:`__getattribute__` is *always* called whenever any attribute is -accessed, while the old :meth:`__getattr__` is only called if ``foo`` isn't +that :meth:`~object.__getattribute__` is *always* called whenever any attribute is +accessed, while the old :meth:`~object.__getattr__` is only called if ``foo`` isn't found in the instance's dictionary. However, Python 2.2's support for :dfn:`properties` will often be a simpler way -to trap attribute references. Writing a :meth:`__getattr__` method is +to trap attribute references. Writing a :meth:`!__getattr__` method is complicated because to avoid recursion you can't use regular attribute accesses inside them, and instead have to mess around with the contents of -:attr:`~object.__dict__`. :meth:`__getattr__` methods also end up being called by Python -when it checks for other methods such as :meth:`__repr__` or :meth:`__coerce__`, +:attr:`~object.__dict__`. :meth:`~object.__getattr__` methods also end up being called by Python +when it checks for other methods such as :meth:`~object.__repr__` or :meth:`!__coerce__`, and so have to be written with this in mind. Finally, calling a function on every attribute access results in a sizable performance loss. :class:`property` is a new built-in type that packages up three functions that get, set, or delete an attribute, and a docstring. For example, if you want to -define a :attr:`size` attribute that's computed, but also settable, you could +define a :attr:`!size` attribute that's computed, but also settable, you could write:: class C(object): @@ -355,9 +355,9 @@ write:: "Storage size of this instance") That is certainly clearer and easier to write than a pair of -:meth:`__getattr__`/:meth:`__setattr__` methods that check for the :attr:`size` +:meth:`!__getattr__`/:meth:`!__setattr__` methods that check for the :attr:`!size` attribute and handle it specially while retrieving all other attributes from the -instance's :attr:`~object.__dict__`. Accesses to :attr:`size` are also the only ones +instance's :attr:`~object.__dict__`. Accesses to :attr:`!size` are also the only ones which have to perform the work of calling a function, so references to other attributes run at their usual speed. @@ -447,7 +447,7 @@ an iterator for the object *obj*, while ``iter(C, sentinel)`` returns an iterator that will invoke the callable object *C* until it returns *sentinel* to signal that the iterator is done. -Python classes can define an :meth:`__iter__` method, which should create and +Python classes can define an :meth:`!__iter__` method, which should create and return a new iterator for the object; if the object is its own iterator, this method can just return ``self``. In particular, iterators will usually be their own iterators. Extension types implemented in C can implement a :c:member:`~PyTypeObject.tp_iter` @@ -478,7 +478,7 @@ there are no more values to be returned, calling :meth:`next` should raise the In 2.2, Python's :keyword:`for` statement no longer expects a sequence; it expects something for which :func:`iter` will return an iterator. For backward compatibility and convenience, an iterator is automatically constructed for -sequences that don't implement :meth:`__iter__` or a :c:member:`~PyTypeObject.tp_iter` slot, so +sequences that don't implement :meth:`!__iter__` or a :c:member:`~PyTypeObject.tp_iter` slot, so ``for i in [1,2,3]`` will still work. Wherever the Python interpreter loops over a sequence, it's been changed to use the iterator protocol. This means you can do things like this:: @@ -510,8 +510,8 @@ Iterator support has been added to some of Python's basic types. Calling Oct 10 That's just the default behaviour. If you want to iterate over keys, values, or -key/value pairs, you can explicitly call the :meth:`iterkeys`, -:meth:`itervalues`, or :meth:`iteritems` methods to get an appropriate iterator. +key/value pairs, you can explicitly call the :meth:`!iterkeys`, +:meth:`!itervalues`, or :meth:`!iteritems` methods to get an appropriate iterator. In a minor related change, the :keyword:`in` operator now works on dictionaries, so ``key in dict`` is now equivalent to ``dict.has_key(key)``. @@ -580,7 +580,7 @@ allowed inside the :keyword:`!try` block of a :keyword:`try`...\ :keyword:`finally` statement; read :pep:`255` for a full explanation of the interaction between :keyword:`!yield` and exceptions.) -Here's a sample usage of the :func:`generate_ints` generator:: +Here's a sample usage of the :func:`!generate_ints` generator:: >>> gen = generate_ints(3) >>> gen @@ -641,7 +641,7 @@ like:: sentence := "Store it in the neighboring harbor" if (i := find("or", sentence)) > 5 then write(i) -In Icon the :func:`find` function returns the indexes at which the substring +In Icon the :func:`!find` function returns the indexes at which the substring "or" is found: 3, 23, 33. In the :keyword:`if` statement, ``i`` is first assigned a value of 3, but 3 is less than 5, so the comparison fails, and Icon retries it with the second value of 23. 23 is greater than 5, so the comparison @@ -671,7 +671,7 @@ PEP 237: Unifying Long Integers and Integers In recent versions, the distinction between regular integers, which are 32-bit values on most machines, and long integers, which can be of arbitrary size, was becoming an annoyance. For example, on platforms that support files larger than -``2**32`` bytes, the :meth:`tell` method of file objects has to return a long +``2**32`` bytes, the :meth:`!tell` method of file objects has to return a long integer. However, there were various bits of Python that expected plain integers and would raise an error if a long integer was provided instead. For example, in Python 1.5, only regular integers could be used as a slice index, and @@ -752,7 +752,7 @@ Here are the changes 2.2 introduces: 0.5. Without the ``__future__`` statement, ``/`` still means classic division. The default meaning of ``/`` will not change until Python 3.0. -* Classes can define methods called :meth:`__truediv__` and :meth:`__floordiv__` +* Classes can define methods called :meth:`~object.__truediv__` and :meth:`~object.__floordiv__` to overload the two division operators. At the C level, there are also slots in the :c:type:`PyNumberMethods` structure so extension types can define the two operators. @@ -785,17 +785,17 @@ support.) When built to use UCS-4 (a "wide Python"), the interpreter can natively handle Unicode characters from U+000000 to U+110000, so the range of legal values for -the :func:`unichr` function is expanded accordingly. Using an interpreter +the :func:`!unichr` function is expanded accordingly. Using an interpreter compiled to use UCS-2 (a "narrow Python"), values greater than 65535 will still -cause :func:`unichr` to raise a :exc:`ValueError` exception. This is all +cause :func:`!unichr` to raise a :exc:`ValueError` exception. This is all described in :pep:`261`, "Support for 'wide' Unicode characters"; consult it for further details. Another change is simpler to explain. Since their introduction, Unicode strings -have supported an :meth:`encode` method to convert the string to a selected +have supported an :meth:`!encode` method to convert the string to a selected encoding such as UTF-8 or Latin-1. A symmetric ``decode([*encoding*])`` method has been added to 8-bit strings (though not to Unicode strings) in 2.2. -:meth:`decode` assumes that the string is in the specified encoding and decodes +:meth:`!decode` assumes that the string is in the specified encoding and decodes it, returning whatever is returned by the codec. Using this new feature, codecs have been added for tasks not directly related to @@ -819,10 +819,10 @@ encoding, and compression with the :mod:`zlib` module:: >>> "sheesh".encode('rot-13') 'furrfu' -To convert a class instance to Unicode, a :meth:`__unicode__` method can be -defined by a class, analogous to :meth:`__str__`. +To convert a class instance to Unicode, a :meth:`!__unicode__` method can be +defined by a class, analogous to :meth:`!__str__`. -:meth:`encode`, :meth:`decode`, and :meth:`__unicode__` were implemented by +:meth:`!encode`, :meth:`!decode`, and :meth:`!__unicode__` were implemented by Marc-André Lemburg. The changes to support using UCS-4 internally were implemented by Fredrik Lundh and Martin von Löwis. @@ -859,7 +859,7 @@ doesn't work:: return g(value-1) + 1 ... -The function :func:`g` will always raise a :exc:`NameError` exception, because +The function :func:`!g` will always raise a :exc:`NameError` exception, because the binding of the name ``g`` isn't in either its local namespace or in the module-level namespace. This isn't much of a problem in practice (how often do you recursively define interior functions like this?), but this also made using @@ -915,7 +915,7 @@ To make the preceding explanation a bit clearer, here's an example:: Line 4 containing the ``exec`` statement is a syntax error, since ``exec`` would define a new local variable named ``x`` whose value should -be accessed by :func:`g`. +be accessed by :func:`!g`. This shouldn't be much of a limitation, since ``exec`` is rarely used in most Python code (and when it is used, it's often a sign of a poor design @@ -933,7 +933,7 @@ anyway). New and Improved Modules ======================== -* The :mod:`xmlrpclib` module was contributed to the standard library by Fredrik +* The :mod:`xmlrpclib <xmlrpc.client>` module was contributed to the standard library by Fredrik Lundh, providing support for writing XML-RPC clients. XML-RPC is a simple remote procedure call protocol built on top of HTTP and XML. For example, the following snippet retrieves a list of RSS channels from the O'Reilly Network, @@ -956,7 +956,7 @@ New and Improved Modules # 'description': 'A utility which converts HTML to XSL FO.', # 'title': 'html2fo 0.3 (Default)'}, ... ] - The :mod:`SimpleXMLRPCServer` module makes it easy to create straightforward + The :mod:`SimpleXMLRPCServer <xmlrpc.server>` module makes it easy to create straightforward XML-RPC servers. See http://xmlrpc.scripting.com/ for more information about XML-RPC. * The new :mod:`hmac` module implements the HMAC algorithm described by @@ -964,9 +964,9 @@ New and Improved Modules * Several functions that originally returned lengthy tuples now return pseudo-sequences that still behave like tuples but also have mnemonic attributes such - as memberst_mtime or :attr:`tm_year`. The enhanced functions include - :func:`stat`, :func:`fstat`, :func:`statvfs`, and :func:`fstatvfs` in the - :mod:`os` module, and :func:`localtime`, :func:`gmtime`, and :func:`strptime` in + as :attr:`!memberst_mtime` or :attr:`~time.struct_time.tm_year`. The enhanced functions include + :func:`~os.stat`, :func:`~os.fstat`, :func:`~os.statvfs`, and :func:`~os.fstatvfs` in the + :mod:`os` module, and :func:`~time.localtime`, :func:`~time.gmtime`, and :func:`~time.strptime` in the :mod:`time` module. For example, to obtain a file's size using the old tuples, you'd end up writing @@ -999,7 +999,7 @@ New and Improved Modules underlying the :mod:`re` module. For example, the :func:`re.sub` and :func:`re.split` functions have been rewritten in C. Another contributed patch speeds up certain Unicode character ranges by a factor of two, and a new - :meth:`finditer` method that returns an iterator over all the non-overlapping + :meth:`~re.finditer` method that returns an iterator over all the non-overlapping matches in a given string. (SRE is maintained by Fredrik Lundh. The BIGCHARSET patch was contributed by Martin von Löwis.) @@ -1012,33 +1012,33 @@ New and Improved Modules new extensions: the NAMESPACE extension defined in :rfc:`2342`, SORT, GETACL and SETACL. (Contributed by Anthony Baxter and Michel Pelletier.) -* The :mod:`rfc822` module's parsing of email addresses is now compliant with +* The :mod:`!rfc822` module's parsing of email addresses is now compliant with :rfc:`2822`, an update to :rfc:`822`. (The module's name is *not* going to be changed to ``rfc2822``.) A new package, :mod:`email`, has also been added for parsing and generating e-mail messages. (Contributed by Barry Warsaw, and arising out of his work on Mailman.) -* The :mod:`difflib` module now contains a new :class:`Differ` class for +* The :mod:`difflib` module now contains a new :class:`!Differ` class for producing human-readable lists of changes (a "delta") between two sequences of - lines of text. There are also two generator functions, :func:`ndiff` and - :func:`restore`, which respectively return a delta from two sequences, or one of + lines of text. There are also two generator functions, :func:`!ndiff` and + :func:`!restore`, which respectively return a delta from two sequences, or one of the original sequences from a delta. (Grunt work contributed by David Goodger, from ndiff.py code by Tim Peters who then did the generatorization.) -* New constants :const:`ascii_letters`, :const:`ascii_lowercase`, and - :const:`ascii_uppercase` were added to the :mod:`string` module. There were - several modules in the standard library that used :const:`string.letters` to +* New constants :const:`!ascii_letters`, :const:`!ascii_lowercase`, and + :const:`!ascii_uppercase` were added to the :mod:`string` module. There were + several modules in the standard library that used :const:`!string.letters` to mean the ranges A-Za-z, but that assumption is incorrect when locales are in - use, because :const:`string.letters` varies depending on the set of legal + use, because :const:`!string.letters` varies depending on the set of legal characters defined by the current locale. The buggy modules have all been fixed - to use :const:`ascii_letters` instead. (Reported by an unknown person; fixed by + to use :const:`!ascii_letters` instead. (Reported by an unknown person; fixed by Fred L. Drake, Jr.) * The :mod:`mimetypes` module now makes it easier to use alternative MIME-type - databases by the addition of a :class:`MimeTypes` class, which takes a list of + databases by the addition of a :class:`~mimetypes.MimeTypes` class, which takes a list of filenames to be parsed. (Contributed by Fred L. Drake, Jr.) -* A :class:`Timer` class was added to the :mod:`threading` module that allows +* A :class:`~threading.Timer` class was added to the :mod:`threading` module that allows scheduling an activity to happen at some future time. (Contributed by Itamar Shtull-Trauring.) @@ -1114,7 +1114,7 @@ code, none of the changes described here will affect you very much. * Two new wrapper functions, :c:func:`PyOS_snprintf` and :c:func:`PyOS_vsnprintf` were added to provide cross-platform implementations for the relatively new :c:func:`snprintf` and :c:func:`vsnprintf` C lib APIs. In contrast to the standard - :c:func:`sprintf` and :c:func:`vsprintf` functions, the Python versions check the + :c:func:`sprintf` and :c:func:`!vsprintf` functions, the Python versions check the bounds of the buffer used to protect against buffer overruns. (Contributed by M.-A. Lemburg.) @@ -1212,12 +1212,12 @@ Some of the more notable changes are: * The :file:`Tools/scripts/ftpmirror.py` script now parses a :file:`.netrc` file, if you have one. (Contributed by Mike Romberg.) -* Some features of the object returned by the :func:`xrange` function are now +* Some features of the object returned by the :func:`!xrange` function are now deprecated, and trigger warnings when they're accessed; they'll disappear in - Python 2.3. :class:`xrange` objects tried to pretend they were full sequence + Python 2.3. :class:`!xrange` objects tried to pretend they were full sequence types by supporting slicing, sequence multiplication, and the :keyword:`in` operator, but these features were rarely used and therefore buggy. The - :meth:`tolist` method and the :attr:`start`, :attr:`stop`, and :attr:`step` + :meth:`!tolist` method and the :attr:`!start`, :attr:`!stop`, and :attr:`!step` attributes are also being deprecated. At the C level, the fourth argument to the :c:func:`!PyRange_New` function, ``repeat``, has also been deprecated. diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index 15d4003622c5065..7e235d4370edaa9 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -1081,7 +1081,7 @@ complete list of changes, or look through the CVS logs for all the details. :func:`nsmallest` that use heaps to find the N largest or smallest values in a dataset without the expense of a full sort. (Contributed by Raymond Hettinger.) -* The :mod:`httplib` module now contains constants for HTTP status codes defined +* The :mod:`httplib <http>` module now contains constants for HTTP status codes defined in various HTTP-related RFC documents. Constants have names such as :const:`OK`, :const:`CREATED`, :const:`CONTINUE`, and :const:`MOVED_PERMANENTLY`; use pydoc to get a full list. (Contributed by @@ -1218,10 +1218,10 @@ complete list of changes, or look through the CVS logs for all the details. now include the string ``'%default'``, which will be replaced by the option's default value. (Contributed by Greg Ward.) -* The long-term plan is to deprecate the :mod:`rfc822` module in some future +* The long-term plan is to deprecate the :mod:`!rfc822` module in some future Python release in favor of the :mod:`email` package. To this end, the - :func:`email.Utils.formatdate` function has been changed to make it usable as a - replacement for :func:`rfc822.formatdate`. You may want to write new e-mail + :func:`email.Utils.formatdate <email.utils.formatdate>` function has been changed to make it usable as a + replacement for :func:`!rfc822.formatdate`. You may want to write new e-mail processing code with this in mind. (Change implemented by Anthony Baxter.) * A new ``urandom(n)`` function was added to the :mod:`os` module, returning @@ -1308,7 +1308,7 @@ complete list of changes, or look through the CVS logs for all the details. sockets, and regular expression pattern objects. (Contributed by Raymond Hettinger.) -* The :mod:`xmlrpclib` module now supports a multi-call extension for +* The :mod:`xmlrpclib <xmlrpc.client>` module now supports a multi-call extension for transmitting multiple XML-RPC calls in a single HTTP operation. (Contributed by Brian Quinlan.) @@ -1323,8 +1323,8 @@ complete list of changes, or look through the CVS logs for all the details. cookielib --------- -The :mod:`cookielib` library supports client-side handling for HTTP cookies, -mirroring the :mod:`Cookie` module's server-side cookie support. Cookies are +The :mod:`cookielib <http.cookiejar>` library supports client-side handling for HTTP cookies, +mirroring the :mod:`Cookie <http.cookies>` module's server-side cookie support. Cookies are stored in cookie jars; the library transparently stores cookies offered by the web server in the cookie jar, and fetches the cookie from the jar when connecting to the server. As in web browsers, policy objects control whether @@ -1335,7 +1335,7 @@ are provided: one that stores cookies in the Netscape format so applications can use the Mozilla or Lynx cookie files, and one that stores cookies in the same format as the Perl libwww library. -:mod:`urllib2` has been changed to interact with :mod:`cookielib`: +:mod:`urllib2 <urllib.request>` has been changed to interact with :mod:`cookielib <http.cookiejar>`: :class:`HTTPCookieProcessor` manages a cookie jar that is used when accessing URLs. diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index f45d70ea5a19a03..2ae26e7a106a0bf 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -1478,8 +1478,8 @@ complete list of changes, or look through the SVN logs for all the details. .. Patch 790710 -* The :mod:`pickle` and :mod:`cPickle` modules no longer accept a return value - of ``None`` from the :meth:`__reduce__` method; the method must return a tuple +* The :mod:`pickle` and :mod:`!cPickle` modules no longer accept a return value + of ``None`` from the :meth:`~object.__reduce__` method; the method must return a tuple of arguments instead. The ability to return ``None`` was deprecated in Python 2.4, so this completes the removal of the feature. @@ -1519,7 +1519,7 @@ complete list of changes, or look through the SVN logs for all the details. .. Patch #1472854 -* The :mod:`SimpleXMLRPCServer` and :mod:`DocXMLRPCServer` classes now have a +* The :mod:`SimpleXMLRPCServer <xmlrpc.server>` and :mod:`DocXMLRPCServer <xmlrpc.server>` classes now have a :attr:`rpc_paths` attribute that constrains XML-RPC operations to a limited set of URL paths; the default is to allow only ``'/'`` and ``'/RPC2'``. Setting :attr:`rpc_paths` to ``None`` or an empty tuple disables this path checking. @@ -1650,9 +1650,9 @@ complete list of changes, or look through the SVN logs for all the details. .. Patch #754022 -* The :mod:`xmlrpclib` module now supports returning :class:`~datetime.datetime` objects - for the XML-RPC date type. Supply ``use_datetime=True`` to the :func:`loads` - function or the :class:`Unmarshaller` class to enable this feature. (Contributed +* The :mod:`xmlrpclib <xmlrpc.client>` module now supports returning :class:`~datetime.datetime` objects + for the XML-RPC date type. Supply ``use_datetime=True`` to the :func:`~xmlrpc.client.loads` + function or the :class:`!Unmarshaller` class to enable this feature. (Contributed by Skip Montanaro.) .. Patch 1120353 @@ -2253,12 +2253,12 @@ code: appeared. In Python 2.5, the argument must be exactly one %char specifier with no surrounding text. -* Library: The :mod:`pickle` and :mod:`cPickle` modules no longer accept a - return value of ``None`` from the :meth:`__reduce__` method; the method must +* Library: The :mod:`pickle` and :mod:`!cPickle` modules no longer accept a + return value of ``None`` from the :meth:`~object.__reduce__` method; the method must return a tuple of arguments instead. The modules also no longer accept the deprecated *bin* keyword parameter. -* Library: The :mod:`SimpleXMLRPCServer` and :mod:`DocXMLRPCServer` classes now +* Library: The :mod:`SimpleXMLRPCServer <xmlrpc.server>` and :mod:`DocXMLRPCServer <xmlrpc.server>` classes now have a :attr:`rpc_paths` attribute that constrains XML-RPC operations to a limited set of URL paths; the default is to allow only ``'/'`` and ``'/RPC2'``. Setting :attr:`rpc_paths` to ``None`` or an empty tuple disables this path diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index c6bab93b7efdda1..e4ade5ecd82b9d4 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -1082,7 +1082,7 @@ the :mod:`io` module: (In Python 2.6, :class:`io.StringIO` is implemented in pure Python, so it's pretty slow. You should therefore stick with the - existing :mod:`StringIO` module or :mod:`cStringIO` for now. At some + existing :mod:`!StringIO` module or :mod:`!cStringIO` for now. At some point Python 3.0's :mod:`io` module will be rewritten into C for speed, and perhaps the C implementation will be backported to the 2.x releases.) @@ -1807,7 +1807,7 @@ changes, or look through the Subversion logs for all the details. Nubis; :issue:`1817`.) The :func:`parse_qs` and :func:`parse_qsl` functions have been - relocated from the :mod:`!cgi` module to the :mod:`urlparse` module. + relocated from the :mod:`!cgi` module to the :mod:`urlparse <urllib.parse>` module. The versions still available in the :mod:`!cgi` module will trigger :exc:`PendingDeprecationWarning` messages in 2.6 (:issue:`600362`). @@ -1895,8 +1895,8 @@ changes, or look through the Subversion logs for all the details. (Contributed by Raymond Hettinger.) -* The :mod:`Cookie` module's :class:`Morsel` objects now support an - :attr:`httponly` attribute. In some browsers. cookies with this attribute +* The :mod:`Cookie <http.cookies>` module's :class:`~http.cookies.Morsel` objects now support an + :attr:`~http.cookies.Morsel.httponly` attribute. In some browsers. cookies with this attribute set cannot be accessed or manipulated by JavaScript code. (Contributed by Arvin Schnell; :issue:`1638033`.) @@ -1987,8 +1987,8 @@ changes, or look through the Subversion logs for all the details. (Contributed by Raymond Hettinger.) * An optional ``timeout`` parameter, specifying a timeout measured in - seconds, was added to the :class:`httplib.HTTPConnection` and - :class:`HTTPSConnection` class constructors. (Added by Facundo + seconds, was added to the :class:`httplib.HTTPConnection <http.client.HTTPConnection>` and + :class:`HTTPSConnection <http.client.HTTPSConnection>` class constructors. (Added by Facundo Batista.) * Most of the :mod:`inspect` module's functions, such as @@ -2371,10 +2371,10 @@ changes, or look through the Subversion logs for all the details. ``socket(socket.AF_INET, ...)`` may be all that's required to make your code work with IPv6. -* The base classes in the :mod:`SocketServer` module now support - calling a :meth:`handle_timeout` method after a span of inactivity - specified by the server's :attr:`timeout` attribute. (Contributed - by Michael Pomraning.) The :meth:`serve_forever` method +* The base classes in the :mod:`SocketServer <socketserver>` module now support + calling a :meth:`~socketserver.BaseServer.handle_timeout` method after a span of inactivity + specified by the server's :attr:`~socketserver.BaseServer.timeout` attribute. (Contributed + by Michael Pomraning.) The :meth:`~socketserver.BaseServer.serve_forever` method now takes an optional poll interval measured in seconds, controlling how often the server will check for a shutdown request. (Contributed by Pedro Werneck and Jeffrey Yasskin; @@ -2388,11 +2388,11 @@ changes, or look through the Subversion logs for all the details. using the format character ``'?'``. (Contributed by David Remahl.) -* The :class:`Popen` objects provided by the :mod:`subprocess` module - now have :meth:`terminate`, :meth:`kill`, and :meth:`send_signal` methods. - On Windows, :meth:`send_signal` only supports the :const:`SIGTERM` +* The :class:`~subprocess.Popen` objects provided by the :mod:`subprocess` module + now have :meth:`~subprocess.Popen.terminate`, :meth:`~subprocess.Popen.kill`, and :meth:`~subprocess.Popen.send_signal` methods. + On Windows, :meth:`!send_signal` only supports the :py:const:`~signal.SIGTERM` signal, and all these methods are aliases for the Win32 API function - :c:func:`TerminateProcess`. + :c:func:`!TerminateProcess`. (Contributed by Christian Heimes.) * A new variable in the :mod:`sys` module, :attr:`float_info`, is an @@ -2478,9 +2478,9 @@ changes, or look through the Subversion logs for all the details. ``with tempfile.NamedTemporaryFile() as tmp: ...``. (Contributed by Alexander Belopolsky; :issue:`2021`.) -* The :mod:`test.test_support` module gained a number +* The :mod:`test.test_support <test.support>` module gained a number of context managers useful for writing tests. - :func:`EnvironmentVarGuard` is a + :func:`~test.support.os_helper.EnvironmentVarGuard` is a context manager that temporarily changes environment variables and automatically restores them to their old values. @@ -2577,9 +2577,9 @@ changes, or look through the Subversion logs for all the details. (:issue:`1513695`) * An optional ``timeout`` parameter was added to the - :func:`urllib.urlopen` function and the + :func:`urllib.urlopen <urllib.request.urlopen>` function and the :class:`urllib.ftpwrapper` class constructor, as well as the - :func:`urllib2.urlopen` function. The parameter specifies a timeout + :func:`urllib2.urlopen <urllib.request.urlopen>` function. The parameter specifies a timeout measured in seconds. For example:: >>> u = urllib2.urlopen("http://slow.example.com", @@ -2604,7 +2604,7 @@ changes, or look through the Subversion logs for all the details. intended for testing purposes that lets you temporarily modify the warning filters and then restore their original values (:issue:`3781`). -* The XML-RPC :class:`SimpleXMLRPCServer` and :class:`DocXMLRPCServer` +* The XML-RPC :class:`SimpleXMLRPCServer <xmlrpc.server>` and :class:`DocXMLRPCServer <xmlrpc.server>` classes can now be prevented from immediately opening and binding to their socket by passing ``False`` as the *bind_and_activate* constructor parameter. This can be used to modify the instance's @@ -2621,11 +2621,11 @@ changes, or look through the Subversion logs for all the details. information. (Contributed by Alan McIntyre as part of his project for Google's Summer of Code 2007.) -* The :mod:`xmlrpclib` module no longer automatically converts +* The :mod:`xmlrpclib <xmlrpc.client>` module no longer automatically converts :class:`datetime.date` and :class:`datetime.time` to the - :class:`xmlrpclib.DateTime` type; the conversion semantics were + :class:`xmlrpclib.DateTime <xmlrpc.client.DateTime>` type; the conversion semantics were not necessarily correct for all applications. Code using - :mod:`xmlrpclib` should convert :class:`date` and :class:`~datetime.time` + :mod:`!xmlrpclib` should convert :class:`date` and :class:`~datetime.time` instances. (:issue:`1330538`) The code can also handle dates before 1900 (contributed by Ralf Schmitt; :issue:`2014`) and 64-bit integers represented by using ``<i8>`` in XML-RPC responses @@ -2992,6 +2992,33 @@ Changes to Python's build process and to the C API include: architectures (x86, PowerPC), 64-bit (x86-64 and PPC-64), or both. (Contributed by Ronald Oussoren.) +* A new function added in Python 2.6.6, :c:func:`!PySys_SetArgvEx`, sets + the value of ``sys.argv`` and can optionally update ``sys.path`` to + include the directory containing the script named by ``sys.argv[0]`` + depending on the value of an *updatepath* parameter. + + This function was added to close a security hole for applications + that embed Python. The old function, :c:func:`!PySys_SetArgv`, would + always update ``sys.path``, and sometimes it would add the current + directory. This meant that, if you ran an application embedding + Python in a directory controlled by someone else, attackers could + put a Trojan-horse module in the directory (say, a file named + :file:`os.py`) that your application would then import and run. + + If you maintain a C/C++ application that embeds Python, check + whether you're calling :c:func:`!PySys_SetArgv` and carefully consider + whether the application should be using :c:func:`!PySys_SetArgvEx` + with *updatepath* set to false. Note that using this function will + break compatibility with Python versions 2.6.5 and earlier; if you + have to continue working with earlier versions, you can leave + the call to :c:func:`!PySys_SetArgv` alone and call + ``PyRun_SimpleString("sys.path.pop(0)\n")`` afterwards to discard + the first ``sys.path`` component. + + Security issue reported as `CVE-2008-5983 + <http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2008-5983>`_; + discussed in :gh:`50003`, and fixed by Antoine Pitrou. + * The BerkeleyDB module now has a C API object, available as ``bsddb.db.api``. This object can be used by other C extensions that wish to use the :mod:`bsddb` module for their own purposes. @@ -3274,11 +3301,11 @@ that may require changes to your code: :exc:`StandardError` but now it is, through :exc:`IOError`. (Implemented by Gregory P. Smith; :issue:`1706815`.) -* The :mod:`xmlrpclib` module no longer automatically converts +* The :mod:`xmlrpclib <xmlrpc.client>` module no longer automatically converts :class:`datetime.date` and :class:`datetime.time` to the - :class:`xmlrpclib.DateTime` type; the conversion semantics were + :class:`xmlrpclib.DateTime <xmlrpc.client.DateTime>` type; the conversion semantics were not necessarily correct for all applications. Code using - :mod:`xmlrpclib` should convert :class:`date` and :class:`~datetime.time` + :mod:`!xmlrpclib` should convert :class:`date` and :class:`~datetime.time` instances. (:issue:`1330538`) * (3.0-warning mode) The :class:`Exception` class now warns @@ -3294,6 +3321,15 @@ that may require changes to your code: scoping rules, also cause warnings because such comparisons are forbidden entirely in 3.0. +For applications that embed Python: + +* The :c:func:`!PySys_SetArgvEx` function was added in Python 2.6.6, + letting applications close a security hole when the existing + :c:func:`!PySys_SetArgv` function was used. Check whether you're + calling :c:func:`!PySys_SetArgv` and carefully consider whether the + application should be using :c:func:`!PySys_SetArgvEx` with + *updatepath* set to false. + .. ====================================================================== diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 524967b45242344..2a42664c02852c2 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -196,7 +196,7 @@ A partial list of 3.1 features that were backported to 2.7: Other new Python3-mode warnings include: -* :func:`operator.isCallable` and :func:`operator.sequenceIncludes`, +* :func:`!operator.isCallable` and :func:`!operator.sequenceIncludes`, which are not supported in 3.x, now trigger warnings. * The :option:`!-3` switch now automatically enables the :option:`!-Qwarn` switch that causes warnings @@ -455,11 +455,11 @@ a varying number of handlers. All this flexibility can require a lot of configuration. You can write Python statements to create objects and set their properties, but a complex set-up requires verbose but boring code. -:mod:`logging` also supports a :func:`~logging.fileConfig` +:mod:`logging` also supports a :func:`~logging.config.fileConfig` function that parses a file, but the file format doesn't support configuring filters, and it's messier to generate programmatically. -Python 2.7 adds a :func:`~logging.dictConfig` function that +Python 2.7 adds a :func:`~logging.config.dictConfig` function that uses a dictionary to configure logging. There are many ways to produce a dictionary from different sources: construct one with code; parse a file containing JSON; or use a YAML parsing library if one is @@ -533,7 +533,7 @@ implemented by Vinay Sajip, are: ``getLogger('app.network.listen')``. * The :class:`~logging.LoggerAdapter` class gained an - :meth:`~logging.LoggerAdapter.isEnabledFor` method that takes a + :meth:`~logging.Logger.isEnabledFor` method that takes a *level* and returns whether the underlying logger would process a message of that level of importance. @@ -554,8 +554,8 @@ called a :dfn:`view` instead of a fully materialized list. It's not possible to change the return values of :meth:`~dict.keys`, :meth:`~dict.values`, and :meth:`~dict.items` in Python 2.7 because too much code would break. Instead the 3.x versions were added -under the new names :meth:`~dict.viewkeys`, :meth:`~dict.viewvalues`, -and :meth:`~dict.viewitems`. +under the new names :meth:`!viewkeys`, :meth:`!viewvalues`, +and :meth:`!viewitems`. :: @@ -720,7 +720,7 @@ Some smaller changes made to the core Python language are: with B() as b: ... suite of statements ... - The :func:`contextlib.nested` function provides a very similar + The :func:`!contextlib.nested` function provides a very similar function, so it's no longer necessary and has been deprecated. (Proposed in https://codereview.appspot.com/53094; implemented by @@ -785,7 +785,7 @@ Some smaller changes made to the core Python language are: implemented by Mark Dickinson; :issue:`1811`.) * Implicit coercion for complex numbers has been removed; the interpreter - will no longer ever attempt to call a :meth:`__coerce__` method on complex + will no longer ever attempt to call a :meth:`!__coerce__` method on complex objects. (Removed by Meador Inge and Mark Dickinson; :issue:`5211`.) * The :meth:`str.format` method now supports automatic numbering of the replacement @@ -817,7 +817,7 @@ Some smaller changes made to the core Python language are: A low-level change: the :meth:`object.__format__` method now triggers a :exc:`PendingDeprecationWarning` if it's passed a format string, - because the :meth:`__format__` method for :class:`object` converts + because the :meth:`!__format__` method for :class:`object` converts the object to a string representation and formats that. Previously the method silently applied the format string to the string representation, but that could hide mistakes in Python code. If @@ -825,7 +825,7 @@ Some smaller changes made to the core Python language are: precision, presumably you're expecting the formatting to be applied in some object-specific way. (Fixed by Eric Smith; :issue:`7994`.) -* The :func:`int` and :func:`long` types gained a ``bit_length`` +* The :func:`int` and :func:`!long` types gained a ``bit_length`` method that returns the number of bits necessary to represent its argument in binary:: @@ -848,8 +848,8 @@ Some smaller changes made to the core Python language are: statements that were only working by accident. (Fixed by Meador Inge; :issue:`7902`.) -* It's now possible for a subclass of the built-in :class:`unicode` type - to override the :meth:`__unicode__` method. (Implemented by +* It's now possible for a subclass of the built-in :class:`!unicode` type + to override the :meth:`!__unicode__` method. (Implemented by Victor Stinner; :issue:`1583863`.) * The :class:`bytearray` type's :meth:`~bytearray.translate` method now accepts @@ -876,7 +876,7 @@ Some smaller changes made to the core Python language are: Forgeot d'Arc in :issue:`1616979`; CP858 contributed by Tim Hatch in :issue:`8016`.) -* The :class:`file` object will now set the :attr:`filename` attribute +* The :class:`!file` object will now set the :attr:`!filename` attribute on the :exc:`IOError` exception when trying to open a directory on POSIX platforms (noted by Jan Kaliszewski; :issue:`4764`), and now explicitly checks for and forbids writing to read-only file objects @@ -915,7 +915,7 @@ used with the :option:`-W` switch, separated by commas. (Contributed by Brian Curtin; :issue:`7301`.) For example, the following setting will print warnings every time -they occur, but turn warnings from the :mod:`Cookie` module into an +they occur, but turn warnings from the :mod:`Cookie <http.cookies>` module into an error. (The exact syntax for setting an environment variable varies across operating systems and shells.) @@ -966,7 +966,7 @@ Several performance enhancements have been added: Apart from the performance improvements this change should be invisible to end users, with one exception: for testing and - debugging purposes there's a new structseq :data:`sys.long_info` that + debugging purposes there's a new structseq :data:`!sys.long_info` that provides information about the internal format, giving the number of bits per digit and the size in bytes of the C type used to store each digit:: @@ -1005,19 +1005,19 @@ Several performance enhancements have been added: conversion function that supports arbitrary bases. (Patch by Gawain Bolton; :issue:`6713`.) -* The :meth:`split`, :meth:`replace`, :meth:`rindex`, - :meth:`rpartition`, and :meth:`rsplit` methods of string-like types +* The :meth:`!split`, :meth:`!replace`, :meth:`!rindex`, + :meth:`!rpartition`, and :meth:`!rsplit` methods of string-like types (strings, Unicode strings, and :class:`bytearray` objects) now use a fast reverse-search algorithm instead of a character-by-character scan. This is sometimes faster by a factor of 10. (Added by Florent Xicluna; :issue:`7462` and :issue:`7622`.) -* The :mod:`pickle` and :mod:`cPickle` modules now automatically +* The :mod:`pickle` and :mod:`!cPickle` modules now automatically intern the strings used for attribute names, reducing memory usage of the objects resulting from unpickling. (Contributed by Jake McGuire; :issue:`5084`.) -* The :mod:`cPickle` module now special-cases dictionaries, +* The :mod:`!cPickle` module now special-cases dictionaries, nearly halving the time required to pickle them. (Contributed by Collin Winter; :issue:`5670`.) @@ -1044,7 +1044,7 @@ changes, or look through the Subversion logs for all the details. used with :class:`memoryview` instances and other similar buffer objects. (Backported from 3.x by Florent Xicluna; :issue:`7703`.) -* Updated module: the :mod:`bsddb` module has been updated from 4.7.2devel9 +* Updated module: the :mod:`!bsddb` module has been updated from 4.7.2devel9 to version 4.8.4 of `the pybsddb package <https://www.jcea.es/programacion/pybsddb.htm>`__. The new version features better Python 3.x compatibility, various bug fixes, @@ -1129,7 +1129,7 @@ changes, or look through the Subversion logs for all the details. (Added by Raymond Hettinger; :issue:`1818`.) - Finally, the :class:`~collections.Mapping` abstract base class now + Finally, the :class:`~collections.abc.Mapping` abstract base class now returns :const:`NotImplemented` if a mapping is compared to another type that isn't a :class:`Mapping`. (Fixed by Daniel Stutzbach; :issue:`8729`.) @@ -1158,12 +1158,12 @@ changes, or look through the Subversion logs for all the details. (Contributed by Mats Kindahl; :issue:`7005`.) -* Deprecated function: :func:`contextlib.nested`, which allows +* Deprecated function: :func:`!contextlib.nested`, which allows handling more than one context manager with a single :keyword:`with` statement, has been deprecated, because the :keyword:`!with` statement now supports multiple context managers. -* The :mod:`cookielib` module now ignores cookies that have an invalid +* The :mod:`cookielib <http.cookiejar>` module now ignores cookies that have an invalid version field, one that doesn't contain an integer value. (Fixed by John J. Lee; :issue:`3924`.) @@ -1184,7 +1184,7 @@ changes, or look through the Subversion logs for all the details. * New method: the :class:`~decimal.Decimal` class gained a :meth:`~decimal.Decimal.from_float` class method that performs an exact - conversion of a floating-point number to a :class:`~decimal.Decimal`. + conversion of a floating-point number to a :class:`!Decimal`. This exact conversion strives for the closest decimal approximation to the floating-point representation's value; the resulting decimal value will therefore still include the inaccuracy, @@ -1198,9 +1198,9 @@ changes, or look through the Subversion logs for all the details. of the operands. Previously such comparisons would fall back to Python's default rules for comparing objects, which produced arbitrary results based on their type. Note that you still cannot combine - :class:`Decimal` and floating-point in other operations such as addition, + :class:`!Decimal` and floating-point in other operations such as addition, since you should be explicitly choosing how to convert between float and - :class:`~decimal.Decimal`. (Fixed by Mark Dickinson; :issue:`2531`.) + :class:`!Decimal`. (Fixed by Mark Dickinson; :issue:`2531`.) The constructor for :class:`~decimal.Decimal` now accepts floating-point numbers (added by Raymond Hettinger; :issue:`8257`) @@ -1218,7 +1218,7 @@ changes, or look through the Subversion logs for all the details. more sensible for numeric types. (Changed by Mark Dickinson; :issue:`6857`.) Comparisons involving a signaling NaN value (or ``sNAN``) now signal - :const:`InvalidOperation` instead of silently returning a true or + :const:`~decimal.InvalidOperation` instead of silently returning a true or false value depending on the comparison operator. Quiet NaN values (or ``NaN``) are now hashable. (Fixed by Mark Dickinson; :issue:`7279`.) @@ -1235,13 +1235,13 @@ changes, or look through the Subversion logs for all the details. created some new files that should be included. (Fixed by Tarek Ziadé; :issue:`8688`.) -* The :mod:`doctest` module's :const:`IGNORE_EXCEPTION_DETAIL` flag +* The :mod:`doctest` module's :const:`~doctest.IGNORE_EXCEPTION_DETAIL` flag will now ignore the name of the module containing the exception being tested. (Patch by Lennart Regebro; :issue:`7490`.) * The :mod:`email` module's :class:`~email.message.Message` class will now accept a Unicode-valued payload, automatically converting the - payload to the encoding specified by :attr:`output_charset`. + payload to the encoding specified by :attr:`!output_charset`. (Added by R. David Murray; :issue:`1368247`.) * The :class:`~fractions.Fraction` class now accepts a single float or @@ -1268,10 +1268,10 @@ changes, or look through the Subversion logs for all the details. :issue:`6845`.) * New class decorator: :func:`~functools.total_ordering` in the :mod:`functools` - module takes a class that defines an :meth:`__eq__` method and one of - :meth:`__lt__`, :meth:`__le__`, :meth:`__gt__`, or :meth:`__ge__`, + module takes a class that defines an :meth:`~object.__eq__` method and one of + :meth:`~object.__lt__`, :meth:`~object.__le__`, :meth:`~object.__gt__`, or :meth:`~object.__ge__`, and generates the missing comparison methods. Since the - :meth:`__cmp__` method is being deprecated in Python 3.x, + :meth:`!__cmp__` method is being deprecated in Python 3.x, this decorator makes it easier to define ordered classes. (Added by Raymond Hettinger; :issue:`5479`.) @@ -1300,17 +1300,17 @@ changes, or look through the Subversion logs for all the details. :mod:`gzip` module will now consume these trailing bytes. (Fixed by Tadek Pietraszek and Brian Curtin; :issue:`2846`.) -* New attribute: the :mod:`hashlib` module now has an :attr:`~hashlib.hashlib.algorithms` +* New attribute: the :mod:`hashlib` module now has an :attr:`!algorithms` attribute containing a tuple naming the supported algorithms. In Python 2.7, ``hashlib.algorithms`` contains ``('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')``. (Contributed by Carl Chenet; :issue:`7418`.) -* The default :class:`~httplib.HTTPResponse` class used by the :mod:`httplib` module now +* The default :class:`~http.client.HTTPResponse` class used by the :mod:`httplib <http>` module now supports buffering, resulting in much faster reading of HTTP responses. (Contributed by Kristján Valur Jónsson; :issue:`4879`.) - The :class:`~httplib.HTTPConnection` and :class:`~httplib.HTTPSConnection` classes + The :class:`~http.client.HTTPConnection` and :class:`~http.client.HTTPSConnection` classes now support a *source_address* parameter, a ``(host, port)`` 2-tuple giving the source address that will be used for the connection. (Contributed by Eldon Ziegler; :issue:`3972`.) @@ -1348,10 +1348,10 @@ changes, or look through the Subversion logs for all the details. * Updated module: The :mod:`io` library has been upgraded to the version shipped with Python 3.1. For 3.1, the I/O library was entirely rewritten in C and is 2 to 20 times faster depending on the task being performed. The - original Python version was renamed to the :mod:`_pyio` module. + original Python version was renamed to the :mod:`!_pyio` module. One minor resulting change: the :class:`io.TextIOBase` class now - has an :attr:`errors` attribute giving the error setting + has an :attr:`~io.TextIOBase.errors` attribute giving the error setting used for encoding and decoding errors (one of ``'strict'``, ``'replace'``, ``'ignore'``). @@ -1423,10 +1423,10 @@ changes, or look through the Subversion logs for all the details. passed to the callable. (Contributed by lekma; :issue:`5585`.) - The :class:`~multiprocessing.Pool` class, which controls a pool of worker processes, + The :class:`~multiprocessing.pool.Pool` class, which controls a pool of worker processes, now has an optional *maxtasksperchild* parameter. Worker processes will perform the specified number of tasks and then exit, causing the - :class:`~multiprocessing.Pool` to start a new worker. This is useful if tasks may leak + :class:`!Pool` to start a new worker. This is useful if tasks may leak memory or other resources, or if some tasks will cause the worker to become very large. (Contributed by Charles Cazabon; :issue:`6963`.) @@ -1498,7 +1498,7 @@ changes, or look through the Subversion logs for all the details. global site-packages directories, :func:`~site.getusersitepackages` returns the path of the user's site-packages directory, and - :func:`~site.getuserbase` returns the value of the :envvar:`USER_BASE` + :func:`~site.getuserbase` returns the value of the :data:`~site.USER_BASE` environment variable, giving the path to a directory that can be used to store data. (Contributed by Tarek Ziadé; :issue:`6693`.) @@ -1518,16 +1518,16 @@ changes, or look through the Subversion logs for all the details. the :class:`bytearray` and :class:`memoryview` objects. (Implemented by Antoine Pitrou; :issue:`8104`.) -* The :mod:`SocketServer` module's :class:`~SocketServer.TCPServer` class now +* The :mod:`SocketServer <socketserver>` module's :class:`~socketserver.TCPServer` class now supports socket timeouts and disabling the Nagle algorithm. - The :attr:`~SocketServer.TCPServer.disable_nagle_algorithm` class attribute + The :attr:`!disable_nagle_algorithm` class attribute defaults to ``False``; if overridden to be true, new request connections will have the TCP_NODELAY option set to prevent buffering many small sends into a single TCP packet. - The :attr:`~SocketServer.BaseServer.timeout` class attribute can hold + The :attr:`~socketserver.BaseServer.timeout` class attribute can hold a timeout in seconds that will be applied to the request socket; if - no request is received within that time, :meth:`~SocketServer.BaseServer.handle_timeout` - will be called and :meth:`~SocketServer.BaseServer.handle_request` will return. + no request is received within that time, :meth:`~socketserver.BaseServer.handle_timeout` + will be called and :meth:`~socketserver.BaseServer.handle_request` will return. (Contributed by Kristján Valur Jónsson; :issue:`6192` and :issue:`6267`.) * Updated module: the :mod:`sqlite3` module has been updated to @@ -1540,11 +1540,11 @@ changes, or look through the Subversion logs for all the details. * The :mod:`ssl` module's :class:`~ssl.SSLSocket` objects now support the buffer API, which fixed a test suite failure (fix by Antoine Pitrou; :issue:`7133`) and automatically set - OpenSSL's :c:macro:`SSL_MODE_AUTO_RETRY`, which will prevent an error + OpenSSL's :c:macro:`!SSL_MODE_AUTO_RETRY`, which will prevent an error code being returned from :meth:`recv` operations that trigger an SSL renegotiation (fix by Antoine Pitrou; :issue:`8222`). - The :func:`ssl.wrap_socket` constructor function now takes a + The :func:`~ssl.SSLContext.wrap_socket` constructor function now takes a *ciphers* argument that's a string listing the encryption algorithms to be allowed; the format of the string is described `in the OpenSSL documentation @@ -1568,8 +1568,8 @@ changes, or look through the Subversion logs for all the details. code (one of ``bBhHiIlLqQ``); it now always raises a :exc:`struct.error` exception. (Changed by Mark Dickinson; :issue:`1523`.) The :func:`~struct.pack` function will also - attempt to use :meth:`__index__` to convert and pack non-integers - before trying the :meth:`__int__` method or reporting an error. + attempt to use :meth:`~object.__index__` to convert and pack non-integers + before trying the :meth:`~object.__int__` method or reporting an error. (Changed by Mark Dickinson; :issue:`8300`.) * New function: the :mod:`subprocess` module's @@ -1590,7 +1590,7 @@ changes, or look through the Subversion logs for all the details. (Contributed by Gregory P. Smith.) The :mod:`subprocess` module will now retry its internal system calls - on receiving an :const:`EINTR` signal. (Reported by several people; final + on receiving an :const:`~errno.EINTR` signal. (Reported by several people; final patch by Gregory P. Smith in :issue:`1068268`.) * New function: :func:`~symtable.Symbol.is_declared_global` in the :mod:`symtable` module @@ -1602,16 +1602,16 @@ changes, or look through the Subversion logs for all the details. identifier instead of the previous default value of ``'python'``. (Changed by Sean Reifschneider; :issue:`8451`.) -* The ``sys.version_info`` value is now a named tuple, with attributes - named :attr:`major`, :attr:`minor`, :attr:`micro`, - :attr:`releaselevel`, and :attr:`serial`. (Contributed by Ross +* The :attr:`sys.version_info` value is now a named tuple, with attributes + named :attr:`!major`, :attr:`!minor`, :attr:`!micro`, + :attr:`!releaselevel`, and :attr:`!serial`. (Contributed by Ross Light; :issue:`4285`.) :func:`sys.getwindowsversion` also returns a named tuple, - with attributes named :attr:`major`, :attr:`minor`, :attr:`build`, - :attr:`platform`, :attr:`service_pack`, :attr:`service_pack_major`, - :attr:`service_pack_minor`, :attr:`suite_mask`, and - :attr:`product_type`. (Contributed by Brian Curtin; :issue:`7766`.) + with attributes named :attr:`!major`, :attr:`!minor`, :attr:`!build`, + :attr:`!platform`, :attr:`!service_pack`, :attr:`!service_pack_major`, + :attr:`!service_pack_minor`, :attr:`!suite_mask`, and + :attr:`!product_type`. (Contributed by Brian Curtin; :issue:`7766`.) * The :mod:`tarfile` module's default error handling has changed, to no longer suppress fatal errors. The default error level was previously 0, @@ -1648,7 +1648,7 @@ changes, or look through the Subversion logs for all the details. and has been updated to version 5.2.0 (updated by Florent Xicluna; :issue:`8024`). -* The :mod:`urlparse` module's :func:`~urlparse.urlsplit` now handles +* The :mod:`urlparse <urllib.parse>` module's :func:`~urllib.parse.urlsplit` now handles unknown URL schemes in a fashion compliant with :rfc:`3986`: if the URL is of the form ``"<something>://..."``, the text before the ``://`` is treated as the scheme, even if it's a made-up scheme that @@ -1675,7 +1675,7 @@ changes, or look through the Subversion logs for all the details. (Python 2.7 actually produces slightly different output, since it returns a named tuple instead of a standard tuple.) - The :mod:`urlparse` module also supports IPv6 literal addresses as defined by + The :mod:`urlparse <urllib.parse>` module also supports IPv6 literal addresses as defined by :rfc:`2732` (contributed by Senthil Kumaran; :issue:`2987`). .. doctest:: @@ -1691,18 +1691,18 @@ changes, or look through the Subversion logs for all the details. (Originally implemented in Python 3.x by Raymond Hettinger, and backported to 2.7 by Michael Foord.) -* The ElementTree library, :mod:`xml.etree`, no longer escapes +* The :mod:`xml.etree.ElementTree` library, no longer escapes ampersands and angle brackets when outputting an XML processing instruction (which looks like ``<?xml-stylesheet href="#style1"?>``) or comment (which looks like ``<!-- comment -->``). (Patch by Neil Muller; :issue:`2746`.) -* The XML-RPC client and server, provided by the :mod:`xmlrpclib` and - :mod:`SimpleXMLRPCServer` modules, have improved performance by +* The XML-RPC client and server, provided by the :mod:`xmlrpclib <xmlrpc.client>` and + :mod:`SimpleXMLRPCServer <xmlrpc.server>` modules, have improved performance by supporting HTTP/1.1 keep-alive and by optionally using gzip encoding to compress the XML being exchanged. The gzip compression is - controlled by the :attr:`encode_threshold` attribute of - :class:`SimpleXMLRPCRequestHandler`, which contains a size in bytes; + controlled by the :attr:`!encode_threshold` attribute of + :class:`~xmlrpc.server.SimpleXMLRPCRequestHandler`, which contains a size in bytes; responses larger than this will be compressed. (Contributed by Kristján Valur Jónsson; :issue:`6267`.) @@ -1713,7 +1713,8 @@ changes, or look through the Subversion logs for all the details. :mod:`zipfile` now also supports archiving empty directories and extracts them correctly. (Fixed by Kuba Wieczorek; :issue:`4710`.) Reading files out of an archive is faster, and interleaving - :meth:`~zipfile.ZipFile.read` and :meth:`~zipfile.ZipFile.readline` now works correctly. + :meth:`read() <io.BufferedIOBase.read>` and + :meth:`readline() <io.IOBase.readline>` now works correctly. (Contributed by Nir Aides; :issue:`7610`.) The :func:`~zipfile.is_zipfile` function now @@ -1807,14 +1808,14 @@ closely resemble the native platform's widgets. This widget set was originally called Tile, but was renamed to Ttk (for "themed Tk") on being added to Tcl/Tck release 8.5. -To learn more, read the :mod:`ttk` module documentation. You may also +To learn more, read the :mod:`~tkinter.ttk` module documentation. You may also wish to read the Tcl/Tk manual page describing the Ttk theme engine, available at -https://www.tcl.tk/man/tcl8.5/TkCmd/ttk_intro.htm. Some +https://www.tcl.tk/man/tcl8.5/TkCmd/ttk_intro.html. Some screenshots of the Python/Ttk code in use are at https://code.google.com/archive/p/python-ttk/wikis/Screenshots.wiki. -The :mod:`ttk` module was written by Guilherme Polo and added in +The :mod:`tkinter.ttk` module was written by Guilherme Polo and added in :issue:`2983`. An alternate version called ``Tile.py``, written by Martin Franklin and maintained by Kevin Walzer, was proposed for inclusion in :issue:`2618`, but the authors argued that Guilherme @@ -1830,7 +1831,7 @@ The :mod:`unittest` module was greatly enhanced; many new features were added. Most of these features were implemented by Michael Foord, unless otherwise noted. The enhanced version of the module is downloadable separately for use with Python versions 2.4 to 2.6, -packaged as the :mod:`unittest2` package, from +packaged as the :mod:`!unittest2` package, from https://pypi.org/project/unittest2. When used from the command line, the module can automatically discover @@ -1938,19 +1939,20 @@ GvR worked on merging them into Python's version of :mod:`unittest`. differences in the two strings. This comparison is now used by default when Unicode strings are compared with :meth:`~unittest.TestCase.assertEqual`. -* :meth:`~unittest.TestCase.assertRegexpMatches` and - :meth:`~unittest.TestCase.assertNotRegexpMatches` checks whether the +* :meth:`assertRegexpMatches() <unittest.TestCase.assertRegex>` and + :meth:`assertNotRegexpMatches() <unittest.TestCase.assertNotRegex>` checks whether the first argument is a string matching or not matching the regular expression provided as the second argument (:issue:`8038`). -* :meth:`~unittest.TestCase.assertRaisesRegexp` checks whether a particular exception +* :meth:`assertRaisesRegexp() <unittest.TestCase.assertRaisesRegex>` checks + whether a particular exception is raised, and then also checks that the string representation of the exception matches the provided regular expression. * :meth:`~unittest.TestCase.assertIn` and :meth:`~unittest.TestCase.assertNotIn` tests whether *first* is or is not in *second*. -* :meth:`~unittest.TestCase.assertItemsEqual` tests whether two provided sequences +* :meth:`assertItemsEqual() <unittest.TestCase.assertCountEqual>` tests whether two provided sequences contain the same elements. * :meth:`~unittest.TestCase.assertSetEqual` compares whether two sets are equal, and @@ -1966,7 +1968,7 @@ GvR worked on merging them into Python's version of :mod:`unittest`. * :meth:`~unittest.TestCase.assertDictEqual` compares two dictionaries and reports the differences; it's now used by default when you compare two dictionaries - using :meth:`~unittest.TestCase.assertEqual`. :meth:`~unittest.TestCase.assertDictContainsSubset` checks whether + using :meth:`~unittest.TestCase.assertEqual`. :meth:`!assertDictContainsSubset` checks whether all of the key/value pairs in *first* are found in *second*. * :meth:`~unittest.TestCase.assertAlmostEqual` and :meth:`~unittest.TestCase.assertNotAlmostEqual` test @@ -2023,8 +2025,8 @@ version 1.3. Some of the new features are: p = ET.XMLParser(encoding='utf-8') t = ET.XML("""<root/>""", parser=p) - Errors in parsing XML now raise a :exc:`ParseError` exception, whose - instances have a :attr:`position` attribute + Errors in parsing XML now raise a :exc:`~xml.etree.ElementTree.ParseError` exception, whose + instances have a :attr:`!position` attribute containing a (*line*, *column*) tuple giving the location of the problem. * ElementTree's code for converting trees to a string has been @@ -2034,7 +2036,8 @@ version 1.3. Some of the new features are: "xml" (the default), "html", or "text". HTML mode will output empty elements as ``<empty></empty>`` instead of ``<empty/>``, and text mode will skip over elements and only output the text chunks. If - you set the :attr:`tag` attribute of an element to ``None`` but + you set the :attr:`~xml.etree.ElementTree.Element.tag` attribute of an + element to ``None`` but leave its children in place, the element will be omitted when the tree is written out, so you don't need to do more extensive rearrangement to remove a single element. @@ -2064,14 +2067,14 @@ version 1.3. Some of the new features are: # Outputs <root><item>1</item>...</root> print ET.tostring(new) -* New :class:`Element` method: +* New :class:`~xml.etree.ElementTree.Element` method: :meth:`~xml.etree.ElementTree.Element.iter` yields the children of the element as a generator. It's also possible to write ``for child in elem:`` to loop over an element's children. The existing method - :meth:`getiterator` is now deprecated, as is :meth:`getchildren` + :meth:`!getiterator` is now deprecated, as is :meth:`!getchildren` which constructs and returns a list of children. -* New :class:`Element` method: +* New :class:`~xml.etree.ElementTree.Element` method: :meth:`~xml.etree.ElementTree.Element.itertext` yields all chunks of text that are descendants of the element. For example:: @@ -2227,7 +2230,7 @@ Changes to Python's build process and to the C API include: (Fixed by Thomas Wouters; :issue:`1590864`.) * The :c:func:`Py_Finalize` function now calls the internal - :func:`threading._shutdown` function; this prevents some exceptions from + :func:`!threading._shutdown` function; this prevents some exceptions from being raised when an interpreter shuts down. (Patch by Adam Olsen; :issue:`1722344`.) @@ -2242,7 +2245,7 @@ Changes to Python's build process and to the C API include: Heller; :issue:`3102`.) * New configure option: the :option:`!--with-system-expat` switch allows - building the :mod:`pyexpat` module to use the system Expat library. + building the :mod:`pyexpat <xml.parsers.expat>` module to use the system Expat library. (Contributed by Arfrever Frehtes Taifersar Arahesis; :issue:`7609`.) * New configure option: the @@ -2329,34 +2332,34 @@ Port-Specific Changes: Windows * The :mod:`msvcrt` module now contains some constants from the :file:`crtassem.h` header file: - :data:`CRT_ASSEMBLY_VERSION`, - :data:`VC_ASSEMBLY_PUBLICKEYTOKEN`, - and :data:`LIBRARIES_ASSEMBLY_NAME_PREFIX`. + :data:`~msvcrt.CRT_ASSEMBLY_VERSION`, + :data:`~msvcrt.VC_ASSEMBLY_PUBLICKEYTOKEN`, + and :data:`~msvcrt.LIBRARIES_ASSEMBLY_NAME_PREFIX`. (Contributed by David Cournapeau; :issue:`4365`.) -* The :mod:`_winreg` module for accessing the registry now implements - the :func:`~_winreg.CreateKeyEx` and :func:`~_winreg.DeleteKeyEx` +* The :mod:`_winreg <winreg>` module for accessing the registry now implements + the :func:`~winreg.CreateKeyEx` and :func:`~winreg.DeleteKeyEx` functions, extended versions of previously supported functions that - take several extra arguments. The :func:`~_winreg.DisableReflectionKey`, - :func:`~_winreg.EnableReflectionKey`, and :func:`~_winreg.QueryReflectionKey` + take several extra arguments. The :func:`~winreg.DisableReflectionKey`, + :func:`~winreg.EnableReflectionKey`, and :func:`~winreg.QueryReflectionKey` were also tested and documented. (Implemented by Brian Curtin: :issue:`7347`.) -* The new :c:func:`_beginthreadex` API is used to start threads, and +* The new :c:func:`!_beginthreadex` API is used to start threads, and the native thread-local storage functions are now used. (Contributed by Kristján Valur Jónsson; :issue:`3582`.) * The :func:`os.kill` function now works on Windows. The signal value - can be the constants :const:`CTRL_C_EVENT`, - :const:`CTRL_BREAK_EVENT`, or any integer. The first two constants + can be the constants :const:`~signal.CTRL_C_EVENT`, + :const:`~signal.CTRL_BREAK_EVENT`, or any integer. The first two constants will send :kbd:`Control-C` and :kbd:`Control-Break` keystroke events to - subprocesses; any other value will use the :c:func:`TerminateProcess` + subprocesses; any other value will use the :c:func:`!TerminateProcess` API. (Contributed by Miki Tebeka; :issue:`1220212`.) * The :func:`os.listdir` function now correctly fails for an empty path. (Fixed by Hirokazu Yamamoto; :issue:`5913`.) -* The :mod:`mimelib` module will now read the MIME database from +* The :mod:`mimetypes` module will now read the MIME database from the Windows registry when initializing. (Patch by Gabriel Genellina; :issue:`4969`.) @@ -2385,7 +2388,7 @@ Port-Specific Changes: Mac OS X Port-Specific Changes: FreeBSD ----------------------------------- -* FreeBSD 7.1's :const:`SO_SETFIB` constant, used with the :func:`~socket.socket` methods +* FreeBSD 7.1's :const:`!SO_SETFIB` constant, used with the :func:`~socket.socket` methods :func:`~socket.socket.getsockopt`/:func:`~socket.socket.setsockopt` to select an alternate routing table, is now available in the :mod:`socket` module. (Added by Kyle VanderBeek; :issue:`8235`.) @@ -2441,7 +2444,7 @@ This section lists previously described changes and other bugfixes that may require changes to your code: * The :func:`range` function processes its arguments more - consistently; it will now call :meth:`__int__` on non-float, + consistently; it will now call :meth:`~object.__int__` on non-float, non-integer arguments that are supplied to it. (Fixed by Alexander Belopolsky; :issue:`1533`.) @@ -2486,13 +2489,13 @@ In the standard library: (or ``NaN``) are now hashable. (Fixed by Mark Dickinson; :issue:`7279`.) -* The ElementTree library, :mod:`xml.etree`, no longer escapes +* The :mod:`xml.etree.ElementTree` library no longer escapes ampersands and angle brackets when outputting an XML processing instruction (which looks like ``<?xml-stylesheet href="#style1"?>``) or comment (which looks like ``<!-- comment -->``). (Patch by Neil Muller; :issue:`2746`.) -* The :meth:`~StringIO.StringIO.readline` method of :class:`~StringIO.StringIO` objects now does +* The :meth:`!readline` method of :class:`~io.StringIO` objects now does nothing when a negative length is requested, as other file-like objects do. (:issue:`7348`). @@ -2508,7 +2511,7 @@ In the standard library: which raises an exception if there's an error. (Changed by Lars Gustäbel; :issue:`7357`.) -* The :mod:`urlparse` module's :func:`~urlparse.urlsplit` now handles +* The :mod:`urlparse <urllib.parse>` module's :func:`~urllib.parse.urlsplit` now handles unknown URL schemes in a fashion compliant with :rfc:`3986`: if the URL is of the form ``"<something>://..."``, the text before the ``://`` is treated as the scheme, even if it's a made-up scheme that @@ -2577,11 +2580,11 @@ Two new environment variables for debug mode -------------------------------------------- In debug mode, the ``[xxx refs]`` statistic is not written by default, the -:envvar:`PYTHONSHOWREFCOUNT` environment variable now must also be set. +:envvar:`!PYTHONSHOWREFCOUNT` environment variable now must also be set. (Contributed by Victor Stinner; :issue:`31733`.) When Python is compiled with ``COUNT_ALLOC`` defined, allocation counts are no -longer dumped by default anymore: the :envvar:`PYTHONSHOWALLOCCOUNT` environment +longer dumped by default anymore: the :envvar:`!PYTHONSHOWALLOCCOUNT` environment variable must now also be set. Moreover, allocation counts are now dumped into stderr, rather than stdout. (Contributed by Victor Stinner; :issue:`31692`.) @@ -2711,8 +2714,9 @@ and :ref:`setuptools-index`. PEP 476: Enabling certificate verification by default for stdlib http clients ----------------------------------------------------------------------------- -:pep:`476` updated :mod:`httplib` and modules which use it, such as -:mod:`urllib2` and :mod:`xmlrpclib`, to now verify that the server +:pep:`476` updated :mod:`httplib <http>` and modules which use it, such as +:mod:`urllib2 <urllib.request>` and :mod:`xmlrpclib <xmlrpc.client>`, to now +verify that the server presents a certificate which is signed by a Certificate Authority in the platform trust store and whose hostname matches the hostname being requested by default, significantly improving security for many applications. This @@ -2753,7 +2757,7 @@ entire Python process back to the default permissive behaviour of Python 2.7.8 and earlier. For cases where the connection establishment code can't be modified, but the -overall application can be, the new :func:`ssl._https_verify_certificates` +overall application can be, the new :func:`!ssl._https_verify_certificates` function can be used to adjust the default behaviour at runtime. diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 1df5209f22c6a5b..888e6279754fc29 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -337,7 +337,7 @@ changed. (However, the standard library remains ASCII-only with the exception of contributor names in comments.) -* The :mod:`StringIO` and :mod:`cStringIO` modules are gone. Instead, +* The :mod:`!StringIO` and :mod:`!cStringIO` modules are gone. Instead, import the :mod:`io` module and use :class:`io.StringIO` or :class:`io.BytesIO` for text and data respectively. @@ -563,7 +563,7 @@ review: removal in Python 3.0 due to lack of use or because a better replacement exists. See :pep:`3108` for an exhaustive list. -* The :mod:`bsddb3` package was removed because its presence in the +* The :mod:`!bsddb3` package was removed because its presence in the core standard library has proved over time to be a particular burden for the core developers due to testing instability and Berkeley DB's release schedule. However, the package is alive and well, @@ -588,40 +588,40 @@ review: * A common pattern in Python 2.x is to have one version of a module implemented in pure Python, with an optional accelerated version implemented as a C extension; for example, :mod:`pickle` and - :mod:`cPickle`. This places the burden of importing the accelerated + :mod:`!cPickle`. This places the burden of importing the accelerated version and falling back on the pure Python version on each user of these modules. In Python 3.0, the accelerated versions are considered implementation details of the pure Python versions. Users should always import the standard version, which attempts to import the accelerated version and falls back to the pure Python - version. The :mod:`pickle` / :mod:`cPickle` pair received this + version. The :mod:`pickle` / :mod:`!cPickle` pair received this treatment. The :mod:`profile` module is on the list for 3.1. The - :mod:`StringIO` module has been turned into a class in the :mod:`io` + :mod:`!StringIO` module has been turned into a class in the :mod:`io` module. * Some related modules have been grouped into packages, and usually the submodule names have been simplified. The resulting new packages are: - * :mod:`dbm` (:mod:`anydbm`, :mod:`dbhash`, :mod:`dbm`, - :mod:`dumbdbm`, :mod:`gdbm`, :mod:`whichdb`). + * :mod:`dbm` (:mod:`!anydbm`, :mod:`!dbhash`, :mod:`!dbm`, + :mod:`!dumbdbm`, :mod:`!gdbm`, :mod:`!whichdb`). - * :mod:`html` (:mod:`HTMLParser`, :mod:`htmlentitydefs`). + * :mod:`html` (:mod:`!HTMLParser`, :mod:`!htmlentitydefs`). - * :mod:`http` (:mod:`httplib`, :mod:`BaseHTTPServer`, - :mod:`CGIHTTPServer`, :mod:`SimpleHTTPServer`, :mod:`Cookie`, - :mod:`cookielib`). + * :mod:`http` (:mod:`!httplib`, :mod:`!BaseHTTPServer`, + :mod:`!CGIHTTPServer`, :mod:`!SimpleHTTPServer`, :mod:`!Cookie`, + :mod:`!cookielib`). * :mod:`tkinter` (all :mod:`Tkinter`-related modules except :mod:`turtle`). The target audience of :mod:`turtle` doesn't really care about :mod:`tkinter`. Also note that as of Python 2.6, the functionality of :mod:`turtle` has been greatly enhanced. - * :mod:`urllib` (:mod:`urllib`, :mod:`urllib2`, :mod:`urlparse`, - :mod:`robotparse`). + * :mod:`urllib` (:mod:`!urllib`, :mod:`!urllib2`, :mod:`!urlparse`, + :mod:`!robotparse`). - * :mod:`xmlrpc` (:mod:`xmlrpclib`, :mod:`DocXMLRPCServer`, - :mod:`SimpleXMLRPCServer`). + * :mod:`xmlrpc` (:mod:`!xmlrpclib`, :mod:`!DocXMLRPCServer`, + :mod:`!SimpleXMLRPCServer`). Some other changes to standard library modules, not covered by :pep:`3108`: @@ -642,9 +642,9 @@ Some other changes to standard library modules, not covered by * Cleanup of the :mod:`operator` module: removed :func:`sequenceIncludes` and :func:`isCallable`. -* Cleanup of the :mod:`thread` module: :func:`acquire_lock` and - :func:`release_lock` are gone; use :func:`acquire` and - :func:`release` instead. +* Cleanup of the :mod:`!thread` module: :func:`!acquire_lock` and + :func:`!release_lock` are gone; use :meth:`~threading.Lock.acquire` and + :meth:`~threading.Lock.release` instead. * Cleanup of the :mod:`random` module: removed the :func:`jumpahead` API. diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index e237179f4b1829f..69b273e58385d27 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -80,6 +80,28 @@ Support was also added for third-party tools like `PyYAML <https://pyyaml.org/>` PEP written by Armin Ronacher and Raymond Hettinger. Implementation written by Raymond Hettinger. +Since an ordered dictionary remembers its insertion order, it can be used +in conjunction with sorting to make a sorted dictionary:: + + >>> # regular unsorted dictionary + >>> d = {'banana': 3, 'apple':4, 'pear': 1, 'orange': 2} + + >>> # dictionary sorted by key + >>> OrderedDict(sorted(d.items(), key=lambda t: t[0])) + OrderedDict([('apple', 4), ('banana', 3), ('orange', 2), ('pear', 1)]) + + >>> # dictionary sorted by value + >>> OrderedDict(sorted(d.items(), key=lambda t: t[1])) + OrderedDict([('pear', 1), ('orange', 2), ('banana', 3), ('apple', 4)]) + + >>> # dictionary sorted by length of the key string + >>> OrderedDict(sorted(d.items(), key=lambda t: len(t[0]))) + OrderedDict([('pear', 1), ('apple', 4), ('orange', 2), ('banana', 3)]) + +The new sorted dictionaries maintain their sort order when entries +are deleted. But when new keys are added, the keys are appended +to the end and the sort is not maintained. + PEP 378: Format Specifier for Thousands Separator ================================================= @@ -152,7 +174,7 @@ Some smaller changes made to the core Python language are: (Contributed by Eric Smith; :issue:`5237`.) -* The :func:`string.maketrans` function is deprecated and is replaced by new +* The :func:`!string.maketrans` function is deprecated and is replaced by new static methods, :meth:`bytes.maketrans` and :meth:`bytearray.maketrans`. This change solves the confusion around which types were supported by the :mod:`string` module. Now, :class:`str`, :class:`bytes`, and @@ -169,7 +191,7 @@ Some smaller changes made to the core Python language are: ... if '<critical>' in line: ... outfile.write(line) - With the new syntax, the :func:`contextlib.nested` function is no longer + With the new syntax, the :func:`!contextlib.nested` function is no longer needed and is now deprecated. (Contributed by Georg Brandl and Mattias Brändström; @@ -359,16 +381,20 @@ New, Improved, and Deprecated Modules x / 0 In addition, several new assertion methods were added including - :func:`assertSetEqual`, :func:`assertDictEqual`, - :func:`assertDictContainsSubset`, :func:`assertListEqual`, - :func:`assertTupleEqual`, :func:`assertSequenceEqual`, - :func:`assertRaisesRegexp`, :func:`assertIsNone`, - and :func:`assertIsNotNone`. + :meth:`~unittest.TestCase.assertSetEqual`, + :meth:`~unittest.TestCase.assertDictEqual`, + :meth:`!assertDictContainsSubset`, + :meth:`~unittest.TestCase.assertListEqual`, + :meth:`~unittest.TestCase.assertTupleEqual`, + :meth:`~unittest.TestCase.assertSequenceEqual`, + :meth:`assertRaisesRegexp() <unittest.TestCase.assertRaisesRegex>`, + :meth:`~unittest.TestCase.assertIsNone`, + and :meth:`~unittest.TestCase.assertIsNotNone`. (Contributed by Benjamin Peterson and Antoine Pitrou.) -* The :mod:`io` module has three new constants for the :meth:`seek` - method :data:`SEEK_SET`, :data:`SEEK_CUR`, and :data:`SEEK_END`. +* The :mod:`io` module has three new constants for the :meth:`~io.IOBase.seek` + method: :data:`~os.SEEK_SET`, :data:`~os.SEEK_CUR`, and :data:`~os.SEEK_END`. * The :data:`sys.version_info` tuple is now a named tuple:: diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 7dc06e9af694d92..9770b12c8af7110 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -1517,6 +1517,13 @@ functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) +The presence of newline or tab characters in parts of a URL allows for some +forms of attacks. Following the WHATWG specification that updates :rfc:`3986`, +ASCII newline ``\n``, ``\r`` and tab ``\t`` characters are stripped from the +URL by the parser in :mod:`urllib.parse` preventing such attacks. The removal +characters are controlled by a new module level variable +``urllib.parse._UNSAFE_URL_BYTES_TO_REMOVE``. (See :gh:`88048`) + xml --- @@ -2315,3 +2322,43 @@ Removed * The ``PyThreadState.use_tracing`` member has been removed to optimize Python. (Contributed by Mark Shannon in :issue:`43760`.) + + +Notable security feature in 3.10.7 +================================== + +Converting between :class:`int` and :class:`str` in bases other than 2 +(binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) +now raises a :exc:`ValueError` if the number of digits in string form is +above a limit to avoid potential denial of service attacks due to the +algorithmic complexity. This is a mitigation for `CVE-2020-10735 +<https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10735>`_. +This limit can be configured or disabled by environment variable, command +line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion +length limitation <int_max_str_digits>` documentation. The default limit +is 4300 digits in string form. + +Notable security feature in 3.10.8 +================================== + +The deprecated :mod:`!mailcap` module now refuses to inject unsafe text +(filenames, MIME types, parameters) into shell commands. Instead of using such +text, it will warn and act as if a match was not found (or for test commands, +as if the test failed). +(Contributed by Petr Viktorin in :gh:`98966`.) + +Notable changes in 3.10.12 +========================== + +tarfile +------- + +* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, + have a new a *filter* argument that allows limiting tar features than may be + surprising or dangerous, such as creating files outside the destination + directory. + See :ref:`tarfile-extraction-filter` for details. + In Python 3.12, use without the *filter* argument will show a + :exc:`DeprecationWarning`. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index a8a7dfda0c5309b..4f4c1de8d8d5964 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2701,4 +2701,30 @@ Removed (Contributed by Inada Naoki in :issue:`44029`.) +Notable changes in 3.11.4 +========================= + +tarfile +------- + +* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, + have a new a *filter* argument that allows limiting tar features than may be + surprising or dangerous, such as creating files outside the destination + directory. + See :ref:`tarfile-extraction-filter` for details. + In Python 3.12, use without the *filter* argument will show a + :exc:`DeprecationWarning`. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) + + +Notable changes in 3.11.5 +========================= + +OpenSSL +------- + +* Windows builds and macOS installers from python.org now use OpenSSL 3.0. + + .. _libb2: https://www.blake2.net/ diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 77b12f9284ba0d9..b986e638498abd3 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -690,7 +690,7 @@ csv * Add :const:`csv.QUOTE_NOTNULL` and :const:`csv.QUOTE_STRINGS` flags to provide finer grained control of ``None`` and empty strings by - :class:`csv.writer` objects. + :class:`~csv.reader` and :class:`~csv.writer` objects. dis --- @@ -1956,6 +1956,8 @@ Build Changes :file:`!configure`. (Contributed by Christian Heimes in :gh:`89886`.) +* Windows builds and macOS installers from python.org now use OpenSSL 3.0. + C API Changes ============= diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 985e34b453f63a5..3a277d7ce1585fc 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -81,6 +81,13 @@ Important deprecations, removals or restrictions: * Python 3.13 and later have two years of full support, followed by three years of security fixes. +Interpreter improvements: + +* A basic :ref:`JIT compiler <whatsnew313-jit-compiler>` was added. + It is currently disabled by default (though we may turn it on later). + Performance improvements are modest -- we expect to be improving this + over the next few releases. + New Features ============ @@ -94,6 +101,17 @@ Improved Error Messages variables. See also :ref:`using-on-controlling-color`. (Contributed by Pablo Galindo Salgado in :gh:`112730`.) +* When an incorrect keyword argument is passed to a function, the error message + now potentially suggests the correct keyword argument. + (Contributed by Pablo Galindo Salgado and Shantanu Jain in :gh:`107944`.) + + >>> "better error messages!".split(max_split=1) + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + "better error messages!".split(max_split=1) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^ + TypeError: split() got an unexpected keyword argument 'max_split'. Did you mean 'maxsplit'? + Other Language Changes ====================== @@ -127,6 +145,11 @@ Other Language Changes is rejected when the global is used in the :keyword:`else` block. (Contributed by Irit Katriel in :gh:`111123`.) +* Many functions now emit a warning if a boolean value is passed as + a file descriptor argument. + This can help catch some errors earlier. + (Contributed by Serhiy Storchaka in :gh:`82626`.) + * Added a new environment variable :envvar:`PYTHON_FROZEN_MODULES`. It determines whether or not frozen modules are ignored by the import machinery, equivalent of the :option:`-X frozen_modules <-X>` command-line option. @@ -137,6 +160,21 @@ Other Language Changes (Contributed by Levi Sabah, Zackery Spytz and Hugo van Kemenade in :gh:`73965`.) +* Add :exc:`PythonFinalizationError` exception. This exception derived from + :exc:`RuntimeError` is raised when an operation is blocked during + the :term:`Python finalization <interpreter shutdown>`. + + The following functions now raise PythonFinalizationError, instead of + :exc:`RuntimeError`: + + * :func:`_thread.start_new_thread`. + * :class:`subprocess.Popen`. + * :func:`os.fork`. + * :func:`os.forkpty`. + + (Contributed by Victor Stinner in :gh:`114570`.) + + New Modules =========== @@ -146,13 +184,14 @@ New Modules Improved Modules ================ -ast ---- +argparse +-------- -* :func:`ast.parse` now accepts an optional argument ``optimize`` - which is passed on to the :func:`compile` built-in. This makes it - possible to obtain an optimized ``AST``. - (Contributed by Irit Katriel in :gh:`108113`). +* Add parameter *deprecated* in methods + :meth:`~argparse.ArgumentParser.add_argument` and :meth:`!add_parser` + which allows to deprecate command-line options, positional arguments and + subcommands. + (Contributed by Serhiy Storchaka in :gh:`83648`). array ----- @@ -161,6 +200,32 @@ array It can be used instead of ``'u'`` type code, which is deprecated. (Contributed by Inada Naoki in :gh:`80480`.) +* Add ``clear()`` method in order to implement ``MutableSequence``. + (Contributed by Mike Zimin in :gh:`114894`.) + +ast +--- + +* The constructors of node types in the :mod:`ast` module are now stricter + in the arguments they accept, and have more intuitive behaviour when + arguments are omitted. + + If an optional field on an AST node is not included as an argument when + constructing an instance, the field will now be set to ``None``. Similarly, + if a list field is omitted, that field will now be set to an empty list. + (Previously, in both cases, the attribute would be missing on the newly + constructed AST node instance.) + + If other arguments are omitted, a :exc:`DeprecationWarning` is emitted. + This will cause an exception in Python 3.15. Similarly, passing a keyword + argument that does not map to a field on the AST node is now deprecated, + and will raise an exception in Python 3.15. + +* :func:`ast.parse` now accepts an optional argument ``optimize`` + which is passed on to the :func:`compile` built-in. This makes it + possible to obtain an optimized ``AST``. + (Contributed by Irit Katriel in :gh:`108113`.) + asyncio ------- @@ -168,6 +233,20 @@ asyncio the Unix socket when the server is closed. (Contributed by Pierre Ossman in :gh:`111246`.) +* :meth:`asyncio.DatagramTransport.sendto` will now send zero-length + datagrams if called with an empty bytes object. The transport flow + control also now accounts for the datagram header when calculating + the buffer size. + (Contributed by Jamie Phan in :gh:`115199`.) + +base64 +--- + +* Add :func:`base64.z85encode` and :func:`base64.z85decode` functions which allow encoding + and decoding z85 data. + See `Z85 specification <https://rfc.zeromq.org/spec/32/>`_ for more information. + (Contributed by Matan Perelman in :gh:`75299`.) + copy ---- @@ -180,6 +259,13 @@ copy any user classes which define the :meth:`!__replace__` method. (Contributed by Serhiy Storchaka in :gh:`108751`.) +dbm +--- + +* Add :meth:`dbm.gnu.gdbm.clear` and :meth:`dbm.ndbm.ndbm.clear` methods that remove all items + from the database. + (Contributed by Donghee Na in :gh:`107122`.) + dis --- @@ -196,6 +282,9 @@ dbm from the database. (Contributed by Donghee Na in :gh:`107122`.) +* Add new :mod:`dbm.sqlite3` backend, and make it the default :mod:`!dbm` backend. + (Contributed by Raymond Hettinger and Erlend E. Aasland in :gh:`100414`.) + doctest ------- @@ -213,7 +302,7 @@ email parameter to these two functions: use ``strict=False`` to get the old behavior, accept malformed inputs. ``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to - check if the *strict* paramater is available. + check if the *strict* parameter is available. (Contributed by Thomas Dwyer and Victor Stinner for :gh:`102988` to improve the CVE-2023-27043 fix.) @@ -223,7 +312,7 @@ fractions * Formatting for objects of type :class:`fractions.Fraction` now supports the standard format specification mini-language rules for fill, alignment, sign handling, minimum width and grouping. (Contributed by Mark Dickinson - in :gh:`111320`) + in :gh:`111320`.) glob ---- @@ -247,6 +336,14 @@ ipaddress * Add the :attr:`ipaddress.IPv4Address.ipv6_mapped` property, which returns the IPv4-mapped IPv6 address. (Contributed by Charles Machalow in :gh:`109466`.) +itertools +--------- + +* Added a ``strict`` option to :func:`itertools.batched`. + This raises a :exc:`ValueError` if the final batch is shorter + than the specified batch size. + (Contributed by Raymond Hettinger in :gh:`113202`.) + marshal ------- @@ -259,8 +356,8 @@ mmap ---- * The :class:`mmap.mmap` class now has an :meth:`~mmap.mmap.seekable` method - that can be used where it requires a file-like object with seekable and - the :meth:`~mmap.mmap.seek` method return the new absolute position. + that can be used when a seekable file-like object is required. + The :meth:`~mmap.mmap.seek` method now returns the new absolute position. (Contributed by Donghee Na and Sylvie Liberman in :gh:`111835`.) * :class:`mmap.mmap` now has a *trackfd* parameter on Unix; if it is ``False``, the file descriptor specified by *fileno* will not be duplicated. @@ -297,17 +394,17 @@ os the new environment variable :envvar:`PYTHON_CPU_COUNT` or the new command-line option :option:`-X cpu_count <-X>`. This option is useful for users who need to limit CPU resources of a container system without having to modify the container (application code). - (Contributed by Donghee Na in :gh:`109595`) + (Contributed by Donghee Na in :gh:`109595`.) * Add support of :func:`os.lchmod` and the *follow_symlinks* argument in :func:`os.chmod` on Windows. Note that the default value of *follow_symlinks* in :func:`!os.lchmod` is ``False`` on Windows. - (Contributed by Serhiy Storchaka in :gh:`59616`) + (Contributed by Serhiy Storchaka in :gh:`59616`.) * Add support of :func:`os.fchmod` and a file descriptor in :func:`os.chmod` on Windows. - (Contributed by Serhiy Storchaka in :gh:`113191`) + (Contributed by Serhiy Storchaka in :gh:`113191`.) * :func:`os.posix_spawn` now accepts ``env=None``, which makes the newly spawned process use the current process environment. @@ -336,7 +433,7 @@ pathlib (Contributed by Barney Gale in :gh:`89812`.) * Add :meth:`pathlib.Path.from_uri`, a new constructor to create a :class:`pathlib.Path` - object from a 'file' URI (``file:/``). + object from a 'file' URI (``file://``). (Contributed by Barney Gale in :gh:`107465`.) * Add :meth:`pathlib.PurePath.full_match` for matching paths with @@ -350,6 +447,11 @@ pathlib (Contributed by Barney Gale in :gh:`77609` and :gh:`105793`, and Kamil Turek in :gh:`107962`). +* Return files and directories from :meth:`pathlib.Path.glob` and + :meth:`~pathlib.Path.rglob` when given a pattern that ends with "``**``". In + earlier versions, only directories were returned. + (Contributed by Barney Gale in :gh:`70303`). + pdb --- @@ -357,7 +459,7 @@ pdb the new ``exceptions [exc_number]`` command for Pdb. (Contributed by Matthias Bussonnier in :gh:`106676`.) -* Expressions/Statements whose prefix is a pdb command are now correctly +* Expressions/statements whose prefix is a pdb command are now correctly identified and executed. (Contributed by Tian Gao in :gh:`108464`.) @@ -366,6 +468,13 @@ pdb command line option or :envvar:`PYTHONSAFEPATH` environment variable). (Contributed by Tian Gao and Christian Walther in :gh:`111762`.) +queue +----- + +* Add :meth:`queue.Queue.shutdown` (along with :exc:`queue.ShutDown`) for queue + termination. + (Contributed by Laurie Opperman and Yves Duprat in :gh:`104750`.) + re -- * Rename :exc:`!re.error` to :exc:`re.PatternError` for improved clarity. @@ -378,6 +487,18 @@ sqlite3 object is not :meth:`closed <sqlite3.Connection.close>` explicitly. (Contributed by Erlend E. Aasland in :gh:`105539`.) +* Add *filter* keyword-only parameter to :meth:`sqlite3.Connection.iterdump` + for filtering database objects to dump. + (Contributed by Mariusz Felisiak in :gh:`91602`.) + +statistics +---------- + +* Add :func:`statistics.kde` for kernel density estimation. + This makes it possible to estimate a continuous probability density function + from a fixed number of discrete samples. + (Contributed by Raymond Hettinger in :gh:`115863`.) + subprocess ---------- @@ -409,6 +530,21 @@ tkinter :meth:`!tk_busy_current`, and :meth:`!tk_busy_status`. (Contributed by Miguel, klappnase and Serhiy Storchaka in :gh:`72684`.) +* The :mod:`tkinter` widget method :meth:`!wm_attributes` now accepts + the attribute name without the minus prefix to get window attributes, + e.g. ``w.wm_attributes('alpha')`` and allows to specify attributes and + values to set as keyword arguments, e.g. ``w.wm_attributes(alpha=0.5)``. + Add new optional keyword-only parameter *return_python_dict*: calling + ``w.wm_attributes(return_python_dict=True)`` returns the attributes as + a dict instead of a tuple. + (Contributed by Serhiy Storchaka in :gh:`43457`.) + +* Add new optional keyword-only parameter *return_ints* in + the :meth:`!Text.count` method. + Passing ``return_ints=True`` makes it always returning the single count + as an integer instead of a 1-tuple or ``None``. + (Contributed by Serhiy Storchaka in :gh:`97928`.) + * Add support of the "vsapi" element type in the :meth:`~tkinter.ttk.Style.element_create` method of :class:`tkinter.ttk.Style`. @@ -460,6 +596,14 @@ warnings warning may also be emitted when a decorated function or class is used at runtime. See :pep:`702`. (Contributed by Jelle Zijlstra in :gh:`104003`.) +xml.etree.ElementTree +--------------------- + +* Add the :meth:`!close` method for the iterator returned by + :func:`~xml.etree.ElementTree.iterparse` for explicit cleaning up. + (Contributed by Serhiy Storchaka in :gh:`69893`.) + + Optimizations ============= @@ -472,6 +616,47 @@ Optimizations FreeBSD and Solaris. See the ``subprocess`` section above for details. (Contributed by Jakub Kulik in :gh:`113117`.) +.. _whatsnew313-jit-compiler: + +Experimental JIT Compiler +========================= + +When CPython is configured using the ``--enable-experimental-jit`` option, +a just-in-time compiler is added which can speed up some Python programs. + +The internal architecture is roughly as follows. + +* We start with specialized *Tier 1 bytecode*. + See :ref:`What's new in 3.11 <whatsnew311-pep659>` for details. + +* When the Tier 1 bytecode gets hot enough, it gets translated + to a new, purely internal *Tier 2 IR*, a.k.a. micro-ops ("uops"). + +* The Tier 2 IR uses the same stack-based VM as Tier 1, but the + instruction format is better suited to translation to machine code. + +* We have several optimization passes for Tier 2 IR, which are applied + before it is interpreted or translated to machine code. + +* There is a Tier 2 interpreter, but it is mostly intended for debugging + the earlier stages of the optimization pipeline. If the JIT is not + enabled, the Tier 2 interpreter can be invoked by passing Python the + ``-X uops`` option or by setting the ``PYTHON_UOPS`` environment + variable to ``1``. + +* When the ``--enable-experimental-jit`` option is used, the optimized + Tier 2 IR is translated to machine code, which is then executed. + This does not require additional runtime options. + +* The machine code translation process uses an architecture called + *copy-and-patch*. It has no runtime dependencies, but there is a new + build-time dependency on LLVM. + +(JIT by Brandt Bucher, inspired by a paper by Haoran Xu and Fredrik Kjolstad. +Tier 2 IR by Mark Shannon and Guido van Rossum. +Tier 2 optimizer by Ken Jin.) + + Deprecated ========== @@ -487,34 +672,69 @@ Deprecated Replace ``ctypes.ARRAY(item_type, size)`` with ``item_type * size``. (Contributed by Victor Stinner in :gh:`105733`.) +* :mod:`decimal`: Deprecate non-standard format specifier "N" for + :class:`decimal.Decimal`. + It was not documented and only supported in the C implementation. + (Contributed by Serhiy Storchaka in :gh:`89902`.) + +* :mod:`dis`: The ``dis.HAVE_ARGUMENT`` separator is deprecated. Check + membership in :data:`~dis.hasarg` instead. + (Contributed by Irit Katriel in :gh:`109319`.) + * :mod:`getopt` and :mod:`optparse` modules: They are now - :term:`soft deprecated`: the :mod:`argparse` should be used for new projects. + :term:`soft deprecated`: the :mod:`argparse` module should be used for new projects. Previously, the :mod:`optparse` module was already deprecated, its removal was not scheduled, and no warnings was emitted: so there is no change in practice. (Contributed by Victor Stinner in :gh:`106535`.) +* :mod:`gettext`: Emit deprecation warning for non-integer numbers in + :mod:`gettext` functions and methods that consider plural forms even if the + translation was not found. + (Contributed by Serhiy Storchaka in :gh:`88434`.) + * :mod:`http.server`: :class:`http.server.CGIHTTPRequestHandler` now emits a - :exc:`DeprecationWarning` as it will be removed in 3.15. Process based CGI - http servers have been out of favor for a very long time. This code was + :exc:`DeprecationWarning` as it will be removed in 3.15. Process-based CGI + HTTP servers have been out of favor for a very long time. This code was outdated, unmaintained, and rarely used. It has a high potential for both security and functionality bugs. This includes removal of the ``--cgi`` flag to the ``python -m http.server`` command line in 3.15. * :mod:`pathlib`: + :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for + removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved + paths on Windows. + +* :mod:`pydoc`: Deprecate undocumented :func:`!pydoc.ispackage` function. + (Contributed by Zackery Spytz in :gh:`64020`.) - * :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for - removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved - paths on Windows. +* :mod:`sqlite3`: Passing more than one positional argument to + :func:`sqlite3.connect` and the :class:`sqlite3.Connection` constructor is + deprecated. The remaining parameters will become keyword-only in Python 3.15. + + Deprecate passing name, number of arguments, and the callable as keyword + arguments for the following :class:`sqlite3.Connection` APIs: + + * :meth:`~sqlite3.Connection.create_function` + * :meth:`~sqlite3.Connection.create_aggregate` + + Deprecate passing the callback callable by keyword for the following + :class:`sqlite3.Connection` APIs: + + * :meth:`~sqlite3.Connection.set_authorizer` + * :meth:`~sqlite3.Connection.set_progress_handler` + * :meth:`~sqlite3.Connection.set_trace_callback` + + The affected parameters will become positional-only in Python 3.15. + + (Contributed by Erlend E. Aasland in :gh:`107948` and :gh:`108278`.) * :mod:`sys`: :func:`sys._enablelegacywindowsfsencoding` function. - Replace it with :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable. + Replace it with the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable. (Contributed by Inada Naoki in :gh:`73427`.) -* :mod:`traceback`: - - * The field *exc_type* of :class:`traceback.TracebackException` is - deprecated. Use *exc_type_str* instead. +* :mod:`traceback`: The field *exc_type* of :class:`traceback.TracebackException` + is deprecated. Use *exc_type_str* instead. * :mod:`typing`: @@ -550,39 +770,6 @@ Deprecated They will be removed in Python 3.15. (Contributed by Victor Stinner in :gh:`105096`.) -* Passing more than one positional argument to :func:`sqlite3.connect` and the - :class:`sqlite3.Connection` constructor is deprecated. The remaining - parameters will become keyword-only in Python 3.15. - - Deprecate passing name, number of arguments, and the callable as keyword - arguments, for the following :class:`sqlite3.Connection` APIs: - - * :meth:`~sqlite3.Connection.create_function` - * :meth:`~sqlite3.Connection.create_aggregate` - - Deprecate passing the callback callable by keyword for the following - :class:`sqlite3.Connection` APIs: - - * :meth:`~sqlite3.Connection.set_authorizer` - * :meth:`~sqlite3.Connection.set_progress_handler` - * :meth:`~sqlite3.Connection.set_trace_callback` - - The affected parameters will become positional-only in Python 3.15. - - (Contributed by Erlend E. Aasland in :gh:`107948` and :gh:`108278`.) - -* The ``dis.HAVE_ARGUMENT`` separator is deprecated. Check membership - in :data:`~dis.hasarg` instead. - (Contributed by Irit Katriel in :gh:`109319`.) - -* Deprecate non-standard format specifier "N" for :class:`decimal.Decimal`. - It was not documented and only supported in the C implementation. - (Contributed by Serhiy Storchaka in :gh:`89902`.) - -* Emit deprecation warning for non-integer numbers in :mod:`gettext` functions - and methods that consider plural forms even if the translation was not found. - (Contributed by Serhiy Storchaka in :gh:`88434`.) - * Calling :meth:`frame.clear` on a suspended frame raises :exc:`RuntimeError` (as has always been the case for an executing frame). (Contributed by Irit Katriel in :gh:`79932`.) @@ -593,9 +780,6 @@ Deprecated coroutine. (Contributed by Irit Katriel in :gh:`81137`.) -* Deprecate undocumented :func:`!pydoc.ispackage` function. - (Contributed by Zackery Spytz in :gh:`64020`.) - Pending Removal in Python 3.14 ------------------------------ @@ -657,11 +841,11 @@ Pending Removal in Python 3.14 :func:`~multiprocessing.set_start_method` APIs to explicitly specify when your code *requires* ``'fork'``. See :ref:`multiprocessing-start-methods`. -* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to`, +* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to` and :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is deprecated. -* :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` +* :mod:`pkgutil`: :func:`~pkgutil.find_loader` and :func:`~pkgutil.get_loader` now raise :exc:`DeprecationWarning`; use :func:`importlib.util.find_spec` instead. (Contributed by Nikita Sobolev in :gh:`97850`.) @@ -719,10 +903,16 @@ Pending Removal in Python 3.15 (Contributed by Hugo van Kemenade in :gh:`111187`.) * :mod:`pathlib`: + :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for + removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved + paths on Windows. - * :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for - removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved - paths on Windows. +* :mod:`threading`: + Passing any arguments to :func:`threading.RLock` is now deprecated. + C version allows any numbers of args and kwargs, + but they are just ignored. Python version does not allow any arguments. + All arguments will be removed from :func:`threading.RLock` in Python 3.15. + (Contributed by Nikita Sobolev in :gh:`102029`.) * :class:`typing.NamedTuple`: @@ -749,12 +939,6 @@ Pending Removal in Python 3.15 They will be removed in Python 3.15. (Contributed by Victor Stinner in :gh:`105096`.) -* Passing any arguments to :func:`threading.RLock` is now deprecated. - C version allows any numbers of args and kwargs, - but they are just ignored. Python version does not allow any arguments. - All arguments will be removed from :func:`threading.RLock` in Python 3.15. - (Contributed by Nikita Sobolev in :gh:`102029`.) - Pending Removal in Python 3.16 ------------------------------ @@ -801,6 +985,9 @@ although there is currently no date scheduled for their removal. :data:`calendar.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) +* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method + instead. + * :mod:`datetime`: * :meth:`~datetime.datetime.utcnow`: @@ -836,11 +1023,13 @@ although there is currently no date scheduled for their removal. underscore. (Contributed by Serhiy Storchaka in :gh:`91760`.) +* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. + * :mod:`ssl` options and protocols: * :class:`ssl.SSLContext` without protocol argument is deprecated. * :class:`ssl.SSLContext`: :meth:`~ssl.SSLContext.set_npn_protocols` and - :meth:`!~ssl.SSLContext.selected_npn_protocol` are deprecated: use ALPN + :meth:`!selected_npn_protocol` are deprecated: use ALPN instead. * ``ssl.OP_NO_SSL*`` options * ``ssl.OP_NO_TLS*`` options @@ -853,13 +1042,6 @@ although there is currently no date scheduled for their removal. * ``ssl.TLSVersion.TLSv1`` * ``ssl.TLSVersion.TLSv1_1`` -* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. - -* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method - instead. - -* :class:`typing.Text` (:gh:`92332`). - * :func:`sysconfig.is_python_build` *check_home* parameter is deprecated and ignored. @@ -874,14 +1056,10 @@ although there is currently no date scheduled for their removal. * :meth:`!threading.currentThread`: use :meth:`threading.current_thread`. * :meth:`!threading.activeCount`: use :meth:`threading.active_count`. -* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value - that is not None from a test case. - -* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and - :class:`~urllib.request.FancyURLopener` style of invoking requests is - deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods. +* :class:`typing.Text` (:gh:`92332`). -* :func:`!urllib.parse.to_bytes`. +* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value + that is not ``None`` from a test case. * :mod:`urllib.parse` deprecated functions: :func:`~urllib.parse.urlparse` instead @@ -895,6 +1073,11 @@ although there is currently no date scheduled for their removal. * ``splittype()`` * ``splituser()`` * ``splitvalue()`` + * ``to_bytes()`` + +* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and + :class:`~urllib.request.FancyURLopener` style of invoking requests is + deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods. * :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial writes. @@ -1180,20 +1363,13 @@ that may require changes to your code. Changes in the Python API ------------------------- -* :meth:`!tkinter.Text.count` now always returns an integer if one or less - counting options are specified. - Previously it could return a single count as a 1-tuple, an integer (only if - option ``"update"`` was specified) or ``None`` if no items found. - The result is now the same if ``wantobjects`` is set to ``0``. - (Contributed by Serhiy Storchaka in :gh:`97928`.) - * Functions :c:func:`PyDict_GetItem`, :c:func:`PyDict_GetItemString`, :c:func:`PyMapping_HasKey`, :c:func:`PyMapping_HasKeyString`, :c:func:`PyObject_HasAttr`, :c:func:`PyObject_HasAttrString`, and - :c:func:`PySys_GetObject`, which clear all errors occurred during calling - the function, report now them using :func:`sys.unraisablehook`. - You can consider to replace these functions with other functions as - recomended in the documentation. + :c:func:`PySys_GetObject`, which clear all errors which occurred when calling + them, now report them using :func:`sys.unraisablehook`. + You may replace them with other functions as + recommended in the documentation. (Contributed by Serhiy Storchaka in :gh:`106672`.) * An :exc:`OSError` is now raised by :func:`getpass.getuser` for any failure to @@ -1202,7 +1378,7 @@ Changes in the Python API * The :mod:`threading` module now expects the :mod:`!_thread` module to have an ``_is_main_interpreter`` attribute. It is a function with no - arguments that returns ``True`` if the current interpreter is the + arguments that return ``True`` if the current interpreter is the main interpreter. Any library or application that provides a custom ``_thread`` module @@ -1213,6 +1389,11 @@ Changes in the Python API * :class:`mailbox.Maildir` now ignores files with a leading dot. (Contributed by Zackery Spytz in :gh:`65559`.) +* :meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob` now return both + files and directories if a pattern that ends with "``**``" is given, rather + than directories only. Users may add a trailing slash to match only + directories. + Build Changes ============= @@ -1225,7 +1406,7 @@ Build Changes (Contributed by Erlend Aasland in :gh:`105875`.) * Python built with :file:`configure` :option:`--with-trace-refs` (tracing - references) is now ABI compatible with Python release build and + references) is now ABI compatible with the Python release build and :ref:`debug build <debug-build>`. (Contributed by Victor Stinner in :gh:`108634`.) @@ -1238,6 +1419,12 @@ Build Changes :ref:`limited C API <limited-c-api>`. (Contributed by Victor Stinner in :gh:`85283`.) +* ``wasm32-wasi`` is now a tier 2 platform. + (Contributed by Brett Cannon in :gh:`115192`.) + +* ``wasm32-emscripten`` is no longer a supported platform. + (Contributed by Brett Cannon in :gh:`115192`.) + C API Changes ============= @@ -1252,7 +1439,7 @@ New Features (Contributed by Inada Naoki in :gh:`104922`.) * The *keywords* parameter of :c:func:`PyArg_ParseTupleAndKeywords` and - :c:func:`PyArg_VaParseTupleAndKeywords` has now type :c:expr:`char * const *` + :c:func:`PyArg_VaParseTupleAndKeywords` now has type :c:expr:`char * const *` in C and :c:expr:`const char * const *` in C++, instead of :c:expr:`char **`. It makes these functions compatible with arguments of type :c:expr:`const char * const *`, :c:expr:`const char **` or @@ -1309,18 +1496,28 @@ New Features always steals a reference to the value. (Contributed by Serhiy Storchaka in :gh:`86493`.) -* Added :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef` +* Add :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef` functions: similar to :c:func:`PyDict_GetItemWithError` but returning a :term:`strong reference` instead of a :term:`borrowed reference`. Moreover, these functions return -1 on error and so checking ``PyErr_Occurred()`` is not needed. (Contributed by Victor Stinner in :gh:`106004`.) -* Added :c:func:`PyDict_ContainsString` function: same as +* Added :c:func:`PyDict_SetDefaultRef`, which is similar to + :c:func:`PyDict_SetDefault` but returns a :term:`strong reference` instead of + a :term:`borrowed reference`. This function returns ``-1`` on error, ``0`` on + insertion, and ``1`` if the key was already present in the dictionary. + (Contributed by Sam Gross in :gh:`112066`.) + +* Add :c:func:`PyDict_ContainsString` function: same as :c:func:`PyDict_Contains`, but *key* is specified as a :c:expr:`const char*` UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. (Contributed by Victor Stinner in :gh:`108314`.) +* Added :c:func:`PyList_GetItemRef` function: similar to + :c:func:`PyList_GetItem` but returns a :term:`strong reference` instead of + a :term:`borrowed reference`. + * Add :c:func:`Py_IsFinalizing` function: check if the main Python interpreter is :term:`shutting down <interpreter shutdown>`. (Contributed by Victor Stinner in :gh:`108014`.) @@ -1374,7 +1571,7 @@ New Features (Contributed by Victor Stinner in :gh:`85283`.) * Add :c:func:`PyErr_FormatUnraisable` function: similar to - :c:func:`PyErr_WriteUnraisable`, but allow to customize the warning mesage. + :c:func:`PyErr_WriteUnraisable`, but allow customizing the warning message. (Contributed by Serhiy Storchaka in :gh:`108082`.) * Add :c:func:`PyList_Extend` and :c:func:`PyList_Clear` functions: similar to @@ -1384,12 +1581,27 @@ New Features * Add :c:func:`PyDict_Pop` and :c:func:`PyDict_PopString` functions: remove a key from a dictionary and optionally return the removed value. This is similar to :meth:`dict.pop`, but without the default value and not raising - :exc:`KeyError` if the key missing. + :exc:`KeyError` if the key is missing. (Contributed by Stefan Behnel and Victor Stinner in :gh:`111262`.) * Add :c:func:`Py_HashPointer` function to hash a pointer. (Contributed by Victor Stinner in :gh:`111545`.) +* Add PyTime C API: + + * :c:type:`PyTime_t` type. + * :c:var:`PyTime_MIN` and :c:var:`PyTime_MAX` constants. + * :c:func:`PyTime_AsSecondsDouble` + :c:func:`PyTime_Monotonic`, :c:func:`PyTime_PerfCounter`, and + :c:func:`PyTime_Time` functions. + + (Contributed by Victor Stinner and Petr Viktorin in :gh:`110850`.) + +* Add :c:func:`PyLong_AsNativeBytes`, :c:func:`PyLong_FromNativeBytes` and + :c:func:`PyLong_FromUnsignedNativeBytes` functions to simplify converting + between native integer types and Python :class:`int` objects. + (Contributed by Steve Dower in :gh:`111140`.) + Porting to Python 3.13 ---------------------- @@ -1449,7 +1661,6 @@ Porting to Python 3.13 platforms, the ``HAVE_STDDEF_H`` macro is only defined on Windows. (Contributed by Victor Stinner in :gh:`108765`.) - Deprecated ---------- @@ -1497,7 +1708,7 @@ Removed ------- * Removed chained :class:`classmethod` descriptors (introduced in - :issue:`19072`). This can no longer be used to wrap other descriptors + :gh:`63272`). This can no longer be used to wrap other descriptors such as :class:`property`. The core design of this feature was flawed and caused a number of downstream problems. To "pass-through" a :class:`classmethod`, consider using the :attr:`!__wrapped__` @@ -1511,14 +1722,14 @@ Removed add ``cc @vstinner`` to the issue to notify Victor Stinner. (Contributed by Victor Stinner in :gh:`106320`.) -* Remove functions deprecated in Python 3.9. +* Remove functions deprecated in Python 3.9: * ``PyEval_CallObject()``, ``PyEval_CallObjectWithKeywords()``: use :c:func:`PyObject_CallNoArgs` or :c:func:`PyObject_Call` instead. Warning: :c:func:`PyObject_Call` positional arguments must be a - :class:`tuple` and must not be *NULL*, keyword arguments must be a - :class:`dict` or *NULL*, whereas removed functions checked arguments type - and accepted *NULL* positional and keyword arguments. + :class:`tuple` and must not be ``NULL``, keyword arguments must be a + :class:`dict` or ``NULL``, whereas removed functions checked arguments type + and accepted ``NULL`` positional and keyword arguments. To replace ``PyEval_CallObjectWithKeywords(func, NULL, kwargs)`` with :c:func:`PyObject_Call`, pass an empty tuple as positional arguments using :c:func:`PyTuple_New(0) <PyTuple_New>`. diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 9834bc03dc4b742..52474517f5facc9 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -344,8 +344,8 @@ aspects that are visible to the programmer: * The :mod:`importlib.abc` module has been updated with new :term:`abstract base classes <abstract base class>` for loading bytecode files. The obsolete - ABCs, :class:`~importlib.abc.PyLoader` and - :class:`~importlib.abc.PyPycLoader`, have been deprecated (instructions on how + ABCs, :class:`!PyLoader` and + :class:`!PyPycLoader`, have been deprecated (instructions on how to stay Python 3.1 compatible are included with the documentation). .. seealso:: @@ -401,7 +401,7 @@ The *native strings* are always of type :class:`str` but are restricted to code points between *U+0000* through *U+00FF* which are translatable to bytes using *Latin-1* encoding. These strings are used for the keys and values in the environment dictionary and for response headers and statuses in the -:func:`start_response` function. They must follow :rfc:`2616` with respect to +:func:`!start_response` function. They must follow :rfc:`2616` with respect to encoding. That is, they must either be *ISO-8859-1* characters or use :rfc:`2047` MIME encoding. @@ -415,8 +415,8 @@ points: encoded in utf-8 was using ``h.encode('utf-8')`` now needs to convert from bytes to native strings using ``h.encode('utf-8').decode('latin-1')``. -* Values yielded by an application or sent using the :meth:`write` method - must be byte strings. The :func:`start_response` function and environ +* Values yielded by an application or sent using the :meth:`!write` method + must be byte strings. The :func:`!start_response` function and environ must use native strings. The two cannot be mixed. For server implementers writing CGI-to-WSGI pathways or other CGI-style @@ -499,7 +499,7 @@ Some smaller changes made to the core Python language are: * The :func:`hasattr` function works by calling :func:`getattr` and detecting whether an exception is raised. This technique allows it to detect methods - created dynamically by :meth:`__getattr__` or :meth:`__getattribute__` which + created dynamically by :meth:`~object.__getattr__` or :meth:`~object.__getattribute__` which would otherwise be absent from the class dictionary. Formerly, *hasattr* would catch any exception, possibly masking genuine errors. Now, *hasattr* has been tightened to only catch :exc:`AttributeError` and let other @@ -620,7 +620,7 @@ Some smaller changes made to the core Python language are: * :class:`range` objects now support *index* and *count* methods. This is part of an effort to make more objects fully implement the - :class:`collections.Sequence` :term:`abstract base class`. As a result, the + :class:`collections.Sequence <collections.abc.Sequence>` :term:`abstract base class`. As a result, the language will have a more uniform API. In addition, :class:`range` objects now support slicing and negative indices, even with values larger than :data:`sys.maxsize`. This makes *range* more interoperable with lists:: @@ -720,7 +720,7 @@ format. elementtree ----------- -The :mod:`xml.etree.ElementTree` package and its :mod:`xml.etree.cElementTree` +The :mod:`xml.etree.ElementTree` package and its :mod:`!xml.etree.cElementTree` counterpart have been updated to version 1.3. Several new and useful functions and methods have been added: @@ -743,8 +743,8 @@ Several new and useful functions and methods have been added: Two methods have been deprecated: -* :meth:`xml.etree.ElementTree.getchildren` use ``list(elem)`` instead. -* :meth:`xml.etree.ElementTree.getiterator` use ``Element.iter`` instead. +* :meth:`!xml.etree.ElementTree.getchildren` use ``list(elem)`` instead. +* :meth:`!xml.etree.ElementTree.getiterator` use ``Element.iter`` instead. For details of the update, see `Introducing ElementTree <https://web.archive.org/web/20200703234532/http://effbot.org/zone/elementtree-13-intro.htm>`_ @@ -1008,13 +1008,13 @@ datetime and time after 1900. The new supported year range is from 1000 to 9999 inclusive. * Whenever a two-digit year is used in a time tuple, the interpretation has been - governed by :data:`time.accept2dyear`. The default is ``True`` which means that + governed by :data:`!time.accept2dyear`. The default is ``True`` which means that for a two-digit year, the century is guessed according to the POSIX rules governing the ``%y`` strptime format. Starting with Py3.2, use of the century guessing heuristic will emit a :exc:`DeprecationWarning`. Instead, it is recommended that - :data:`time.accept2dyear` be set to ``False`` so that large date ranges + :data:`!time.accept2dyear` be set to ``False`` so that large date ranges can be used without guesswork:: >>> import time, warnings @@ -1032,7 +1032,7 @@ datetime and time 'Fri Jan 1 12:34:56 11' Several functions now have significantly expanded date ranges. When - :data:`time.accept2dyear` is false, the :func:`time.asctime` function will + :data:`!time.accept2dyear` is false, the :func:`time.asctime` function will accept any year that fits in a C int, while the :func:`time.mktime` and :func:`time.strftime` functions will accept the full range supported by the corresponding operating system functions. @@ -1148,15 +1148,15 @@ for slice notation are well-suited to in-place editing:: reprlib ------- -When writing a :meth:`__repr__` method for a custom container, it is easy to +When writing a :meth:`~object.__repr__` method for a custom container, it is easy to forget to handle the case where a member refers back to the container itself. Python's builtin objects such as :class:`list` and :class:`set` handle self-reference by displaying "..." in the recursive part of the representation string. -To help write such :meth:`__repr__` methods, the :mod:`reprlib` module has a new +To help write such :meth:`~object.__repr__` methods, the :mod:`reprlib` module has a new decorator, :func:`~reprlib.recursive_repr`, for detecting recursive calls to -:meth:`__repr__` and substituting a placeholder string instead:: +:meth:`!__repr__` and substituting a placeholder string instead:: >>> class MyList(list): ... @recursive_repr() @@ -1308,7 +1308,7 @@ used for the imaginary part of a number: >>> sys.hash_info # doctest: +SKIP sys.hash_info(width=64, modulus=2305843009213693951, inf=314159, nan=0, imag=1000003) -An early decision to limit the inter-operability of various numeric types has +An early decision to limit the interoperability of various numeric types has been relaxed. It is still unsupported (and ill-advised) to have implicit mixing in arithmetic expressions such as ``Decimal('1.1') + float('1.1')`` because the latter loses information in the process of constructing the binary @@ -1336,7 +1336,7 @@ Decimal('1.100000000000000088817841970012523233890533447265625') Fraction(2476979795053773, 2251799813685248) Another useful change for the :mod:`decimal` module is that the -:attr:`Context.clamp` attribute is now public. This is useful in creating +:attr:`Context.clamp <decimal.Context.clamp>` attribute is now public. This is useful in creating contexts that correspond to the decimal interchange formats specified in IEEE 754 (see :issue:`8540`). @@ -1428,7 +1428,7 @@ before compressing and decompressing: Aides and Brian Curtin in :issue:`9962`, :issue:`1675951`, :issue:`7471` and :issue:`2846`.) -Also, the :class:`zipfile.ZipExtFile` class was reworked internally to represent +Also, the :class:`zipfile.ZipExtFile <zipfile.ZipFile.open>` class was reworked internally to represent files stored inside an archive. The new implementation is significantly faster and can be wrapped in an :class:`io.BufferedReader` object for more speedups. It also solves an issue where interleaved calls to *read* and *readline* gave the @@ -1596,7 +1596,7 @@ sqlite3 The :mod:`sqlite3` module was updated to pysqlite version 2.6.0. It has two new capabilities. -* The :attr:`sqlite3.Connection.in_transit` attribute is true if there is an +* The :attr:`!sqlite3.Connection.in_transit` attribute is true if there is an active transaction for uncommitted changes. * The :meth:`sqlite3.Connection.enable_load_extension` and @@ -1643,11 +1643,11 @@ for secure (encrypted, authenticated) internet connections: other options. It includes a :meth:`~ssl.SSLContext.wrap_socket` for creating an SSL socket from an SSL context. -* A new function, :func:`ssl.match_hostname`, supports server identity +* A new function, :func:`!ssl.match_hostname`, supports server identity verification for higher-level protocols by implementing the rules of HTTPS (from :rfc:`2818`) which are also suitable for other protocols. -* The :func:`ssl.wrap_socket` constructor function now takes a *ciphers* +* The :func:`ssl.wrap_socket() <ssl.SSLContext.wrap_socket>` constructor function now takes a *ciphers* argument. The *ciphers* string lists the allowed encryption algorithms using the format described in the `OpenSSL documentation <https://www.openssl.org/docs/man1.0.2/man1/ciphers.html#CIPHER-LIST-FORMAT>`__. @@ -1759,7 +1759,7 @@ names. (Contributed by Michael Foord.) * Experimentation at the interactive prompt is now easier because the - :class:`unittest.case.TestCase` class can now be instantiated without + :class:`unittest.TestCase` class can now be instantiated without arguments: >>> from unittest import TestCase @@ -1797,7 +1797,7 @@ names. * In addition, the method names in the module have undergone a number of clean-ups. For example, :meth:`~unittest.TestCase.assertRegex` is the new name for - :meth:`~unittest.TestCase.assertRegexpMatches` which was misnamed because the + :meth:`!assertRegexpMatches` which was misnamed because the test uses :func:`re.search`, not :func:`re.match`. Other methods using regular expressions are now named using short form "Regex" in preference to "Regexp" -- this matches the names used in other unittest implementations, @@ -1812,11 +1812,11 @@ names. =============================== ============================== Old Name Preferred Name =============================== ============================== - :meth:`assert_` :meth:`.assertTrue` - :meth:`assertEquals` :meth:`.assertEqual` - :meth:`assertNotEquals` :meth:`.assertNotEqual` - :meth:`assertAlmostEquals` :meth:`.assertAlmostEqual` - :meth:`assertNotAlmostEquals` :meth:`.assertNotAlmostEqual` + :meth:`!assert_` :meth:`.assertTrue` + :meth:`!assertEquals` :meth:`.assertEqual` + :meth:`!assertNotEquals` :meth:`.assertNotEqual` + :meth:`!assertAlmostEquals` :meth:`.assertAlmostEqual` + :meth:`!assertNotAlmostEquals` :meth:`.assertNotAlmostEqual` =============================== ============================== Likewise, the ``TestCase.fail*`` methods deprecated in Python 3.1 are expected @@ -1824,7 +1824,7 @@ names. (Contributed by Ezio Melotti; :issue:`9424`.) -* The :meth:`~unittest.TestCase.assertDictContainsSubset` method was deprecated +* The :meth:`!assertDictContainsSubset` method was deprecated because it was misimplemented with the arguments in the wrong order. This created hard-to-debug optical illusions where tests like ``TestCase().assertDictContainsSubset({'a':1, 'b':2}, {'a':1})`` would fail. @@ -1997,7 +1997,7 @@ under-the-hood. dbm --- -All database modules now support the :meth:`get` and :meth:`setdefault` methods. +All database modules now support the :meth:`!get` and :meth:`!setdefault` methods. (Suggested by Ray Allen in :issue:`9523`.) @@ -2118,7 +2118,7 @@ The :mod:`pdb` debugger module gained a number of usability improvements: :file:`.pdbrc` script file. * A :file:`.pdbrc` script file can contain ``continue`` and ``next`` commands that continue debugging. -* The :class:`Pdb` class constructor now accepts a *nosigint* argument. +* The :class:`~pdb.Pdb` class constructor now accepts a *nosigint* argument. * New commands: ``l(list)``, ``ll(long list)`` and ``source`` for listing source code. * New commands: ``display`` and ``undisplay`` for showing or hiding @@ -2394,11 +2394,11 @@ A number of small performance enhancements have been added: (Contributed by Antoine Pitrou; :issue:`3001`.) -* The fast-search algorithm in stringlib is now used by the :meth:`split`, - :meth:`rsplit`, :meth:`splitlines` and :meth:`replace` methods on +* The fast-search algorithm in stringlib is now used by the :meth:`~str.split`, + :meth:`~str.rsplit`, :meth:`~str.splitlines` and :meth:`~str.replace` methods on :class:`bytes`, :class:`bytearray` and :class:`str` objects. Likewise, the - algorithm is also used by :meth:`rfind`, :meth:`rindex`, :meth:`rsplit` and - :meth:`rpartition`. + algorithm is also used by :meth:`~str.rfind`, :meth:`~str.rindex`, :meth:`~str.rsplit` and + :meth:`~str.rpartition`. (Patch by Florent Xicluna in :issue:`7622` and :issue:`7462`.) @@ -2410,8 +2410,8 @@ A number of small performance enhancements have been added: There were several other minor optimizations. Set differencing now runs faster when one operand is much larger than the other (patch by Andress Bennetts in -:issue:`8685`). The :meth:`array.repeat` method has a faster implementation -(:issue:`1569291` by Alexander Belopolsky). The :class:`BaseHTTPRequestHandler` +:issue:`8685`). The :meth:`!array.repeat` method has a faster implementation +(:issue:`1569291` by Alexander Belopolsky). The :class:`~http.server.BaseHTTPRequestHandler` has more efficient buffering (:issue:`3709` by Andrew Schaaf). The :func:`operator.attrgetter` function has been sped-up (:issue:`10160` by Christos Georgiou). And :class:`~configparser.ConfigParser` loads multi-line arguments a bit @@ -2562,11 +2562,11 @@ Changes to Python's build process and to the C API include: (Suggested by Raymond Hettinger and implemented by Benjamin Peterson; :issue:`9778`.) -* A new macro :c:macro:`Py_VA_COPY` copies the state of the variable argument +* A new macro :c:macro:`!Py_VA_COPY` copies the state of the variable argument list. It is equivalent to C99 *va_copy* but available on all Python platforms (:issue:`2443`). -* A new C API function :c:func:`PySys_SetArgvEx` allows an embedded interpreter +* A new C API function :c:func:`!PySys_SetArgvEx` allows an embedded interpreter to set :data:`sys.argv` without also modifying :data:`sys.path` (:issue:`5753`). @@ -2650,8 +2650,9 @@ require changes to your code: * :class:`bytearray` objects can no longer be used as filenames; instead, they should be converted to :class:`bytes`. -* The :meth:`array.tostring` and :meth:`array.fromstring` have been renamed to - :meth:`array.tobytes` and :meth:`array.frombytes` for clarity. The old names +* The :meth:`!array.tostring` and :meth:`!array.fromstring` have been renamed to + :meth:`array.tobytes() <array.array.tobytes>` and + :meth:`array.frombytes() <array.array.frombytes>` for clarity. The old names have been deprecated. (See :issue:`8990`.) * ``PyArg_Parse*()`` functions: @@ -2664,7 +2665,7 @@ require changes to your code: instead; the new type has a well-defined interface for passing typing safety information and a less complicated signature for calling a destructor. -* The :func:`sys.setfilesystemencoding` function was removed because +* The :func:`!sys.setfilesystemencoding` function was removed because it had a flawed design. * The :func:`random.seed` function and method now salt string seeds with an @@ -2672,7 +2673,7 @@ require changes to your code: reproduce Python 3.1 sequences, set the *version* argument to *1*, ``random.seed(s, version=1)``. -* The previously deprecated :func:`string.maketrans` function has been removed +* The previously deprecated :func:`!string.maketrans` function has been removed in favor of the static methods :meth:`bytes.maketrans` and :meth:`bytearray.maketrans`. This change solves the confusion around which types were supported by the :mod:`string` module. Now, :class:`str`, @@ -2682,7 +2683,7 @@ require changes to your code: (Contributed by Georg Brandl; :issue:`5675`.) -* The previously deprecated :func:`contextlib.nested` function has been removed +* The previously deprecated :func:`!contextlib.nested` function has been removed in favor of a plain :keyword:`with` statement which can accept multiple context managers. The latter technique is faster (because it is built-in), and it does a better job finalizing multiple context managers when one of them diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 1c7a9270af0aabc..5c2ec230441b426 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -2418,7 +2418,7 @@ Changes in the Python API (Contributed by Victor Stinner in :issue:`21205`.) * The deprecated "strict" mode and argument of :class:`~html.parser.HTMLParser`, - :meth:`HTMLParser.error`, and the :exc:`HTMLParserError` exception have been + :meth:`!HTMLParser.error`, and the :exc:`!HTMLParserError` exception have been removed. (Contributed by Ezio Melotti in :issue:`15114`.) The *convert_charrefs* argument of :class:`~html.parser.HTMLParser` is now ``True`` by default. (Contributed by Berker Peksag in :issue:`21047`.) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 11e1d73232a96db..d62beb0bdc86720 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -1472,6 +1472,10 @@ Server and client-side specific TLS protocols for :class:`~ssl.SSLContext` were added. (Contributed by Christian Heimes in :issue:`28085`.) +Added :attr:`ssl.SSLContext.post_handshake_auth` to enable and +:meth:`ssl.SSLSocket.verify_client_post_handshake` to initiate TLS 1.3 +post-handshake authentication. +(Contributed by Christian Heimes in :gh:`78851`.) statistics ---------- @@ -2063,6 +2067,15 @@ connected to and thus what Python interpreter will be used by the virtual environment. (Contributed by Brett Cannon in :issue:`25154`.) +xml +--- + +* As mitigation against DTD and external entity retrieval, the + :mod:`xml.dom.minidom` and :mod:`xml.sax` modules no longer process + external entities by default. + (Contributed by Christian Heimes in :gh:`61441`.) + + Deprecated functions and types of the C API ------------------------------------------- @@ -2430,9 +2443,13 @@ The :func:`locale.localeconv` function now sets temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` locale in some cases. (Contributed by Victor Stinner in :issue:`31900`.) + Notable changes in Python 3.6.7 =============================== +:mod:`xml.dom.minidom` and :mod:`xml.sax` modules no longer process +external entities by default. See also :gh:`61441`. + In 3.6.7 the :mod:`tokenize` module now implicitly emits a ``NEWLINE`` token when provided with input that does not have a trailing new line. This behavior now matches what the C tokenizer does internally. @@ -2460,3 +2477,19 @@ separator key, with ``&`` as the default. This change also affects functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) + +Notable changes in Python 3.6.14 +================================ + +A security fix alters the :class:`ftplib.FTP` behavior to not trust the +IPv4 address sent from the remote server when setting up a passive data +channel. We reuse the ftp server IP address instead. For unusual code +requiring the old behavior, set a ``trust_server_pasv_ipv4_address`` +attribute on your FTP instance to ``True``. (See :gh:`87451`) + +The presence of newline or tab characters in parts of a URL allows for some +forms of attacks. Following the WHATWG specification that updates RFC 3986, +ASCII newline ``\n``, ``\r`` and tab ``\t`` characters are stripped from the +URL by the parser :func:`urllib.parse` preventing such attacks. The removal +characters are controlled by a new module level variable +``urllib.parse._UNSAFE_URL_BYTES_TO_REMOVE``. (See :gh:`88048`) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 402b15a277e53db..8122e0ee129b0d2 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -1380,6 +1380,10 @@ Supported protocols are indicated by several new flags, such as :data:`~ssl.HAS_TLSv1_1`. (Contributed by Christian Heimes in :issue:`32609`.) +Added :attr:`ssl.SSLContext.post_handshake_auth` to enable and +:meth:`ssl.SSLSocket.verify_client_post_handshake` to initiate TLS 1.3 +post-handshake authentication. +(Contributed by Christian Heimes in :gh:`78851`.) string ------ @@ -1599,6 +1603,15 @@ at the interactive prompt. See :ref:`whatsnew37-pep565` for details. (Contributed by Nick Coghlan in :issue:`31975`.) +xml +--- + +As mitigation against DTD and external entity retrieval, the +:mod:`xml.dom.minidom` and :mod:`xml.sax` modules no longer process +external entities by default. +(Contributed by Christian Heimes in :gh:`61441`.) + + xml.etree --------- @@ -2571,3 +2584,34 @@ separator key, with ``&`` as the default. This change also affects functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) + +Notable changes in Python 3.7.11 +================================ + +A security fix alters the :class:`ftplib.FTP` behavior to not trust the +IPv4 address sent from the remote server when setting up a passive data +channel. We reuse the ftp server IP address instead. For unusual code +requiring the old behavior, set a ``trust_server_pasv_ipv4_address`` +attribute on your FTP instance to ``True``. (See :gh:`87451`) + + +The presence of newline or tab characters in parts of a URL allows for some +forms of attacks. Following the WHATWG specification that updates RFC 3986, +ASCII newline ``\n``, ``\r`` and tab ``\t`` characters are stripped from the +URL by the parser :func:`urllib.parse` preventing such attacks. The removal +characters are controlled by a new module level variable +``urllib.parse._UNSAFE_URL_BYTES_TO_REMOVE``. (See :gh:`88048`) + +Notable security feature in 3.7.14 +================================== + +Converting between :class:`int` and :class:`str` in bases other than 2 +(binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) +now raises a :exc:`ValueError` if the number of digits in string form is +above a limit to avoid potential denial of service attacks due to the +algorithmic complexity. This is a mitigation for `CVE-2020-10735 +<https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10735>`_. +This limit can be configured or disabled by environment variable, command +line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion +length limitation <int_max_str_digits>` documentation. The default limit +is 4300 digits in string form. diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 304d1b4ef4efe80..9a2652f5e336051 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -754,8 +754,8 @@ datetime -------- Added new alternate constructors :meth:`datetime.date.fromisocalendar` and -:meth:`datetime.datetime.fromisocalendar`, which construct :class:`date` and -:class:`datetime` objects respectively from ISO year, week number, and weekday; +:meth:`datetime.datetime.fromisocalendar`, which construct :class:`~datetime.date` and +:class:`~datetime.datetime` objects respectively from ISO year, week number, and weekday; these are the inverse of each class's ``isocalendar`` method. (Contributed by Paul Ganssle in :issue:`36004`.) @@ -2243,6 +2243,21 @@ details, see the documentation for ``loop.create_datagram_endpoint()``. (Contributed by Kyle Stanley, Antoine Pitrou, and Yury Selivanov in :issue:`37228`.) +Notable changes in Python 3.8.2 +=============================== + +Fixed a regression with the ``ignore`` callback of :func:`shutil.copytree`. +The argument types are now str and List[str] again. +(Contributed by Manuel Barkhau and Giampaolo Rodola in :gh:`83571`.) + +Notable changes in Python 3.8.3 +=============================== + +The constant values of future flags in the :mod:`__future__` module +are updated in order to prevent collision with compiler flags. Previously +``PyCF_ALLOW_TOP_LEVEL_AWAIT`` was clashing with ``CO_FUTURE_DIVISION``. +(Contributed by Batuhan Taskaya in :gh:`83743`) + Notable changes in Python 3.8.8 =============================== @@ -2256,9 +2271,55 @@ functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) +Notable changes in Python 3.8.9 +=============================== + +A security fix alters the :class:`ftplib.FTP` behavior to not trust the +IPv4 address sent from the remote server when setting up a passive data +channel. We reuse the ftp server IP address instead. For unusual code +requiring the old behavior, set a ``trust_server_pasv_ipv4_address`` +attribute on your FTP instance to ``True``. (See :gh:`87451`) + +Notable changes in Python 3.8.10 +================================ + +macOS 11.0 (Big Sur) and Apple Silicon Mac support +-------------------------------------------------- + +As of 3.8.10, Python now supports building and running on macOS 11 +(Big Sur) and on Apple Silicon Macs (based on the ``ARM64`` architecture). +A new universal build variant, ``universal2``, is now available to natively +support both ``ARM64`` and ``Intel 64`` in one set of executables. +Note that support for "weaklinking", building binaries targeted for newer +versions of macOS that will also run correctly on older versions by +testing at runtime for missing features, is not included in this backport +from Python 3.9; to support a range of macOS versions, continue to target +for and build on the oldest version in the range. + +(Originally contributed by Ronald Oussoren and Lawrence D'Anna in :gh:`85272`, +with fixes by FX Coudert and Eli Rykoff, and backported to 3.8 by Maxime Bélanger +and Ned Deily) + +Notable changes in Python 3.8.10 +================================ + +urllib.parse +------------ + +The presence of newline or tab characters in parts of a URL allows for some +forms of attacks. Following the WHATWG specification that updates :rfc:`3986`, +ASCII newline ``\n``, ``\r`` and tab ``\t`` characters are stripped from the +URL by the parser in :mod:`urllib.parse` preventing such attacks. The removal +characters are controlled by a new module level variable +``urllib.parse._UNSAFE_URL_BYTES_TO_REMOVE``. (See :issue:`43882`) + + Notable changes in Python 3.8.12 ================================ +Changes in the Python API +------------------------- + Starting with Python 3.8.12 the :mod:`ipaddress` module no longer accepts any leading zeros in IPv4 address strings. Leading zeros are ambiguous and interpreted as octal notation by some libraries. For example the legacy @@ -2268,3 +2329,33 @@ any leading zeros. (Originally contributed by Christian Heimes in :issue:`36384`, and backported to 3.8 by Achraf Merzouki.) + +Notable security feature in 3.8.14 +================================== + +Converting between :class:`int` and :class:`str` in bases other than 2 +(binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) +now raises a :exc:`ValueError` if the number of digits in string form is +above a limit to avoid potential denial of service attacks due to the +algorithmic complexity. This is a mitigation for `CVE-2020-10735 +<https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10735>`_. +This limit can be configured or disabled by environment variable, command +line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion +length limitation <int_max_str_digits>` documentation. The default limit +is 4300 digits in string form. + +Notable changes in 3.8.17 +========================= + +tarfile +------- + +* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, + have a new a *filter* argument that allows limiting tar features than may be + surprising or dangerous, such as creating files outside the destination + directory. + See :ref:`tarfile-extraction-filter` for details. + In Python 3.12, use without the *filter* argument will show a + :exc:`DeprecationWarning`. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index f7ad4372325ccb1..b24c13813be2209 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -1562,3 +1562,55 @@ separator key, with ``&`` as the default. This change also affects functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) + +Notable changes in Python 3.9.3 +=============================== + +A security fix alters the :class:`ftplib.FTP` behavior to not trust the +IPv4 address sent from the remote server when setting up a passive data +channel. We reuse the ftp server IP address instead. For unusual code +requiring the old behavior, set a ``trust_server_pasv_ipv4_address`` +attribute on your FTP instance to ``True``. (See :gh:`87451`) + +Notable changes in Python 3.9.5 +=============================== + +urllib.parse +------------ + +The presence of newline or tab characters in parts of a URL allows for some +forms of attacks. Following the WHATWG specification that updates :rfc:`3986`, +ASCII newline ``\n``, ``\r`` and tab ``\t`` characters are stripped from the +URL by the parser in :mod:`urllib.parse` preventing such attacks. The removal +characters are controlled by a new module level variable +``urllib.parse._UNSAFE_URL_BYTES_TO_REMOVE``. (See :gh:`88048`) + +Notable security feature in 3.9.14 +================================== + +Converting between :class:`int` and :class:`str` in bases other than 2 +(binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) +now raises a :exc:`ValueError` if the number of digits in string form is +above a limit to avoid potential denial of service attacks due to the +algorithmic complexity. This is a mitigation for `CVE-2020-10735 +<https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10735>`_. +This limit can be configured or disabled by environment variable, command +line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion +length limitation <int_max_str_digits>` documentation. The default limit +is 4300 digits in string form. + +Notable changes in 3.9.17 +========================= + +tarfile +------- + +* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, + have a new a *filter* argument that allows limiting tar features than may be + surprising or dangerous, such as creating files outside the destination + directory. + See :ref:`tarfile-extraction-filter` for details. + In Python 3.12, use without the *filter* argument will show a + :exc:`DeprecationWarning`. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) diff --git a/Grammar/python.gram b/Grammar/python.gram index 174b4dbb6f78427..797c195a0a91ba7 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -393,7 +393,7 @@ for_stmt[stmt_ty]: with_stmt[stmt_ty]: | invalid_with_stmt_indent | 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' tc=[TYPE_COMMENT] b=block { - CHECK_VERSION(stmt_ty, 9, "Parenthesized context managers are", _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA)) } + _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } | 'with' a[asdl_withitem_seq*]=','.with_item+ ':' tc=[TYPE_COMMENT] b=block { _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } | 'async' 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' b=block { diff --git a/Include/Python.h b/Include/Python.h index 196751c3201e620..01fc45137a17bbc 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -97,6 +97,7 @@ #include "weakrefobject.h" #include "structseq.h" #include "cpython/picklebufobject.h" +#include "cpython/pytime.h" #include "codecs.h" #include "pyerrors.h" #include "pythread.h" diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 944965fb9e5351a..35b6a822a0dfffd 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -17,6 +17,9 @@ typedef struct { /* Dictionary version: globally unique, value change each time the dictionary is modified */ #ifdef Py_BUILD_CORE + /* Bits 0-7 are for dict watchers. + * Bits 8-11 are for the watched mutation counter (used by tier2 optimization) + * The remaining bits (12-63) are the actual version tag. */ uint64_t ma_version_tag; #else Py_DEPRECATED(3.12) uint64_t ma_version_tag; @@ -38,6 +41,16 @@ PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *); PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); +// Inserts `key` with a value `default_value`, if `key` is not already present +// in the dictionary. If `result` is not NULL, then the value associated +// with `key` is returned in `*result` (either the existing value, or the now +// inserted `default_value`). +// Returns: +// -1 on error +// 0 if `key` was not present and `default_value` was inserted +// 1 if `key` was present and `default_value` was not inserted +PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); + /* Get the number of items of a dictionary. */ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyDictObject *mp; diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index f5ccbb704e8bb8e..f037c7bb90deda5 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -62,21 +62,32 @@ typedef long stwodigits; /* signed variant of twodigits */ #define PyLong_MASK ((digit)(PyLong_BASE - 1)) /* Long integer representation. + + Long integers are made up of a number of 30- or 15-bit digits, depending on + the platform. The number of digits (ndigits) is stored in the high bits of + the lv_tag field (lvtag >> _PyLong_NON_SIZE_BITS). + The absolute value of a number is equal to - SUM(for i=0 through abs(ob_size)-1) ob_digit[i] * 2**(SHIFT*i) - Negative numbers are represented with ob_size < 0; - zero is represented by ob_size == 0. - In a normalized number, ob_digit[abs(ob_size)-1] (the most significant + SUM(for i=0 through ndigits-1) ob_digit[i] * 2**(PyLong_SHIFT*i) + + The sign of the value is stored in the lower 2 bits of lv_tag. + + - 0: Positive + - 1: Zero + - 2: Negative + + The third lowest bit of lv_tag is reserved for an immortality flag, but is + not currently used. + + In a normalized number, ob_digit[ndigits-1] (the most significant digit) is never zero. Also, in all cases, for all valid i, - 0 <= ob_digit[i] <= MASK. + 0 <= ob_digit[i] <= PyLong_MASK. + The allocation function takes care of allocating extra memory - so that ob_digit[0] ... ob_digit[abs(ob_size)-1] are actually available. + so that ob_digit[0] ... ob_digit[ndigits-1] are actually available. We always allocate memory for at least one digit, so accessing ob_digit[0] - is always safe. However, in the case ob_size == 0, the contents of + is always safe. However, in the case ndigits == 0, the contents of ob_digit[0] may be undefined. - - CAUTION: Generic code manipulating subtypes of PyVarObject has to - aware that ints abuse ob_size's sign bit. */ typedef struct _PyLongValue { diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index fd1be29ed397d18..07251db6bcc203c 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -4,6 +4,40 @@ PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); +/* PyLong_AsNativeBytes: Copy the integer value to a native variable. + buffer points to the first byte of the variable. + n_bytes is the number of bytes available in the buffer. Pass 0 to request + the required size for the value. + endianness is -1 for native endian, 0 for big endian or 1 for little. + Big endian mode will write the most significant byte into the address + directly referenced by buffer; little endian will write the least significant + byte into that address. + + If an exception is raised, returns a negative value. + Otherwise, returns the number of bytes that are required to store the value. + To check that the full value is represented, ensure that the return value is + equal or less than n_bytes. + All n_bytes are guaranteed to be written (unless an exception occurs), and + so ignoring a positive return value is the equivalent of a downcast in C. + In cases where the full value could not be represented, the returned value + may be larger than necessary - this function is not an accurate way to + calculate the bit length of an integer object. + */ +PyAPI_FUNC(Py_ssize_t) PyLong_AsNativeBytes(PyObject* v, void* buffer, + Py_ssize_t n_bytes, int endianness); + +/* PyLong_FromNativeBytes: Create an int value from a native integer + n_bytes is the number of bytes to read from the buffer. Passing 0 will + always produce the zero int. + PyLong_FromUnsignedNativeBytes always produces a non-negative int. + endianness is -1 for native endian, 0 for big endian or 1 for little. + + Returns the int object, or NULL with an exception set. */ +PyAPI_FUNC(PyObject*) PyLong_FromNativeBytes(const void* buffer, size_t n_bytes, + int endianness); +PyAPI_FUNC(PyObject*) PyLong_FromUnsignedNativeBytes(const void* buffer, + size_t n_bytes, int endianness); + PyAPI_FUNC(int) PyUnstable_Long_IsCompact(const PyLongObject* op); PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op); @@ -50,7 +84,7 @@ PyAPI_FUNC(PyObject *) _PyLong_FromByteArray( */ PyAPI_FUNC(int) _PyLong_AsByteArray(PyLongObject* v, unsigned char* bytes, size_t n, - int little_endian, int is_signed); + int little_endian, int is_signed, int with_exceptions); /* For use by the gcd function in mathmodule.c */ PyAPI_FUNC(PyObject *) _PyLong_GCD(PyObject *, PyObject *); diff --git a/Include/cpython/object.h b/Include/cpython/object.h index c93931634fee051..7512bb70c760fdd 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -229,6 +229,7 @@ struct _typeobject { /* bitset of which type-watchers care about this type */ unsigned char tp_watched; + uint16_t tp_versions_used; }; /* This struct is used by the specializer diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h index 58a30aeea6ac642..e0c2ce286f13ce3 100644 --- a/Include/cpython/objimpl.h +++ b/Include/cpython/objimpl.h @@ -85,3 +85,20 @@ PyAPI_FUNC(PyObject **) PyObject_GET_WEAKREFS_LISTPTR(PyObject *op); PyAPI_FUNC(PyObject *) PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *, size_t); + + +/* Visit all live GC-capable objects, similar to gc.get_objects(None). The + * supplied callback is called on every such object with the void* arg set + * to the supplied arg. Returning 0 from the callback ends iteration, returning + * 1 allows iteration to continue. Returning any other value may result in + * undefined behaviour. + * + * If new objects are (de)allocated by the callback it is undefined if they + * will be visited. + + * Garbage collection is disabled during operation. Explicitly running a + * collection in the callback may lead to undefined behaviour e.g. visiting the + * same objects multiple times or not at all. + */ +typedef int (*gcvisitobjects_t)(PyObject*, void*); +PyAPI_FUNC(void) PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void* arg); diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 96e829f8fbe97db..8fc9fb62aebdb41 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -24,28 +24,46 @@ typedef struct { uint8_t opcode; uint8_t oparg; uint8_t valid; - uint8_t linked; + int index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). _PyBloomFilter bloom; _PyExecutorLinkListNode links; + PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR). } _PyVMData; typedef struct { uint16_t opcode; uint16_t oparg; - uint32_t target; + union { + uint32_t target; + uint32_t exit_index; + }; uint64_t operand; // A cache entry } _PyUOpInstruction; +typedef struct _exit_data { + uint32_t target; + int16_t temperature; + const struct _PyExecutorObject *executor; +} _PyExitData; + typedef struct _PyExecutorObject { PyObject_VAR_HEAD + const _PyUOpInstruction *trace; _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ - _PyUOpInstruction trace[1]; + uint32_t exit_count; + uint32_t code_size; + size_t jit_size; + void *jit_code; + _PyExitData exits[1]; } _PyExecutorObject; typedef struct _PyOptimizerObject _PyOptimizerObject; /* Should return > 0 if a new executor is created. O if no executor is produced and < 0 if an error occurred. */ -typedef int (*optimize_func)(_PyOptimizerObject* self, PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject **, int curr_stackentries); +typedef int (*optimize_func)( + _PyOptimizerObject* self, struct _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, + int curr_stackentries); typedef struct _PyOptimizerObject { PyObject_HEAD @@ -53,6 +71,7 @@ typedef struct _PyOptimizerObject { /* These thresholds are treated as signed so do not exceed INT16_MAX * Use INT16_MAX to indicate that the optimizer should never be called */ uint16_t resume_threshold; + uint16_t side_threshold; uint16_t backedge_threshold; /* Data needed by the optimizer goes here, but is opaque to the VM */ } _PyOptimizerObject; @@ -65,24 +84,24 @@ typedef struct { PyAPI_FUNC(int) PyUnstable_Replace_Executor(PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject *executor); -PyAPI_FUNC(void) PyUnstable_SetOptimizer(_PyOptimizerObject* optimizer); +_PyOptimizerObject *_Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject* optimizer); + +PyAPI_FUNC(int) PyUnstable_SetOptimizer(_PyOptimizerObject* optimizer); PyAPI_FUNC(_PyOptimizerObject *) PyUnstable_GetOptimizer(void); PyAPI_FUNC(_PyExecutorObject *) PyUnstable_GetExecutor(PyCodeObject *code, int offset); int -_PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer); - -extern _PyOptimizerObject _PyOptimizer_Default; +_PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer, _PyExecutorObject **exec_ptr); -void _Py_ExecutorInit(_PyExecutorObject *, _PyBloomFilter *); +void _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); void _Py_ExecutorClear(_PyExecutorObject *); void _Py_BloomFilter_Init(_PyBloomFilter *); void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); -PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj); -extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp); +PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation); +extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation); /* For testing */ PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void); @@ -90,7 +109,14 @@ PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void); #define OPTIMIZER_BITS_IN_COUNTER 4 /* Minimum of 16 additional executions before retry */ -#define MINIMUM_TIER2_BACKOFF 4 +#define MIN_TIER2_BACKOFF 4 +#define MAX_TIER2_BACKOFF (15 - OPTIMIZER_BITS_IN_COUNTER) +#define OPTIMIZER_BITS_MASK ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1) +/* A value <= UINT16_MAX but large enough that when shifted is > UINT16_MAX */ +#define OPTIMIZER_UNREACHABLE_THRESHOLD UINT16_MAX + +#define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 +#define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 #ifdef __cplusplus } diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 5314a70436bfc39..c3e132d3877ca51 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -360,6 +360,8 @@ _Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj); static inline void * _Py_atomic_load_ptr_relaxed(const void *obj); +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj); // --- _Py_atomic_store ------------------------------------------------------ // Atomically performs `*obj = value` (sequential consistency) @@ -452,6 +454,10 @@ _Py_atomic_store_ptr_relaxed(void *obj, void *value); static inline void _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value); +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value); + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -463,12 +469,27 @@ _Py_atomic_load_ptr_acquire(const void *obj); static inline void _Py_atomic_store_ptr_release(void *obj, void *value); +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); + static inline void _Py_atomic_store_int_release(int *obj, int value); static inline int _Py_atomic_load_int_acquire(const int *obj); +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value); + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj); + +static inline uint32_t +_Py_atomic_load_uint32_acquire(const uint32_t *obj); + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); + // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 70f2b7e1b5706a9..0b40f81bd8736de 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -358,6 +358,10 @@ static inline void * _Py_atomic_load_ptr_relaxed(const void *obj) { return (void *)__atomic_load_n((const void **)obj, __ATOMIC_RELAXED); } +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + // --- _Py_atomic_store ------------------------------------------------------ @@ -476,6 +480,11 @@ static inline void _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELAXED); } +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -491,10 +500,29 @@ static inline void _Py_atomic_store_int_release(int *obj, int value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline int _Py_atomic_load_int_acquire(const int *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +static inline uint32_t +_Py_atomic_load_uint32_acquire(const uint32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 601a0cf65afc1c5..3205e253b285466 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -712,6 +712,12 @@ _Py_atomic_load_ptr_relaxed(const void *obj) return *(void * volatile *)obj; } +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj) +{ + return *(volatile unsigned long long *)obj; +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -886,6 +892,14 @@ _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) *(volatile Py_ssize_t *)obj = value; } +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value) +{ + *(volatile unsigned long long *)obj = value; +} + + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ static inline void * @@ -925,6 +939,18 @@ _Py_atomic_store_int_release(int *obj, int value) #endif } +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(Py_ssize_t volatile *)obj = value; +#elif defined(_M_ARM64) + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_ssize_release" +#endif +} + static inline int _Py_atomic_load_int_acquire(const int *obj) { @@ -938,6 +964,55 @@ _Py_atomic_load_int_acquire(const int *obj) #endif } +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uint64_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_uint64_release" +#endif +} + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uint64_t volatile *)obj; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (uint64_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint64_acquire" +#endif +} + +static inline uint32_t +_Py_atomic_load_uint32_acquire(const uint32_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uint32_t volatile *)obj; +#elif defined(_M_ARM64) + return (uint32_t)__ldar32((uint32_t volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint32_acquire" +#endif +} + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(Py_ssize_t volatile *)obj; +#elif defined(_M_ARM64) + return (Py_ssize_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_ssize_acquire" +#endif +} // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index a05bfaec47e89d5..f3970a45df24f91 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -619,6 +619,14 @@ _Py_atomic_load_ptr_relaxed(const void *obj) memory_order_relaxed); } +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned long long)*)obj, + memory_order_relaxed); +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -835,6 +843,15 @@ _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) memory_order_relaxed); } +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned long long)*)obj, value, + memory_order_relaxed); +} + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -862,6 +879,14 @@ _Py_atomic_store_int_release(int *obj, int value) memory_order_release); } +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(Py_ssize_t)*)obj, value, + memory_order_release); +} + static inline int _Py_atomic_load_int_acquire(const int *obj) { @@ -870,6 +895,35 @@ _Py_atomic_load_int_acquire(const int *obj) memory_order_acquire); } +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint64_t)*)obj, value, + memory_order_release); +} + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint64_t)*)obj, + memory_order_acquire); +} + +static inline uint32_t +_Py_atomic_load_uint32_acquire(const uint32_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint32_t)*)obj, +} + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj, +} // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 479b908fb7058ac..32c5884cd21341c 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -122,4 +122,6 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalErrorFunc( PyAPI_FUNC(void) PyErr_FormatUnraisable(const char *, ...); +PyAPI_DATA(PyObject *) PyExc_PythonFinalizationError; + #define Py_FatalError(message) _Py_FatalErrorFunc(__func__, (message)) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 1dbf97660f382fd..ac7ff83748dbfc8 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -68,6 +68,11 @@ struct _ts { PyThreadState *next; PyInterpreterState *interp; + /* The global instrumentation version in high bits, plus flags indicating + when to break out of the interpreter loop in lower bits. See details in + pycore_ceval.h. */ + uintptr_t eval_breaker; + struct { /* Has been initialized to a safe state. @@ -212,6 +217,8 @@ struct _ts { /* The thread's exception stack entry. (Always the last entry.) */ _PyErr_StackItem exc_state; + PyObject *previous_executor; + }; #ifdef Py_DEBUG @@ -229,7 +236,7 @@ struct _ts { #elif defined(__s390x__) # define Py_C_RECURSION_LIMIT 800 #elif defined(_WIN32) -# define Py_C_RECURSION_LIMIT 4000 +# define Py_C_RECURSION_LIMIT 3000 #elif defined(_Py_ADDRESS_SANITIZER) # define Py_C_RECURSION_LIMIT 4000 #else diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index bf0cfe4cb695b45..887fbbedf88502b 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -115,11 +115,15 @@ typedef struct _optimization_stats { uint64_t inner_loop; uint64_t recursive_call; uint64_t low_confidence; + uint64_t executors_invalidated; UOpStats opcode[512]; uint64_t unsupported_opcode[256]; uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; uint64_t trace_run_length_hist[_Py_UOP_HIST_SIZE]; uint64_t optimized_trace_length_hist[_Py_UOP_HIST_SIZE]; + uint64_t optimizer_attempts; + uint64_t optimizer_successes; + uint64_t optimizer_failure_reason_no_memory; } OptimizationStats; typedef struct _rare_event_stats { @@ -133,6 +137,9 @@ typedef struct _rare_event_stats { uint64_t builtin_dict; /* Modifying a function, e.g. func.__defaults__ = ..., etc. */ uint64_t func_modification; + /* Modifying a dict that is being watched */ + uint64_t watched_dict_modification; + uint64_t watched_globals_modification; } RareEventStats; typedef struct _stats { diff --git a/Include/cpython/pytime.h b/Include/cpython/pytime.h new file mode 100644 index 000000000000000..d8244700d614ceb --- /dev/null +++ b/Include/cpython/pytime.h @@ -0,0 +1,23 @@ +// PyTime_t C API: see Doc/c-api/time.rst for the documentation. + +#ifndef Py_LIMITED_API +#ifndef Py_PYTIME_H +#define Py_PYTIME_H +#ifdef __cplusplus +extern "C" { +#endif + +typedef int64_t PyTime_t; +#define PyTime_MIN INT64_MIN +#define PyTime_MAX INT64_MAX + +PyAPI_FUNC(double) PyTime_AsSecondsDouble(PyTime_t t); +PyAPI_FUNC(int) PyTime_Monotonic(PyTime_t *result); +PyAPI_FUNC(int) PyTime_PerfCounter(PyTime_t *result); +PyAPI_FUNC(int) PyTime_Time(PyTime_t *result); + +#ifdef __cplusplus +} +#endif +#endif /* Py_PYTIME_H */ +#endif /* Py_LIMITED_API */ diff --git a/Include/internal/mimalloc/mimalloc/prim.h b/Include/internal/mimalloc/mimalloc/prim.h index 4b9e4dc4194d776..8a60d528458e6c2 100644 --- a/Include/internal/mimalloc/mimalloc/prim.h +++ b/Include/internal/mimalloc/mimalloc/prim.h @@ -131,7 +131,13 @@ extern bool _mi_process_is_initialized; // has mi_process_init been static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept; -#if defined(_WIN32) +#ifdef MI_PRIM_THREAD_ID + +static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { + return MI_PRIM_THREAD_ID(); +} + +#elif defined(_WIN32) #define WIN32_LEAN_AND_MEAN #include <windows.h> diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h index b8cae24507fc5e0..ed93e45062c2dbd 100644 --- a/Include/internal/mimalloc/mimalloc/types.h +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -312,6 +312,7 @@ typedef struct mi_page_s { uint8_t is_committed : 1; // `true` if the page virtual memory is committed uint8_t is_zero_init : 1; // `true` if the page was initially zero initialized uint8_t tag : 4; // tag from the owning heap + uint8_t debug_offset; // number of bytes to preserve when filling freed or uninitialized memory // layout like this to optimize access in `mi_malloc` and `mi_free` uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear` @@ -553,6 +554,7 @@ struct mi_heap_s { mi_heap_t* next; // list of heaps per thread bool no_reclaim; // `true` if this heap should not reclaim abandoned pages uint8_t tag; // custom identifier for this heap + uint8_t debug_offset; // number of bytes to preserve when filling freed or uninitialized memory }; diff --git a/Include/internal/pycore_brc.h b/Include/internal/pycore_brc.h new file mode 100644 index 000000000000000..3453d83b57ca97e --- /dev/null +++ b/Include/internal/pycore_brc.h @@ -0,0 +1,74 @@ +#ifndef Py_INTERNAL_BRC_H +#define Py_INTERNAL_BRC_H + +#include <stdint.h> +#include "pycore_llist.h" // struct llist_node +#include "pycore_lock.h" // PyMutex +#include "pycore_object_stack.h" // _PyObjectStack + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_GIL_DISABLED + +// Prime number to avoid correlations with memory addresses. +#define _Py_BRC_NUM_BUCKETS 257 + +// Hash table bucket +struct _brc_bucket { + // Mutex protects both the bucket and thread state queues in this bucket. + PyMutex mutex; + + // Linked list of _PyThreadStateImpl objects hashed to this bucket. + struct llist_node root; +}; + +// Per-interpreter biased reference counting state +struct _brc_state { + // Hash table of thread states by thread-id. Thread states within a bucket + // are chained using a doubly-linked list. + struct _brc_bucket table[_Py_BRC_NUM_BUCKETS]; +}; + +// Per-thread biased reference counting state +struct _brc_thread_state { + // Linked-list of thread states per hash bucket + struct llist_node bucket_node; + + // Thread-id as determined by _PyThread_Id() + uintptr_t tid; + + // Objects with refcounts to be merged (protected by bucket mutex) + _PyObjectStack objects_to_merge; + + // Local stack of objects to be merged (not accessed by other threads) + _PyObjectStack local_objects_to_merge; +}; + +// Initialize/finalize the per-thread biased reference counting state +void _Py_brc_init_thread(PyThreadState *tstate); +void _Py_brc_remove_thread(PyThreadState *tstate); + +// Initialize per-interpreter state +void _Py_brc_init_state(PyInterpreterState *interp); + +void _Py_brc_after_fork(PyInterpreterState *interp); + +// Enqueues an object to be merged by it's owning thread (tid). This +// steals a reference to the object. +void _Py_brc_queue_object(PyObject *ob); + +// Merge the refcounts of queued objects for the current thread. +void _Py_brc_merge_refcounts(PyThreadState *tstate); + +#endif /* Py_GIL_DISABLED */ + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_BRC_H */ diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a66af1389541dd7..bf77526cf75cc14 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -42,7 +42,7 @@ PyAPI_FUNC(int) _PyEval_MakePendingCalls(PyThreadState *); extern void _Py_FinishPendingCalls(PyThreadState *tstate); extern void _PyEval_InitState(PyInterpreterState *); -extern void _PyEval_SignalReceived(PyInterpreterState *interp); +extern void _PyEval_SignalReceived(void); // bitwise flags: #define _Py_PENDING_MAINTHREADONLY 1 @@ -55,7 +55,6 @@ PyAPI_FUNC(int) _PyEval_AddPendingCall( void *arg, int flags); -extern void _PyEval_SignalAsyncExc(PyInterpreterState *interp); #ifdef HAVE_FORK extern PyStatus _PyEval_ReInitThreads(PyThreadState *tstate); #endif @@ -200,39 +199,43 @@ int _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int a void _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); -#define _PY_GIL_DROP_REQUEST_BIT 0 -#define _PY_SIGNALS_PENDING_BIT 1 -#define _PY_CALLS_TO_DO_BIT 2 -#define _PY_ASYNC_EXCEPTION_BIT 3 -#define _PY_GC_SCHEDULED_BIT 4 -#define _PY_EVAL_PLEASE_STOP_BIT 5 +/* Bits that can be set in PyThreadState.eval_breaker */ +#define _PY_GIL_DROP_REQUEST_BIT (1U << 0) +#define _PY_SIGNALS_PENDING_BIT (1U << 1) +#define _PY_CALLS_TO_DO_BIT (1U << 2) +#define _PY_ASYNC_EXCEPTION_BIT (1U << 3) +#define _PY_GC_SCHEDULED_BIT (1U << 4) +#define _PY_EVAL_PLEASE_STOP_BIT (1U << 5) +#define _PY_EVAL_EXPLICIT_MERGE_BIT (1U << 6) /* Reserve a few bits for future use */ #define _PY_EVAL_EVENTS_BITS 8 #define _PY_EVAL_EVENTS_MASK ((1 << _PY_EVAL_EVENTS_BITS)-1) static inline void -_Py_set_eval_breaker_bit(PyInterpreterState *interp, uint32_t bit, uint32_t set) +_Py_set_eval_breaker_bit(PyThreadState *tstate, uintptr_t bit) { - assert(set == 0 || set == 1); - uintptr_t to_set = set << bit; - uintptr_t mask = ((uintptr_t)1) << bit; - uintptr_t old = _Py_atomic_load_uintptr(&interp->ceval.eval_breaker); - if ((old & mask) == to_set) { - return; - } - uintptr_t new; - do { - new = (old & ~mask) | to_set; - } while (!_Py_atomic_compare_exchange_uintptr(&interp->ceval.eval_breaker, &old, new)); + _Py_atomic_or_uintptr(&tstate->eval_breaker, bit); +} + +static inline void +_Py_unset_eval_breaker_bit(PyThreadState *tstate, uintptr_t bit) +{ + _Py_atomic_and_uintptr(&tstate->eval_breaker, ~bit); } -static inline bool -_Py_eval_breaker_bit_is_set(PyInterpreterState *interp, int32_t bit) +static inline int +_Py_eval_breaker_bit_is_set(PyThreadState *tstate, uintptr_t bit) { - return _Py_atomic_load_uintptr_relaxed(&interp->ceval.eval_breaker) & (((uintptr_t)1) << bit); + uintptr_t b = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + return (b & bit) != 0; } +// Free-threaded builds use these functions to set or unset a bit on all +// threads in the given interpreter. +void _Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); +void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 28738980eb49be8..b453328f15649eb 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -78,13 +78,10 @@ struct _ceval_runtime_state { struct _ceval_state { - /* This single variable consolidates all requests to break out of - * the fast path in the eval loop. - * It is by far the hottest field in this struct and - * should be placed at the beginning. */ - uintptr_t eval_breaker; - /* Avoid false sharing */ - int64_t padding[7]; + /* This variable holds the global instrumentation version. When a thread is + running, this value is overlaid onto PyThreadState.eval_breaker so that + changes in the instrumentation version will trigger the eval breaker. */ + uintptr_t instrumentation_version; int recursion_limit; struct _gil_runtime_state *gil; int own_gil; diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index fdd5918228455d7..855361621320729 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -8,6 +8,15 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif + +// We hide some of the newer PyCodeObject fields behind macros. +// This helps with backporting certain changes to 3.12. +#define _PyCode_HAS_EXECUTORS(CODE) \ + (CODE->co_executors != NULL) +#define _PyCode_HAS_INSTRUMENTATION(CODE) \ + (CODE->_co_instrumentation_version > 0) + + #define CODE_MAX_WATCHERS 8 /* PEP 659 diff --git a/Include/internal/pycore_condvar.h b/Include/internal/pycore_condvar.h index 34c21aaad43197b..ee9533484e80488 100644 --- a/Include/internal/pycore_condvar.h +++ b/Include/internal/pycore_condvar.h @@ -35,14 +35,14 @@ #include <windows.h> // CRITICAL_SECTION /* options */ -/* non-emulated condition variables are provided for those that want - * to target Windows Vista. Modify this macro to enable them. +/* emulated condition variables are provided for those that want + * to target Windows XP or earlier. Modify this macro to enable them. */ #ifndef _PY_EMULATED_WIN_CV -#define _PY_EMULATED_WIN_CV 1 /* use emulated condition variables */ +#define _PY_EMULATED_WIN_CV 0 /* use non-emulated condition variables */ #endif -/* fall back to emulation if not targeting Vista */ +/* fall back to emulation if targeting earlier than Vista */ #if !defined NTDDI_VISTA || NTDDI_VERSION < NTDDI_VISTA #undef _PY_EMULATED_WIN_CV #define _PY_EMULATED_WIN_CV 1 @@ -77,7 +77,7 @@ typedef struct _PyCOND_T #else /* !_PY_EMULATED_WIN_CV */ -/* Use native Win7 primitives if build target is Win7 or higher */ +/* Use native Windows primitives if build target is Vista or higher */ /* SRWLOCK is faster and better than CriticalSection */ typedef SRWLOCK PyMUTEX_T; diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index 3284efba2b6f4cb..ae5c47f195eb7f0 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -14,7 +14,6 @@ extern PyTypeObject _PyContextTokenMissing_Type; /* runtime lifecycle */ PyStatus _PyContext_Init(PyInterpreterState *); -void _PyContext_Fini(_PyFreeListState *); /* other API */ diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index bf2bbfffc38bd0f..9163b5cf0f2e8a8 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -96,6 +96,15 @@ extern "C" { _PyCriticalSection_End(&_cs); \ } +# define Py_XBEGIN_CRITICAL_SECTION(op) \ + { \ + _PyCriticalSection _cs_opt = {0}; \ + _PyCriticalSection_XBegin(&_cs_opt, _PyObject_CAST(op)) + +# define Py_XEND_CRITICAL_SECTION() \ + _PyCriticalSection_XEnd(&_cs_opt); \ + } + # define Py_BEGIN_CRITICAL_SECTION2(a, b) \ { \ _PyCriticalSection2 _cs2; \ @@ -104,12 +113,39 @@ extern "C" { # define Py_END_CRITICAL_SECTION2() \ _PyCriticalSection2_End(&_cs2); \ } + +// Asserts that the mutex is locked. The mutex must be held by the +// top-most critical section otherwise there's the possibility +// that the mutex would be swalled out in some code paths. +#define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) \ + _PyCriticalSection_AssertHeld(mutex) + +// Asserts that the mutex for the given object is locked. The mutex must +// be held by the top-most critical section otherwise there's the +// possibility that the mutex would be swalled out in some code paths. +#ifdef Py_DEBUG + +#define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) \ + if (Py_REFCNT(op) != 1) { \ + _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&_PyObject_CAST(op)->ob_mutex); \ + } + +#else /* Py_DEBUG */ + +#define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) + +#endif /* Py_DEBUG */ + #else /* !Py_GIL_DISABLED */ // The critical section APIs are no-ops with the GIL. # define Py_BEGIN_CRITICAL_SECTION(op) # define Py_END_CRITICAL_SECTION() +# define Py_XBEGIN_CRITICAL_SECTION(op) +# define Py_XEND_CRITICAL_SECTION() # define Py_BEGIN_CRITICAL_SECTION2(a, b) # define Py_END_CRITICAL_SECTION2() +# define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) +# define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) #endif /* !Py_GIL_DISABLED */ typedef struct { @@ -162,6 +198,16 @@ _PyCriticalSection_Begin(_PyCriticalSection *c, PyMutex *m) } } +static inline void +_PyCriticalSection_XBegin(_PyCriticalSection *c, PyObject *op) +{ +#ifdef Py_GIL_DISABLED + if (op != NULL) { + _PyCriticalSection_Begin(c, &_PyObject_CAST(op)->ob_mutex); + } +#endif +} + // Removes the top-most critical section from the thread's stack of critical // sections. If the new top-most critical section is inactive, then it is // resumed. @@ -184,6 +230,14 @@ _PyCriticalSection_End(_PyCriticalSection *c) _PyCriticalSection_Pop(c); } +static inline void +_PyCriticalSection_XEnd(_PyCriticalSection *c) +{ + if (c->mutex) { + _PyCriticalSection_End(c); + } +} + static inline void _PyCriticalSection2_Begin(_PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) { @@ -236,6 +290,28 @@ _PyCriticalSection2_End(_PyCriticalSection2 *c) PyAPI_FUNC(void) _PyCriticalSection_SuspendAll(PyThreadState *tstate); +#ifdef Py_GIL_DISABLED + +static inline void +_PyCriticalSection_AssertHeld(PyMutex *mutex) +{ +#ifdef Py_DEBUG + PyThreadState *tstate = _PyThreadState_GET(); + uintptr_t prev = tstate->critical_section; + if (prev & _Py_CRITICAL_SECTION_TWO_MUTEXES) { + _PyCriticalSection2 *cs = (_PyCriticalSection2 *)(prev & ~_Py_CRITICAL_SECTION_MASK); + assert(cs != NULL && (cs->base.mutex == mutex || cs->mutex2 == mutex)); + } + else { + _PyCriticalSection *cs = (_PyCriticalSection *)(tstate->critical_section & ~_Py_CRITICAL_SECTION_MASK); + assert(cs != NULL && cs->mutex == mutex); + } + +#endif +} + +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index d6e297a7e8e6dbc..63abef864ff87fe 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -87,6 +87,11 @@ struct _xid { PyAPI_FUNC(_PyCrossInterpreterData *) _PyCrossInterpreterData_New(void); PyAPI_FUNC(void) _PyCrossInterpreterData_Free(_PyCrossInterpreterData *data); +#define _PyCrossInterpreterData_DATA(DATA) ((DATA)->data) +#define _PyCrossInterpreterData_OBJ(DATA) ((DATA)->obj) +#define _PyCrossInterpreterData_INTERPID(DATA) ((DATA)->interpid) +// Users should not need getters for "new_object" or "free". + /* defining cross-interpreter data */ @@ -101,6 +106,25 @@ PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( PyInterpreterState *, _PyCrossInterpreterData *); +// Normally the Init* functions are sufficient. The only time +// additional initialization might be needed is to set the "free" func, +// though that should be infrequent. +#define _PyCrossInterpreterData_SET_FREE(DATA, FUNC) \ + do { \ + (DATA)->free = (FUNC); \ + } while (0) +// Additionally, some shareable types are essentially light wrappers +// around other shareable types. The crossinterpdatafunc of the wrapper +// can often be implemented by calling the wrapped object's +// crossinterpdatafunc and then changing the "new_object" function. +// We have _PyCrossInterpreterData_SET_NEW_OBJECT() here for that, +// but might be better to have a function like +// _PyCrossInterpreterData_AdaptToWrapper() instead. +#define _PyCrossInterpreterData_SET_NEW_OBJECT(DATA, FUNC) \ + do { \ + (DATA)->new_object = (FUNC); \ + } while (0) + /* using cross-interpreter data */ @@ -170,6 +194,8 @@ extern void _PyXI_Fini(PyInterpreterState *interp); extern PyStatus _PyXI_InitTypes(PyInterpreterState *interp); extern void _PyXI_FiniTypes(PyInterpreterState *interp); +#define _PyInterpreterState_GetXIState(interp) (&(interp)->xi) + /***************************/ /* short-term data sharing */ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index d96870e9197bbf0..d1a0010b9a81dd7 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -9,6 +9,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // _PyFreeListState #include "pycore_identifier.h" // _Py_Identifier #include "pycore_object.h" // PyDictOrValues @@ -66,12 +67,6 @@ typedef struct { extern PyObject* _PyDictView_New(PyObject *, PyTypeObject *); extern PyObject* _PyDictView_Intersect(PyObject* self, PyObject *other); - -/* runtime lifecycle */ - -extern void _PyDict_Fini(PyInterpreterState *interp); - - /* other API */ typedef struct { @@ -141,6 +136,11 @@ struct _dictkeysobject { /* Kind of keys */ uint8_t dk_kind; +#ifdef Py_GIL_DISABLED + /* Lock used to protect shared keys */ + PyMutex dk_mutex; +#endif + /* Version number -- Reset to 0 by any modification to keys */ uint32_t dk_version; @@ -150,6 +150,7 @@ struct _dictkeysobject { /* Number of used entries in dk_entries. */ Py_ssize_t dk_nentries; + /* Actual hash table of dk_size entries. It holds indices in dk_entries, or DKIX_EMPTY(-1) or DKIX_DUMMY(-2). @@ -206,11 +207,21 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DK_IS_UNICODE(dk) ((dk)->dk_kind != DICT_KEYS_GENERAL) -#define DICT_VERSION_INCREMENT (1 << DICT_MAX_WATCHERS) -#define DICT_VERSION_MASK (DICT_VERSION_INCREMENT - 1) +#define DICT_VERSION_INCREMENT (1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) +#define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1) +#define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1) + +#define DICT_VALUES_SIZE(values) ((uint8_t *)values)[-1] +#define DICT_VALUES_USED_SIZE(values) ((uint8_t *)values)[-2] + +#ifdef Py_GIL_DISABLED +#define DICT_NEXT_VERSION(INTERP) \ + (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT) +#else #define DICT_NEXT_VERSION(INTERP) \ ((INTERP)->dict_state.global_version += DICT_VERSION_INCREMENT) +#endif void _PyDict_SendEvent(int watcher_bits, @@ -227,12 +238,12 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, PyObject *value) { assert(Py_REFCNT((PyObject*)mp) > 0); - int watcher_bits = mp->ma_version_tag & DICT_VERSION_MASK; + int watcher_bits = mp->ma_version_tag & DICT_WATCHER_MASK; if (watcher_bits) { + RARE_EVENT_STAT_INC(watched_dict_modification); _PyDict_SendEvent(watcher_bits, event, mp, key, value); - return DICT_NEXT_VERSION(interp) | watcher_bits; } - return DICT_NEXT_VERSION(interp); + return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK); } extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); @@ -248,7 +259,7 @@ _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix) assert(ix < SHARED_KEYS_MAX_SIZE); uint8_t *size_ptr = ((uint8_t *)values)-2; int size = *size_ptr; - assert(size+2 < ((uint8_t *)values)[-1]); + assert(size+2 < DICT_VALUES_SIZE(values)); size++; size_ptr[-size] = (uint8_t)ix; *size_ptr = size; diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index ece0f10ca251707..1a44755c7a01a3a 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -8,17 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif - -#ifndef WITH_FREELISTS -// without freelists -# define PyDict_MAXFREELIST 0 -#endif - -#ifndef PyDict_MAXFREELIST -# define PyDict_MAXFREELIST 80 -#endif - #define DICT_MAX_WATCHERS 8 +#define DICT_WATCHED_MUTATION_BITS 4 struct _Py_dict_state { /*Global counter used to set ma_version_tag field of dictionary. @@ -26,15 +17,6 @@ struct _Py_dict_state { * time that a dictionary is modified. */ uint64_t global_version; uint32_t next_keys_version; - -#if PyDict_MAXFREELIST > 0 - /* Dictionary reuse scheme to save calls to malloc and free */ - PyDictObject *free_list[PyDict_MAXFREELIST]; - PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST]; - int numfree; - int keys_numfree; -#endif - PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 8143ee7a5c83cc7..0a3fd991988e363 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -15,7 +15,6 @@ extern "C" { extern void _PyFloat_InitState(PyInterpreterState *); extern PyStatus _PyFloat_InitTypes(PyInterpreterState *); -extern void _PyFloat_Fini(_PyFreeListState *); extern void _PyFloat_FiniType(PyInterpreterState *); diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h index b91d2bc066b783b..e684e084b8bef8e 100644 --- a/Include/internal/pycore_freelist.h +++ b/Include/internal/pycore_freelist.h @@ -17,6 +17,7 @@ extern "C" { # define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE # define PyTuple_MAXFREELIST 2000 # define PyList_MAXFREELIST 80 +# define PyDict_MAXFREELIST 80 # define PyFloat_MAXFREELIST 100 # define PyContext_MAXFREELIST 255 # define _PyAsyncGen_MAXFREELIST 80 @@ -25,20 +26,21 @@ extern "C" { # define PyTuple_NFREELISTS 0 # define PyTuple_MAXFREELIST 0 # define PyList_MAXFREELIST 0 +# define PyDict_MAXFREELIST 0 # define PyFloat_MAXFREELIST 0 # define PyContext_MAXFREELIST 0 # define _PyAsyncGen_MAXFREELIST 0 # define _PyObjectStackChunk_MAXFREELIST 0 #endif -struct _Py_list_state { +struct _Py_list_freelist { #ifdef WITH_FREELISTS - PyListObject *free_list[PyList_MAXFREELIST]; + PyListObject *items[PyList_MAXFREELIST]; int numfree; #endif }; -struct _Py_tuple_state { +struct _Py_tuple_freelist { #if WITH_FREELISTS /* There is one freelist for each size from 1 to PyTuple_MAXSAVESIZE. The empty tuple is handled separately. @@ -48,24 +50,40 @@ struct _Py_tuple_state { object is used as the linked list node, with its first item (ob_item[0]) pointing to the next node (i.e. the previous head). Each linked list is initially NULL. */ - PyTupleObject *free_list[PyTuple_NFREELISTS]; + PyTupleObject *items[PyTuple_NFREELISTS]; int numfree[PyTuple_NFREELISTS]; #else char _unused; // Empty structs are not allowed. #endif }; -struct _Py_float_state { +struct _Py_float_freelist { #ifdef WITH_FREELISTS /* Special free list free_list is a singly-linked list of available PyFloatObjects, linked via abuse of their ob_type members. */ int numfree; - PyFloatObject *free_list; + PyFloatObject *items; #endif }; -struct _Py_slice_state { +struct _Py_dict_freelist { +#ifdef WITH_FREELISTS + /* Dictionary reuse scheme to save calls to malloc and free */ + PyDictObject *items[PyDict_MAXFREELIST]; + int numfree; +#endif +}; + +struct _Py_dictkeys_freelist { +#ifdef WITH_FREELISTS + /* Dictionary keys reuse scheme to save calls to malloc and free */ + PyDictKeysObject *items[PyDict_MAXFREELIST]; + int numfree; +#endif +}; + +struct _Py_slice_freelist { #ifdef WITH_FREELISTS /* Using a cache is very effective since typically only a single slice is created and then deleted again. */ @@ -73,44 +91,61 @@ struct _Py_slice_state { #endif }; -struct _Py_context_state { +struct _Py_context_freelist { #ifdef WITH_FREELISTS // List of free PyContext objects - PyContext *freelist; + PyContext *items; int numfree; #endif }; -struct _Py_async_gen_state { +struct _Py_async_gen_freelist { #ifdef WITH_FREELISTS /* Freelists boost performance 6-10%; they also reduce memory fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend are short-living objects that are instantiated for every __anext__() call. */ - struct _PyAsyncGenWrappedValue* value_freelist[_PyAsyncGen_MAXFREELIST]; - int value_numfree; + struct _PyAsyncGenWrappedValue* items[_PyAsyncGen_MAXFREELIST]; + int numfree; +#endif +}; - struct PyAsyncGenASend* asend_freelist[_PyAsyncGen_MAXFREELIST]; - int asend_numfree; +struct _Py_async_gen_asend_freelist { +#ifdef WITH_FREELISTS + struct PyAsyncGenASend* items[_PyAsyncGen_MAXFREELIST]; + int numfree; #endif }; struct _PyObjectStackChunk; -struct _Py_object_stack_state { - struct _PyObjectStackChunk *free_list; +struct _Py_object_stack_freelist { + struct _PyObjectStackChunk *items; Py_ssize_t numfree; }; -typedef struct _Py_freelist_state { - struct _Py_float_state floats; - struct _Py_tuple_state tuples; - struct _Py_list_state lists; - struct _Py_slice_state slices; - struct _Py_context_state contexts; - struct _Py_async_gen_state async_gens; - struct _Py_object_stack_state object_stacks; -} _PyFreeListState; +struct _Py_object_freelists { + struct _Py_float_freelist floats; + struct _Py_tuple_freelist tuples; + struct _Py_list_freelist lists; + struct _Py_dict_freelist dicts; + struct _Py_dictkeys_freelist dictkeys; + struct _Py_slice_freelist slices; + struct _Py_context_freelist contexts; + struct _Py_async_gen_freelist async_gens; + struct _Py_async_gen_asend_freelist async_gen_asends; + struct _Py_object_stack_freelist object_stacks; +}; + +extern void _PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); #ifdef __cplusplus } diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index b362a294a59042a..40414a868518bb6 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -43,6 +43,7 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) { # define _PyGC_BITS_FINALIZED (2) # define _PyGC_BITS_UNREACHABLE (4) # define _PyGC_BITS_FROZEN (8) +# define _PyGC_BITS_SHARED (16) #endif /* True if the object is currently tracked by the GC. */ @@ -68,6 +69,22 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) { return 1; } +#ifdef Py_GIL_DISABLED + +/* True if an object is shared between multiple threads and + * needs special purpose when freeing to do the possibility + * of in-flight lock-free reads occuring */ +static inline int _PyObject_GC_IS_SHARED(PyObject *op) { + return (op->ob_gc_bits & _PyGC_BITS_SHARED) != 0; +} +#define _PyObject_GC_IS_SHARED(op) _PyObject_GC_IS_SHARED(_Py_CAST(PyObject*, op)) + +static inline void _PyObject_GC_SET_SHARED(PyObject *op) { + op->ob_gc_bits |= _PyGC_BITS_SHARED; +} +#define _PyObject_GC_SET_SHARED(op) _PyObject_GC_SET_SHARED(_Py_CAST(PyObject*, op)) + +#endif /* Bit flags for _gc_prev */ /* Bit 0 is set when tp_finalize is called */ @@ -243,6 +260,13 @@ struct _gc_runtime_state { Py_ssize_t long_lived_pending; }; +#ifdef Py_GIL_DISABLED +struct _gc_thread_state { + /* Thread-local allocation count. */ + Py_ssize_t alloc_count; +}; +#endif + extern void _PyGC_InitState(struct _gc_runtime_state *); @@ -262,15 +286,7 @@ extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); // Functions to clear types free lists extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); -extern void _Py_ClearFreeLists(_PyFreeListState *state, int is_finalization); -extern void _PyTuple_ClearFreeList(_PyFreeListState *state, int is_finalization); -extern void _PyFloat_ClearFreeList(_PyFreeListState *state, int is_finalization); -extern void _PyList_ClearFreeList(_PyFreeListState *state, int is_finalization); -extern void _PySlice_ClearCache(_PyFreeListState *state); -extern void _PyDict_ClearFreeList(PyInterpreterState *interp); -extern void _PyAsyncGen_ClearFreeLists(_PyFreeListState *state, int is_finalization); -extern void _PyContext_ClearFreeList(_PyFreeListState *state, int is_finalization); -extern void _Py_ScheduleGC(PyInterpreterState *interp); +extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); #ifdef __cplusplus diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index 5ad63658051e86d..b2aa017598409f7 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -26,10 +26,6 @@ extern PyTypeObject _PyCoroWrapper_Type; extern PyTypeObject _PyAsyncGenWrappedValue_Type; extern PyTypeObject _PyAsyncGenAThrow_Type; -/* runtime lifecycle */ - -extern void _PyAsyncGen_Fini(_PyFreeListState *); - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index e92707051c12b7f..e8b62bd0dcd3691 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -742,6 +742,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abc_impl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abstract_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_active)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_align_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_annotation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_anonymous_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_)); @@ -752,6 +753,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_check_retval_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_dealloc_warn)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_feature_version)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_field_types)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_fields_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_finalizing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load)); @@ -876,12 +878,14 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(d)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(data)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(database)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(day)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(default)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(defaultaction)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(delete)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(depth)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(desired_access)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(detect_types)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(deterministic)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(device)); @@ -939,12 +943,14 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fileno)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filepath)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fillvalue)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filter)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filters)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(final)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(find_class)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fix_imports)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(flags)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(flush)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fold)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(follow_symlinks)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(from_param)); @@ -970,11 +976,13 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(groups)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(h)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(handle)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(handle_seq)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hash_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(header)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(headers)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hi)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hour)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); @@ -986,9 +994,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(indexgroup)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inf)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(infer_variance)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inherit_handle)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inheritable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial_bytes)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial_owner)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial_state)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initval)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inner_size)); @@ -1044,11 +1055,13 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(locals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(logoption)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(loop)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(manual_reset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mapping)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(match)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_length)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxdigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxevents)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxlen)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxmem)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxsplit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxvalue)); @@ -1058,13 +1071,18 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metaclass)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(method)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(microsecond)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(milliseconds)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minute)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mod)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module_globals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(modules)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(month)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mro)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(msg)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mutex)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mycmp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_arg)); @@ -1167,6 +1185,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(salt)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sched_priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(second)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(security_attributes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seek)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seekable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(selectors)); @@ -1243,6 +1263,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(type)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tz)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tzinfo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tzname)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(uid)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(unlink)); @@ -1253,6 +1274,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(values)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(version)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(volume)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wait_all)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnings)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnoptions)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wbits)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index eb60b80c964d423..b41f5c8952021e4 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -231,6 +231,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_abc_impl) STRUCT_FOR_ID(_abstract_) STRUCT_FOR_ID(_active) + STRUCT_FOR_ID(_align_) STRUCT_FOR_ID(_annotation) STRUCT_FOR_ID(_anonymous_) STRUCT_FOR_ID(_argtypes_) @@ -241,6 +242,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_check_retval_) STRUCT_FOR_ID(_dealloc_warn) STRUCT_FOR_ID(_feature_version) + STRUCT_FOR_ID(_field_types) STRUCT_FOR_ID(_fields_) STRUCT_FOR_ID(_finalizing) STRUCT_FOR_ID(_find_and_load) @@ -365,12 +367,14 @@ struct _Py_global_strings { STRUCT_FOR_ID(d) STRUCT_FOR_ID(data) STRUCT_FOR_ID(database) + STRUCT_FOR_ID(day) STRUCT_FOR_ID(decode) STRUCT_FOR_ID(decoder) STRUCT_FOR_ID(default) STRUCT_FOR_ID(defaultaction) STRUCT_FOR_ID(delete) STRUCT_FOR_ID(depth) + STRUCT_FOR_ID(desired_access) STRUCT_FOR_ID(detect_types) STRUCT_FOR_ID(deterministic) STRUCT_FOR_ID(device) @@ -428,12 +432,14 @@ struct _Py_global_strings { STRUCT_FOR_ID(fileno) STRUCT_FOR_ID(filepath) STRUCT_FOR_ID(fillvalue) + STRUCT_FOR_ID(filter) STRUCT_FOR_ID(filters) STRUCT_FOR_ID(final) STRUCT_FOR_ID(find_class) STRUCT_FOR_ID(fix_imports) STRUCT_FOR_ID(flags) STRUCT_FOR_ID(flush) + STRUCT_FOR_ID(fold) STRUCT_FOR_ID(follow_symlinks) STRUCT_FOR_ID(format) STRUCT_FOR_ID(from_param) @@ -459,11 +465,13 @@ struct _Py_global_strings { STRUCT_FOR_ID(groups) STRUCT_FOR_ID(h) STRUCT_FOR_ID(handle) + STRUCT_FOR_ID(handle_seq) STRUCT_FOR_ID(hash_name) STRUCT_FOR_ID(header) STRUCT_FOR_ID(headers) STRUCT_FOR_ID(hi) STRUCT_FOR_ID(hook) + STRUCT_FOR_ID(hour) STRUCT_FOR_ID(id) STRUCT_FOR_ID(ident) STRUCT_FOR_ID(identity_hint) @@ -475,9 +483,12 @@ struct _Py_global_strings { STRUCT_FOR_ID(indexgroup) STRUCT_FOR_ID(inf) STRUCT_FOR_ID(infer_variance) + STRUCT_FOR_ID(inherit_handle) STRUCT_FOR_ID(inheritable) STRUCT_FOR_ID(initial) STRUCT_FOR_ID(initial_bytes) + STRUCT_FOR_ID(initial_owner) + STRUCT_FOR_ID(initial_state) STRUCT_FOR_ID(initial_value) STRUCT_FOR_ID(initval) STRUCT_FOR_ID(inner_size) @@ -533,11 +544,13 @@ struct _Py_global_strings { STRUCT_FOR_ID(locals) STRUCT_FOR_ID(logoption) STRUCT_FOR_ID(loop) + STRUCT_FOR_ID(manual_reset) STRUCT_FOR_ID(mapping) STRUCT_FOR_ID(match) STRUCT_FOR_ID(max_length) STRUCT_FOR_ID(maxdigits) STRUCT_FOR_ID(maxevents) + STRUCT_FOR_ID(maxlen) STRUCT_FOR_ID(maxmem) STRUCT_FOR_ID(maxsplit) STRUCT_FOR_ID(maxvalue) @@ -547,13 +560,18 @@ struct _Py_global_strings { STRUCT_FOR_ID(metaclass) STRUCT_FOR_ID(metadata) STRUCT_FOR_ID(method) + STRUCT_FOR_ID(microsecond) + STRUCT_FOR_ID(milliseconds) + STRUCT_FOR_ID(minute) STRUCT_FOR_ID(mod) STRUCT_FOR_ID(mode) STRUCT_FOR_ID(module) STRUCT_FOR_ID(module_globals) STRUCT_FOR_ID(modules) + STRUCT_FOR_ID(month) STRUCT_FOR_ID(mro) STRUCT_FOR_ID(msg) + STRUCT_FOR_ID(mutex) STRUCT_FOR_ID(mycmp) STRUCT_FOR_ID(n) STRUCT_FOR_ID(n_arg) @@ -656,6 +674,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(salt) STRUCT_FOR_ID(sched_priority) STRUCT_FOR_ID(scheduler) + STRUCT_FOR_ID(second) + STRUCT_FOR_ID(security_attributes) STRUCT_FOR_ID(seek) STRUCT_FOR_ID(seekable) STRUCT_FOR_ID(selectors) @@ -732,6 +752,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(type) STRUCT_FOR_ID(type_params) STRUCT_FOR_ID(tz) + STRUCT_FOR_ID(tzinfo) STRUCT_FOR_ID(tzname) STRUCT_FOR_ID(uid) STRUCT_FOR_ID(unlink) @@ -742,6 +763,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(values) STRUCT_FOR_ID(version) STRUCT_FOR_ID(volume) + STRUCT_FOR_ID(wait_all) STRUCT_FOR_ID(warnings) STRUCT_FOR_ID(warnoptions) STRUCT_FOR_ID(wbits) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index c84f87a831bf384..eb8a9a0db46c221 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -11,7 +11,6 @@ extern "C" { #include "pycore_lock.h" // PyMutex #include "pycore_hashtable.h" // _Py_hashtable_t -#include "pycore_time.h" // _PyTime_t extern int _PyImport_IsInitialized(PyInterpreterState *); @@ -103,7 +102,7 @@ struct _import_state { /* diagnostic info in PyImport_ImportModuleLevelObject() */ struct { int import_level; - _PyTime_t accumulated; + PyTime_t accumulated; int header; } find_and_load; }; diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 04e75940dcb5734..6a00aafea73779a 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -30,6 +30,7 @@ extern "C" { #include "pycore_mimalloc.h" // struct _mimalloc_interp_state #include "pycore_object_state.h" // struct _py_object_state #include "pycore_obmalloc.h" // struct _obmalloc_state +#include "pycore_qsbr.h" // struct _qsbr_state #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_tuple.h" // struct _Py_tuple_state #include "pycore_typeobject.h" // struct types_state @@ -71,7 +72,6 @@ typedef struct _rare_events { uint8_t set_eval_frame_func; /* Modifying the builtins, __builtins__.__dict__[var] = ... */ uint8_t builtin_dict; - int builtins_dict_watcher_id; /* Modifying a function, e.g. func.__defaults__ = ..., etc. */ uint8_t func_modification; } _rare_events; @@ -112,7 +112,7 @@ struct _is { /* The thread currently executing in the __main__ module, if any. */ PyThreadState *main; /* Used in Modules/_threadmodule.c. */ - long count; + Py_ssize_t count; /* Support for runtime thread stack size tuning. A value of 0 means using the platform's default stack size or the size specified by the THREAD_STACK_SIZE macro. */ @@ -198,9 +198,11 @@ struct _is { struct _warnings_runtime_state warnings; struct atexit_state atexit; struct _stoptheworld_state stoptheworld; + struct _qsbr_shared qsbr; #if defined(Py_GIL_DISABLED) struct _mimalloc_interp_state mimalloc; + struct _brc_state brc; // biased reference counting state #endif // Per-interpreter state for the obmalloc allocator. For the main @@ -221,28 +223,31 @@ struct _is { // One bit is set for each non-NULL entry in code_watchers uint8_t active_code_watchers; -#if !defined(Py_GIL_DISABLED) - struct _Py_freelist_state freelist_state; -#endif struct _py_object_state object_state; struct _Py_unicode_state unicode; struct _Py_long_state long_state; struct _dtoa_state dtoa; struct _py_func_state func_state; - struct _Py_tuple_state tuple; struct _Py_dict_state dict_state; struct _Py_exc_state exc_state; + struct _Py_mem_interp_free_queue mem_free_queue; struct ast_state ast; struct types_state types; struct callable_cache callable_cache; _PyOptimizerObject *optimizer; _PyExecutorObject *executor_list_head; - uint16_t optimizer_resume_threshold; - uint16_t optimizer_backedge_threshold; + + /* These two values are shifted and offset to speed up check in JUMP_BACKWARD */ + uint32_t optimizer_resume_threshold; + uint32_t optimizer_backedge_threshold; + + uint16_t optimizer_side_threshold; + uint32_t next_func_version; _rare_events rare_events; + PyDict_WatchCallback builtins_dict_watcher; _Py_GlobalMonitors monitors; bool sys_profile_initialized; diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h new file mode 100644 index 000000000000000..17bd23f0752be20 --- /dev/null +++ b/Include/internal/pycore_jit.h @@ -0,0 +1,25 @@ +#ifndef Py_INTERNAL_JIT_H +#define Py_INTERNAL_JIT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef _Py_JIT + +typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate); + +int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length); +void _PyJIT_Free(_PyExecutorObject *executor); + +#endif // _Py_JIT + +#ifdef __cplusplus +} +#endif + +#endif // !Py_INTERNAL_JIT_H diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 6c29d882335512e..50dc13c4da4487c 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -13,23 +13,18 @@ extern "C" { extern PyObject* _PyList_Extend(PyListObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); - -/* runtime lifecycle */ - -extern void _PyList_Fini(_PyFreeListState *); - - #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item) extern int _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem); +// In free-threaded build: self should be locked by the caller, if it should be thread-safe. static inline int _PyList_AppendTakeRef(PyListObject *self, PyObject *newitem) { assert(self != NULL && newitem != NULL); assert(PyList_Check(self)); - Py_ssize_t len = PyList_GET_SIZE(self); + Py_ssize_t len = Py_SIZE(self); Py_ssize_t allocated = self->allocated; assert((size_t)len + 1 < PY_SSIZE_T_MAX); if (allocated > len) { diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index 18a8896d97a5481..c89159b55e130f6 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -13,8 +13,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_time.h" // _PyTime_t - // A mutex that occupies one byte. The lock can be zero initialized. // @@ -28,6 +26,9 @@ extern "C" { // Typical initialization: // PyMutex m = (PyMutex){0}; // +// Or initialize as global variables: +// static PyMutex m; +// // Typical usage: // PyMutex_Lock(&m); // ... @@ -113,7 +114,7 @@ typedef enum _PyLockFlags { // Lock a mutex with an optional timeout and additional options. See // _PyLockFlags for details. extern PyLockStatus -_PyMutex_LockTimed(PyMutex *m, _PyTime_t timeout_ns, _PyLockFlags flags); +_PyMutex_LockTimed(PyMutex *m, PyTime_t timeout_ns, _PyLockFlags flags); // Lock a mutex with aditional options. See _PyLockFlags for details. static inline void @@ -146,7 +147,7 @@ PyAPI_FUNC(void) PyEvent_Wait(PyEvent *evt); // Wait for the event to be set, or until the timeout expires. If the event is // already set, then this returns immediately. Returns 1 if the event was set, // and 0 if the timeout expired or thread was interrupted. -PyAPI_FUNC(int) PyEvent_WaitTimed(PyEvent *evt, _PyTime_t timeout_ns); +PyAPI_FUNC(int) PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns); // _PyRawMutex implements a word-sized mutex that that does not depend on the @@ -251,6 +252,39 @@ PyAPI_FUNC(void) _PyRWMutex_RUnlock(_PyRWMutex *rwmutex); PyAPI_FUNC(void) _PyRWMutex_Lock(_PyRWMutex *rwmutex); PyAPI_FUNC(void) _PyRWMutex_Unlock(_PyRWMutex *rwmutex); +// Similar to linux seqlock: https://en.wikipedia.org/wiki/Seqlock +// We use a sequence number to lock the writer, an even sequence means we're unlocked, an odd +// sequence means we're locked. Readers will read the sequence before attempting to read the +// underlying data and then read the sequence number again after reading the data. If the +// sequence has not changed the data is valid. +// +// Differs a little bit in that we use CAS on sequence as the lock, instead of a seperate spin lock. +// The writer can also detect that the undelering data has not changed and abandon the write +// and restore the previous sequence. +typedef struct { + uint32_t sequence; +} _PySeqLock; + +// Lock the sequence lock for the writer +PyAPI_FUNC(void) _PySeqLock_LockWrite(_PySeqLock *seqlock); + +// Unlock the sequence lock and move to the next sequence number. +PyAPI_FUNC(void) _PySeqLock_UnlockWrite(_PySeqLock *seqlock); + +// Abandon the current update indicating that no mutations have occured +// and restore the previous sequence value. +PyAPI_FUNC(void) _PySeqLock_AbandonWrite(_PySeqLock *seqlock); + +// Begin a read operation and return the current sequence number. +PyAPI_FUNC(uint32_t) _PySeqLock_BeginRead(_PySeqLock *seqlock); + +// End the read operation and confirm that the sequence number has not changed. +// Returns 1 if the read was successful or 0 if the read should be re-tried. +PyAPI_FUNC(uint32_t) _PySeqLock_EndRead(_PySeqLock *seqlock, uint32_t previous); + +// Check if the lock was held during a fork and clear the lock. Returns 1 +// if the lock was held and any associated datat should be cleared. +PyAPI_FUNC(uint32_t) _PySeqLock_AfterFork(_PySeqLock *seqlock); #ifdef __cplusplus } diff --git a/Include/internal/pycore_mimalloc.h b/Include/internal/pycore_mimalloc.h index 1e7ed5a4ca62e20..14c98108ec131e0 100644 --- a/Include/internal/pycore_mimalloc.h +++ b/Include/internal/pycore_mimalloc.h @@ -20,9 +20,12 @@ typedef enum { #include "pycore_pymem.h" #ifdef WITH_MIMALLOC -#define MI_DEBUG_UNINIT PYMEM_CLEANBYTE -#define MI_DEBUG_FREED PYMEM_DEADBYTE -#define MI_DEBUG_PADDING PYMEM_FORBIDDENBYTE +# ifdef Py_GIL_DISABLED +# define MI_PRIM_THREAD_ID _Py_ThreadId +# endif +# define MI_DEBUG_UNINIT PYMEM_CLEANBYTE +# define MI_DEBUG_FREED PYMEM_DEADBYTE +# define MI_DEBUG_PADDING PYMEM_FORBIDDENBYTE #ifdef Py_DEBUG # define MI_DEBUG 1 #else diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 4e52ffc77c5956b..34a83ea228e8b10 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -178,7 +178,7 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) } _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG - _Py_DEC_REFTOTAL(_PyInterpreterState_GET()); + _Py_DEC_REFTOTAL(PyInterpreterState_Get()); #endif if (--op->ob_refcnt != 0) { assert(op->ob_refcnt > 0); @@ -199,7 +199,7 @@ _Py_DECREF_NO_DEALLOC(PyObject *op) } _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG - _Py_DEC_REFTOTAL(_PyInterpreterState_GET()); + _Py_DEC_REFTOTAL(PyInterpreterState_Get()); #endif op->ob_refcnt--; #ifdef Py_DEBUG @@ -315,16 +315,15 @@ static inline void _PyObject_GC_TRACK( _PyObject_ASSERT_FROM(op, !_PyObject_GC_IS_TRACKED(op), "object already tracked by the garbage collector", filename, lineno, __func__); - +#ifdef Py_GIL_DISABLED + op->ob_gc_bits |= _PyGC_BITS_TRACKED; +#else PyGC_Head *gc = _Py_AS_GC(op); _PyObject_ASSERT_FROM(op, (gc->_gc_prev & _PyGC_PREV_MASK_COLLECTING) == 0, "object is in generation which is garbage collected", filename, lineno, __func__); -#ifdef Py_GIL_DISABLED - op->ob_gc_bits |= _PyGC_BITS_TRACKED; -#else PyInterpreterState *interp = _PyInterpreterState_GET(); PyGC_Head *generation0 = interp->gc.generation0; PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); @@ -594,8 +593,12 @@ _PyObject_IS_GC(PyObject *obj) static inline size_t _PyType_PreHeaderSize(PyTypeObject *tp) { - return _PyType_IS_GC(tp) * sizeof(PyGC_Head) + - _PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER) * 2 * sizeof(PyObject *); + return ( +#ifndef Py_GIL_DISABLED + _PyType_IS_GC(tp) * sizeof(PyGC_Head) + +#endif + _PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER) * 2 * sizeof(PyObject *) + ); } void _PyObject_GC_Link(PyObject *op); @@ -625,6 +628,14 @@ extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name); +#ifdef Py_GIL_DISABLED +# define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-1) +# define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-2) +#else +# define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-3) +# define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-4) +#endif + typedef union { PyObject *dict; /* Use a char* to generate a warning if directly assigning a PyDictValues */ @@ -635,7 +646,7 @@ static inline PyDictOrValues * _PyObject_DictOrValuesPointer(PyObject *obj) { assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - return ((PyDictOrValues *)obj)-3; + return (PyDictOrValues *)((char *)obj + MANAGED_DICT_OFFSET); } static inline int @@ -664,8 +675,6 @@ _PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values) ptr->values = ((char *)values) - 1; } -#define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-4) - extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); diff --git a/Include/internal/pycore_object_stack.h b/Include/internal/pycore_object_stack.h index 1dc1c1591525ded..fc130b1e9920b41 100644 --- a/Include/internal/pycore_object_stack.h +++ b/Include/internal/pycore_object_stack.h @@ -1,6 +1,8 @@ #ifndef Py_INTERNAL_OBJECT_STACK_H #define Py_INTERNAL_OBJECT_STACK_H +#include "pycore_freelist.h" // _PyFreeListState + #ifdef __cplusplus extern "C" { #endif @@ -32,9 +34,6 @@ _PyObjectStackChunk_New(void); extern void _PyObjectStackChunk_Free(_PyObjectStackChunk *); -extern void -_PyObjectStackChunk_ClearFreeList(_PyFreeListState *state, int is_finalization); - // Push an item onto the stack. Return -1 on allocation failure, 0 on success. static inline int _PyObjectStack_Push(_PyObjectStack *stack, PyObject *obj) @@ -74,6 +73,10 @@ _PyObjectStack_Pop(_PyObjectStack *stack) return obj; } +// Merge src into dst, leaving src empty +extern void +_PyObjectStack_Merge(_PyObjectStack *dst, _PyObjectStack *src); + // Remove all items from the stack extern void _PyObjectStack_Clear(_PyObjectStack *stack); diff --git a/Include/internal/pycore_object_state.h b/Include/internal/pycore_object_state.h index 9eac27b1a9a4e3e..cd7c9335b3e611c 100644 --- a/Include/internal/pycore_object_state.h +++ b/Include/internal/pycore_object_state.h @@ -8,6 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // _PyObject_freelists #include "pycore_hashtable.h" // _Py_hashtable_t struct _py_object_runtime_state { @@ -18,6 +19,9 @@ struct _py_object_runtime_state { }; struct _py_object_state { +#if !defined(Py_GIL_DISABLED) + struct _Py_object_freelists freelists; +#endif #ifdef Py_REF_DEBUG Py_ssize_t reftotal; #endif diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 75d7f44025328ee..ab34366ab1066c4 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -519,7 +519,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CALL_ALLOC_AND_ENTER_INIT: return 1; case CALL_BOUND_METHOD_EXACT_ARGS: - return ((0) ? 1 : 0); + return 0; case CALL_BUILTIN_CLASS: return 1; case CALL_BUILTIN_FAST: @@ -551,7 +551,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CALL_METHOD_DESCRIPTOR_O: return 1; case CALL_PY_EXACT_ARGS: - return ((0) ? 1 : 0); + return 0; case CALL_PY_WITH_DEFAULTS: return 1; case CALL_STR_1: @@ -697,23 +697,23 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case LOAD_ATTR_CLASS: return 1 + (oparg & 1); case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_ATTR_INSTANCE_VALUE: return 1 + (oparg & 1); case LOAD_ATTR_METHOD_LAZY_DICT: - return 1 + ((1) ? 1 : 0); + return 2; case LOAD_ATTR_METHOD_NO_DICT: - return 1 + ((1) ? 1 : 0); + return 2; case LOAD_ATTR_METHOD_WITH_VALUES: - return 1 + ((1) ? 1 : 0); + return 2; case LOAD_ATTR_MODULE: return 1 + (oparg & 1); case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_ATTR_PROPERTY: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_ATTR_SLOT: return 1 + (oparg & 1); case LOAD_ATTR_WITH_HINT: @@ -749,7 +749,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case LOAD_SUPER_ATTR: return 1 + (oparg & 1); case LOAD_SUPER_ATTR_ATTR: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_SUPER_ATTR_METHOD: return 2; case MAKE_CELL: @@ -909,8 +909,10 @@ enum InstructionFormat { #define HAS_DEOPT_FLAG (128) #define HAS_ERROR_FLAG (256) #define HAS_ESCAPES_FLAG (512) -#define HAS_PURE_FLAG (1024) -#define HAS_PASSTHROUGH_FLAG (2048) +#define HAS_EXIT_FLAG (1024) +#define HAS_PURE_FLAG (2048) +#define HAS_PASSTHROUGH_FLAG (4096) +#define HAS_OPARG_AND_1_FLAG (8192) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -921,8 +923,10 @@ enum InstructionFormat { #define OPCODE_HAS_DEOPT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_DEOPT_FLAG)) #define OPCODE_HAS_ERROR(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_FLAG)) #define OPCODE_HAS_ESCAPES(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ESCAPES_FLAG)) +#define OPCODE_HAS_EXIT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EXIT_FLAG)) #define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) #define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) +#define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) #define OPARG_FULL 0 #define OPARG_CACHE_1 1 @@ -945,14 +949,14 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -995,9 +999,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, @@ -1060,16 +1064,16 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1094,7 +1098,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, - [NOP] = { true, INSTR_FMT_IX, 0 }, + [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, @@ -1118,8 +1122,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -1133,12 +1137,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, @@ -1156,10 +1160,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [LOAD_SUPER_METHOD] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ZERO_SUPER_ATTR] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ZERO_SUPER_METHOD] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [POP_BLOCK] = { true, -1, 0 }, - [SETUP_CLEANUP] = { true, -1, HAS_ARG_FLAG }, - [SETUP_FINALLY] = { true, -1, HAS_ARG_FLAG }, - [SETUP_WITH] = { true, -1, HAS_ARG_FLAG }, + [POP_BLOCK] = { true, -1, HAS_PURE_FLAG }, + [SETUP_CLEANUP] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, + [SETUP_FINALLY] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, + [SETUP_WITH] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, }; #endif @@ -1217,9 +1221,9 @@ _PyOpcode_macro_expansion[256] = { [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { _CHECK_EG_MATCH, 0, 0 } } }, [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { _CHECK_EXC_MATCH, 0, 0 } } }, [COMPARE_OP] = { .nuops = 1, .uops = { { _COMPARE_OP, 0, 0 } } }, - [COMPARE_OP_FLOAT] = { .nuops = 1, .uops = { { _COMPARE_OP_FLOAT, 0, 0 } } }, - [COMPARE_OP_INT] = { .nuops = 1, .uops = { { _COMPARE_OP_INT, 0, 0 } } }, - [COMPARE_OP_STR] = { .nuops = 1, .uops = { { _COMPARE_OP_STR, 0, 0 } } }, + [COMPARE_OP_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _COMPARE_OP_FLOAT, 0, 0 } } }, + [COMPARE_OP_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _COMPARE_OP_INT, 0, 0 } } }, + [COMPARE_OP_STR] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _COMPARE_OP_STR, 0, 0 } } }, [CONTAINS_OP] = { .nuops = 1, .uops = { { _CONTAINS_OP, 0, 0 } } }, [CONVERT_VALUE] = { .nuops = 1, .uops = { { _CONVERT_VALUE, 0, 0 } } }, [COPY] = { .nuops = 1, .uops = { { _COPY, 0, 0 } } }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 31f30c673f207a1..614850468ec1d38 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -8,8 +8,17 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -int _Py_uop_analyze_and_optimize(PyCodeObject *code, - _PyUOpInstruction *trace, int trace_len, int curr_stackentries); +#include "pycore_uop_ids.h" +#include <stdbool.h> + +// This is the length of the trace we project initially. +#define UOP_MAX_TRACE_LENGTH 512 + +#define TRACE_STACK_SIZE 5 + +int _Py_uop_analyze_and_optimize(_PyInterpreterFrame *frame, + _PyUOpInstruction *trace, int trace_len, int curr_stackentries, + _PyBloomFilter *dependencies); extern PyTypeObject _PyCounterExecutor_Type; extern PyTypeObject _PyCounterOptimizer_Type; @@ -17,6 +26,91 @@ extern PyTypeObject _PyDefaultOptimizer_Type; extern PyTypeObject _PyUOpExecutor_Type; extern PyTypeObject _PyUOpOptimizer_Type; +/* Symbols */ +/* See explanation in optimizer_symbols.c */ + +struct _Py_UopsSymbol { + int flags; // 0 bits: Top; 2 or more bits: Bottom + PyTypeObject *typ; // Borrowed reference + PyObject *const_val; // Owned reference (!) +}; + +// Holds locals, stack, locals, stack ... co_consts (in that order) +#define MAX_ABSTRACT_INTERP_SIZE 4096 + +#define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5) + +// Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) +#define MAX_ABSTRACT_FRAME_DEPTH (TRACE_STACK_SIZE + 2) + +typedef struct _Py_UopsSymbol _Py_UopsSymbol; + +struct _Py_UOpsAbstractFrame { + // Max stacklen + int stack_len; + int locals_len; + + _Py_UopsSymbol **stack_pointer; + _Py_UopsSymbol **stack; + _Py_UopsSymbol **locals; +}; + +typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; + +typedef struct ty_arena { + int ty_curr_number; + int ty_max_number; + _Py_UopsSymbol arena[TY_ARENA_SIZE]; +} ty_arena; + +struct _Py_UOpsContext { + PyObject_HEAD + // The current "executing" frame. + _Py_UOpsAbstractFrame *frame; + _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; + int curr_frame_depth; + + // Arena for the symbolic types. + ty_arena t_arena; + + _Py_UopsSymbol **n_consumed; + _Py_UopsSymbol **limit; + _Py_UopsSymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; +}; + +typedef struct _Py_UOpsContext _Py_UOpsContext; + +extern bool _Py_uop_sym_is_null(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_is_not_null(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_is_const(_Py_UopsSymbol *sym); +extern PyObject *_Py_uop_sym_get_const(_Py_UopsSymbol *sym); +extern _Py_UopsSymbol *_Py_uop_sym_new_unknown(_Py_UOpsContext *ctx); +extern _Py_UopsSymbol *_Py_uop_sym_new_not_null(_Py_UOpsContext *ctx); +extern _Py_UopsSymbol *_Py_uop_sym_new_type( + _Py_UOpsContext *ctx, PyTypeObject *typ); +extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val); +extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx); +extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ); +extern void _Py_uop_sym_set_null(_Py_UopsSymbol *sym); +extern void _Py_uop_sym_set_non_null(_Py_UopsSymbol *sym); +extern void _Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *typ); +extern void _Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val); +extern bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym); + + +extern int _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx); +extern void _Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx); + +extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( + _Py_UOpsContext *ctx, + PyCodeObject *co, + _Py_UopsSymbol **localsplus_start, + int n_locals_already_filled, + int curr_stackentries); +extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx); + +PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_parking_lot.h b/Include/internal/pycore_parking_lot.h index f444da730055e84..8c9260e2636fbc7 100644 --- a/Include/internal/pycore_parking_lot.h +++ b/Include/internal/pycore_parking_lot.h @@ -18,8 +18,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_time.h" // _PyTime_t - enum { // The thread was unparked by another thread. @@ -61,7 +59,7 @@ enum { // } PyAPI_FUNC(int) _PyParkingLot_Park(const void *address, const void *expected, - size_t address_size, _PyTime_t timeout_ns, + size_t address_size, PyTime_t timeout_ns, void *park_arg, int detach); // Callback for _PyParkingLot_Unpark: diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h new file mode 100644 index 000000000000000..e441600d54e1aac --- /dev/null +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -0,0 +1,43 @@ +// This header file provides wrappers around the atomic operations found in +// `pyatomic.h` that are only atomic in free-threaded builds. +// +// These are intended to be used in places where atomics are required in +// free-threaded builds, but not in the default build, and we don't want to +// introduce the potential performance overhead of an atomic operation in the +// default build. +// +// All usages of these macros should be replaced with unconditionally atomic or +// non-atomic versions, and this file should be removed, once the dust settles +// on free threading. +#ifndef Py_ATOMIC_FT_WRAPPERS_H +#define Py_ATOMIC_FT_WRAPPERS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +#error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_GIL_DISABLED +#define FT_ATOMIC_LOAD_SSIZE(value) _Py_atomic_load_ssize(&value) +#define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) \ + _Py_atomic_load_ssize_relaxed(&value) +#define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ + _Py_atomic_store_ptr_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \ + _Py_atomic_store_ptr_release(&value, new_value) +#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ + _Py_atomic_store_ssize_relaxed(&value, new_value) +#else +#define FT_ATOMIC_LOAD_SSIZE(value) value +#define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value +#define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_ATOMIC_FT_WRAPPERS_H */ diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 1a72d07b50b738e..1aea91abc5d69f4 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -1,6 +1,7 @@ #ifndef Py_INTERNAL_PYMEM_H #define Py_INTERNAL_PYMEM_H +#include "pycore_llist.h" // struct llist_node #include "pycore_lock.h" // PyMutex #ifdef __cplusplus @@ -48,6 +49,11 @@ struct _pymem_allocators { PyObjectArenaAllocator obj_arena; }; +struct _Py_mem_interp_free_queue { + int has_work; // true if the queue is not empty + PyMutex mutex; // protects the queue + struct llist_node head; // queue of _mem_work_chunk items +}; /* Set the memory allocator of the specified domain to the default. Save the old allocator into *old_alloc if it's non-NULL. @@ -110,6 +116,19 @@ extern int _PyMem_SetupAllocators(PyMemAllocatorName allocator); /* Is the debug allocator enabled? */ extern int _PyMem_DebugEnabled(void); +// Enqueue a pointer to be freed possibly after some delay. +extern void _PyMem_FreeDelayed(void *ptr); + +// Periodically process delayed free requests. +extern void _PyMem_ProcessDelayed(PyThreadState *tstate); + +// Abandon all thread-local delayed free requests and push them to the +// interpreter's queue. +extern void _PyMem_AbandonDelayed(PyThreadState *tstate); + +// On interpreter shutdown, frees all delayed free requests. +extern void _PyMem_FiniDelayed(PyInterpreterState *interp); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pymem_init.h b/Include/internal/pycore_pymem_init.h index 96c49ed7338d6d6..c593edc86d9952a 100644 --- a/Include/internal/pycore_pymem_init.h +++ b/Include/internal/pycore_pymem_init.h @@ -92,6 +92,11 @@ extern void _PyMem_ArenaFree(void *, void *, size_t); { NULL, _PyMem_ArenaAlloc, _PyMem_ArenaFree } +#define _Py_mem_free_queue_INIT(queue) \ + { \ + .head = LLIST_INIT(queue.head), \ + } + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 289ef28f0dd9a9c..6f9e6a332a7830b 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -268,7 +268,7 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); // See also PyInterpreterState_Get() and _PyInterpreterState_GET(). extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void); -static inline _PyFreeListState* _PyFreeListState_GET(void) +static inline struct _Py_object_freelists* _Py_object_freelists_GET(void) { PyThreadState *tstate = _PyThreadState_GET(); #ifdef Py_DEBUG @@ -276,9 +276,9 @@ static inline _PyFreeListState* _PyFreeListState_GET(void) #endif #ifdef Py_GIL_DISABLED - return &((_PyThreadStateImpl*)tstate)->freelist_state; + return &((_PyThreadStateImpl*)tstate)->freelists; #else - return &tstate->interp->freelist_state; + return &tstate->interp->object_state.freelists; #endif } diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index 9c9a09f60f34418..d017d4ff308aa8d 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "dynamic_annotations.h" // _Py_ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX +#include "pycore_llist.h" // struct llist_node // Get _POSIX_THREADS and _POSIX_SEMAPHORES macros if available #if (defined(HAVE_UNISTD_H) && !defined(_POSIX_THREADS) \ @@ -75,19 +76,27 @@ struct _pythread_runtime_state { struct py_stub_tls_entry tls_entries[PTHREAD_KEYS_MAX]; } stubs; #endif + + // Linked list of ThreadHandleObjects + struct llist_node handles; }; +#define _pythread_RUNTIME_INIT(pythread) \ + { \ + .handles = LLIST_INIT(pythread.handles), \ + } #ifdef HAVE_FORK /* Private function to reinitialize a lock at fork in the child process. Reset the lock to the unlocked state. Return 0 on success, return -1 on error. */ extern int _PyThread_at_fork_reinit(PyThread_type_lock *lock); +extern void _PyThread_AfterFork(struct _pythread_runtime_state *state); #endif /* HAVE_FORK */ // unset: -1 seconds, in nanoseconds -#define PyThread_UNSET_TIMEOUT ((_PyTime_t)(-1 * 1000 * 1000 * 1000)) +#define PyThread_UNSET_TIMEOUT ((PyTime_t)(-1 * 1000 * 1000 * 1000)) // Exported for the _xxinterpchannels module. PyAPI_FUNC(int) PyThread_ParseTimeoutArg( @@ -143,12 +152,6 @@ PyAPI_FUNC(int) PyThread_join_thread(PyThread_handle_t); */ PyAPI_FUNC(int) PyThread_detach_thread(PyThread_handle_t); -/* - * Obtain the new thread ident and handle in a forked child process. - */ -PyAPI_FUNC(void) PyThread_update_thread_after_fork(PyThread_ident_t* ident, - PyThread_handle_t* handle); - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h new file mode 100644 index 000000000000000..475f00deedc2265 --- /dev/null +++ b/Include/internal/pycore_qsbr.h @@ -0,0 +1,139 @@ +// The QSBR APIs (quiescent state-based reclamation) provide a mechanism for +// the free-threaded build to safely reclaim memory when there may be +// concurrent accesses. +// +// Many operations in the free-threaded build are protected by locks. However, +// in some cases, we want to allow reads to happen concurrently with updates. +// In this case, we need to delay freeing ("reclaiming") any memory that may be +// concurrently accessed by a reader. The QSBR APIs provide a way to do this. +#ifndef Py_INTERNAL_QSBR_H +#define Py_INTERNAL_QSBR_H + +#include <stdbool.h> +#include <stdint.h> +#include "pycore_lock.h" // PyMutex + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// The shared write sequence is always odd and incremented by two. Detached +// threads are indicated by a read sequence of zero. This avoids collisions +// between the offline state and any valid sequence number even if the +// sequences numbers wrap around. +#define QSBR_OFFLINE 0 +#define QSBR_INITIAL 1 +#define QSBR_INCR 2 + +struct _qsbr_shared; +struct _PyThreadStateImpl; // forward declare to avoid circular dependency + +// Per-thread state +struct _qsbr_thread_state { + // Last observed write sequence (or 0 if detached) + uint64_t seq; + + // Shared (per-interpreter) QSBR state + struct _qsbr_shared *shared; + + // Thread state (or NULL) + PyThreadState *tstate; + + // Used to defer advancing write sequence a fixed number of times + int deferrals; + + // Is this thread state allocated? + bool allocated; + struct _qsbr_thread_state *freelist_next; +}; + +// Padding to avoid false sharing +struct _qsbr_pad { + struct _qsbr_thread_state qsbr; + char __padding[64 - sizeof(struct _qsbr_thread_state)]; +}; + +// Per-interpreter state +struct _qsbr_shared { + // Write sequence: always odd, incremented by two + uint64_t wr_seq; + + // Minimum observed read sequence of all QSBR thread states + uint64_t rd_seq; + + // Array of QSBR thread states. + struct _qsbr_pad *array; + Py_ssize_t size; + + // Freelist of unused _qsbr_thread_states (protected by mutex) + PyMutex mutex; + struct _qsbr_thread_state *freelist; +}; + +static inline uint64_t +_Py_qsbr_shared_current(struct _qsbr_shared *shared) +{ + return _Py_atomic_load_uint64_acquire(&shared->wr_seq); +} + +// Reports a quiescent state: the caller no longer holds any pointer to shared +// data not protected by locks or reference counts. +static inline void +_Py_qsbr_quiescent_state(struct _qsbr_thread_state *qsbr) +{ + uint64_t seq = _Py_qsbr_shared_current(qsbr->shared); + _Py_atomic_store_uint64_release(&qsbr->seq, seq); +} + +// Advance the write sequence and return the new goal. This should be called +// after data is removed. The returned goal is used with `_Py_qsbr_poll()` to +// determine when it is safe to reclaim (free) the memory. +extern uint64_t +_Py_qsbr_advance(struct _qsbr_shared *shared); + +// Batches requests to advance the write sequence. This advances the write +// sequence every N calls, which reduces overhead but increases time to +// reclamation. Returns the new goal. +extern uint64_t +_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr); + +// Have the read sequences advanced to the given goal? If this returns true, +// it safe to reclaim any memory tagged with the goal (or earlier goal). +extern bool +_Py_qsbr_poll(struct _qsbr_thread_state *qsbr, uint64_t goal); + +// Called when thread attaches to interpreter +extern void +_Py_qsbr_attach(struct _qsbr_thread_state *qsbr); + +// Called when thread detaches from interpreter +extern void +_Py_qsbr_detach(struct _qsbr_thread_state *qsbr); + +// Reserves (allocates) a QSBR state and returns its index. +extern Py_ssize_t +_Py_qsbr_reserve(PyInterpreterState *interp); + +// Associates a PyThreadState with the QSBR state at the given index +extern void +_Py_qsbr_register(struct _PyThreadStateImpl *tstate, + PyInterpreterState *interp, Py_ssize_t index); + +// Disassociates a PyThreadState from the QSBR state and frees the QSBR state. +extern void +_Py_qsbr_unregister(struct _PyThreadStateImpl *tstate); + +extern void +_Py_qsbr_fini(PyInterpreterState *interp); + +extern void +_Py_qsbr_after_fork(struct _PyThreadStateImpl *tstate); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_QSBR_H */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 02ab22b967b38ff..dc6f6f100f7a922 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -55,74 +55,81 @@ typedef struct _Py_DebugOffsets { uint64_t version; // Runtime state offset; struct _runtime_state { - off_t finalizing; - off_t interpreters_head; + uint64_t finalizing; + uint64_t interpreters_head; } runtime_state; // Interpreter state offset; struct _interpreter_state { - off_t next; - off_t threads_head; - off_t gc; - off_t imports_modules; - off_t sysdict; - off_t builtins; - off_t ceval_gil; - off_t gil_runtime_state_locked; - off_t gil_runtime_state_holder; + uint64_t next; + uint64_t threads_head; + uint64_t gc; + uint64_t imports_modules; + uint64_t sysdict; + uint64_t builtins; + uint64_t ceval_gil; + uint64_t gil_runtime_state_locked; + uint64_t gil_runtime_state_holder; } interpreter_state; // Thread state offset; struct _thread_state{ - off_t prev; - off_t next; - off_t interp; - off_t current_frame; - off_t thread_id; - off_t native_thread_id; + uint64_t prev; + uint64_t next; + uint64_t interp; + uint64_t current_frame; + uint64_t thread_id; + uint64_t native_thread_id; } thread_state; // InterpreterFrame offset; struct _interpreter_frame { - off_t previous; - off_t executable; - off_t instr_ptr; - off_t localsplus; - off_t owner; + uint64_t previous; + uint64_t executable; + uint64_t instr_ptr; + uint64_t localsplus; + uint64_t owner; } interpreter_frame; // CFrame offset; struct _cframe { - off_t current_frame; - off_t previous; + uint64_t current_frame; + uint64_t previous; } cframe; // Code object offset; struct _code_object { - off_t filename; - off_t name; - off_t linetable; - off_t firstlineno; - off_t argcount; - off_t localsplusnames; - off_t localspluskinds; - off_t co_code_adaptive; + uint64_t filename; + uint64_t name; + uint64_t linetable; + uint64_t firstlineno; + uint64_t argcount; + uint64_t localsplusnames; + uint64_t localspluskinds; + uint64_t co_code_adaptive; } code_object; // PyObject offset; struct _pyobject { - off_t ob_type; + uint64_t ob_type; } pyobject; // PyTypeObject object offset; struct _type_object { - off_t tp_name; + uint64_t tp_name; } type_object; // PyTuple object offset; struct _tuple_object { - off_t ob_item; + uint64_t ob_item; } tuple_object; + + // Unicode object offset; + struct _unicode_object { + uint64_t state; + uint64_t length; + size_t asciiobject_size; + } unicode_object; } _Py_DebugOffsets; /* Full Python runtime state */ @@ -191,7 +198,10 @@ typedef struct pyruntimestate { int64_t next_id; } interpreters; + /* Platform-specific identifier and PyThreadState, respectively, for the + main thread in the main interpreter. */ unsigned long main_thread; + PyThreadState *main_tstate; /* ---------- IMPORTANT --------------------------- The fields above this line are declared as early as @@ -268,7 +278,7 @@ typedef struct pyruntimestate { a pointer type. */ - /* PyInterpreterState.interpreters.main */ + /* _PyRuntimeState.interpreters.main */ PyInterpreterState _main_interpreter; #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 0a5c92bb84b524f..cc47b9a82e28792 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -16,6 +16,8 @@ extern "C" { #include "pycore_parser.h" // _parser_runtime_state_INIT #include "pycore_pyhash.h" // pyhash_state_INIT #include "pycore_pymem_init.h" // _pymem_allocators_standard_INIT +#include "pycore_pythread.h" // _pythread_RUNTIME_INIT +#include "pycore_qsbr.h" // QSBR_INITIAL #include "pycore_runtime_init_generated.h" // _Py_bytes_characters_INIT #include "pycore_signal.h" // _signals_RUNTIME_INIT #include "pycore_tracemalloc.h" // _tracemalloc_runtime_state_INIT @@ -81,6 +83,11 @@ extern PyTypeObject _PyExc_MemoryError; .tuple_object = { \ .ob_item = offsetof(PyTupleObject, ob_item), \ }, \ + .unicode_object = { \ + .state = offsetof(PyUnicodeObject, _base._base.state), \ + .length = offsetof(PyUnicodeObject, _base._base.length), \ + .asciiobject_size = sizeof(PyASCIIObject), \ + }, \ }, \ .allocators = { \ .standard = _pymem_allocators_standard_INIT(runtime), \ @@ -90,6 +97,7 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .obmalloc = _obmalloc_global_state_INIT, \ .pyhash_state = pyhash_state_INIT, \ + .threads = _pythread_RUNTIME_INIT(runtime.threads), \ .signals = _signals_RUNTIME_INIT, \ .interpreters = { \ /* This prevents interpreters from getting created \ @@ -167,9 +175,13 @@ extern PyTypeObject _PyExc_MemoryError; { .threshold = 10, }, \ }, \ }, \ - .object_state = _py_object_state_INIT(INTERP), \ + .qsbr = { \ + .wr_seq = QSBR_INITIAL, \ + .rd_seq = QSBR_INITIAL, \ + }, \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ .dict_state = _dict_state_INIT, \ + .mem_free_queue = _Py_mem_free_queue_INIT(INTERP.mem_free_queue), \ .func_state = { \ .next_version = 1, \ }, \ @@ -204,16 +216,6 @@ extern PyTypeObject _PyExc_MemoryError; .context_ver = 1, \ } -#ifdef Py_TRACE_REFS -# define _py_object_state_INIT(INTERP) \ - { \ - .refchain = NULL, \ - } -#else -# define _py_object_state_INIT(INTERP) \ - { 0 } -#endif - // global objects diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 9b39de1d69c6c7f..016eae02a2103d1 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -740,6 +740,7 @@ extern "C" { INIT_ID(_abc_impl), \ INIT_ID(_abstract_), \ INIT_ID(_active), \ + INIT_ID(_align_), \ INIT_ID(_annotation), \ INIT_ID(_anonymous_), \ INIT_ID(_argtypes_), \ @@ -750,6 +751,7 @@ extern "C" { INIT_ID(_check_retval_), \ INIT_ID(_dealloc_warn), \ INIT_ID(_feature_version), \ + INIT_ID(_field_types), \ INIT_ID(_fields_), \ INIT_ID(_finalizing), \ INIT_ID(_find_and_load), \ @@ -874,12 +876,14 @@ extern "C" { INIT_ID(d), \ INIT_ID(data), \ INIT_ID(database), \ + INIT_ID(day), \ INIT_ID(decode), \ INIT_ID(decoder), \ INIT_ID(default), \ INIT_ID(defaultaction), \ INIT_ID(delete), \ INIT_ID(depth), \ + INIT_ID(desired_access), \ INIT_ID(detect_types), \ INIT_ID(deterministic), \ INIT_ID(device), \ @@ -937,12 +941,14 @@ extern "C" { INIT_ID(fileno), \ INIT_ID(filepath), \ INIT_ID(fillvalue), \ + INIT_ID(filter), \ INIT_ID(filters), \ INIT_ID(final), \ INIT_ID(find_class), \ INIT_ID(fix_imports), \ INIT_ID(flags), \ INIT_ID(flush), \ + INIT_ID(fold), \ INIT_ID(follow_symlinks), \ INIT_ID(format), \ INIT_ID(from_param), \ @@ -968,11 +974,13 @@ extern "C" { INIT_ID(groups), \ INIT_ID(h), \ INIT_ID(handle), \ + INIT_ID(handle_seq), \ INIT_ID(hash_name), \ INIT_ID(header), \ INIT_ID(headers), \ INIT_ID(hi), \ INIT_ID(hook), \ + INIT_ID(hour), \ INIT_ID(id), \ INIT_ID(ident), \ INIT_ID(identity_hint), \ @@ -984,9 +992,12 @@ extern "C" { INIT_ID(indexgroup), \ INIT_ID(inf), \ INIT_ID(infer_variance), \ + INIT_ID(inherit_handle), \ INIT_ID(inheritable), \ INIT_ID(initial), \ INIT_ID(initial_bytes), \ + INIT_ID(initial_owner), \ + INIT_ID(initial_state), \ INIT_ID(initial_value), \ INIT_ID(initval), \ INIT_ID(inner_size), \ @@ -1042,11 +1053,13 @@ extern "C" { INIT_ID(locals), \ INIT_ID(logoption), \ INIT_ID(loop), \ + INIT_ID(manual_reset), \ INIT_ID(mapping), \ INIT_ID(match), \ INIT_ID(max_length), \ INIT_ID(maxdigits), \ INIT_ID(maxevents), \ + INIT_ID(maxlen), \ INIT_ID(maxmem), \ INIT_ID(maxsplit), \ INIT_ID(maxvalue), \ @@ -1056,13 +1069,18 @@ extern "C" { INIT_ID(metaclass), \ INIT_ID(metadata), \ INIT_ID(method), \ + INIT_ID(microsecond), \ + INIT_ID(milliseconds), \ + INIT_ID(minute), \ INIT_ID(mod), \ INIT_ID(mode), \ INIT_ID(module), \ INIT_ID(module_globals), \ INIT_ID(modules), \ + INIT_ID(month), \ INIT_ID(mro), \ INIT_ID(msg), \ + INIT_ID(mutex), \ INIT_ID(mycmp), \ INIT_ID(n), \ INIT_ID(n_arg), \ @@ -1165,6 +1183,8 @@ extern "C" { INIT_ID(salt), \ INIT_ID(sched_priority), \ INIT_ID(scheduler), \ + INIT_ID(second), \ + INIT_ID(security_attributes), \ INIT_ID(seek), \ INIT_ID(seekable), \ INIT_ID(selectors), \ @@ -1241,6 +1261,7 @@ extern "C" { INIT_ID(type), \ INIT_ID(type_params), \ INIT_ID(tz), \ + INIT_ID(tzinfo), \ INIT_ID(tzname), \ INIT_ID(uid), \ INIT_ID(unlink), \ @@ -1251,6 +1272,7 @@ extern "C" { INIT_ID(values), \ INIT_ID(version), \ INIT_ID(volume), \ + INIT_ID(wait_all), \ INIT_ID(warnings), \ INIT_ID(warnoptions), \ INIT_ID(wbits), \ diff --git a/Include/internal/pycore_semaphore.h b/Include/internal/pycore_semaphore.h index 4c37df7b39a48a3..ffcc6d80344d6ea 100644 --- a/Include/internal/pycore_semaphore.h +++ b/Include/internal/pycore_semaphore.h @@ -8,7 +8,6 @@ #endif #include "pycore_pythread.h" // _POSIX_SEMAPHORES -#include "pycore_time.h" // _PyTime_t #ifdef MS_WINDOWS # define WIN32_LEAN_AND_MEAN @@ -48,7 +47,7 @@ typedef struct _PySemaphore { // If `detach` is true, then the thread will detach/release the GIL while // sleeping. PyAPI_FUNC(int) -_PySemaphore_Wait(_PySemaphore *sema, _PyTime_t timeout_ns, int detach); +_PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout_ns, int detach); // Wakes up a single thread waiting on sema. Note that _PySemaphore_Wakeup() // can be called before _PySemaphore_Wait(). diff --git a/Include/internal/pycore_sliceobject.h b/Include/internal/pycore_sliceobject.h index 0c72d3ee6225c51..89086f67683a2f5 100644 --- a/Include/internal/pycore_sliceobject.h +++ b/Include/internal/pycore_sliceobject.h @@ -11,8 +11,6 @@ extern "C" { /* runtime lifecycle */ -extern void _PySlice_Fini(_PyFreeListState *); - extern PyObject * _PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop); diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 1d782ca2c96e055..b44393b56446735 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -109,18 +109,18 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); /* Flags for def-use information */ -#define DEF_GLOBAL 1 /* global stmt */ -#define DEF_LOCAL 2 /* assignment in code block */ -#define DEF_PARAM 2<<1 /* formal parameter */ -#define DEF_NONLOCAL 2<<2 /* nonlocal stmt */ -#define USE 2<<3 /* name is used */ -#define DEF_FREE 2<<4 /* name used but not defined in nested block */ -#define DEF_FREE_CLASS 2<<5 /* free variable from class's method */ -#define DEF_IMPORT 2<<6 /* assignment occurred via import */ -#define DEF_ANNOT 2<<7 /* this name is annotated */ -#define DEF_COMP_ITER 2<<8 /* this name is a comprehension iteration variable */ -#define DEF_TYPE_PARAM 2<<9 /* this name is a type parameter */ -#define DEF_COMP_CELL 2<<10 /* this name is a cell in an inlined comprehension */ +#define DEF_GLOBAL 1 /* global stmt */ +#define DEF_LOCAL 2 /* assignment in code block */ +#define DEF_PARAM (2<<1) /* formal parameter */ +#define DEF_NONLOCAL (2<<2) /* nonlocal stmt */ +#define USE (2<<3) /* name is used */ +#define DEF_FREE (2<<4) /* name used but not defined in nested block */ +#define DEF_FREE_CLASS (2<<5) /* free variable from class's method */ +#define DEF_IMPORT (2<<6) /* assignment occurred via import */ +#define DEF_ANNOT (2<<7) /* this name is annotated */ +#define DEF_COMP_ITER (2<<8) /* this name is a comprehension iteration variable */ +#define DEF_TYPE_PARAM (2<<9) /* this name is a type parameter */ +#define DEF_COMP_CELL (2<<10) /* this name is a cell in an inlined comprehension */ #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT) diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index dabbd7b41556cdf..40b28e0ba221aee 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -1,34 +1,39 @@ -// The _PyTime_t API is written to use timestamp and timeout values stored in -// various formats and to read clocks. +// Internal PyTime_t C API: see Doc/c-api/time.rst for the documentation. // -// The _PyTime_t type is an integer to support directly common arithmetic -// operations like t1 + t2. +// The PyTime_t type is an integer to support directly common arithmetic +// operations such as t1 + t2. // -// The _PyTime_t API supports a resolution of 1 nanosecond. The _PyTime_t type -// is signed to support negative timestamps. The supported range is around -// [-292.3 years; +292.3 years]. Using the Unix epoch (January 1st, 1970), the -// supported date range is around [1677-09-21; 2262-04-11]. +// Time formats: // -// Formats: +// * Seconds. +// * Seconds as a floating point number (C double). +// * Milliseconds (10^-3 seconds). +// * Microseconds (10^-6 seconds). +// * 100 nanoseconds (10^-7 seconds), used on Windows. +// * Nanoseconds (10^-9 seconds). +// * timeval structure, 1 microsecond (10^-6 seconds). +// * timespec structure, 1 nanosecond (10^-9 seconds). // -// * seconds -// * seconds as a floating pointer number (C double) -// * milliseconds (10^-3 seconds) -// * microseconds (10^-6 seconds) -// * 100 nanoseconds (10^-7 seconds) -// * nanoseconds (10^-9 seconds) -// * timeval structure, 1 microsecond resolution (10^-6 seconds) -// * timespec structure, 1 nanosecond resolution (10^-9 seconds) +// Note that PyTime_t is now specified as int64_t, in nanoseconds. +// (If we need to change this, we'll need new public API with new names.) +// Previously, PyTime_t was configurable (in theory); some comments and code +// might still allude to that. // // Integer overflows are detected and raise OverflowError. Conversion to a -// resolution worse than 1 nanosecond is rounded correctly with the requested -// rounding mode. There are 4 rounding modes: floor (towards -inf), ceiling -// (towards +inf), half even and up (away from zero). +// resolution larger than 1 nanosecond is rounded correctly with the requested +// rounding mode. Available rounding modes: // -// Some functions clamp the result in the range [_PyTime_MIN; _PyTime_MAX], so -// the caller doesn't have to handle errors and doesn't need to hold the GIL. -// For example, _PyTime_Add(t1, t2) computes t1+t2 and clamp the result on -// overflow. +// * Round towards minus infinity (-inf). For example, used to read a clock. +// * Round towards infinity (+inf). For example, used for timeout to wait "at +// least" N seconds. +// * Round to nearest with ties going to nearest even integer. For example, used +// to round from a Python float. +// * Round away from zero. For example, used for timeout. +// +// Some functions clamp the result in the range [PyTime_MIN; PyTime_MAX]. The +// caller doesn't have to handle errors and so doesn't need to hold the GIL to +// handle exceptions. For example, _PyTime_Add(t1, t2) computes t1+t2 and +// clamps the result on overflow. // // Clocks: // @@ -36,10 +41,11 @@ // * Monotonic clock // * Performance counter // -// Operations like (t * k / q) with integers are implemented in a way to reduce -// the risk of integer overflow. Such operation is used to convert a clock -// value expressed in ticks with a frequency to _PyTime_t, like -// QueryPerformanceCounter() with QueryPerformanceFrequency(). +// Internally, operations like (t * k / q) with integers are implemented in a +// way to reduce the risk of integer overflow. Such operation is used to convert a +// clock value expressed in ticks with a frequency to PyTime_t, like +// QueryPerformanceCounter() with QueryPerformanceFrequency() on Windows. + #ifndef Py_INTERNAL_TIME_H #define Py_INTERNAL_TIME_H @@ -56,14 +62,6 @@ extern "C" { struct timeval; #endif -// _PyTime_t: Python timestamp with subsecond precision. It can be used to -// store a duration, and so indirectly a date (related to another date, like -// UNIX epoch). -typedef int64_t _PyTime_t; -// _PyTime_MIN nanoseconds is around -292.3 years -#define _PyTime_MIN INT64_MIN -// _PyTime_MAX nanoseconds is around +292.3 years -#define _PyTime_MAX INT64_MAX #define _SIZEOF_PYTIME_T 8 typedef enum { @@ -131,76 +129,64 @@ PyAPI_FUNC(int) _PyTime_ObjectToTimespec( // Create a timestamp from a number of seconds. // Export for '_socket' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_FromSeconds(int seconds); +PyAPI_FUNC(PyTime_t) _PyTime_FromSeconds(int seconds); // Create a timestamp from a number of seconds in double. // Export for '_socket' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round); +PyAPI_FUNC(PyTime_t) _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round); // Macro to create a timestamp from a number of seconds, no integer overflow. // Only use the macro for small values, prefer _PyTime_FromSeconds(). #define _PYTIME_FROMSECONDS(seconds) \ - ((_PyTime_t)(seconds) * (1000 * 1000 * 1000)) - -// Create a timestamp from a number of nanoseconds. -// Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_FromNanoseconds(_PyTime_t ns); + ((PyTime_t)(seconds) * (1000 * 1000 * 1000)) // Create a timestamp from a number of microseconds. -// Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. -extern _PyTime_t _PyTime_FromMicrosecondsClamp(_PyTime_t us); +// Clamp to [PyTime_MIN; PyTime_MAX] on overflow. +extern PyTime_t _PyTime_FromMicrosecondsClamp(PyTime_t us); -// Create a timestamp from nanoseconds (Python int). +// Create a timestamp from a Python int object (number of nanoseconds). // Export for '_lsprof' shared extension. -PyAPI_FUNC(int) _PyTime_FromNanosecondsObject(_PyTime_t *t, +PyAPI_FUNC(int) _PyTime_FromLong(PyTime_t *t, PyObject *obj); // Convert a number of seconds (Python float or int) to a timestamp. // Raise an exception and return -1 on error, return 0 on success. // Export for '_socket' shared extension. -PyAPI_FUNC(int) _PyTime_FromSecondsObject(_PyTime_t *t, +PyAPI_FUNC(int) _PyTime_FromSecondsObject(PyTime_t *t, PyObject *obj, _PyTime_round_t round); // Convert a number of milliseconds (Python float or int, 10^-3) to a timestamp. // Raise an exception and return -1 on error, return 0 on success. // Export for 'select' shared extension. -PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(_PyTime_t *t, +PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(PyTime_t *t, PyObject *obj, _PyTime_round_t round); -// Convert a timestamp to a number of seconds as a C double. -// Export for '_socket' shared extension. -PyAPI_FUNC(double) _PyTime_AsSecondsDouble(_PyTime_t t); - // Convert timestamp to a number of milliseconds (10^-3 seconds). // Export for '_ssl' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_AsMilliseconds(_PyTime_t t, +PyAPI_FUNC(PyTime_t) _PyTime_AsMilliseconds(PyTime_t t, _PyTime_round_t round); // Convert timestamp to a number of microseconds (10^-6 seconds). // Export for '_queue' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_AsMicroseconds(_PyTime_t t, +PyAPI_FUNC(PyTime_t) _PyTime_AsMicroseconds(PyTime_t t, _PyTime_round_t round); -// Convert timestamp to a number of nanoseconds (10^-9 seconds). -extern _PyTime_t _PyTime_AsNanoseconds(_PyTime_t t); - #ifdef MS_WINDOWS // Convert timestamp to a number of 100 nanoseconds (10^-7 seconds). -extern _PyTime_t _PyTime_As100Nanoseconds(_PyTime_t t, +extern PyTime_t _PyTime_As100Nanoseconds(PyTime_t t, _PyTime_round_t round); #endif -// Convert timestamp to a number of nanoseconds (10^-9 seconds) as a Python int -// object. +// Convert a timestamp (number of nanoseconds) as a Python int object. // Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(PyObject*) _PyTime_AsNanosecondsObject(_PyTime_t t); +PyAPI_FUNC(PyObject*) _PyTime_AsLong(PyTime_t t); #ifndef MS_WINDOWS // Create a timestamp from a timeval structure. // Raise an exception and return -1 on overflow, return 0 on success. -extern int _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv); +extern int _PyTime_FromTimeval(PyTime_t *tp, struct timeval *tv); #endif // Convert a timestamp to a timeval structure (microsecond resolution). @@ -208,14 +194,14 @@ extern int _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv); // Raise an exception and return -1 if the conversion overflowed, // return 0 on success. // Export for 'select' shared extension. -PyAPI_FUNC(int) _PyTime_AsTimeval(_PyTime_t t, +PyAPI_FUNC(int) _PyTime_AsTimeval(PyTime_t t, struct timeval *tv, _PyTime_round_t round); // Similar to _PyTime_AsTimeval() but don't raise an exception on overflow. -// On overflow, clamp tv_sec to _PyTime_t min/max. +// On overflow, clamp tv_sec to PyTime_t min/max. // Export for 'select' shared extension. -PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(_PyTime_t t, +PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(PyTime_t t, struct timeval *tv, _PyTime_round_t round); @@ -227,7 +213,7 @@ PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(_PyTime_t t, // return 0 on success. // Export for '_datetime' shared extension. PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( - _PyTime_t t, + PyTime_t t, time_t *secs, int *us, _PyTime_round_t round); @@ -235,23 +221,23 @@ PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) // Create a timestamp from a timespec structure. // Raise an exception and return -1 on overflow, return 0 on success. -extern int _PyTime_FromTimespec(_PyTime_t *tp, const struct timespec *ts); +extern int _PyTime_FromTimespec(PyTime_t *tp, const struct timespec *ts); // Convert a timestamp to a timespec structure (nanosecond resolution). // tv_nsec is always positive. // Raise an exception and return -1 on error, return 0 on success. // Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts); +PyAPI_FUNC(int) _PyTime_AsTimespec(PyTime_t t, struct timespec *ts); // Similar to _PyTime_AsTimespec() but don't raise an exception on overflow. -// On overflow, clamp tv_sec to _PyTime_t min/max. +// On overflow, clamp tv_sec to PyTime_t min/max. // Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts); +PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(PyTime_t t, struct timespec *ts); #endif -// Compute t1 + t2. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. -extern _PyTime_t _PyTime_Add(_PyTime_t t1, _PyTime_t t2); +// Compute t1 + t2. Clamp to [PyTime_MIN; PyTime_MAX] on overflow. +extern PyTime_t _PyTime_Add(PyTime_t t1, PyTime_t t2); // Structure used by time.get_clock_info() typedef struct { @@ -261,35 +247,28 @@ typedef struct { double resolution; } _Py_clock_info_t; -// Get the current time from the system clock. +// Similar to PyTime_Time() but silently ignore the error and return 0 if the +// internal clock fails. // -// If the internal clock fails, silently ignore the error and return 0. -// On integer overflow, silently ignore the overflow and clamp the clock to -// [_PyTime_MIN; _PyTime_MAX]. -// -// Use _PyTime_GetSystemClockWithInfo() to check for failure. +// Use _PyTime_TimeWithInfo() or the public PyTime_Time() to check +// for failure. // Export for '_random' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); +PyAPI_FUNC(PyTime_t) _PyTime_TimeUnchecked(void); // Get the current time from the system clock. // On success, set *t and *info (if not NULL), and return 0. // On error, raise an exception and return -1. -extern int _PyTime_GetSystemClockWithInfo( - _PyTime_t *t, +extern int _PyTime_TimeWithInfo( + PyTime_t *t, _Py_clock_info_t *info); -// Get the time of a monotonic clock, i.e. a clock that cannot go backwards. -// The clock is not affected by system clock updates. The reference point of -// the returned value is undefined, so that only the difference between the -// results of consecutive calls is valid. +// Similar to PyTime_Monotonic() but silently ignore the error and return 0 if +// the internal clock fails. // -// If the internal clock fails, silently ignore the error and return 0. -// On integer overflow, silently ignore the overflow and clamp the clock to -// [_PyTime_MIN; _PyTime_MAX]. -// -// Use _PyTime_GetMonotonicClockWithInfo() to check for failure. +// Use _PyTime_MonotonicWithInfo() or the public PyTime_Monotonic() +// to check for failure. // Export for '_random' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); +PyAPI_FUNC(PyTime_t) _PyTime_MonotonicUnchecked(void); // Get the time of a monotonic clock, i.e. a clock that cannot go backwards. // The clock is not affected by system clock updates. The reference point of @@ -300,8 +279,8 @@ PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); // // Return 0 on success, raise an exception and return -1 on error. // Export for '_testsinglephase' shared extension. -PyAPI_FUNC(int) _PyTime_GetMonotonicClockWithInfo( - _PyTime_t *t, +PyAPI_FUNC(int) _PyTime_MonotonicWithInfo( + PyTime_t *t, _Py_clock_info_t *info); @@ -315,16 +294,14 @@ PyAPI_FUNC(int) _PyTime_localtime(time_t t, struct tm *tm); // Export for '_datetime' shared extension. PyAPI_FUNC(int) _PyTime_gmtime(time_t t, struct tm *tm); -// Get the performance counter: clock with the highest available resolution to -// measure a short duration. -// -// If the internal clock fails, silently ignore the error and return 0. -// On integer overflow, silently ignore the overflow and clamp the clock to -// [_PyTime_MIN; _PyTime_MAX]. +// Similar to PyTime_PerfCounter() but silently ignore the error and return 0 +// if the internal clock fails. // -// Use _PyTime_GetPerfCounterWithInfo() to check for failure. +// Use _PyTime_PerfCounterWithInfo() or the public PyTime_PerfCounter() to +// check for failure. // Export for '_lsprof' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_GetPerfCounter(void); +PyAPI_FUNC(PyTime_t) _PyTime_PerfCounterUnchecked(void); + // Get the performance counter: clock with the highest available resolution to // measure a short duration. @@ -332,27 +309,29 @@ PyAPI_FUNC(_PyTime_t) _PyTime_GetPerfCounter(void); // Fill info (if set) with information of the function used to get the time. // // Return 0 on success, raise an exception and return -1 on error. -extern int _PyTime_GetPerfCounterWithInfo( - _PyTime_t *t, +extern int _PyTime_PerfCounterWithInfo( + PyTime_t *t, _Py_clock_info_t *info); +// --- _PyDeadline ----------------------------------------------------------- + // Create a deadline. -// Pseudo code: _PyTime_GetMonotonicClock() + timeout. +// Pseudo code: _PyTime_MonotonicUnchecked() + timeout. // Export for '_ssl' shared extension. -PyAPI_FUNC(_PyTime_t) _PyDeadline_Init(_PyTime_t timeout); +PyAPI_FUNC(PyTime_t) _PyDeadline_Init(PyTime_t timeout); // Get remaining time from a deadline. -// Pseudo code: deadline - _PyTime_GetMonotonicClock(). +// Pseudo code: deadline - _PyTime_MonotonicUnchecked(). // Export for '_ssl' shared extension. -PyAPI_FUNC(_PyTime_t) _PyDeadline_Get(_PyTime_t deadline); +PyAPI_FUNC(PyTime_t) _PyDeadline_Get(PyTime_t deadline); // --- _PyTimeFraction ------------------------------------------------------- typedef struct { - _PyTime_t numer; - _PyTime_t denom; + PyTime_t numer; + PyTime_t denom; } _PyTimeFraction; // Set a fraction. @@ -360,13 +339,13 @@ typedef struct { // Return -1 if the fraction is invalid. extern int _PyTimeFraction_Set( _PyTimeFraction *frac, - _PyTime_t numer, - _PyTime_t denom); + PyTime_t numer, + PyTime_t denom); // Compute ticks * frac.numer / frac.denom. -// Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. -extern _PyTime_t _PyTimeFraction_Mul( - _PyTime_t ticks, +// Clamp to [PyTime_MIN; PyTime_MAX] on overflow. +extern PyTime_t _PyTimeFraction_Mul( + PyTime_t ticks, const _PyTimeFraction *frac); // Compute a clock resolution: frac.numer / frac.denom / 1e9. diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 472fa08154e8f92..e268e6fbbb087b2 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -8,8 +8,17 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_brc.h" // struct _brc_thread_state #include "pycore_freelist.h" // struct _Py_freelist_state #include "pycore_mimalloc.h" // struct _mimalloc_thread_state +#include "pycore_qsbr.h" // struct qsbr + + +static inline void +_PyThreadState_SetWhence(PyThreadState *tstate, int whence) +{ + tstate->_whence = whence; +} // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The @@ -19,9 +28,14 @@ typedef struct _PyThreadStateImpl { // semi-public fields are in PyThreadState. PyThreadState base; + struct _qsbr_thread_state *qsbr; // only used by free-threaded build + struct llist_node mem_free_queue; // delayed free queue + #ifdef Py_GIL_DISABLED + struct _gc_thread_state gc; struct _mimalloc_thread_state mimalloc; - struct _Py_freelist_state freelist_state; + struct _Py_object_freelists freelists; + struct _brc_thread_state brc; #endif } _PyThreadStateImpl; diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index b348339a505b0f7..4605f355ccbc38f 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -14,7 +14,6 @@ extern void _PyTuple_DebugMallocStats(FILE *out); /* runtime lifecycle */ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *); -extern void _PyTuple_Fini(_PyFreeListState *); /* other API */ diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index c03c3d766bef616..9134ab45cd00395 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_moduleobject.h" // PyModuleObject +#include "pycore_lock.h" // PyMutex /* state */ @@ -28,6 +29,9 @@ struct _types_runtime_state { // see _PyType_Lookup(). struct type_cache_entry { unsigned int version; // initialized from type->tp_version_tag +#ifdef Py_GIL_DISABLED + _PySeqLock sequence; +#endif PyObject *name; // reference to exactly a str or None PyObject *value; // borrowed reference or NULL }; @@ -66,6 +70,7 @@ struct types_state { struct type_cache type_cache; size_t num_builtins_initialized; static_builtin_state builtins[_Py_MAX_STATIC_BUILTIN_TYPES]; + PyMutex mutex; }; @@ -74,7 +79,7 @@ struct types_state { extern PyStatus _PyTypes_InitTypes(PyInterpreterState *); extern void _PyTypes_FiniTypes(PyInterpreterState *); extern void _PyTypes_Fini(PyInterpreterState *); - +extern void _PyTypes_AfterFork(void); /* other API */ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 898d386f4cfd05d..64c4cf8c0770562 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -534,6 +534,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(_active); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(_align_); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(_annotation); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -564,6 +567,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(_feature_version); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(_field_types); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(_fields_); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -936,6 +942,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(database); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(day); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(decode); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -954,6 +963,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(depth); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(desired_access); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(detect_types); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1125,6 +1137,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(fillvalue); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(filter); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(filters); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1143,6 +1158,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(flush); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(fold); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(follow_symlinks); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1218,6 +1236,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(handle); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(handle_seq); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(hash_name); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1233,6 +1254,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(hook); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(hour); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(id); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1266,6 +1290,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(infer_variance); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(inherit_handle); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(inheritable); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1275,6 +1302,12 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(initial_bytes); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(initial_owner); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(initial_state); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(initial_value); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1440,6 +1473,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(loop); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(manual_reset); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(mapping); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1455,6 +1491,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(maxevents); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(maxlen); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(maxmem); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1482,6 +1521,15 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(method); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(microsecond); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(milliseconds); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(minute); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(mod); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1497,12 +1545,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(modules); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(month); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(mro); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(msg); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(mutex); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(mycmp); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1809,6 +1863,12 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(scheduler); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(second); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(security_attributes); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(seek); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -2037,6 +2097,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(tz); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(tzinfo); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(tzname); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -2067,6 +2130,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(volume); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(wait_all); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(warnings); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Include/internal/pycore_unionobject.h b/Include/internal/pycore_unionobject.h index 87264635b6e1cfe..6ece7134cdeca03 100644 --- a/Include/internal/pycore_unionobject.h +++ b/Include/internal/pycore_unionobject.h @@ -8,9 +8,11 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -extern PyTypeObject _PyUnion_Type; +// For extensions created by test_peg_generator +PyAPI_DATA(PyTypeObject) _PyUnion_Type; +PyAPI_FUNC(PyObject *) _Py_union_type_or(PyObject *, PyObject *); + #define _PyUnion_Check(op) Py_IS_TYPE((op), &_PyUnion_Type) -extern PyObject *_Py_union_type_or(PyObject *, PyObject *); #define _PyGenericAlias_Check(op) PyObject_TypeCheck((op), &Py_GenericAliasType) extern PyObject *_Py_subs_parameters(PyObject *, PyObject *, PyObject *, PyObject *); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index a7056586ff04c01..3c133d97b2f03eb 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -11,229 +11,263 @@ extern "C" { #define _EXIT_TRACE 300 #define _SET_IP 301 -#define _NOP NOP -#define _RESUME_CHECK RESUME_CHECK -#define _INSTRUMENTED_RESUME INSTRUMENTED_RESUME -#define _LOAD_FAST_CHECK LOAD_FAST_CHECK -#define _LOAD_FAST LOAD_FAST -#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST -#define _LOAD_CONST LOAD_CONST -#define _STORE_FAST STORE_FAST -#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST -#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST -#define _POP_TOP POP_TOP -#define _PUSH_NULL PUSH_NULL -#define _END_SEND END_SEND -#define _UNARY_NEGATIVE UNARY_NEGATIVE -#define _UNARY_NOT UNARY_NOT -#define _TO_BOOL 302 -#define _TO_BOOL_BOOL TO_BOOL_BOOL -#define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST TO_BOOL_LIST -#define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR TO_BOOL_STR -#define _TO_BOOL_ALWAYS_TRUE TO_BOOL_ALWAYS_TRUE -#define _UNARY_INVERT UNARY_INVERT -#define _GUARD_BOTH_INT 303 -#define _BINARY_OP_MULTIPLY_INT 304 -#define _BINARY_OP_ADD_INT 305 -#define _BINARY_OP_SUBTRACT_INT 306 -#define _GUARD_BOTH_FLOAT 307 -#define _BINARY_OP_MULTIPLY_FLOAT 308 -#define _BINARY_OP_ADD_FLOAT 309 -#define _BINARY_OP_SUBTRACT_FLOAT 310 -#define _GUARD_BOTH_UNICODE 311 -#define _BINARY_OP_ADD_UNICODE 312 -#define _BINARY_SUBSCR 313 +#define _BEFORE_ASYNC_WITH BEFORE_ASYNC_WITH +#define _BEFORE_WITH BEFORE_WITH +#define _BINARY_OP 302 +#define _BINARY_OP_ADD_FLOAT 303 +#define _BINARY_OP_ADD_INT 304 +#define _BINARY_OP_ADD_UNICODE 305 +#define _BINARY_OP_MULTIPLY_FLOAT 306 +#define _BINARY_OP_MULTIPLY_INT 307 +#define _BINARY_OP_SUBTRACT_FLOAT 308 +#define _BINARY_OP_SUBTRACT_INT 309 #define _BINARY_SLICE BINARY_SLICE -#define _STORE_SLICE STORE_SLICE +#define _BINARY_SUBSCR 310 +#define _BINARY_SUBSCR_DICT BINARY_SUBSCR_DICT +#define _BINARY_SUBSCR_GETITEM BINARY_SUBSCR_GETITEM #define _BINARY_SUBSCR_LIST_INT BINARY_SUBSCR_LIST_INT #define _BINARY_SUBSCR_STR_INT BINARY_SUBSCR_STR_INT #define _BINARY_SUBSCR_TUPLE_INT BINARY_SUBSCR_TUPLE_INT -#define _BINARY_SUBSCR_DICT BINARY_SUBSCR_DICT -#define _BINARY_SUBSCR_GETITEM BINARY_SUBSCR_GETITEM -#define _LIST_APPEND LIST_APPEND -#define _SET_ADD SET_ADD -#define _STORE_SUBSCR 314 -#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT -#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT -#define _DELETE_SUBSCR DELETE_SUBSCR +#define _BUILD_CONST_KEY_MAP BUILD_CONST_KEY_MAP +#define _BUILD_LIST BUILD_LIST +#define _BUILD_MAP BUILD_MAP +#define _BUILD_SET BUILD_SET +#define _BUILD_SLICE BUILD_SLICE +#define _BUILD_STRING BUILD_STRING +#define _BUILD_TUPLE BUILD_TUPLE +#define _CALL 311 +#define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT +#define _CALL_BUILTIN_CLASS CALL_BUILTIN_CLASS +#define _CALL_BUILTIN_FAST CALL_BUILTIN_FAST +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS CALL_BUILTIN_FAST_WITH_KEYWORDS +#define _CALL_BUILTIN_O CALL_BUILTIN_O +#define _CALL_FUNCTION_EX CALL_FUNCTION_EX #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 -#define _POP_FRAME 315 -#define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE -#define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST +#define _CALL_ISINSTANCE CALL_ISINSTANCE +#define _CALL_KW CALL_KW +#define _CALL_LEN CALL_LEN +#define _CALL_METHOD_DESCRIPTOR_FAST CALL_METHOD_DESCRIPTOR_FAST +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS +#define _CALL_METHOD_DESCRIPTOR_NOARGS CALL_METHOD_DESCRIPTOR_NOARGS +#define _CALL_METHOD_DESCRIPTOR_O CALL_METHOD_DESCRIPTOR_O +#define _CALL_PY_WITH_DEFAULTS CALL_PY_WITH_DEFAULTS +#define _CALL_STR_1 CALL_STR_1 +#define _CALL_TUPLE_1 CALL_TUPLE_1 +#define _CALL_TYPE_1 CALL_TYPE_1 +#define _CHECK_ATTR_CLASS 312 +#define _CHECK_ATTR_METHOD_LAZY_DICT 313 +#define _CHECK_ATTR_MODULE 314 +#define _CHECK_ATTR_WITH_HINT 315 +#define _CHECK_BUILTINS 316 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 317 +#define _CHECK_EG_MATCH CHECK_EG_MATCH +#define _CHECK_EXC_MATCH CHECK_EXC_MATCH +#define _CHECK_FUNCTION_EXACT_ARGS 318 +#define _CHECK_GLOBALS 319 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 320 +#define _CHECK_PEP_523 321 +#define _CHECK_STACK_SPACE 322 +#define _CHECK_VALIDITY 323 +#define _CHECK_VALIDITY_AND_SET_IP 324 +#define _COLD_EXIT 325 +#define _COMPARE_OP 326 +#define _COMPARE_OP_FLOAT 327 +#define _COMPARE_OP_INT 328 +#define _COMPARE_OP_STR 329 +#define _CONTAINS_OP CONTAINS_OP +#define _CONVERT_VALUE CONVERT_VALUE +#define _COPY COPY +#define _COPY_FREE_VARS COPY_FREE_VARS +#define _DELETE_ATTR DELETE_ATTR +#define _DELETE_DEREF DELETE_DEREF +#define _DELETE_FAST DELETE_FAST +#define _DELETE_GLOBAL DELETE_GLOBAL +#define _DELETE_NAME DELETE_NAME +#define _DELETE_SUBSCR DELETE_SUBSCR +#define _DICT_MERGE DICT_MERGE +#define _DICT_UPDATE DICT_UPDATE +#define _END_SEND END_SEND +#define _EXIT_INIT_CHECK EXIT_INIT_CHECK +#define _FATAL_ERROR 330 +#define _FORMAT_SIMPLE FORMAT_SIMPLE +#define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC +#define _FOR_ITER 331 +#define _FOR_ITER_GEN FOR_ITER_GEN +#define _FOR_ITER_TIER_TWO 332 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE -#define _SEND 316 -#define _SEND_GEN SEND_GEN +#define _GET_ITER GET_ITER +#define _GET_LEN GET_LEN +#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER +#define _GUARD_BOTH_FLOAT 333 +#define _GUARD_BOTH_INT 334 +#define _GUARD_BOTH_UNICODE 335 +#define _GUARD_BUILTINS_VERSION 336 +#define _GUARD_DORV_VALUES 337 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 338 +#define _GUARD_GLOBALS_VERSION 339 +#define _GUARD_IS_FALSE_POP 340 +#define _GUARD_IS_NONE_POP 341 +#define _GUARD_IS_NOT_NONE_POP 342 +#define _GUARD_IS_TRUE_POP 343 +#define _GUARD_KEYS_VERSION 344 +#define _GUARD_NOT_EXHAUSTED_LIST 345 +#define _GUARD_NOT_EXHAUSTED_RANGE 346 +#define _GUARD_NOT_EXHAUSTED_TUPLE 347 +#define _GUARD_TYPE_VERSION 348 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 349 +#define _INIT_CALL_PY_EXACT_ARGS 350 +#define _INIT_CALL_PY_EXACT_ARGS_0 351 +#define _INIT_CALL_PY_EXACT_ARGS_1 352 +#define _INIT_CALL_PY_EXACT_ARGS_2 353 +#define _INIT_CALL_PY_EXACT_ARGS_3 354 +#define _INIT_CALL_PY_EXACT_ARGS_4 355 +#define _INSTRUMENTED_CALL INSTRUMENTED_CALL +#define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX +#define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW +#define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER +#define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION +#define _INSTRUMENTED_JUMP_BACKWARD INSTRUMENTED_JUMP_BACKWARD +#define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD +#define _INSTRUMENTED_LOAD_SUPER_ATTR INSTRUMENTED_LOAD_SUPER_ATTR +#define _INSTRUMENTED_POP_JUMP_IF_FALSE INSTRUMENTED_POP_JUMP_IF_FALSE +#define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE +#define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE +#define _INSTRUMENTED_RESUME INSTRUMENTED_RESUME +#define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST +#define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _POP_EXCEPT POP_EXCEPT +#define _INTERNAL_INCREMENT_OPT_COUNTER 356 +#define _IS_NONE 357 +#define _IS_OP IS_OP +#define _ITER_CHECK_LIST 358 +#define _ITER_CHECK_RANGE 359 +#define _ITER_CHECK_TUPLE 360 +#define _ITER_JUMP_LIST 361 +#define _ITER_JUMP_RANGE 362 +#define _ITER_JUMP_TUPLE 363 +#define _ITER_NEXT_LIST 364 +#define _ITER_NEXT_RANGE 365 +#define _ITER_NEXT_TUPLE 366 +#define _JUMP_TO_TOP 367 +#define _LIST_APPEND LIST_APPEND +#define _LIST_EXTEND LIST_EXTEND #define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR +#define _LOAD_ATTR 368 +#define _LOAD_ATTR_CLASS 369 +#define _LOAD_ATTR_CLASS_0 370 +#define _LOAD_ATTR_CLASS_1 371 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN +#define _LOAD_ATTR_INSTANCE_VALUE 372 +#define _LOAD_ATTR_INSTANCE_VALUE_0 373 +#define _LOAD_ATTR_INSTANCE_VALUE_1 374 +#define _LOAD_ATTR_METHOD_LAZY_DICT 375 +#define _LOAD_ATTR_METHOD_NO_DICT 376 +#define _LOAD_ATTR_METHOD_WITH_VALUES 377 +#define _LOAD_ATTR_MODULE 378 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 379 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 380 +#define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY +#define _LOAD_ATTR_SLOT 381 +#define _LOAD_ATTR_SLOT_0 382 +#define _LOAD_ATTR_SLOT_1 383 +#define _LOAD_ATTR_WITH_HINT 384 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _STORE_NAME STORE_NAME -#define _DELETE_NAME DELETE_NAME -#define _UNPACK_SEQUENCE 317 -#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE -#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE -#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST -#define _UNPACK_EX UNPACK_EX -#define _STORE_ATTR 318 -#define _DELETE_ATTR DELETE_ATTR -#define _STORE_GLOBAL STORE_GLOBAL -#define _DELETE_GLOBAL DELETE_GLOBAL -#define _LOAD_LOCALS LOAD_LOCALS +#define _LOAD_CONST LOAD_CONST +#define _LOAD_CONST_INLINE 385 +#define _LOAD_CONST_INLINE_BORROW 386 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 387 +#define _LOAD_CONST_INLINE_WITH_NULL 388 +#define _LOAD_DEREF LOAD_DEREF +#define _LOAD_FAST 389 +#define _LOAD_FAST_0 390 +#define _LOAD_FAST_1 391 +#define _LOAD_FAST_2 392 +#define _LOAD_FAST_3 393 +#define _LOAD_FAST_4 394 +#define _LOAD_FAST_5 395 +#define _LOAD_FAST_6 396 +#define _LOAD_FAST_7 397 +#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR +#define _LOAD_FAST_CHECK LOAD_FAST_CHECK +#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST +#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS +#define _LOAD_GLOBAL 398 +#define _LOAD_GLOBAL_BUILTINS 399 +#define _LOAD_GLOBAL_MODULE 400 +#define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_GLOBAL 319 -#define _GUARD_GLOBALS_VERSION 320 -#define _GUARD_BUILTINS_VERSION 321 -#define _LOAD_GLOBAL_MODULE 322 -#define _LOAD_GLOBAL_BUILTINS 323 -#define _DELETE_FAST DELETE_FAST -#define _MAKE_CELL MAKE_CELL -#define _DELETE_DEREF DELETE_DEREF -#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF -#define _LOAD_DEREF LOAD_DEREF -#define _STORE_DEREF STORE_DEREF -#define _COPY_FREE_VARS COPY_FREE_VARS -#define _BUILD_STRING BUILD_STRING -#define _BUILD_TUPLE BUILD_TUPLE -#define _BUILD_LIST BUILD_LIST -#define _LIST_EXTEND LIST_EXTEND -#define _SET_UPDATE SET_UPDATE -#define _BUILD_SET BUILD_SET -#define _BUILD_MAP BUILD_MAP -#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS -#define _BUILD_CONST_KEY_MAP BUILD_CONST_KEY_MAP -#define _DICT_UPDATE DICT_UPDATE -#define _DICT_MERGE DICT_MERGE -#define _MAP_ADD MAP_ADD -#define _INSTRUMENTED_LOAD_SUPER_ATTR INSTRUMENTED_LOAD_SUPER_ATTR #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _LOAD_ATTR 324 -#define _GUARD_TYPE_VERSION 325 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 326 -#define _LOAD_ATTR_INSTANCE_VALUE 327 -#define _CHECK_ATTR_MODULE 328 -#define _LOAD_ATTR_MODULE 329 -#define _CHECK_ATTR_WITH_HINT 330 -#define _LOAD_ATTR_WITH_HINT 331 -#define _LOAD_ATTR_SLOT 332 -#define _CHECK_ATTR_CLASS 333 -#define _LOAD_ATTR_CLASS 334 -#define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _GUARD_DORV_VALUES 335 -#define _STORE_ATTR_INSTANCE_VALUE 336 -#define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT -#define _STORE_ATTR_SLOT 337 -#define _COMPARE_OP 338 -#define _COMPARE_OP_FLOAT COMPARE_OP_FLOAT -#define _COMPARE_OP_INT COMPARE_OP_INT -#define _COMPARE_OP_STR COMPARE_OP_STR -#define _IS_OP IS_OP -#define _CONTAINS_OP CONTAINS_OP -#define _CHECK_EG_MATCH CHECK_EG_MATCH -#define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _JUMP_BACKWARD JUMP_BACKWARD -#define _POP_JUMP_IF_FALSE 339 -#define _POP_JUMP_IF_TRUE 340 -#define _IS_NONE 341 -#define _GET_LEN GET_LEN +#define _MAKE_CELL MAKE_CELL +#define _MAKE_FUNCTION MAKE_FUNCTION +#define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS +#define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MATCH_KEYS MATCH_KEYS -#define _GET_ITER GET_ITER -#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _FOR_ITER 342 -#define _FOR_ITER_TIER_TWO 343 -#define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER -#define _ITER_CHECK_LIST 344 -#define _ITER_JUMP_LIST 345 -#define _GUARD_NOT_EXHAUSTED_LIST 346 -#define _ITER_NEXT_LIST 347 -#define _ITER_CHECK_TUPLE 348 -#define _ITER_JUMP_TUPLE 349 -#define _GUARD_NOT_EXHAUSTED_TUPLE 350 -#define _ITER_NEXT_TUPLE 351 -#define _ITER_CHECK_RANGE 352 -#define _ITER_JUMP_RANGE 353 -#define _GUARD_NOT_EXHAUSTED_RANGE 354 -#define _ITER_NEXT_RANGE 355 -#define _FOR_ITER_GEN FOR_ITER_GEN -#define _BEFORE_ASYNC_WITH BEFORE_ASYNC_WITH -#define _BEFORE_WITH BEFORE_WITH -#define _WITH_EXCEPT_START WITH_EXCEPT_START +#define _NOP NOP +#define _POP_EXCEPT POP_EXCEPT +#define _POP_FRAME 401 +#define _POP_JUMP_IF_FALSE 402 +#define _POP_JUMP_IF_TRUE 403 +#define _POP_TOP POP_TOP #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 356 -#define _GUARD_KEYS_VERSION 357 -#define _LOAD_ATTR_METHOD_WITH_VALUES 358 -#define _LOAD_ATTR_METHOD_NO_DICT 359 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 360 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 361 -#define _CHECK_ATTR_METHOD_LAZY_DICT 362 -#define _LOAD_ATTR_METHOD_LAZY_DICT 363 -#define _INSTRUMENTED_CALL INSTRUMENTED_CALL -#define _CALL 364 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 365 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 366 -#define _CHECK_PEP_523 367 -#define _CHECK_FUNCTION_EXACT_ARGS 368 -#define _CHECK_STACK_SPACE 369 -#define _INIT_CALL_PY_EXACT_ARGS 370 -#define _PUSH_FRAME 371 -#define _CALL_PY_WITH_DEFAULTS CALL_PY_WITH_DEFAULTS -#define _CALL_TYPE_1 CALL_TYPE_1 -#define _CALL_STR_1 CALL_STR_1 -#define _CALL_TUPLE_1 CALL_TUPLE_1 -#define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT -#define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _CALL_BUILTIN_CLASS CALL_BUILTIN_CLASS -#define _CALL_BUILTIN_O CALL_BUILTIN_O -#define _CALL_BUILTIN_FAST CALL_BUILTIN_FAST -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS CALL_BUILTIN_FAST_WITH_KEYWORDS -#define _CALL_LEN CALL_LEN -#define _CALL_ISINSTANCE CALL_ISINSTANCE -#define _CALL_METHOD_DESCRIPTOR_O CALL_METHOD_DESCRIPTOR_O -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS -#define _CALL_METHOD_DESCRIPTOR_NOARGS CALL_METHOD_DESCRIPTOR_NOARGS -#define _CALL_METHOD_DESCRIPTOR_FAST CALL_METHOD_DESCRIPTOR_FAST -#define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW -#define _CALL_KW CALL_KW -#define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX -#define _CALL_FUNCTION_EX CALL_FUNCTION_EX -#define _MAKE_FUNCTION MAKE_FUNCTION +#define _PUSH_FRAME 404 +#define _PUSH_NULL PUSH_NULL +#define _RESUME_CHECK RESUME_CHECK +#define _SAVE_RETURN_OFFSET 405 +#define _SEND 406 +#define _SEND_GEN SEND_GEN +#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS +#define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE -#define _BUILD_SLICE BUILD_SLICE -#define _CONVERT_VALUE CONVERT_VALUE -#define _FORMAT_SIMPLE FORMAT_SIMPLE -#define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _COPY COPY -#define _BINARY_OP 372 +#define _SET_UPDATE SET_UPDATE +#define _START_EXECUTOR 407 +#define _STORE_ATTR 408 +#define _STORE_ATTR_INSTANCE_VALUE 409 +#define _STORE_ATTR_SLOT 410 +#define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT +#define _STORE_DEREF STORE_DEREF +#define _STORE_FAST 411 +#define _STORE_FAST_0 412 +#define _STORE_FAST_1 413 +#define _STORE_FAST_2 414 +#define _STORE_FAST_3 415 +#define _STORE_FAST_4 416 +#define _STORE_FAST_5 417 +#define _STORE_FAST_6 418 +#define _STORE_FAST_7 419 +#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST +#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST +#define _STORE_GLOBAL STORE_GLOBAL +#define _STORE_NAME STORE_NAME +#define _STORE_SLICE STORE_SLICE +#define _STORE_SUBSCR 420 +#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT +#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION -#define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD -#define _INSTRUMENTED_JUMP_BACKWARD INSTRUMENTED_JUMP_BACKWARD -#define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _INSTRUMENTED_POP_JUMP_IF_FALSE INSTRUMENTED_POP_JUMP_IF_FALSE -#define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE -#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE -#define _GUARD_IS_TRUE_POP 373 -#define _GUARD_IS_FALSE_POP 374 -#define _GUARD_IS_NONE_POP 375 -#define _GUARD_IS_NOT_NONE_POP 376 -#define _JUMP_TO_TOP 377 -#define _SAVE_RETURN_OFFSET 378 -#define _CHECK_VALIDITY 379 -#define _LOAD_CONST_INLINE 380 -#define _LOAD_CONST_INLINE_BORROW 381 -#define _INTERNAL_INCREMENT_OPT_COUNTER 382 -#define MAX_UOP_ID 382 +#define _TO_BOOL 421 +#define _TO_BOOL_ALWAYS_TRUE TO_BOOL_ALWAYS_TRUE +#define _TO_BOOL_BOOL TO_BOOL_BOOL +#define _TO_BOOL_INT TO_BOOL_INT +#define _TO_BOOL_LIST TO_BOOL_LIST +#define _TO_BOOL_NONE TO_BOOL_NONE +#define _TO_BOOL_STR TO_BOOL_STR +#define _UNARY_INVERT UNARY_INVERT +#define _UNARY_NEGATIVE UNARY_NEGATIVE +#define _UNARY_NOT UNARY_NOT +#define _UNPACK_EX UNPACK_EX +#define _UNPACK_SEQUENCE 422 +#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST +#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE +#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE +#define _WITH_EXCEPT_START WITH_EXCEPT_START +#define MAX_UOP_ID 422 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 14d3382e895cdf4..35340fe9ee1b634 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -12,17 +12,34 @@ extern "C" { #include <stdint.h> #include "pycore_uop_ids.h" extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; +extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1]; extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; #ifdef NEED_OPCODE_METADATA const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { - [_NOP] = 0, + [_NOP] = HAS_PURE_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, + [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_3] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_4] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_5] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_6] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_0] = HAS_LOCAL_FLAG, + [_STORE_FAST_1] = HAS_LOCAL_FLAG, + [_STORE_FAST_2] = HAS_LOCAL_FLAG, + [_STORE_FAST_3] = HAS_LOCAL_FLAG, + [_STORE_FAST_4] = HAS_LOCAL_FLAG, + [_STORE_FAST_5] = HAS_LOCAL_FLAG, + [_STORE_FAST_6] = HAS_LOCAL_FLAG, + [_STORE_FAST_7] = HAS_LOCAL_FLAG, [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, @@ -32,22 +49,22 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_TO_BOOL_BOOL] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_TO_BOOL_INT] = HAS_DEOPT_FLAG, - [_TO_BOOL_LIST] = HAS_DEOPT_FLAG, - [_TO_BOOL_NONE] = HAS_DEOPT_FLAG, - [_TO_BOOL_STR] = HAS_DEOPT_FLAG, - [_TO_BOOL_ALWAYS_TRUE] = HAS_DEOPT_FLAG, + [_TO_BOOL_BOOL] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_TO_BOOL_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_TO_BOOL_LIST] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_TO_BOOL_NONE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_TO_BOOL_STR] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_TO_BOOL_ALWAYS_TRUE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_GUARD_BOTH_FLOAT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_FLOAT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_BINARY_OP_MULTIPLY_FLOAT] = HAS_PURE_FLAG, [_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG, - [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -112,23 +129,29 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE_0] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE_1] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, [_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_ATTR_SLOT_0] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_SLOT_1] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, [_CHECK_ATTR_CLASS] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG, + [_LOAD_ATTR_CLASS_0] = 0, + [_LOAD_ATTR_CLASS_1] = 0, + [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG, [_GUARD_DORV_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = HAS_ESCAPES_FLAG, [_STORE_ATTR_SLOT] = HAS_ESCAPES_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_COMPARE_OP_STR] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_COMPARE_OP_STR] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_IS_OP] = HAS_ARG_FLAG, [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EG_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -168,6 +191,11 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_PEP_523] = HAS_DEOPT_FLAG, [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_0] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_1] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_2] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_PUSH_FRAME] = HAS_ESCAPES_FLAG, [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, @@ -193,18 +221,32 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG, - [_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG, - [_GUARD_IS_NONE_POP] = HAS_DEOPT_FLAG, - [_GUARD_IS_NOT_NONE_POP] = HAS_DEOPT_FLAG, + [_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_GUARD_IS_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_GUARD_IS_NOT_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, [_JUMP_TO_TOP] = HAS_EVAL_BREAK_FLAG, - [_SET_IP] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_SET_IP] = 0, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, - [_EXIT_TRACE] = HAS_DEOPT_FLAG, + [_EXIT_TRACE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, - [_LOAD_CONST_INLINE] = 0, - [_LOAD_CONST_INLINE_BORROW] = 0, + [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE_WITH_NULL] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG, + [_CHECK_GLOBALS] = HAS_DEOPT_FLAG, + [_CHECK_BUILTINS] = HAS_DEOPT_FLAG, [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, + [_COLD_EXIT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_START_EXECUTOR] = 0, + [_FATAL_ERROR] = HAS_ESCAPES_FLAG, + [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, +}; + +const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { + [_LOAD_FAST] = 8, + [_STORE_FAST] = 8, + [_INIT_CALL_PY_EXACT_ARGS] = 5, }; const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { @@ -250,14 +292,18 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", [_CHECK_ATTR_MODULE] = "_CHECK_ATTR_MODULE", [_CHECK_ATTR_WITH_HINT] = "_CHECK_ATTR_WITH_HINT", + [_CHECK_BUILTINS] = "_CHECK_BUILTINS", [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", [_CHECK_EG_MATCH] = "_CHECK_EG_MATCH", [_CHECK_EXC_MATCH] = "_CHECK_EXC_MATCH", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", + [_CHECK_GLOBALS] = "_CHECK_GLOBALS", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", [_CHECK_PEP_523] = "_CHECK_PEP_523", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", + [_CHECK_VALIDITY_AND_SET_IP] = "_CHECK_VALIDITY_AND_SET_IP", + [_COLD_EXIT] = "_COLD_EXIT", [_COMPARE_OP] = "_COMPARE_OP", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", [_COMPARE_OP_INT] = "_COMPARE_OP_INT", @@ -277,6 +323,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_END_SEND] = "_END_SEND", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", [_EXIT_TRACE] = "_EXIT_TRACE", + [_FATAL_ERROR] = "_FATAL_ERROR", [_FORMAT_SIMPLE] = "_FORMAT_SIMPLE", [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", [_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO", @@ -304,6 +351,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", + [_INIT_CALL_PY_EXACT_ARGS_0] = "_INIT_CALL_PY_EXACT_ARGS_0", + [_INIT_CALL_PY_EXACT_ARGS_1] = "_INIT_CALL_PY_EXACT_ARGS_1", + [_INIT_CALL_PY_EXACT_ARGS_2] = "_INIT_CALL_PY_EXACT_ARGS_2", + [_INIT_CALL_PY_EXACT_ARGS_3] = "_INIT_CALL_PY_EXACT_ARGS_3", + [_INIT_CALL_PY_EXACT_ARGS_4] = "_INIT_CALL_PY_EXACT_ARGS_4", [_INTERNAL_INCREMENT_OPT_COUNTER] = "_INTERNAL_INCREMENT_OPT_COUNTER", [_IS_NONE] = "_IS_NONE", [_IS_OP] = "_IS_OP", @@ -319,7 +371,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_ASSERTION_ERROR] = "_LOAD_ASSERTION_ERROR", [_LOAD_ATTR] = "_LOAD_ATTR", [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", + [_LOAD_ATTR_CLASS_0] = "_LOAD_ATTR_CLASS_0", + [_LOAD_ATTR_CLASS_1] = "_LOAD_ATTR_CLASS_1", [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", + [_LOAD_ATTR_INSTANCE_VALUE_0] = "_LOAD_ATTR_INSTANCE_VALUE_0", + [_LOAD_ATTR_INSTANCE_VALUE_1] = "_LOAD_ATTR_INSTANCE_VALUE_1", [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", @@ -327,13 +383,25 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", + [_LOAD_ATTR_SLOT_0] = "_LOAD_ATTR_SLOT_0", + [_LOAD_ATTR_SLOT_1] = "_LOAD_ATTR_SLOT_1", [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", [_LOAD_CONST] = "_LOAD_CONST", [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE", [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", + [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = "_LOAD_CONST_INLINE_BORROW_WITH_NULL", + [_LOAD_CONST_INLINE_WITH_NULL] = "_LOAD_CONST_INLINE_WITH_NULL", [_LOAD_DEREF] = "_LOAD_DEREF", [_LOAD_FAST] = "_LOAD_FAST", + [_LOAD_FAST_0] = "_LOAD_FAST_0", + [_LOAD_FAST_1] = "_LOAD_FAST_1", + [_LOAD_FAST_2] = "_LOAD_FAST_2", + [_LOAD_FAST_3] = "_LOAD_FAST_3", + [_LOAD_FAST_4] = "_LOAD_FAST_4", + [_LOAD_FAST_5] = "_LOAD_FAST_5", + [_LOAD_FAST_6] = "_LOAD_FAST_6", + [_LOAD_FAST_7] = "_LOAD_FAST_7", [_LOAD_FAST_AND_CLEAR] = "_LOAD_FAST_AND_CLEAR", [_LOAD_FAST_CHECK] = "_LOAD_FAST_CHECK", [_LOAD_FAST_LOAD_FAST] = "_LOAD_FAST_LOAD_FAST", @@ -367,11 +435,20 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", [_SET_IP] = "_SET_IP", [_SET_UPDATE] = "_SET_UPDATE", + [_START_EXECUTOR] = "_START_EXECUTOR", [_STORE_ATTR] = "_STORE_ATTR", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", [_STORE_DEREF] = "_STORE_DEREF", [_STORE_FAST] = "_STORE_FAST", + [_STORE_FAST_0] = "_STORE_FAST_0", + [_STORE_FAST_1] = "_STORE_FAST_1", + [_STORE_FAST_2] = "_STORE_FAST_2", + [_STORE_FAST_3] = "_STORE_FAST_3", + [_STORE_FAST_4] = "_STORE_FAST_4", + [_STORE_FAST_5] = "_STORE_FAST_5", + [_STORE_FAST_6] = "_STORE_FAST_6", + [_STORE_FAST_7] = "_STORE_FAST_7", [_STORE_FAST_LOAD_FAST] = "_STORE_FAST_LOAD_FAST", [_STORE_FAST_STORE_FAST] = "_STORE_FAST_STORE_FAST", [_STORE_GLOBAL] = "_STORE_GLOBAL", diff --git a/Include/listobject.h b/Include/listobject.h index 6b7041ba0b05d59..4e4084b43483a2d 100644 --- a/Include/listobject.h +++ b/Include/listobject.h @@ -29,6 +29,7 @@ PyAPI_FUNC(PyObject *) PyList_New(Py_ssize_t size); PyAPI_FUNC(Py_ssize_t) PyList_Size(PyObject *); PyAPI_FUNC(PyObject *) PyList_GetItem(PyObject *, Py_ssize_t); +PyAPI_FUNC(PyObject *) PyList_GetItemRef(PyObject *, Py_ssize_t); PyAPI_FUNC(int) PyList_SetItem(PyObject *, Py_ssize_t, PyObject *); PyAPI_FUNC(int) PyList_Insert(PyObject *, Py_ssize_t, PyObject *); PyAPI_FUNC(int) PyList_Append(PyObject *, PyObject *); diff --git a/Include/methodobject.h b/Include/methodobject.h index 2381e8482b82a8b..39272815b127f4e 100644 --- a/Include/methodobject.h +++ b/Include/methodobject.h @@ -17,15 +17,22 @@ PyAPI_DATA(PyTypeObject) PyCFunction_Type; #define PyCFunction_Check(op) PyObject_TypeCheck((op), &PyCFunction_Type) typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); -typedef PyObject *(*_PyCFunctionFast) (PyObject *, PyObject *const *, Py_ssize_t); +typedef PyObject *(*PyCFunctionFast) (PyObject *, PyObject *const *, Py_ssize_t); typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *, PyObject *); -typedef PyObject *(*_PyCFunctionFastWithKeywords) (PyObject *, - PyObject *const *, Py_ssize_t, - PyObject *); +typedef PyObject *(*PyCFunctionFastWithKeywords) (PyObject *, + PyObject *const *, Py_ssize_t, + PyObject *); typedef PyObject *(*PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +// For backwards compatibility. `METH_FASTCALL` was added to the stable API in +// 3.10 alongside `_PyCFunctionFastWithKeywords` and `_PyCFunctionFast`. +// Note that the underscore-prefixed names were documented in public docs; +// people may be using them. +typedef PyCFunctionFast _PyCFunctionFast; +typedef PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords; + // Cast an function to the PyCFunction type to use it with PyMethodDef. // // This macro can be used to prevent compiler warnings if the first parameter diff --git a/Include/object.h b/Include/object.h index ef3fb721c2b0122..05187fe5dc4f20d 100644 --- a/Include/object.h +++ b/Include/object.h @@ -212,8 +212,9 @@ struct _object { struct _PyMutex { uint8_t v; }; struct _object { - // ob_tid stores the thread id (or zero). It is also used by the GC to - // store linked lists and the computed "gc_refs" refcount. + // ob_tid stores the thread id (or zero). It is also used by the GC and the + // trashcan mechanism as a linked list pointer and by the GC to store the + // computed "gc_refs" refcount. uintptr_t ob_tid; uint16_t _padding; struct _PyMutex ob_mutex; // per-object lock @@ -428,7 +429,11 @@ static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { assert(ob->ob_base.ob_type != &PyLong_Type); assert(ob->ob_base.ob_type != &PyBool_Type); +#ifdef Py_GIL_DISABLED + _Py_atomic_store_ssize_relaxed(&ob->ob_size, size); +#else ob->ob_size = size; +#endif } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SET_SIZE(ob, size) Py_SET_SIZE(_PyVarObject_CAST(ob), (size)) diff --git a/Include/objimpl.h b/Include/objimpl.h index ff5fa7a8c1d3d8d..56472a72e42d341 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -153,25 +153,6 @@ PyAPI_FUNC(int) PyGC_Enable(void); PyAPI_FUNC(int) PyGC_Disable(void); PyAPI_FUNC(int) PyGC_IsEnabled(void); - -#if !defined(Py_LIMITED_API) -/* Visit all live GC-capable objects, similar to gc.get_objects(None). The - * supplied callback is called on every such object with the void* arg set - * to the supplied arg. Returning 0 from the callback ends iteration, returning - * 1 allows iteration to continue. Returning any other value may result in - * undefined behaviour. - * - * If new objects are (de)allocated by the callback it is undefined if they - * will be visited. - - * Garbage collection is disabled during operation. Explicitly running a - * collection in the callback may lead to undefined behaviour e.g. visiting the - * same objects multiple times or not at all. - */ -typedef int (*gcvisitobjects_t)(PyObject*, void*); -PyAPI_FUNC(void) PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void* arg); -#endif - /* Test if a type has a GC head */ #define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 5f9b720f8b16715..ae2d22f80817d5d 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 13 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 3 +#define PY_RELEASE_SERIAL 4 /* Version as a string */ -#define PY_VERSION "3.13.0a3+" +#define PY_VERSION "3.13.0a4+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 5d0028c116e2d86..68d7985dac8876b 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -108,6 +108,7 @@ PyAPI_DATA(PyObject *) PyExc_NotImplementedError; PyAPI_DATA(PyObject *) PyExc_SyntaxError; PyAPI_DATA(PyObject *) PyExc_IndentationError; PyAPI_DATA(PyObject *) PyExc_TabError; +PyAPI_DATA(PyObject *) PyExc_IncompleteInputError; PyAPI_DATA(PyObject *) PyExc_ReferenceError; PyAPI_DATA(PyObject *) PyExc_SystemError; PyAPI_DATA(PyObject *) PyExc_SystemExit; diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index 355145387e355bb..b7d569cc41740e0 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -556,10 +556,6 @@ def _check_tzinfo_arg(tz): if tz is not None and not isinstance(tz, tzinfo): raise TypeError("tzinfo argument must be None or of a tzinfo subclass") -def _cmperror(x, y): - raise TypeError("can't compare '%s' to '%s'" % ( - type(x).__name__, type(y).__name__)) - def _divide_and_round(a, b): """divide a by b and round result to the nearest integer @@ -1113,32 +1109,33 @@ def replace(self, year=None, month=None, day=None): # Comparisons of date objects with other. def __eq__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) == 0 return NotImplemented def __le__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) <= 0 return NotImplemented def __lt__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) < 0 return NotImplemented def __ge__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) >= 0 return NotImplemented def __gt__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) > 0 return NotImplemented def _cmp(self, other): assert isinstance(other, date) + assert not isinstance(other, datetime) y, m, d = self._year, self._month, self._day y2, m2, d2 = other._year, other._month, other._day return _cmp((y, m, d), (y2, m2, d2)) @@ -1809,7 +1806,7 @@ def fromtimestamp(cls, timestamp, tz=None): def utcfromtimestamp(cls, t): """Construct a naive UTC datetime from a POSIX timestamp.""" import warnings - warnings.warn("datetime.utcfromtimestamp() is deprecated and scheduled " + warnings.warn("datetime.datetime.utcfromtimestamp() is deprecated and scheduled " "for removal in a future version. Use timezone-aware " "objects to represent datetimes in UTC: " "datetime.datetime.fromtimestamp(t, datetime.UTC).", @@ -1827,8 +1824,8 @@ def now(cls, tz=None): def utcnow(cls): "Construct a UTC datetime from time.time()." import warnings - warnings.warn("datetime.utcnow() is deprecated and scheduled for " - "removal in a future version. Instead, Use timezone-aware " + warnings.warn("datetime.datetime.utcnow() is deprecated and scheduled for " + "removal in a future version. Use timezone-aware " "objects to represent datetimes in UTC: " "datetime.datetime.now(datetime.UTC).", DeprecationWarning, @@ -2137,42 +2134,32 @@ def dst(self): def __eq__(self, other): if isinstance(other, datetime): return self._cmp(other, allow_mixed=True) == 0 - elif not isinstance(other, date): - return NotImplemented else: - return False + return NotImplemented def __le__(self, other): if isinstance(other, datetime): return self._cmp(other) <= 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def __lt__(self, other): if isinstance(other, datetime): return self._cmp(other) < 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def __ge__(self, other): if isinstance(other, datetime): return self._cmp(other) >= 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def __gt__(self, other): if isinstance(other, datetime): return self._cmp(other) > 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def _cmp(self, other, allow_mixed=False): assert isinstance(other, datetime) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index df2c29bfa9caeed..a3fede699218a15 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -1197,7 +1197,8 @@ def _readinto(self, buf, read1): return written def tell(self): - return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos + # GH-95782: Keep return value non-negative + return max(_BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos, 0) def seek(self, pos, whence=0): if whence not in valid_seek_flags: @@ -1495,6 +1496,11 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if isinstance(file, float): raise TypeError('integer argument expected, got float') if isinstance(file, int): + if isinstance(file, bool): + import warnings + warnings.warn("bool is used as a file descriptor", + RuntimeWarning, stacklevel=2) + file = int(file) fd = file if fd < 0: raise ValueError('negative file descriptor') diff --git a/Lib/argparse.py b/Lib/argparse.py index a32884db80d1ea1..4200dd5e334ad5c 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -223,7 +223,8 @@ def format_help(self): # add the heading if the section was non-empty if self.heading is not SUPPRESS and self.heading is not None: current_indent = self.formatter._current_indent - heading = '%*s%s:\n' % (current_indent, '', self.heading) + heading_text = _('%(heading)s:') % dict(heading=self.heading) + heading = '%*s%s\n' % (current_indent, '', heading_text) else: heading = '' @@ -413,6 +414,8 @@ def _format_actions_usage(self, actions, groups): suppressed_actions_count += 1 exposed_actions_count = group_action_count - suppressed_actions_count + if not exposed_actions_count: + continue if not group.required: if start in inserts: @@ -564,22 +567,18 @@ def _format_action_invocation(self, action): return metavar else: - parts = [] # if the Optional doesn't take a value, format is: # -s, --long if action.nargs == 0: - parts.extend(action.option_strings) + return ', '.join(action.option_strings) # if the Optional takes a value, format is: - # -s ARGS, --long ARGS + # -s, --long ARGS else: default = self._get_default_metavar_for_optional(action) args_string = self._format_args(action, default) - for option_string in action.option_strings: - parts.append('%s %s' % (option_string, args_string)) - - return ', '.join(parts) + return ', '.join(action.option_strings) + ' ' + args_string def _metavar_formatter(self, action, default_metavar): if action.metavar is not None: @@ -702,14 +701,6 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter): """ def _get_help_string(self, action): - """ - Add the default value to the option help message. - - ArgumentDefaultsHelpFormatter and BooleanOptionalAction when it isn't - already present. This code will do that, detecting cornercases to - prevent duplicates or cases where it wouldn't make sense to the end - user. - """ help = action.help if help is None: help = '' @@ -718,7 +709,7 @@ def _get_help_string(self, action): if action.default is not SUPPRESS: defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] if action.option_strings or action.nargs in defaulting_nargs: - help += ' (default: %(default)s)' + help += _(' (default: %(default)s)') return help @@ -847,7 +838,8 @@ def __init__(self, choices=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): self.option_strings = option_strings self.dest = dest self.nargs = nargs @@ -858,6 +850,7 @@ def __init__(self, self.required = required self.help = help self.metavar = metavar + self.deprecated = deprecated def _get_kwargs(self): names = [ @@ -871,6 +864,7 @@ def _get_kwargs(self): 'required', 'help', 'metavar', + 'deprecated', ] return [(name, getattr(self, name)) for name in names] @@ -893,7 +887,8 @@ def __init__(self, choices=_deprecated_default, required=False, help=None, - metavar=_deprecated_default): + metavar=_deprecated_default, + deprecated=False): _option_strings = [] for option_string in option_strings: @@ -931,7 +926,8 @@ def __init__(self, choices=choices, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): @@ -954,7 +950,8 @@ def __init__(self, choices=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): if nargs == 0: raise ValueError('nargs for store actions must be != 0; if you ' 'have nothing to store, actions such as store ' @@ -971,7 +968,8 @@ def __init__(self, choices=choices, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, values) @@ -986,7 +984,8 @@ def __init__(self, default=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): super(_StoreConstAction, self).__init__( option_strings=option_strings, dest=dest, @@ -994,7 +993,8 @@ def __init__(self, const=const, default=default, required=required, - help=help) + help=help, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, self.const) @@ -1007,14 +1007,16 @@ def __init__(self, dest, default=False, required=False, - help=None): + help=None, + deprecated=False): super(_StoreTrueAction, self).__init__( option_strings=option_strings, dest=dest, const=True, - default=default, + deprecated=deprecated, required=required, - help=help) + help=help, + default=default) class _StoreFalseAction(_StoreConstAction): @@ -1024,14 +1026,16 @@ def __init__(self, dest, default=True, required=False, - help=None): + help=None, + deprecated=False): super(_StoreFalseAction, self).__init__( option_strings=option_strings, dest=dest, const=False, default=default, required=required, - help=help) + help=help, + deprecated=deprecated) class _AppendAction(Action): @@ -1046,7 +1050,8 @@ def __init__(self, choices=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): if nargs == 0: raise ValueError('nargs for append actions must be != 0; if arg ' 'strings are not supplying the value to append, ' @@ -1063,7 +1068,8 @@ def __init__(self, choices=choices, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): items = getattr(namespace, self.dest, None) @@ -1081,7 +1087,8 @@ def __init__(self, default=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): super(_AppendConstAction, self).__init__( option_strings=option_strings, dest=dest, @@ -1090,7 +1097,8 @@ def __init__(self, default=default, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): items = getattr(namespace, self.dest, None) @@ -1106,14 +1114,16 @@ def __init__(self, dest, default=None, required=False, - help=None): + help=None, + deprecated=False): super(_CountAction, self).__init__( option_strings=option_strings, dest=dest, nargs=0, default=default, required=required, - help=help) + help=help, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): count = getattr(namespace, self.dest, None) @@ -1128,13 +1138,15 @@ def __init__(self, option_strings, dest=SUPPRESS, default=SUPPRESS, - help=None): + help=None, + deprecated=False): super(_HelpAction, self).__init__( option_strings=option_strings, dest=dest, default=default, nargs=0, - help=help) + help=help, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): parser.print_help() @@ -1148,7 +1160,10 @@ def __init__(self, version=None, dest=SUPPRESS, default=SUPPRESS, - help="show program's version number and exit"): + help=None, + deprecated=False): + if help is None: + help = _("show program's version number and exit") super(_VersionAction, self).__init__( option_strings=option_strings, dest=dest, @@ -1192,6 +1207,7 @@ def __init__(self, self._parser_class = parser_class self._name_parser_map = {} self._choices_actions = [] + self._deprecated = set() super(_SubParsersAction, self).__init__( option_strings=option_strings, @@ -1202,7 +1218,7 @@ def __init__(self, help=help, metavar=metavar) - def add_parser(self, name, **kwargs): + def add_parser(self, name, *, deprecated=False, **kwargs): # set prog from the existing prefix if kwargs.get('prog') is None: kwargs['prog'] = '%s %s' % (self._prog_prefix, name) @@ -1230,6 +1246,10 @@ def add_parser(self, name, **kwargs): for alias in aliases: self._name_parser_map[alias] = parser + if deprecated: + self._deprecated.add(name) + self._deprecated.update(aliases) + return parser def _get_subactions(self): @@ -1245,13 +1265,17 @@ def __call__(self, parser, namespace, values, option_string=None): # select the parser try: - parser = self._name_parser_map[parser_name] + subparser = self._name_parser_map[parser_name] except KeyError: args = {'parser_name': parser_name, 'choices': ', '.join(self._name_parser_map)} msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args raise ArgumentError(self, msg) + if parser_name in self._deprecated: + parser._warning(_("command '%(parser_name)s' is deprecated") % + {'parser_name': parser_name}) + # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top # level parser can decide what to do with them @@ -1259,7 +1283,7 @@ def __call__(self, parser, namespace, values, option_string=None): # In case this subparser defines new defaults, we parse them # in a new namespace object and then update the original # namespace for the relevant parts. - subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) + subnamespace, arg_strings = subparser.parse_known_args(arg_strings, None) for key, value in vars(subnamespace).items(): setattr(namespace, key, value) @@ -1979,6 +2003,7 @@ def _parse_known_args(self, arg_strings, namespace): # converts arg strings to the appropriate and then takes the action seen_actions = set() seen_non_default_actions = set() + warned = set() def take_action(action, argument_strings, option_string=None): seen_actions.add(action) @@ -2005,7 +2030,7 @@ def consume_optional(start_index): # get the optional identified at this index option_tuple = option_string_indices[start_index] - action, option_string, explicit_arg = option_tuple + action, option_string, sep, explicit_arg = option_tuple # identify additional optionals in the same arg string # (e.g. -xyz is the same as -x -y -z if no args are required) @@ -2032,18 +2057,27 @@ def consume_optional(start_index): and option_string[1] not in chars and explicit_arg != '' ): + if sep or explicit_arg[0] in chars: + msg = _('ignored explicit argument %r') + raise ArgumentError(action, msg % explicit_arg) action_tuples.append((action, [], option_string)) char = option_string[0] option_string = char + explicit_arg[0] - new_explicit_arg = explicit_arg[1:] or None optionals_map = self._option_string_actions if option_string in optionals_map: action = optionals_map[option_string] - explicit_arg = new_explicit_arg + explicit_arg = explicit_arg[1:] + if not explicit_arg: + sep = explicit_arg = None + elif explicit_arg[0] == '=': + sep = '=' + explicit_arg = explicit_arg[1:] + else: + sep = '' else: - msg = _('ignored explicit argument %r') - raise ArgumentError(action, msg % explicit_arg) - + extras.append(char + explicit_arg) + stop = start_index + 1 + break # if the action expect exactly one argument, we've # successfully matched the option; exit the loop elif arg_count == 1: @@ -2074,6 +2108,10 @@ def consume_optional(start_index): # the Optional's string args stopped assert action_tuples for action, args, option_string in action_tuples: + if action.deprecated and option_string not in warned: + self._warning(_("option '%(option)s' is deprecated") % + {'option': option_string}) + warned.add(option_string) take_action(action, args, option_string) return stop @@ -2093,6 +2131,10 @@ def consume_positionals(start_index): for action, arg_count in zip(positionals, arg_counts): args = arg_strings[start_index: start_index + arg_count] start_index += arg_count + if args and action.deprecated and action.dest not in warned: + self._warning(_("argument '%(argument_name)s' is deprecated") % + {'argument_name': action.dest}) + warned.add(action.dest) take_action(action, args) # slice off the Positionals that we just parsed and return the @@ -2263,18 +2305,17 @@ def _parse_optional(self, arg_string): # if the option string is present in the parser, return the action if arg_string in self._option_string_actions: action = self._option_string_actions[arg_string] - return action, arg_string, None + return action, arg_string, None, None # if it's just a single character, it was meant to be positional if len(arg_string) == 1: return None # if the option string before the "=" is present, return the action - if '=' in arg_string: - option_string, explicit_arg = arg_string.split('=', 1) - if option_string in self._option_string_actions: - action = self._option_string_actions[option_string] - return action, option_string, explicit_arg + option_string, sep, explicit_arg = arg_string.partition('=') + if sep and option_string in self._option_string_actions: + action = self._option_string_actions[option_string] + return action, option_string, sep, explicit_arg # search through all possible prefixes of the option string # and all actions in the parser for possible interpretations @@ -2283,7 +2324,7 @@ def _parse_optional(self, arg_string): # if multiple actions match, the option string was ambiguous if len(option_tuples) > 1: options = ', '.join([option_string - for action, option_string, explicit_arg in option_tuples]) + for action, option_string, sep, explicit_arg in option_tuples]) args = {'option': arg_string, 'matches': options} msg = _('ambiguous option: %(option)s could match %(matches)s') self.error(msg % args) @@ -2307,7 +2348,7 @@ def _parse_optional(self, arg_string): # it was meant to be an optional but there is no such option # in this parser (though it might be a valid option in a subparser) - return None, arg_string, None + return None, arg_string, None, None def _get_option_tuples(self, option_string): result = [] @@ -2317,15 +2358,13 @@ def _get_option_tuples(self, option_string): chars = self.prefix_chars if option_string[0] in chars and option_string[1] in chars: if self.allow_abbrev: - if '=' in option_string: - option_prefix, explicit_arg = option_string.split('=', 1) - else: - option_prefix = option_string - explicit_arg = None + option_prefix, sep, explicit_arg = option_string.partition('=') + if not sep: + sep = explicit_arg = None for option_string in self._option_string_actions: if option_string.startswith(option_prefix): action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg + tup = action, option_string, sep, explicit_arg result.append(tup) # single character options can be concatenated with their arguments @@ -2333,18 +2372,17 @@ def _get_option_tuples(self, option_string): # separate elif option_string[0] in chars and option_string[1] not in chars: option_prefix = option_string - explicit_arg = None short_option_prefix = option_string[:2] short_explicit_arg = option_string[2:] for option_string in self._option_string_actions: if option_string == short_option_prefix: action = self._option_string_actions[option_string] - tup = action, option_string, short_explicit_arg + tup = action, option_string, '', short_explicit_arg result.append(tup) elif option_string.startswith(option_prefix): action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg + tup = action, option_string, None, None result.append(tup) # shouldn't ever get here @@ -2489,7 +2527,7 @@ def parse_known_intermixed_args(self, args=None, namespace=None): # ======================== def _get_values(self, action, arg_strings): # for everything but PARSER, REMAINDER args, strip out first '--' - if action.nargs not in [PARSER, REMAINDER]: + if not action.option_strings and action.nargs not in [PARSER, REMAINDER]: try: arg_strings.remove('--') except ValueError: @@ -2654,3 +2692,7 @@ def error(self, message): self.print_usage(_sys.stderr) args = {'prog': self.prog, 'message': message} self.exit(2, _('%(prog)s: error: %(message)s\n') % args) + + def _warning(self, message): + args = {'prog': self.prog, 'message': message} + self._print_message(_('%(prog)s: warning: %(message)s\n') % args, _sys.stderr) diff --git a/Lib/ast.py b/Lib/ast.py index 43703a8325cc5ec..b8c4ce6f919e6b4 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1269,14 +1269,18 @@ def visit_JoinedStr(self, node): quote_type = quote_types[0] self.write(f"{quote_type}{value}{quote_type}") - def _write_fstring_inner(self, node, escape_newlines=False): + def _write_fstring_inner(self, node, is_format_spec=False): if isinstance(node, JoinedStr): # for both the f-string itself, and format_spec for value in node.values: - self._write_fstring_inner(value, escape_newlines=escape_newlines) + self._write_fstring_inner(value, is_format_spec=is_format_spec) elif isinstance(node, Constant) and isinstance(node.value, str): value = node.value.replace("{", "{{").replace("}", "}}") - if escape_newlines: + + if is_format_spec: + value = value.replace("\\", "\\\\") + value = value.replace("'", "\\'") + value = value.replace('"', '\\"') value = value.replace("\n", "\\n") self.write(value) elif isinstance(node, FormattedValue): @@ -1300,10 +1304,7 @@ def unparse_inner(inner): self.write(f"!{chr(node.conversion)}") if node.format_spec: self.write(":") - self._write_fstring_inner( - node.format_spec, - escape_newlines=True - ) + self._write_fstring_inner(node.format_spec, is_format_spec=True) def visit_Name(self, node): self.write(node.id) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index c60d7688ef8c772..6c5cf28e7c59d42 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -45,6 +45,7 @@ from . import sslproto from . import staggered from . import tasks +from . import timeouts from . import transports from . import trsock from .log import logger @@ -598,23 +599,24 @@ async def shutdown_default_executor(self, timeout=None): thread = threading.Thread(target=self._do_shutdown, args=(future,)) thread.start() try: - await future - finally: - thread.join(timeout) - - if thread.is_alive(): + async with timeouts.timeout(timeout): + await future + except TimeoutError: warnings.warn("The executor did not finishing joining " - f"its threads within {timeout} seconds.", - RuntimeWarning, stacklevel=2) + f"its threads within {timeout} seconds.", + RuntimeWarning, stacklevel=2) self._default_executor.shutdown(wait=False) + else: + thread.join() def _do_shutdown(self, future): try: self._default_executor.shutdown(wait=True) if not self.is_closed(): - self.call_soon_threadsafe(future.set_result, None) + self.call_soon_threadsafe(futures._set_result_unless_cancelled, + future, None) except Exception as ex: - if not self.is_closed(): + if not self.is_closed() and not future.cancelled(): self.call_soon_threadsafe(future.set_exception, ex) def _check_running(self): @@ -1340,9 +1342,9 @@ async def create_datagram_endpoint(self, protocol_factory, allow_broadcast=None, sock=None): """Create datagram connection.""" if sock is not None: - if sock.type != socket.SOCK_DGRAM: + if sock.type == socket.SOCK_STREAM: raise ValueError( - f'A UDP Socket was expected, got {sock!r}') + f'A datagram socket was expected, got {sock!r}') if (local_addr or remote_addr or family or proto or flags or reuse_port or allow_broadcast): diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 072a99fee123c39..680749325025db3 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -54,7 +54,8 @@ def _repr_info(self): info.append('cancelled') if self._callback is not None: info.append(format_helpers._format_callback_source( - self._callback, self._args)) + self._callback, self._args, + debug=self._loop.get_debug())) if self._source_traceback: frame = self._source_traceback[-1] info.append(f'created at {frame[0]}:{frame[1]}') @@ -90,7 +91,8 @@ def _run(self): raise except BaseException as exc: cb = format_helpers._format_callback_source( - self._callback, self._args) + self._callback, self._args, + debug=self._loop.get_debug()) msg = f'Exception in callback {cb}' context = { 'message': msg, diff --git a/Lib/asyncio/format_helpers.py b/Lib/asyncio/format_helpers.py index 27d11fd4fa9553e..93737b7708a67b6 100644 --- a/Lib/asyncio/format_helpers.py +++ b/Lib/asyncio/format_helpers.py @@ -19,19 +19,26 @@ def _get_function_source(func): return None -def _format_callback_source(func, args): - func_repr = _format_callback(func, args, None) +def _format_callback_source(func, args, *, debug=False): + func_repr = _format_callback(func, args, None, debug=debug) source = _get_function_source(func) if source: func_repr += f' at {source[0]}:{source[1]}' return func_repr -def _format_args_and_kwargs(args, kwargs): +def _format_args_and_kwargs(args, kwargs, *, debug=False): """Format function arguments and keyword arguments. Special case for a single parameter: ('hello',) is formatted as ('hello'). + + Note that this function only returns argument details when + debug=True is specified, as arguments may contain sensitive + information. """ + if not debug: + return '()' + # use reprlib to limit the length of the output items = [] if args: @@ -41,10 +48,11 @@ def _format_args_and_kwargs(args, kwargs): return '({})'.format(', '.join(items)) -def _format_callback(func, args, kwargs, suffix=''): +def _format_callback(func, args, kwargs, *, debug=False, suffix=''): if isinstance(func, functools.partial): - suffix = _format_args_and_kwargs(args, kwargs) + suffix - return _format_callback(func.func, func.args, func.keywords, suffix) + suffix = _format_args_and_kwargs(args, kwargs, debug=debug) + suffix + return _format_callback(func.func, func.args, func.keywords, + debug=debug, suffix=suffix) if hasattr(func, '__qualname__') and func.__qualname__: func_repr = func.__qualname__ @@ -53,7 +61,7 @@ def _format_callback(func, args, kwargs, suffix=''): else: func_repr = repr(func) - func_repr += _format_args_and_kwargs(args, kwargs) + func_repr += _format_args_and_kwargs(args, kwargs, debug=debug) if suffix: func_repr += suffix return func_repr diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 04158e667a895fc..aaee8ff07029233 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -24,25 +24,23 @@ class Lock(_ContextManagerMixin, mixins._LoopBoundMixin): """Primitive lock objects. A primitive lock is a synchronization primitive that is not owned - by a particular coroutine when locked. A primitive lock is in one + by a particular task when locked. A primitive lock is in one of two states, 'locked' or 'unlocked'. It is created in the unlocked state. It has two basic methods, acquire() and release(). When the state is unlocked, acquire() changes the state to locked and returns immediately. When the state is locked, acquire() blocks until a call to release() in - another coroutine changes it to unlocked, then the acquire() call + another task changes it to unlocked, then the acquire() call resets it to locked and returns. The release() method should only be called in the locked state; it changes the state to unlocked and returns immediately. If an attempt is made to release an unlocked lock, a RuntimeError will be raised. - When more than one coroutine is blocked in acquire() waiting for - the state to turn to unlocked, only one coroutine proceeds when a - release() call resets the state to unlocked; first coroutine which - is blocked in acquire() is being processed. - - acquire() is a coroutine and should be called with 'await'. + When more than one task is blocked in acquire() waiting for + the state to turn to unlocked, only one task proceeds when a + release() call resets the state to unlocked; successive release() + calls will unblock tasks in FIFO order. Locks also support the asynchronous context management protocol. 'async with lock' statement should be used. @@ -130,7 +128,7 @@ def release(self): """Release a lock. When the lock is locked, reset it to unlocked, and return. - If any other coroutines are blocked waiting for the lock to become + If any other tasks are blocked waiting for the lock to become unlocked, allow exactly one of them to proceed. When invoked on an unlocked lock, a RuntimeError is raised. @@ -182,8 +180,8 @@ def is_set(self): return self._value def set(self): - """Set the internal flag to true. All coroutines waiting for it to - become true are awakened. Coroutine that call wait() once the flag is + """Set the internal flag to true. All tasks waiting for it to + become true are awakened. Tasks that call wait() once the flag is true will not block at all. """ if not self._value: @@ -194,7 +192,7 @@ def set(self): fut.set_result(True) def clear(self): - """Reset the internal flag to false. Subsequently, coroutines calling + """Reset the internal flag to false. Subsequently, tasks calling wait() will block until set() is called to set the internal flag to true again.""" self._value = False @@ -203,7 +201,7 @@ async def wait(self): """Block until the internal flag is true. If the internal flag is true on entry, return True - immediately. Otherwise, block until another coroutine calls + immediately. Otherwise, block until another task calls set() to set the flag to true, then return True. """ if self._value: @@ -222,8 +220,8 @@ class Condition(_ContextManagerMixin, mixins._LoopBoundMixin): """Asynchronous equivalent to threading.Condition. This class implements condition variable objects. A condition variable - allows one or more coroutines to wait until they are notified by another - coroutine. + allows one or more tasks to wait until they are notified by another + task. A new Lock object is created and used as the underlying lock. """ @@ -250,50 +248,64 @@ def __repr__(self): async def wait(self): """Wait until notified. - If the calling coroutine has not acquired the lock when this + If the calling task has not acquired the lock when this method is called, a RuntimeError is raised. This method releases the underlying lock, and then blocks until it is awakened by a notify() or notify_all() call for - the same condition variable in another coroutine. Once + the same condition variable in another task. Once awakened, it re-acquires the lock and returns True. + + This method may return spuriously, + which is why the caller should always + re-check the state and be prepared to wait() again. """ if not self.locked(): raise RuntimeError('cannot wait on un-acquired lock') + fut = self._get_loop().create_future() self.release() try: - fut = self._get_loop().create_future() - self._waiters.append(fut) try: - await fut - return True - finally: - self._waiters.remove(fut) - - finally: - # Must re-acquire lock even if wait is cancelled. - # We only catch CancelledError here, since we don't want any - # other (fatal) errors with the future to cause us to spin. - err = None - while True: - try: - await self.acquire() - break - except exceptions.CancelledError as e: - err = e - - if err: + self._waiters.append(fut) try: - raise err # Re-raise most recent exception instance. + await fut + return True finally: - err = None # Break reference cycles. + self._waiters.remove(fut) + + finally: + # Must re-acquire lock even if wait is cancelled. + # We only catch CancelledError here, since we don't want any + # other (fatal) errors with the future to cause us to spin. + err = None + while True: + try: + await self.acquire() + break + except exceptions.CancelledError as e: + err = e + + if err is not None: + try: + raise err # Re-raise most recent exception instance. + finally: + err = None # Break reference cycles. + except BaseException: + # Any error raised out of here _may_ have occurred after this Task + # believed to have been successfully notified. + # Make sure to notify another Task instead. This may result + # in a "spurious wakeup", which is allowed as part of the + # Condition Variable protocol. + self._notify(1) + raise async def wait_for(self, predicate): """Wait until a predicate becomes true. - The predicate should be a callable which result will be - interpreted as a boolean value. The final predicate value is + The predicate should be a callable whose result will be + interpreted as a boolean value. The method will repeatedly + wait() until it evaluates to true. The final predicate value is the return value. """ result = predicate() @@ -303,20 +315,22 @@ async def wait_for(self, predicate): return result def notify(self, n=1): - """By default, wake up one coroutine waiting on this condition, if any. - If the calling coroutine has not acquired the lock when this method + """By default, wake up one task waiting on this condition, if any. + If the calling task has not acquired the lock when this method is called, a RuntimeError is raised. - This method wakes up at most n of the coroutines waiting for the - condition variable; it is a no-op if no coroutines are waiting. + This method wakes up n of the tasks waiting for the condition + variable; if fewer than n are waiting, they are all awoken. - Note: an awakened coroutine does not actually return from its + Note: an awakened task does not actually return from its wait() call until it can reacquire the lock. Since notify() does not release the lock, its caller should. """ if not self.locked(): raise RuntimeError('cannot notify on un-acquired lock') + self._notify(n) + def _notify(self, n): idx = 0 for fut in self._waiters: if idx >= n: @@ -374,7 +388,7 @@ async def acquire(self): If the internal counter is larger than zero on entry, decrement it by one and return True immediately. If it is - zero on entry, block, waiting until some other coroutine has + zero on entry, block, waiting until some other task has called release() to make it larger than 0, and then return True. """ @@ -414,8 +428,8 @@ async def acquire(self): def release(self): """Release a semaphore, incrementing the internal counter by one. - When it was zero on entry and another coroutine is waiting for it to - become larger than zero again, wake up that coroutine. + When it was zero on entry and another task is waiting for it to + become larger than zero again, wake up that task. """ self._value += 1 self._wake_up_next() diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 1e2a730cf368a99..a512db6367b20ac 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -487,9 +487,6 @@ def sendto(self, data, addr=None): raise TypeError('data argument must be bytes-like object (%r)', type(data)) - if not data: - return - if self._address is not None and addr not in (None, self._address): raise ValueError( f'Invalid address: must be None or {self._address}') @@ -502,7 +499,7 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + self._buffer_size += len(data) + 8 # include header bytes if self._write_fut is None: # No current write operations are active, kick one off diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index dcd5e0aa345029a..8e888d26ea07377 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -235,6 +235,10 @@ async def _accept_connection2( await waiter except BaseException: transport.close() + # gh-109534: When an exception is raised by the SSLProtocol object the + # exception set in this future can keep the protocol object alive and + # cause a reference cycle. + waiter = None raise # It's now up to the protocol to handle the connection. @@ -1237,8 +1241,6 @@ def sendto(self, data, addr=None): if not isinstance(data, (bytes, bytearray, memoryview)): raise TypeError(f'data argument must be a bytes-like object, ' f'not {type(data).__name__!r}') - if not data: - return if self._address: if addr not in (None, self._address): @@ -1274,7 +1276,7 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + self._buffer_size += len(data) + 8 # include header bytes self._maybe_pause_protocol() def _sendto_ready(self): diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 599e91ba0003d10..fa99d4533aa0a6a 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -579,6 +579,7 @@ def _on_handshake_complete(self, handshake_exc): peercert = sslobj.getpeercert() except Exception as exc: + handshake_exc = None self._set_state(SSLProtocolState.UNWRAPPED) if isinstance(exc, ssl.CertificateError): msg = 'SSL handshake failed on verifying the certificate' diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index df58b7a799a5add..3fe52dbac25c916 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -201,7 +201,6 @@ def __init__(self, stream_reader, client_connected_cb=None, loop=None): # is established. self._strong_reader = stream_reader self._reject_connection = False - self._stream_writer = None self._task = None self._transport = None self._client_connected_cb = client_connected_cb @@ -214,10 +213,8 @@ def _stream_reader(self): return None return self._stream_reader_wr() - def _replace_writer(self, writer): + def _replace_transport(self, transport): loop = self._loop - transport = writer.transport - self._stream_writer = writer self._transport = transport self._over_ssl = transport.get_extra_info('sslcontext') is not None @@ -239,11 +236,8 @@ def connection_made(self, transport): reader.set_transport(transport) self._over_ssl = transport.get_extra_info('sslcontext') is not None if self._client_connected_cb is not None: - self._stream_writer = StreamWriter(transport, self, - reader, - self._loop) - res = self._client_connected_cb(reader, - self._stream_writer) + writer = StreamWriter(transport, self, reader, self._loop) + res = self._client_connected_cb(reader, writer) if coroutines.iscoroutine(res): def callback(task): if task.cancelled(): @@ -405,7 +399,7 @@ async def start_tls(self, sslcontext, *, ssl_handshake_timeout=ssl_handshake_timeout, ssl_shutdown_timeout=ssl_shutdown_timeout) self._transport = new_transport - protocol._replace_writer(self) + protocol._replace_transport(new_transport) def __del__(self, warnings=warnings): if not self._transport.is_closing(): diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index e1c56d140bef7de..f322b1f6653f6a2 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -132,7 +132,7 @@ async def __aexit__(self, et, exc, tb): # Propagate CancelledError if there is one, except if there # are other errors -- those have priority. - if propagate_cancellation_error and not self._errors: + if propagate_cancellation_error is not None and not self._errors: raise propagate_cancellation_error if et is not None and not issubclass(et, exceptions.CancelledError): diff --git a/Lib/asyncio/transports.py b/Lib/asyncio/transports.py index 30fd41d49af71ff..34c7ad44ffd8ab3 100644 --- a/Lib/asyncio/transports.py +++ b/Lib/asyncio/transports.py @@ -181,6 +181,8 @@ def sendto(self, data, addr=None): to be sent out asynchronously. addr is target socket address. If addr is None use target address pointed on transport creation. + If data is an empty bytes object a zero-length datagram will be + sent. """ raise NotImplementedError diff --git a/Lib/base64.py b/Lib/base64.py index e3e983b3064fe7b..25164d1a1df4fc8 100755 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -18,7 +18,7 @@ 'b64encode', 'b64decode', 'b32encode', 'b32decode', 'b32hexencode', 'b32hexdecode', 'b16encode', 'b16decode', # Base85 and Ascii85 encodings - 'b85encode', 'b85decode', 'a85encode', 'a85decode', + 'b85encode', 'b85decode', 'a85encode', 'a85decode', 'z85encode', 'z85decode', # Standard Base64 encoding 'standard_b64encode', 'standard_b64decode', # Some common Base64 alternatives. As referenced by RFC 3458, see thread @@ -497,6 +497,33 @@ def b85decode(b): result = result[:-padding] return result +_z85alphabet = (b'0123456789abcdefghijklmnopqrstuvwxyz' + b'ABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#') +# Translating b85 valid but z85 invalid chars to b'\x00' is required +# to prevent them from being decoded as b85 valid chars. +_z85_b85_decode_diff = b';_`|~' +_z85_decode_translation = bytes.maketrans( + _z85alphabet + _z85_b85_decode_diff, + _b85alphabet + b'\x00' * len(_z85_b85_decode_diff) +) +_z85_encode_translation = bytes.maketrans(_b85alphabet, _z85alphabet) + +def z85encode(s): + """Encode bytes-like object b in z85 format and return a bytes object.""" + return b85encode(s).translate(_z85_encode_translation) + +def z85decode(s): + """Decode the z85-encoded bytes-like object or ASCII string b + + The result is returned as a bytes object. + """ + s = _bytes_from_decode_data(s) + s = s.translate(_z85_decode_translation) + try: + return b85decode(s) + except ValueError as e: + raise ValueError(e.args[0].replace('base85', 'z85')) from None + # Legacy interface. This code could be cleaned up since I don't believe # binascii has any line length limitations. It just doesn't seem worth it # though. The files should be opened in binary mode. diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 135a12c3965c00e..9c132372dc4ee04 100755 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -41,7 +41,9 @@ class Profile(_lsprof.Profiler): def print_stats(self, sort=-1): import pstats - pstats.Stats(self).strip_dirs().sort_stats(sort).print_stats() + if not isinstance(sort, tuple): + sort = (sort,) + pstats.Stats(self).strip_dirs().sort_stats(*sort).print_stats() def dump_stats(self, file): import marshal diff --git a/Lib/calendar.py b/Lib/calendar.py index 03469d8ac96bcd9..833ce331b14a0cc 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -734,10 +734,15 @@ def main(args=None): choices=("text", "html"), help="output type (text or html)" ) + parser.add_argument( + "-f", "--first-weekday", + type=int, default=0, + help="weekday (0 is Monday, 6 is Sunday) to start each week (default 0)" + ) parser.add_argument( "year", nargs='?', type=int, - help="year number (1-9999)" + help="year number" ) parser.add_argument( "month", @@ -761,6 +766,7 @@ def main(args=None): cal = LocaleHTMLCalendar(locale=locale) else: cal = HTMLCalendar() + cal.setfirstweekday(options.first_weekday) encoding = options.encoding if encoding is None: encoding = sys.getdefaultencoding() @@ -775,6 +781,7 @@ def main(args=None): cal = LocaleTextCalendar(locale=locale) else: cal = TextCalendar() + cal.setfirstweekday(options.first_weekday) optdict = dict(w=options.width, l=options.lines) if options.month is None: optdict["c"] = options.spacing diff --git a/Lib/codeop.py b/Lib/codeop.py index 91146be2c438e2c..6ad60e7f85098d8 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -65,9 +65,10 @@ def _maybe_compile(compiler, source, filename, symbol): try: compiler(source + "\n", filename, symbol) return None + except IncompleteInputError as e: + return None except SyntaxError as e: - if "incomplete input" in str(e): - return None + pass # fallthrough return compiler(source, filename, symbol, incomplete_input=False) diff --git a/Lib/csv.py b/Lib/csv.py index a079279b8b8cbce..75e35b23236795f 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -113,8 +113,8 @@ def _validate(self): try: _Dialect(self) except TypeError as e: - # We do this for compatibility with py2.3 - raise Error(str(e)) + # Re-raise to get a traceback showing more user code. + raise Error(str(e)) from None class excel(Dialect): """Describe the usual properties of Excel-generated CSV files.""" diff --git a/Lib/dbm/__init__.py b/Lib/dbm/__init__.py index 8055d3769f9dd00..4fdbc54e74cfb64 100644 --- a/Lib/dbm/__init__.py +++ b/Lib/dbm/__init__.py @@ -5,7 +5,7 @@ import dbm d = dbm.open(file, 'w', 0o666) -The returned object is a dbm.gnu, dbm.ndbm or dbm.dumb object, dependent on the +The returned object is a dbm.sqlite3, dbm.gnu, dbm.ndbm or dbm.dumb database object, dependent on the type of database being opened (determined by the whichdb function) in the case of an existing dbm. If the dbm does not exist and the create or new flag ('c' or 'n') was specified, the dbm type will be determined by the availability of @@ -38,7 +38,7 @@ class error(Exception): pass -_names = ['dbm.gnu', 'dbm.ndbm', 'dbm.dumb'] +_names = ['dbm.sqlite3', 'dbm.gnu', 'dbm.ndbm', 'dbm.dumb'] _defaultmod = None _modules = {} @@ -164,6 +164,10 @@ def whichdb(filename): if len(s) != 4: return "" + # Check for SQLite3 header string. + if s16 == b"SQLite format 3\0": + return "dbm.sqlite3" + # Convert to 4-byte int in native byte order -- return "" if impossible try: (magic,) = struct.unpack("=l", s) diff --git a/Lib/dbm/dumb.py b/Lib/dbm/dumb.py index 754624ccc8f5008..def120ffc3778b6 100644 --- a/Lib/dbm/dumb.py +++ b/Lib/dbm/dumb.py @@ -98,7 +98,8 @@ def _update(self, flag): except OSError: if flag not in ('c', 'n'): raise - self._modified = True + with self._io.open(self._dirfile, 'w', encoding="Latin-1") as f: + self._chmod(self._dirfile) else: with f: for line in f: @@ -134,6 +135,7 @@ def _commit(self): # position; UTF-8, though, does care sometimes. entry = "%r, %r\n" % (key.decode('Latin-1'), pos_and_siz_pair) f.write(entry) + self._modified = False sync = _commit diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py new file mode 100644 index 000000000000000..74c9d9b7e2f1d86 --- /dev/null +++ b/Lib/dbm/sqlite3.py @@ -0,0 +1,141 @@ +import os +import sqlite3 +import sys +from pathlib import Path +from contextlib import suppress, closing +from collections.abc import MutableMapping + +BUILD_TABLE = """ + CREATE TABLE IF NOT EXISTS Dict ( + key BLOB UNIQUE NOT NULL, + value BLOB NOT NULL + ) +""" +GET_SIZE = "SELECT COUNT (key) FROM Dict" +LOOKUP_KEY = "SELECT value FROM Dict WHERE key = CAST(? AS BLOB)" +STORE_KV = "REPLACE INTO Dict (key, value) VALUES (CAST(? AS BLOB), CAST(? AS BLOB))" +DELETE_KEY = "DELETE FROM Dict WHERE key = CAST(? AS BLOB)" +ITER_KEYS = "SELECT key FROM Dict" + + +class error(OSError): + pass + + +_ERR_CLOSED = "DBM object has already been closed" +_ERR_REINIT = "DBM object does not support reinitialization" + + +def _normalize_uri(path): + path = Path(path) + uri = path.absolute().as_uri() + while "//" in uri: + uri = uri.replace("//", "/") + return uri + + +class _Database(MutableMapping): + + def __init__(self, path, /, *, flag, mode): + if hasattr(self, "_cx"): + raise error(_ERR_REINIT) + + path = os.fsdecode(path) + match flag: + case "r": + flag = "ro" + case "w": + flag = "rw" + case "c": + flag = "rwc" + Path(path).touch(mode=mode, exist_ok=True) + case "n": + flag = "rwc" + Path(path).unlink(missing_ok=True) + Path(path).touch(mode=mode) + case _: + raise ValueError("Flag must be one of 'r', 'w', 'c', or 'n', " + f"not {flag!r}") + + # We use the URI format when opening the database. + uri = _normalize_uri(path) + uri = f"{uri}?mode={flag}" + + try: + self._cx = sqlite3.connect(uri, autocommit=True, uri=True) + except sqlite3.Error as exc: + raise error(str(exc)) + + # This is an optimization only; it's ok if it fails. + with suppress(sqlite3.OperationalError): + self._cx.execute("PRAGMA journal_mode = wal") + + if flag == "rwc": + self._execute(BUILD_TABLE) + + def _execute(self, *args, **kwargs): + if not self._cx: + raise error(_ERR_CLOSED) + try: + return closing(self._cx.execute(*args, **kwargs)) + except sqlite3.Error as exc: + raise error(str(exc)) + + def __len__(self): + with self._execute(GET_SIZE) as cu: + row = cu.fetchone() + return row[0] + + def __getitem__(self, key): + with self._execute(LOOKUP_KEY, (key,)) as cu: + row = cu.fetchone() + if not row: + raise KeyError(key) + return row[0] + + def __setitem__(self, key, value): + self._execute(STORE_KV, (key, value)) + + def __delitem__(self, key): + with self._execute(DELETE_KEY, (key,)) as cu: + if not cu.rowcount: + raise KeyError(key) + + def __iter__(self): + try: + with self._execute(ITER_KEYS) as cu: + for row in cu: + yield row[0] + except sqlite3.Error as exc: + raise error(str(exc)) + + def close(self): + if self._cx: + self._cx.close() + self._cx = None + + def keys(self): + return list(super().keys()) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +def open(filename, /, flag="r", mode=0o666): + """Open a dbm.sqlite3 database and return the dbm object. + + The 'filename' parameter is the name of the database file. + + The optional 'flag' parameter can be one of ...: + 'r' (default): open an existing database for read only access + 'w': open an existing database for read/write access + 'c': create a database if it does not exist; open for read/write access + 'n': always create a new, empty database; open for read/write access + + The optional 'mode' parameter is the Unix file access mode of the database; + only used when creating a new database. Default: 0o666. + """ + return _Database(filename, flag=flag, mode=mode) diff --git a/Lib/dis.py b/Lib/dis.py index f05ea1a24f45a71..d146bcbb5097efd 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -505,6 +505,20 @@ def __init__(self, co_consts=None, names=None, varname_from_oparg=None, labels_m self.varname_from_oparg = varname_from_oparg self.labels_map = labels_map or {} + def offset_from_jump_arg(self, op, arg, offset): + deop = _deoptop(op) + if deop in hasjabs: + return arg * 2 + elif deop in hasjrel: + signed_arg = -arg if _is_backward_jump(deop) else arg + argval = offset + 2 + signed_arg*2 + caches = _get_cache_size(_all_opname[deop]) + argval += 2 * caches + if deop == ENTER_EXECUTOR: + argval += 2 + return argval + return None + def get_label_for_offset(self, offset): return self.labels_map.get(offset, None) @@ -536,17 +550,11 @@ def get_argval_argrepr(self, op, arg, offset): argrepr = f"{argrepr} + NULL|self" else: argval, argrepr = _get_name_info(arg, get_name) - elif deop in hasjabs: - argval = arg*2 - argrepr = f"to L{self.labels_map[argval]}" - elif deop in hasjrel: - signed_arg = -arg if _is_backward_jump(deop) else arg - argval = offset + 2 + signed_arg*2 - caches = _get_cache_size(_all_opname[deop]) - argval += 2 * caches - if deop == ENTER_EXECUTOR: - argval += 2 - argrepr = f"to L{self.labels_map[argval]}" + elif deop in hasjump or deop in hasexc: + argval = self.offset_from_jump_arg(op, arg, offset) + lbl = self.get_label_for_offset(argval) + assert lbl is not None + argrepr = f"to L{lbl}" elif deop in (LOAD_FAST_LOAD_FAST, STORE_FAST_LOAD_FAST, STORE_FAST_STORE_FAST): arg1 = arg >> 4 arg2 = arg & 15 diff --git a/Lib/doctest.py b/Lib/doctest.py index 114aac62a34e95b..6049423b5147a5e 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1140,7 +1140,7 @@ def _find_lineno(self, obj, source_lines): obj = obj.fget if inspect.isfunction(obj) and getattr(obj, '__doc__', None): # We don't use `docstring` var here, because `obj` can be changed. - obj = obj.__code__ + obj = inspect.unwrap(obj).__code__ if inspect.istraceback(obj): obj = obj.tb_frame if inspect.isframe(obj): obj = obj.f_code if inspect.iscode(obj): @@ -2225,13 +2225,13 @@ def __init__(self, test, optionflags=0, setUp=None, tearDown=None, unittest.TestCase.__init__(self) self._dt_optionflags = optionflags self._dt_checker = checker - self._dt_globs = test.globs.copy() self._dt_test = test self._dt_setUp = setUp self._dt_tearDown = tearDown def setUp(self): test = self._dt_test + self._dt_globs = test.globs.copy() if self._dt_setUp is not None: self._dt_setUp(test) diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 5b653f66c18554c..e4a342d446f6a38 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -949,6 +949,7 @@ class _InvalidEwError(errors.HeaderParseError): # up other parse trees. Maybe should have tests for that, too. DOT = ValueTerminal('.', 'dot') ListSeparator = ValueTerminal(',', 'list-separator') +ListSeparator.as_ew_allowed = False RouteComponentMarker = ValueTerminal('@', 'route-component-marker') # @@ -2022,7 +2023,7 @@ def get_address_list(value): address_list.defects.append(errors.InvalidHeaderDefect( "invalid address in address-list")) if value: # Must be a , at this point. - address_list.append(ValueTerminal(',', 'list-separator')) + address_list.append(ListSeparator) value = value[1:] return address_list, value diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index febe411355d6be3..0f1bf8e4253ec4b 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -13,7 +13,7 @@ 'quote', ] -import time, calendar +import time SPACE = ' ' EMPTYSTRING = '' @@ -194,6 +194,9 @@ def mktime_tz(data): # No zone info, so localtime is better assumption than GMT return time.mktime(data[:8] + (-1,)) else: + # Delay the import, since mktime_tz is rarely used + import calendar + t = calendar.timegm(data) return t - data[9] diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py index d85afd6d5cf704b..6a5ca046b5eb6cd 100644 --- a/Lib/encodings/aliases.py +++ b/Lib/encodings/aliases.py @@ -209,6 +209,7 @@ 'ms932' : 'cp932', 'mskanji' : 'cp932', 'ms_kanji' : 'cp932', + 'windows_31j' : 'cp932', # cp949 codec '949' : 'cp949', diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index a09bf3201e1fb7d..e8dd253bb555209 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -1,78 +1,64 @@ -import collections import os -import os.path import subprocess import sys import sysconfig import tempfile +from contextlib import nullcontext from importlib import resources +from pathlib import Path +from shutil import copy2 __all__ = ["version", "bootstrap"] -_PACKAGE_NAMES = ('pip',) -_PIP_VERSION = "23.3.2" -_PROJECTS = [ - ("pip", _PIP_VERSION, "py3"), -] - -# Packages bundled in ensurepip._bundled have wheel_name set. -# Packages from WHEEL_PKG_DIR have wheel_path set. -_Package = collections.namedtuple('Package', - ('version', 'wheel_name', 'wheel_path')) +_PIP_VERSION = "24.0" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora # installs wheel packages in the /usr/share/python-wheels/ directory and don't # install the ensurepip._bundled package. -_WHEEL_PKG_DIR = sysconfig.get_config_var('WHEEL_PKG_DIR') +if (_pkg_dir := sysconfig.get_config_var('WHEEL_PKG_DIR')) is not None: + _WHEEL_PKG_DIR = Path(_pkg_dir).resolve() +else: + _WHEEL_PKG_DIR = None + +def _find_wheel_pkg_dir_pip(): + if _WHEEL_PKG_DIR is None: + # NOTE: The compile-time `WHEEL_PKG_DIR` is unset so there is no place + # NOTE: for looking up the wheels. + return None -def _find_packages(path): - packages = {} + dist_matching_wheels = _WHEEL_PKG_DIR.glob('pip-*.whl') try: - filenames = os.listdir(path) - except OSError: - # Ignore: path doesn't exist or permission error - filenames = () - # Make the code deterministic if a directory contains multiple wheel files - # of the same package, but don't attempt to implement correct version - # comparison since this case should not happen. - filenames = sorted(filenames) - for filename in filenames: - # filename is like 'pip-21.2.4-py3-none-any.whl' - if not filename.endswith(".whl"): - continue - for name in _PACKAGE_NAMES: - prefix = name + '-' - if filename.startswith(prefix): - break - else: - continue - - # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl' - version = filename.removeprefix(prefix).partition('-')[0] - wheel_path = os.path.join(path, filename) - packages[name] = _Package(version, None, wheel_path) - return packages - - -def _get_packages(): - global _PACKAGES, _WHEEL_PKG_DIR - if _PACKAGES is not None: - return _PACKAGES - - packages = {} - for name, version, py_tag in _PROJECTS: - wheel_name = f"{name}-{version}-{py_tag}-none-any.whl" - packages[name] = _Package(version, wheel_name, None) - if _WHEEL_PKG_DIR: - dir_packages = _find_packages(_WHEEL_PKG_DIR) - # only used the wheel package directory if all packages are found there - if all(name in dir_packages for name in _PACKAGE_NAMES): - packages = dir_packages - _PACKAGES = packages - return packages -_PACKAGES = None + last_matching_dist_wheel = sorted(dist_matching_wheels)[-1] + except IndexError: + # NOTE: `WHEEL_PKG_DIR` does not contain any wheel files for `pip`. + return None + + return nullcontext(last_matching_dist_wheel) + + +def _get_pip_whl_path_ctx(): + # Prefer pip from the wheel package directory, if present. + if (alternative_pip_wheel_path := _find_wheel_pkg_dir_pip()) is not None: + return alternative_pip_wheel_path + + return resources.as_file( + resources.files('ensurepip') + / '_bundled' + / f'pip-{_PIP_VERSION}-py3-none-any.whl' + ) + + +def _get_pip_version(): + with _get_pip_whl_path_ctx() as bundled_wheel_path: + wheel_name = bundled_wheel_path.name + return ( + # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl' + wheel_name. + removeprefix('pip-'). + partition('-')[0] + ) def _run_pip(args, additional_paths=None): @@ -105,7 +91,7 @@ def version(): """ Returns a string specifying the bundled version of pip. """ - return _get_packages()['pip'].version + return _get_pip_version() def _disable_pip_configuration_settings(): @@ -167,24 +153,10 @@ def _bootstrap(*, root=None, upgrade=False, user=False, with tempfile.TemporaryDirectory() as tmpdir: # Put our bundled wheels into a temporary directory and construct the # additional paths that need added to sys.path - additional_paths = [] - for name, package in _get_packages().items(): - if package.wheel_name: - # Use bundled wheel package - wheel_name = package.wheel_name - wheel_path = resources.files("ensurepip") / "_bundled" / wheel_name - whl = wheel_path.read_bytes() - else: - # Use the wheel package directory - with open(package.wheel_path, "rb") as fp: - whl = fp.read() - wheel_name = os.path.basename(package.wheel_path) - - filename = os.path.join(tmpdir, wheel_name) - with open(filename, "wb") as fp: - fp.write(whl) - - additional_paths.append(filename) + tmpdir_path = Path(tmpdir) + with _get_pip_whl_path_ctx() as bundled_wheel_path: + tmp_wheel_path = tmpdir_path / bundled_wheel_path.name + copy2(bundled_wheel_path, tmp_wheel_path) # Construct the arguments to be passed to the pip command args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir] @@ -197,7 +169,8 @@ def _bootstrap(*, root=None, upgrade=False, user=False, if verbosity: args += ["-" + "v" * verbosity] - return _run_pip([*args, *_PACKAGE_NAMES], additional_paths) + return _run_pip([*args, "pip"], [os.fsdecode(tmp_wheel_path)]) + def _uninstall_helper(*, verbosity=0): """Helper to support a clean default uninstall process on Windows @@ -227,7 +200,7 @@ def _uninstall_helper(*, verbosity=0): if verbosity: args += ["-" + "v" * verbosity] - return _run_pip([*args, *reversed(_PACKAGE_NAMES)]) + return _run_pip([*args, "pip"]) def _main(argv=None): diff --git a/Lib/ensurepip/_bundled/pip-23.3.2-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl similarity index 83% rename from Lib/ensurepip/_bundled/pip-23.3.2-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl index ae78b8a6ce07379..2e6aa9d2cb99232 100644 Binary files a/Lib/ensurepip/_bundled/pip-23.3.2-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl differ diff --git a/Lib/enum.py b/Lib/enum.py index a8a50a583803758..22963cca4466f2d 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -279,9 +279,10 @@ def __set_name__(self, enum_class, member_name): enum_member._sort_order_ = len(enum_class._member_names_) if Flag is not None and issubclass(enum_class, Flag): - enum_class._flag_mask_ |= value - if _is_single_bit(value): - enum_class._singles_mask_ |= value + if isinstance(value, int): + enum_class._flag_mask_ |= value + if _is_single_bit(value): + enum_class._singles_mask_ |= value enum_class._all_bits_ = 2 ** ((enum_class._flag_mask_).bit_length()) - 1 # If another member with the same value was already defined, the @@ -309,6 +310,7 @@ def __set_name__(self, enum_class, member_name): elif ( Flag is not None and issubclass(enum_class, Flag) + and isinstance(value, int) and _is_single_bit(value) ): # no other instances found, record this member in _member_names_ @@ -409,10 +411,11 @@ def __setitem__(self, key, value): if isinstance(value, auto): single = True value = (value, ) - if type(value) is tuple and any(isinstance(v, auto) for v in value): + if isinstance(value, tuple) and any(isinstance(v, auto) for v in value): # insist on an actual tuple, no subclasses, in keeping with only supporting # top-level auto() usage (not contained in any other data structure) auto_valued = [] + t = type(value) for v in value: if isinstance(v, auto): non_auto_store = False @@ -427,7 +430,12 @@ def __setitem__(self, key, value): if single: value = auto_valued[0] else: - value = tuple(auto_valued) + try: + # accepts iterable as multiple arguments? + value = t(auto_valued) + except TypeError: + # then pass them in singlely + value = t(*auto_valued) self._member_names[key] = None if non_auto_store: self._last_values.append(value) @@ -539,7 +547,10 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_inverted_'] = None try: exc = None + classdict['_%s__in_progress' % cls] = True enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) + classdict['_%s__in_progress' % cls] = False + delattr(enum_class, '_%s__in_progress' % cls) except Exception as e: # since 3.12 the line "Error calling __set_name__ on '_proto_member' instance ..." # is tacked on to the error instead of raising a RuntimeError @@ -1147,6 +1158,8 @@ def __new__(cls, value): # still not found -- verify that members exist, in-case somebody got here mistakenly # (such as via super when trying to override __new__) if not cls._member_map_: + if getattr(cls, '_%s__in_progress' % cls.__name__, False): + raise TypeError('do not use `super().__new__; call the appropriate __new__ directly') from None raise TypeError("%r has no members defined" % cls) # # still not found -- try _missing_ hook @@ -1552,37 +1565,50 @@ def __str__(self): def __bool__(self): return bool(self._value_) + def _get_value(self, flag): + if isinstance(flag, self.__class__): + return flag._value_ + elif self._member_type_ is not object and isinstance(flag, self._member_type_): + return flag + return NotImplemented + def __or__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with |") value = self._value_ - return self.__class__(value | other) + return self.__class__(value | other_value) def __and__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with &") value = self._value_ - return self.__class__(value & other) + return self.__class__(value & other_value) def __xor__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with ^") value = self._value_ - return self.__class__(value ^ other) + return self.__class__(value ^ other_value) def __invert__(self): + if self._get_value(self) is None: + raise TypeError(f"'{self}' cannot be inverted") + if self._inverted_ is None: if self._boundary_ in (EJECT, KEEP): self._inverted_ = self.__class__(~self._value_) diff --git a/Lib/fractions.py b/Lib/fractions.py index 389ab386b6a8a4d..f8c6c9c438c7379 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -579,7 +579,8 @@ def __format__(self, format_spec, /): f"for object of type {type(self).__name__!r}" ) - def _operator_fallbacks(monomorphic_operator, fallback_operator): + def _operator_fallbacks(monomorphic_operator, fallback_operator, + handle_complex=True): """Generates forward and reverse operators given a purely-rational operator and a function from the operator module. @@ -666,7 +667,7 @@ def forward(a, b): return monomorphic_operator(a, Fraction(b)) elif isinstance(b, float): return fallback_operator(float(a), b) - elif isinstance(b, complex): + elif handle_complex and isinstance(b, complex): return fallback_operator(complex(a), b) else: return NotImplemented @@ -679,7 +680,7 @@ def reverse(b, a): return monomorphic_operator(Fraction(a), b) elif isinstance(a, numbers.Real): return fallback_operator(float(a), float(b)) - elif isinstance(a, numbers.Complex): + elif handle_complex and isinstance(a, numbers.Complex): return fallback_operator(complex(a), complex(b)) else: return NotImplemented @@ -830,7 +831,7 @@ def _floordiv(a, b): """a // b""" return (a.numerator * b.denominator) // (a.denominator * b.numerator) - __floordiv__, __rfloordiv__ = _operator_fallbacks(_floordiv, operator.floordiv) + __floordiv__, __rfloordiv__ = _operator_fallbacks(_floordiv, operator.floordiv, False) def _divmod(a, b): """(a // b, a % b)""" @@ -838,14 +839,14 @@ def _divmod(a, b): div, n_mod = divmod(a.numerator * db, da * b.numerator) return div, Fraction(n_mod, da * db) - __divmod__, __rdivmod__ = _operator_fallbacks(_divmod, divmod) + __divmod__, __rdivmod__ = _operator_fallbacks(_divmod, divmod, False) def _mod(a, b): """a % b""" da, db = a.denominator, b.denominator return Fraction((a.numerator * db) % (b.numerator * da), da * db) - __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod) + __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod, False) def __pow__(a, b): """a ** b diff --git a/Lib/functools.py b/Lib/functools.py index 55990e742bf23fd..601cb8e7c0b74b1 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -303,13 +303,13 @@ def __call__(self, /, *args, **keywords): @recursive_repr() def __repr__(self): - qualname = type(self).__qualname__ + cls = type(self) + qualname = cls.__qualname__ + module = cls.__module__ args = [repr(self.func)] args.extend(repr(x) for x in self.args) args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items()) - if type(self).__module__ == "functools": - return f"functools.{qualname}({', '.join(args)})" - return f"{qualname}({', '.join(args)})" + return f"{module}.{qualname}({', '.join(args)})" def __reduce__(self): return type(self), (self.func,), (self.func, self.args, @@ -388,7 +388,7 @@ def _method(cls_or_self, /, *args, **keywords): keywords = {**self.keywords, **keywords} return self.func(cls_or_self, *self.args, *args, **keywords) _method.__isabstractmethod__ = self.__isabstractmethod__ - _method._partialmethod = self + _method.__partialmethod__ = self return _method def __get__(self, obj, cls=None): @@ -424,6 +424,17 @@ def _unwrap_partial(func): func = func.func return func +def _unwrap_partialmethod(func): + prev = None + while func is not prev: + prev = func + while isinstance(getattr(func, "__partialmethod__", None), partialmethod): + func = func.__partialmethod__ + while isinstance(func, partialmethod): + func = getattr(func, 'func') + func = _unwrap_partial(func) + return func + ################################################################################ ### LRU Cache function decorator ################################################################################ @@ -907,7 +918,6 @@ def wrapper(*args, **kw): if not args: raise TypeError(f'{funcname} requires at least ' '1 positional argument') - return dispatch(args[0].__class__)(*args, **kw) funcname = getattr(func, '__name__', 'singledispatch function') @@ -957,7 +967,11 @@ def __get__(self, obj, cls=None): return _method dispatch = self.dispatcher.dispatch + funcname = getattr(self.func, '__name__', 'singledispatchmethod method') def _method(*args, **kwargs): + if not args: + raise TypeError(f'{funcname} requires at least ' + '1 positional argument') return dispatch(args[0].__class__).__get__(obj, cls)(*args, **kwargs) _method.__isabstractmethod__ = self.__isabstractmethod__ diff --git a/Lib/glob.py b/Lib/glob.py index 4a335a10766cf40..343be78a73b20a3 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -132,7 +132,8 @@ def glob1(dirname, pattern): def _glob2(dirname, pattern, dir_fd, dironly, include_hidden=False): assert _isrecursive(pattern) - yield pattern[:0] + if not dirname or _isdir(dirname, dir_fd): + yield pattern[:0] yield from _rlistdir(dirname, dir_fd, dironly, include_hidden=include_hidden) diff --git a/Lib/http/client.py b/Lib/http/client.py index 5eebfccafbca59b..a353716a8506e65 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -936,17 +936,23 @@ def _get_hostport(self, host, port): host = host[:i] else: port = self.default_port - if host and host[0] == '[' and host[-1] == ']': - host = host[1:-1] + if host and host[0] == '[' and host[-1] == ']': + host = host[1:-1] return (host, port) def set_debuglevel(self, level): self.debuglevel = level + def _wrap_ipv6(self, ip): + if b':' in ip and ip[0] != b'['[0]: + return b"[" + ip + b"]" + return ip + def _tunnel(self): connect = b"CONNECT %s:%d %s\r\n" % ( - self._tunnel_host.encode("idna"), self._tunnel_port, + self._wrap_ipv6(self._tunnel_host.encode("idna")), + self._tunnel_port, self._http_vsn_str.encode("ascii")) headers = [connect] for header, value in self._tunnel_headers.items(): @@ -1221,9 +1227,8 @@ def putrequest(self, method, url, skip_host=False, # As per RFC 273, IPv6 address should be wrapped with [] # when used as Host header - + host_enc = self._wrap_ipv6(host_enc) if ":" in host: - host_enc = b'[' + host_enc + b']' host_enc = _strip_ipv6_iface(host_enc) if port == self.default_port: diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 6ad383f460c7ee2..7bfa0932500d81c 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -166,8 +166,9 @@ def __init__(self, flist=None, filename=None, key=None, root=None): text.bind("<3>",self.right_menu_event) text.bind('<MouseWheel>', wheel_event) - text.bind('<Button-4>', wheel_event) - text.bind('<Button-5>', wheel_event) + if text._windowingsystem == 'x11': + text.bind('<Button-4>', wheel_event) + text.bind('<Button-5>', wheel_event) text.bind('<Configure>', self.handle_winconfig) text.bind("<<cut>>", self.cut) text.bind("<<copy>>", self.copy) @@ -1043,7 +1044,9 @@ def open_recent_file(fn_closure=file_name): def saved_change_hook(self): short = self.short_title() long = self.long_title() - if short and long: + if short and long and not macosx.isCocoaTk(): + # Don't use both values on macOS because + # that doesn't match platform conventions. title = short + " - " + long + _py_version elif short: title = short @@ -1058,6 +1061,13 @@ def saved_change_hook(self): self.top.wm_title(title) self.top.wm_iconname(icon) + if macosx.isCocoaTk(): + # Add a proxy icon to the window title + self.top.wm_attributes("-titlepath", long) + + # Maintain the modification status for the window + self.top.wm_attributes("-modified", not self.get_saved()) + def get_saved(self): return self.undo.get_saved() diff --git a/Lib/idlelib/idle_test/test_sidebar.py b/Lib/idlelib/idle_test/test_sidebar.py index fb52b3a0179553d..605e7a892570d7d 100644 --- a/Lib/idlelib/idle_test/test_sidebar.py +++ b/Lib/idlelib/idle_test/test_sidebar.py @@ -690,16 +690,22 @@ def test_mousewheel(self): last_lineno = get_end_linenumber(text) self.assertIsNotNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) - # Scroll up using the <MouseWheel> event. - # The meaning of delta is platform-dependent. - delta = -1 if sys.platform == 'darwin' else 120 - sidebar.canvas.event_generate('<MouseWheel>', x=0, y=0, delta=delta) + # Delta for <MouseWheel>, whose meaning is platform-dependent. + delta = 1 if sidebar.canvas._windowingsystem == 'aqua' else 120 + + # Scroll up. + if sidebar.canvas._windowingsystem == 'x11': + sidebar.canvas.event_generate('<Button-4>', x=0, y=0) + else: + sidebar.canvas.event_generate('<MouseWheel>', x=0, y=0, delta=delta) yield - if sys.platform != 'darwin': # .update_idletasks() does not work. - self.assertIsNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) + self.assertIsNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) - # Scroll back down using the <Button-5> event. - sidebar.canvas.event_generate('<Button-5>', x=0, y=0) + # Scroll back down. + if sidebar.canvas._windowingsystem == 'x11': + sidebar.canvas.event_generate('<Button-5>', x=0, y=0) + else: + sidebar.canvas.event_generate('<MouseWheel>', x=0, y=0, delta=-delta) yield self.assertIsNotNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) diff --git a/Lib/idlelib/sidebar.py b/Lib/idlelib/sidebar.py index ff77b568a786e0f..aa19a24e3edef28 100644 --- a/Lib/idlelib/sidebar.py +++ b/Lib/idlelib/sidebar.py @@ -27,7 +27,7 @@ def get_displaylines(text, index): """Display height, in lines, of a logical line in a Tk text widget.""" return text.count(f"{index} linestart", f"{index} lineend", - "displaylines") + "displaylines", return_ints=True) def get_widget_padding(widget): """Get the total padding of a Tk widget, including its border.""" diff --git a/Lib/idlelib/tree.py b/Lib/idlelib/tree.py index 9c2eb47b24aec94..0726d7e23660f64 100644 --- a/Lib/idlelib/tree.py +++ b/Lib/idlelib/tree.py @@ -285,8 +285,9 @@ def drawtext(self): self.label.bind("<1>", self.select_or_edit) self.label.bind("<Double-1>", self.flip) self.label.bind("<MouseWheel>", lambda e: wheel_event(e, self.canvas)) - self.label.bind("<Button-4>", lambda e: wheel_event(e, self.canvas)) - self.label.bind("<Button-5>", lambda e: wheel_event(e, self.canvas)) + if self.label._windowingsystem == 'x11': + self.label.bind("<Button-4>", lambda e: wheel_event(e, self.canvas)) + self.label.bind("<Button-5>", lambda e: wheel_event(e, self.canvas)) self.text_id = id def select_or_edit(self, event=None): @@ -460,8 +461,9 @@ def __init__(self, master, **opts): self.canvas.bind("<Key-Up>", self.unit_up) self.canvas.bind("<Key-Down>", self.unit_down) self.canvas.bind("<MouseWheel>", wheel_event) - self.canvas.bind("<Button-4>", wheel_event) - self.canvas.bind("<Button-5>", wheel_event) + if self.canvas._windowingsystem == 'x11': + self.canvas.bind("<Button-4>", wheel_event) + self.canvas.bind("<Button-5>", wheel_event) #if isinstance(master, Toplevel) or isinstance(master, Tk): self.canvas.bind("<Alt-Key-2>", self.zoom_height) self.canvas.focus_set() diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index d942045f3de666f..6d6292f95592534 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -53,7 +53,7 @@ def _new_module(name): # For a list that can have a weakref to it. class _List(list): - pass + __slots__ = ("__weakref__",) # Copied from weakref.py with some simplifications and modifications unique to diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index 7b142e786e829ef..c612fbefee2e802 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -1,7 +1,6 @@ import os import re import abc -import csv import sys import json import email @@ -478,6 +477,10 @@ def make_file(name, hash=None, size_str=None): @pass_none def make_files(lines): + # Delay csv import, since Distribution.files is not as widely used + # as other parts of importlib.metadata + import csv + return starmap(make_file, csv.reader(lines)) @pass_none diff --git a/Lib/importlib/resources/simple.py b/Lib/importlib/resources/simple.py index 7770c922c84fabe..96f117fec62c102 100644 --- a/Lib/importlib/resources/simple.py +++ b/Lib/importlib/resources/simple.py @@ -88,7 +88,7 @@ def is_dir(self): def open(self, mode='r', *args, **kwargs): stream = self.parent.reader.open_binary(self.name) if 'b' not in mode: - stream = io.TextIOWrapper(*args, **kwargs) + stream = io.TextIOWrapper(stream, *args, **kwargs) return stream def joinpath(self, name): diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 3ad71d31c2f4384..ff4f12fb1af70cd 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -13,6 +13,7 @@ import _imp import sys +import threading import types @@ -171,36 +172,54 @@ class _LazyModule(types.ModuleType): def __getattribute__(self, attr): """Trigger the load of the module and return the attribute.""" - # All module metadata must be garnered from __spec__ in order to avoid - # using mutated values. - # Stop triggering this method. - self.__class__ = types.ModuleType - # Get the original name to make sure no object substitution occurred - # in sys.modules. - original_name = self.__spec__.name - # Figure out exactly what attributes were mutated between the creation - # of the module and now. - attrs_then = self.__spec__.loader_state['__dict__'] - attrs_now = self.__dict__ - attrs_updated = {} - for key, value in attrs_now.items(): - # Code that set the attribute may have kept a reference to the - # assigned object, making identity more important than equality. - if key not in attrs_then: - attrs_updated[key] = value - elif id(attrs_now[key]) != id(attrs_then[key]): - attrs_updated[key] = value - self.__spec__.loader.exec_module(self) - # If exec_module() was used directly there is no guarantee the module - # object was put into sys.modules. - if original_name in sys.modules: - if id(self) != id(sys.modules[original_name]): - raise ValueError(f"module object for {original_name!r} " - "substituted in sys.modules during a lazy " - "load") - # Update after loading since that's what would happen in an eager - # loading situation. - self.__dict__.update(attrs_updated) + __spec__ = object.__getattribute__(self, '__spec__') + loader_state = __spec__.loader_state + with loader_state['lock']: + # Only the first thread to get the lock should trigger the load + # and reset the module's class. The rest can now getattr(). + if object.__getattribute__(self, '__class__') is _LazyModule: + # The first thread comes here multiple times as it descends the + # call stack. The first time, it sets is_loading and triggers + # exec_module(), which will access module.__dict__, module.__name__, + # and/or module.__spec__, reentering this method. These accesses + # need to be allowed to proceed without triggering the load again. + if loader_state['is_loading'] and attr.startswith('__') and attr.endswith('__'): + return object.__getattribute__(self, attr) + loader_state['is_loading'] = True + + __dict__ = object.__getattribute__(self, '__dict__') + + # All module metadata must be gathered from __spec__ in order to avoid + # using mutated values. + # Get the original name to make sure no object substitution occurred + # in sys.modules. + original_name = __spec__.name + # Figure out exactly what attributes were mutated between the creation + # of the module and now. + attrs_then = loader_state['__dict__'] + attrs_now = __dict__ + attrs_updated = {} + for key, value in attrs_now.items(): + # Code that set an attribute may have kept a reference to the + # assigned object, making identity more important than equality. + if key not in attrs_then: + attrs_updated[key] = value + elif id(attrs_now[key]) != id(attrs_then[key]): + attrs_updated[key] = value + __spec__.loader.exec_module(self) + # If exec_module() was used directly there is no guarantee the module + # object was put into sys.modules. + if original_name in sys.modules: + if id(self) != id(sys.modules[original_name]): + raise ValueError(f"module object for {original_name!r} " + "substituted in sys.modules during a lazy " + "load") + # Update after loading since that's what would happen in an eager + # loading situation. + __dict__.update(attrs_updated) + # Finally, stop triggering this method. + self.__class__ = types.ModuleType + return getattr(self, attr) def __delattr__(self, attr): @@ -244,5 +263,7 @@ def exec_module(self, module): loader_state = {} loader_state['__dict__'] = module.__dict__.copy() loader_state['__class__'] = module.__class__ + loader_state['lock'] = threading.RLock() + loader_state['is_loading'] = False module.__spec__.loader_state = loader_state module.__class__ = _LazyModule diff --git a/Lib/inspect.py b/Lib/inspect.py index f0b72662a9a0b21..9191d4761b2c768 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -383,8 +383,10 @@ def isfunction(object): def _has_code_flag(f, flag): """Return true if ``f`` is a function (or a method or functools.partial - wrapper wrapping a function) whose code object has the given ``flag`` + wrapper wrapping a function or a functools.partialmethod wrapping a + function) whose code object has the given ``flag`` set in its flags.""" + f = functools._unwrap_partialmethod(f) while ismethod(f): f = f.__func__ f = functools._unwrap_partial(f) @@ -760,18 +762,14 @@ def unwrap(func, *, stop=None): :exc:`ValueError` is raised if a cycle is encountered. """ - if stop is None: - def _is_wrapper(f): - return hasattr(f, '__wrapped__') - else: - def _is_wrapper(f): - return hasattr(f, '__wrapped__') and not stop(f) f = func # remember the original func for error reporting # Memoise by id to tolerate non-hashable objects, but store objects to # ensure they aren't destroyed, which would allow their IDs to be reused. memo = {id(f): f} recursion_limit = sys.getrecursionlimit() - while _is_wrapper(func): + while not isinstance(func, type) and hasattr(func, '__wrapped__'): + if stop is not None and stop(func): + break func = func.__wrapped__ id_func = id(func) if (id_func in memo) or (len(memo) >= recursion_limit): @@ -832,9 +830,8 @@ def _finddoc(obj): cls = self.__class__ # Should be tested before isdatadescriptor(). elif isinstance(obj, property): - func = obj.fget - name = func.__name__ - cls = _findclass(func) + name = obj.__name__ + cls = _findclass(obj.fget) if cls is None or getattr(cls, name) is not obj: return None elif ismethoddescriptor(obj) or isdatadescriptor(obj): @@ -2561,7 +2558,7 @@ def _signature_from_callable(obj, *, return sig try: - partialmethod = obj._partialmethod + partialmethod = obj.__partialmethod__ except AttributeError: pass else: diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 45f547741885a86..597849eca0524a3 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -174,7 +174,7 @@ def default(self, o): else: return list(iterable) # Let the base class default method raise the TypeError - return JSONEncoder.default(self, o) + return super().default(o) """ raise TypeError(f'Object of type {o.__class__.__name__} ' diff --git a/Lib/linecache.py b/Lib/linecache.py index 329a14053458b7b..04c8f45a6c60ca2 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -166,13 +166,11 @@ def lazycache(filename, module_globals): return False # Try for a __loader__, if available if module_globals and '__name__' in module_globals: - name = module_globals['__name__'] - if (loader := module_globals.get('__loader__')) is None: - if spec := module_globals.get('__spec__'): - try: - loader = spec.loader - except AttributeError: - pass + spec = module_globals.get('__spec__') + name = getattr(spec, 'name', None) or module_globals['__name__'] + loader = getattr(spec, 'loader', None) + if loader is None: + loader = module_globals.get('__loader__') get_source = getattr(loader, 'get_source', None) if name and get_source: diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 684b58d5548f91f..fcec9e76b986618 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -1949,18 +1949,11 @@ def hasHandlers(self): """ return self.logger.hasHandlers() - def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False): + def _log(self, level, msg, args, **kwargs): """ Low-level log implementation, proxied to allow nested logger adapters. """ - return self.logger._log( - level, - msg, - args, - exc_info=exc_info, - extra=extra, - stack_info=stack_info, - ) + return self.logger._log(level, msg, args, **kwargs) @property def manager(self): diff --git a/Lib/logging/config.py b/Lib/logging/config.py index de06090942d9658..ea37dd7544564a2 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -667,10 +667,9 @@ def configure_formatter(self, config): except TypeError as te: if "'format'" not in str(te): raise - #Name of parameter changed from fmt to format. - #Retry with old name. - #This is so that code can be used with older Python versions - #(e.g. by Django) + # logging.Formatter and its subclasses expect the `fmt` + # parameter instead of `format`. Retry passing configuration + # with `fmt`. config['fmt'] = config.pop('format') config['()'] = factory result = self.configure_custom(config) diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 9840b7b0aeba884..895f11c3392253e 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -369,16 +369,20 @@ def getFilesToDelete(self): dirName, baseName = os.path.split(self.baseFilename) fileNames = os.listdir(dirName) result = [] - # See bpo-44753: Don't use the extension when computing the prefix. - n, e = os.path.splitext(baseName) - prefix = n + '.' - plen = len(prefix) - for fileName in fileNames: - if self.namer is None: - # Our files will always start with baseName - if not fileName.startswith(baseName): - continue - else: + if self.namer is None: + prefix = baseName + '.' + plen = len(prefix) + for fileName in fileNames: + if fileName[:plen] == prefix: + suffix = fileName[plen:] + if self.extMatch.match(suffix): + result.append(os.path.join(dirName, fileName)) + else: + # See bpo-44753: Don't use the extension when computing the prefix. + n, e = os.path.splitext(baseName) + prefix = n + '.' + plen = len(prefix) + for fileName in fileNames: # Our files could be just about anything after custom naming, but # likely candidates are of the form # foo.log.DATETIME_SUFFIX or foo.DATETIME_SUFFIX.log @@ -386,15 +390,16 @@ def getFilesToDelete(self): len(fileName) > (plen + 1) and not fileName[plen+1].isdigit()): continue - if fileName[:plen] == prefix: - suffix = fileName[plen:] - # See bpo-45628: The date/time suffix could be anywhere in the - # filename - parts = suffix.split('.') - for part in parts: - if self.extMatch.match(part): - result.append(os.path.join(dirName, fileName)) - break + if fileName[:plen] == prefix: + suffix = fileName[plen:] + # See bpo-45628: The date/time suffix could be anywhere in the + # filename + + parts = suffix.split('.') + for part in parts: + if self.extMatch.match(part): + result.append(os.path.join(dirName, fileName)) + break if len(result) < self.backupCount: result = [] else: @@ -1586,6 +1591,7 @@ def stop(self): Note that if you don't call this before your application exits, there may be some records still left on the queue, which won't be processed. """ - self.enqueue_sentinel() - self._thread.join() - self._thread = None + if self._thread: # see gh-114706 - allow calling this more than once + self.enqueue_sentinel() + self._thread.join() + self._thread = None diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 81ea210cf815a48..746811bd559412d 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -830,10 +830,11 @@ def get_message(self, key): """Return a Message representation or raise a KeyError.""" start, stop = self._lookup(key) self._file.seek(start) - from_line = self._file.readline().replace(linesep, b'') + from_line = self._file.readline().replace(linesep, b'').decode('ascii') string = self._file.read(stop - self._file.tell()) msg = self._message_factory(string.replace(linesep, b'\n')) - msg.set_from(from_line[5:].decode('ascii')) + msg.set_unixfrom(from_line) + msg.set_from(from_line[5:]) return msg def get_string(self, key, from_=False): diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 37228de4828de59..51b99701c9d727b 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -528,6 +528,7 @@ def _default_mime_types(): '.tiff' : 'image/tiff', '.tif' : 'image/tiff', '.ico' : 'image/vnd.microsoft.icon', + '.webp' : 'image/webp', '.ras' : 'image/x-cmu-raster', '.pnm' : 'image/x-portable-anymap', '.pbm' : 'image/x-portable-bitmap', @@ -587,7 +588,6 @@ def _default_mime_types(): '.pict': 'image/pict', '.pct' : 'image/pict', '.pic' : 'image/pict', - '.webp': 'image/webp', '.xul' : 'text/xul', } diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index dbbf106f6809649..b7e1e132172d020 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -19,7 +19,6 @@ import tempfile import itertools -import _multiprocessing from . import util @@ -28,6 +27,7 @@ _ForkingPickler = reduction.ForkingPickler try: + import _multiprocessing import _winapi from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE except ImportError: @@ -476,8 +476,9 @@ def accept(self): ''' if self._listener is None: raise OSError('listener is closed') + c = self._listener.accept() - if self._authkey: + if self._authkey is not None: deliver_challenge(c, self._authkey) answer_challenge(c, self._authkey) return c @@ -1011,8 +1012,20 @@ def _exhaustive_wait(handles, timeout): # returning the first signalled might create starvation issues.) L = list(handles) ready = [] + # Windows limits WaitForMultipleObjects at 64 handles, and we use a + # few for synchronisation, so we switch to batched waits at 60. + if len(L) > 60: + try: + res = _winapi.BatchedWaitForMultipleObjects(L, False, timeout) + except TimeoutError: + return [] + ready.extend(L[i] for i in res) + if res: + L = [h for i, h in enumerate(L) if i > res[0] & i not in res] + timeout = 0 while L: - res = _winapi.WaitForMultipleObjects(L, False, timeout) + short_L = L[:60] if len(L) > 60 else L + res = _winapi.WaitForMultipleObjects(short_L, False, timeout) if res == WAIT_TIMEOUT: break elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L): diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 852ae87b2768612..925f043900004e7 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -20,8 +20,6 @@ from queue import Empty, Full -import _multiprocessing - from . import connection from . import context _ForkingPickler = context.reduction.ForkingPickler diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 8e41f461cc934e0..20ddd9c50e3d884 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -29,8 +29,12 @@ _HAVE_SIGMASK = hasattr(signal, 'pthread_sigmask') _IGNORED_SIGNALS = (signal.SIGINT, signal.SIGTERM) +def cleanup_noop(name): + raise RuntimeError('noop should never be registered or cleaned up') + _CLEANUP_FUNCS = { - 'noop': lambda: None, + 'noop': cleanup_noop, + 'dummy': lambda name: None, # Dummy resource used in tests } if os.name == 'posix': @@ -61,6 +65,7 @@ def __init__(self): self._lock = threading.RLock() self._fd = None self._pid = None + self._exitcode = None def _reentrant_call_error(self): # gh-109629: this happens if an explicit call to the ResourceTracker @@ -84,9 +89,16 @@ def _stop(self): os.close(self._fd) self._fd = None - os.waitpid(self._pid, 0) + _, status = os.waitpid(self._pid, 0) + self._pid = None + try: + self._exitcode = os.waitstatus_to_exitcode(status) + except ValueError: + # os.waitstatus_to_exitcode may raise an exception for invalid values + self._exitcode = None + def getfd(self): self.ensure_running() return self._fd @@ -119,6 +131,7 @@ def ensure_running(self): pass self._fd = None self._pid = None + self._exitcode = None warnings.warn('resource_tracker: process died unexpectedly, ' 'relaunching. Some resources might leak.') @@ -221,6 +234,8 @@ def main(fd): pass cache = {rtype: set() for rtype in _CLEANUP_FUNCS.keys()} + exit_code = 0 + try: # keep track of registered/unregistered resources with open(fd, 'rb') as f: @@ -242,6 +257,7 @@ def main(fd): else: raise RuntimeError('unrecognized command %r' % cmd) except Exception: + exit_code = 3 try: sys.excepthook(*sys.exc_info()) except: @@ -251,10 +267,17 @@ def main(fd): for rtype, rtype_cache in cache.items(): if rtype_cache: try: - warnings.warn( - f'resource_tracker: There appear to be {len(rtype_cache)} ' - f'leaked {rtype} objects to clean up at shutdown: {rtype_cache}' - ) + exit_code = 1 + if rtype == 'dummy': + # The test 'dummy' resource is expected to leak. + # We skip the warning (and *only* the warning) for it. + pass + else: + warnings.warn( + f'resource_tracker: There appear to be ' + f'{len(rtype_cache)} leaked {rtype} objects to ' + f'clean up at shutdown: {rtype_cache}' + ) except Exception: pass for name in rtype_cache: @@ -265,6 +288,9 @@ def main(fd): try: _CLEANUP_FUNCS[rtype](name) except Exception as e: + exit_code = 2 warnings.warn('resource_tracker: %r: %s' % (name, e)) finally: pass + + sys.exit(exit_code) diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index cc159edab5796f5..46834b1a76a6ebd 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -465,14 +465,6 @@ def _pattern_stack(self): elif pattern[-1] in (self.pathmod.sep, self.pathmod.altsep): # GH-65238: pathlib doesn't preserve trailing slash. Add it back. parts.append('') - elif parts[-1] == '**': - # GH-70303: '**' only matches directories. Add trailing slash. - warnings.warn( - "Pattern ending '**' will match files and directories in a " - "future Python release. Add a trailing slash to match only " - "directories and remove this warning.", - FutureWarning, 4) - parts.append('') parts.reverse() return parts @@ -522,9 +514,8 @@ class Path(_abc.PathBase, PurePath): as_uri = PurePath.as_uri @classmethod - def _unsupported(cls, method_name): - msg = f"{cls.__name__}.{method_name}() is unsupported on this system" - raise UnsupportedOperation(msg) + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported on this system" def __init__(self, *args, **kwargs): if kwargs: @@ -596,9 +587,13 @@ def iterdir(self): def _scandir(self): return os.scandir(self) - def _make_child_entry(self, entry): + def _direntry_str(self, entry): + # Transform an entry yielded from _scandir() into a path string. + return entry.name if str(self) == '.' else entry.path + + def _make_child_direntry(self, entry): # Transform an entry yielded from _scandir() into a path object. - path_str = entry.name if str(self) == '.' else entry.path + path_str = self._direntry_str(entry) path = self.with_segments(path_str) path._str = path_str path._drv = self.drive diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index ad5684829ebc800..44fea525b6cac74 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -86,19 +86,29 @@ def _select_children(parent_paths, dir_only, follow_symlinks, match): continue except OSError: continue - if match(entry.name): - yield parent_path._make_child_entry(entry) + # Avoid cost of making a path object for non-matching paths by + # matching against the os.DirEntry.name string. + if match is None or match(entry.name): + yield parent_path._make_child_direntry(entry) -def _select_recursive(parent_paths, dir_only, follow_symlinks): - """Yield given paths and all their subdirectories, recursively.""" +def _select_recursive(parent_paths, dir_only, follow_symlinks, match): + """Yield given paths and all their children, recursively, filtering by + string and type. + """ if follow_symlinks is None: follow_symlinks = False for parent_path in parent_paths: - paths = [parent_path] + if match is not None: + # If we're filtering paths through a regex, record the length of + # the parent path. We'll pass it to match(path, pos=...) later. + parent_len = len(str(parent_path._make_child_relpath('_'))) - 1 + paths = [parent_path._make_child_relpath('')] while paths: path = paths.pop() - yield path + if match is None or match(str(path), parent_len): + # Yield *directory* path that matches pattern (if any). + yield path try: # We must close the scandir() object before proceeding to # avoid exhausting file descriptors when globbing deep trees. @@ -108,14 +118,22 @@ def _select_recursive(parent_paths, dir_only, follow_symlinks): pass else: for entry in entries: + # Handle directory entry. try: if entry.is_dir(follow_symlinks=follow_symlinks): - paths.append(path._make_child_entry(entry)) + # Recurse into this directory. + paths.append(path._make_child_direntry(entry)) continue except OSError: pass + + # Handle file entry. if not dir_only: - yield path._make_child_entry(entry) + # Avoid cost of making a path object for non-matching + # files by matching against the os.DirEntry object. + if match is None or match(path._direntry_str(entry), parent_len): + # Yield *file* path that matches pattern (if any). + yield path._make_child_direntry(entry) def _select_unique(paths): @@ -149,39 +167,39 @@ class PathModuleBase: """ @classmethod - def _unsupported(cls, attr): - raise UnsupportedOperation(f"{cls.__name__}.{attr} is unsupported") + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported" @property def sep(self): """The character used to separate path components.""" - self._unsupported('sep') + raise UnsupportedOperation(self._unsupported_msg('sep')) def join(self, path, *paths): """Join path segments.""" - self._unsupported('join()') + raise UnsupportedOperation(self._unsupported_msg('join()')) def split(self, path): """Split the path into a pair (head, tail), where *head* is everything before the final path separator, and *tail* is everything after. Either part may be empty. """ - self._unsupported('split()') + raise UnsupportedOperation(self._unsupported_msg('split()')) def splitdrive(self, path): """Split the path into a 2-item tuple (drive, tail), where *drive* is a device name or mount point, and *tail* is everything after the drive. Either part may be empty.""" - self._unsupported('splitdrive()') + raise UnsupportedOperation(self._unsupported_msg('splitdrive()')) def normcase(self, path): """Normalize the case of the path.""" - self._unsupported('normcase()') + raise UnsupportedOperation(self._unsupported_msg('normcase()')) def isabs(self, path): """Returns whether the path is absolute, i.e. unaffected by the current directory or drive.""" - self._unsupported('isabs()') + raise UnsupportedOperation(self._unsupported_msg('isabs()')) class PurePathBase: @@ -207,6 +225,9 @@ class PurePathBase: def __init__(self, path, *paths): self._raw_path = self.pathmod.join(path, *paths) if paths else path + if not isinstance(self._raw_path, str): + raise TypeError( + f"path should be a str, not {type(self._raw_path).__name__!r}") self._resolving = False def with_segments(self, *pathsegments): @@ -292,17 +313,28 @@ def with_name(self, name): def with_stem(self, stem): """Return a new path with the stem changed.""" - return self.with_name(stem + self.suffix) + suffix = self.suffix + if not suffix: + return self.with_name(stem) + elif not stem: + # If the suffix is non-empty, we can't make the stem empty. + raise ValueError(f"{self!r} has a non-empty suffix") + else: + return self.with_name(stem + suffix) def with_suffix(self, suffix): """Return a new path with the file suffix changed. If the path has no suffix, add given suffix. If the given suffix is an empty string, remove the suffix from the path. """ + stem = self.stem if not suffix: - return self.with_name(self.stem) + return self.with_name(stem) + elif not stem: + # If the stem is empty, we can't make the suffix non-empty. + raise ValueError(f"{self!r} has an empty name") elif suffix.startswith('.') and len(suffix) > 1: - return self.with_name(self.stem + suffix) + return self.with_name(stem + suffix) else: raise ValueError(f"Invalid suffix {suffix!r}") @@ -318,8 +350,6 @@ def relative_to(self, other, *, walk_up=False): other = self.with_segments(other) anchor0, parts0 = self._stack anchor1, parts1 = other._stack - if isinstance(anchor0, str) != isinstance(anchor1, str): - raise TypeError(f"{self._raw_path!r} and {other._raw_path!r} have different types") if anchor0 != anchor1: raise ValueError(f"{self._raw_path!r} and {other._raw_path!r} have different anchors") while parts0 and parts1 and parts0[-1] == parts1[-1]: @@ -343,8 +373,6 @@ def is_relative_to(self, other): other = self.with_segments(other) anchor0, parts0 = self._stack anchor1, parts1 = other._stack - if isinstance(anchor0, str) != isinstance(anchor1, str): - raise TypeError(f"{self._raw_path!r} and {other._raw_path!r} have different types") if anchor0 != anchor1: return False while parts0 and parts1 and parts0[-1] == parts1[-1]: @@ -502,16 +530,15 @@ class PathBase(PurePathBase): _max_symlinks = 40 @classmethod - def _unsupported(cls, method_name): - msg = f"{cls.__name__}.{method_name}() is unsupported" - raise UnsupportedOperation(msg) + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported" def stat(self, *, follow_symlinks=True): """ Return the result of the stat() system call on this path, like os.stat() does. """ - self._unsupported("stat") + raise UnsupportedOperation(self._unsupported_msg('stat()')) def lstat(self): """ @@ -700,7 +727,7 @@ def open(self, mode='r', buffering=-1, encoding=None, Open the file pointed by this path and return a file object, as the built-in open() function does. """ - self._unsupported("open") + raise UnsupportedOperation(self._unsupported_msg('open()')) def read_bytes(self): """ @@ -741,7 +768,7 @@ def iterdir(self): The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. """ - self._unsupported("iterdir") + raise UnsupportedOperation(self._unsupported_msg('iterdir()')) def _scandir(self): # Emulate os.scandir(), which returns an object that can be used as a @@ -749,8 +776,14 @@ def _scandir(self): from contextlib import nullcontext return nullcontext(self.iterdir()) - def _make_child_entry(self, entry): + def _direntry_str(self, entry): + # Transform an entry yielded from _scandir() into a path string. + # PathBase._scandir() yields PathBase objects, so use str(). + return str(entry) + + def _make_child_direntry(self, entry): # Transform an entry yielded from _scandir() into a path object. + # PathBase._scandir() yields PathBase objects, so this is a no-op. return entry def _make_child_relpath(self, name): @@ -768,43 +801,49 @@ def glob(self, pattern, *, case_sensitive=None, follow_symlinks=None): stack = pattern._pattern_stack specials = ('', '.', '..') - filter_paths = False deduplicate_paths = False sep = self.pathmod.sep paths = iter([self] if self.is_dir() else []) while stack: part = stack.pop() if part in specials: + # Join special component (e.g. '..') onto paths. paths = _select_special(paths, part) + elif part == '**': - # Consume adjacent '**' components. + # Consume following '**' components, which have no effect. while stack and stack[-1] == '**': stack.pop() - # Consume adjacent non-special components and enable post-walk - # regex filtering, provided we're treating symlinks consistently. + # Consume following non-special components, provided we're + # treating symlinks consistently. Each component is joined + # onto 'part', which is used to generate an re.Pattern object. if follow_symlinks is not None: while stack and stack[-1] not in specials: - filter_paths = True - stack.pop() + part += sep + stack.pop() + + # If the previous loop consumed pattern components, compile an + # re.Pattern object based on those components. + match = _compile_pattern(part, sep, case_sensitive) if part != '**' else None + + # Recursively walk directories, filtering by type and regex. + paths = _select_recursive(paths, bool(stack), follow_symlinks, match) - dir_only = bool(stack) - paths = _select_recursive(paths, dir_only, follow_symlinks) + # De-duplicate if we've already seen a '**' component. if deduplicate_paths: - # De-duplicate if we've already seen a '**' component. paths = _select_unique(paths) deduplicate_paths = True + elif '**' in part: raise ValueError("Invalid pattern: '**' can only be an entire path component") + else: - dir_only = bool(stack) - match = _compile_pattern(part, sep, case_sensitive) - paths = _select_children(paths, dir_only, follow_symlinks, match) - if filter_paths: - # Filter out paths that don't match pattern. - prefix_len = len(str(self._make_child_relpath('_'))) - 1 - match = _compile_pattern(pattern._pattern_str, sep, case_sensitive) - paths = (path for path in paths if match(path._pattern_str, prefix_len)) + # If the pattern component isn't '*', compile an re.Pattern + # object based on the component. + match = _compile_pattern(part, sep, case_sensitive) if part != '*' else None + + # Iterate over directories' children filtering by type and regex. + paths = _select_children(paths, bool(stack), follow_symlinks, match) return paths def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=None): @@ -853,7 +892,7 @@ def walk(self, top_down=True, on_error=None, follow_symlinks=False): if is_dir: if not top_down: - paths.append(path._make_child_entry(entry)) + paths.append(path._make_child_direntry(entry)) dirnames.append(entry.name) else: filenames.append(entry.name) @@ -868,7 +907,7 @@ def absolute(self): Use resolve() to resolve symlinks and remove '..' segments. """ - self._unsupported("absolute") + raise UnsupportedOperation(self._unsupported_msg('absolute()')) @classmethod def cwd(cls): @@ -883,7 +922,7 @@ def expanduser(self): """ Return a new path with expanded ~ and ~user constructs (as returned by os.path.expanduser) """ - self._unsupported("expanduser") + raise UnsupportedOperation(self._unsupported_msg('expanduser()')) @classmethod def home(cls): @@ -895,7 +934,7 @@ def readlink(self): """ Return the path to which the symbolic link points. """ - self._unsupported("readlink") + raise UnsupportedOperation(self._unsupported_msg('readlink()')) readlink._supported = False def resolve(self, strict=False): @@ -970,7 +1009,7 @@ def symlink_to(self, target, target_is_directory=False): Make this path a symlink pointing to the target path. Note the order of arguments (link, target) is the reverse of os.symlink. """ - self._unsupported("symlink_to") + raise UnsupportedOperation(self._unsupported_msg('symlink_to()')) def hardlink_to(self, target): """ @@ -978,19 +1017,19 @@ def hardlink_to(self, target): Note the order of arguments (self, target) is the reverse of os.link's. """ - self._unsupported("hardlink_to") + raise UnsupportedOperation(self._unsupported_msg('hardlink_to()')) def touch(self, mode=0o666, exist_ok=True): """ Create this file with the given access mode, if it doesn't exist. """ - self._unsupported("touch") + raise UnsupportedOperation(self._unsupported_msg('touch()')) def mkdir(self, mode=0o777, parents=False, exist_ok=False): """ Create a new directory at this given path. """ - self._unsupported("mkdir") + raise UnsupportedOperation(self._unsupported_msg('mkdir()')) def rename(self, target): """ @@ -1002,7 +1041,7 @@ def rename(self, target): Returns the new Path instance pointing to the target path. """ - self._unsupported("rename") + raise UnsupportedOperation(self._unsupported_msg('rename()')) def replace(self, target): """ @@ -1014,13 +1053,13 @@ def replace(self, target): Returns the new Path instance pointing to the target path. """ - self._unsupported("replace") + raise UnsupportedOperation(self._unsupported_msg('replace()')) def chmod(self, mode, *, follow_symlinks=True): """ Change the permissions of the path, like os.chmod(). """ - self._unsupported("chmod") + raise UnsupportedOperation(self._unsupported_msg('chmod()')) def lchmod(self, mode): """ @@ -1034,31 +1073,31 @@ def unlink(self, missing_ok=False): Remove this file or link. If the path is a directory, use rmdir() instead. """ - self._unsupported("unlink") + raise UnsupportedOperation(self._unsupported_msg('unlink()')) def rmdir(self): """ Remove this directory. The directory must be empty. """ - self._unsupported("rmdir") + raise UnsupportedOperation(self._unsupported_msg('rmdir()')) def owner(self, *, follow_symlinks=True): """ Return the login name of the file owner. """ - self._unsupported("owner") + raise UnsupportedOperation(self._unsupported_msg('owner()')) def group(self, *, follow_symlinks=True): """ Return the group name of the file gid. """ - self._unsupported("group") + raise UnsupportedOperation(self._unsupported_msg('group()')) @classmethod def from_uri(cls, uri): """Return a new path from the given 'file' URI.""" - cls._unsupported("from_uri") + raise UnsupportedOperation(cls._unsupported_msg('from_uri()')) def as_uri(self): """Return the path as a URI.""" - self._unsupported("as_uri") + raise UnsupportedOperation(self._unsupported_msg('as_uri()')) diff --git a/Lib/pdb.py b/Lib/pdb.py index 6f7719eb9ba6c5b..0754e8b628cf570 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -97,17 +97,47 @@ class Restart(Exception): __all__ = ["run", "pm", "Pdb", "runeval", "runctx", "runcall", "set_trace", "post_mortem", "help"] + +def find_first_executable_line(code): + """ Try to find the first executable line of the code object. + + Equivalently, find the line number of the instruction that's + after RESUME + + Return code.co_firstlineno if no executable line is found. + """ + prev = None + for instr in dis.get_instructions(code): + if prev is not None and prev.opname == 'RESUME': + if instr.positions.lineno is not None: + return instr.positions.lineno + return code.co_firstlineno + prev = instr + return code.co_firstlineno + def find_function(funcname, filename): cre = re.compile(r'def\s+%s\s*[(]' % re.escape(funcname)) try: fp = tokenize.open(filename) except OSError: return None + funcdef = "" + funcstart = None # consumer of this info expects the first line to be 1 with fp: for lineno, line in enumerate(fp, start=1): if cre.match(line): - return funcname, filename, lineno + funcstart, funcdef = lineno, line + elif funcdef: + funcdef += line + + if funcdef: + try: + funccode = compile(funcdef, filename, 'exec').co_consts[0] + except SyntaxError: + continue + lineno_offset = find_first_executable_line(funccode) + return funcname, filename, funcstart + lineno_offset - 1 return None def lasti2lineno(code, lasti): @@ -975,7 +1005,7 @@ def do_break(self, arg, temporary = 0): #use co_name to identify the bkpt (function names #could be aliased, but co_name is invariant) funcname = code.co_name - lineno = self._find_first_executable_line(code) + lineno = find_first_executable_line(code) filename = code.co_filename except: # last thing to try @@ -1078,23 +1108,6 @@ def checkline(self, filename, lineno): return 0 return lineno - def _find_first_executable_line(self, code): - """ Try to find the first executable line of the code object. - - Equivalently, find the line number of the instruction that's - after RESUME - - Return code.co_firstlineno if no executable line is found. - """ - prev = None - for instr in dis.get_instructions(code): - if prev is not None and prev.opname == 'RESUME': - if instr.positions.lineno is not None: - return instr.positions.lineno - return code.co_firstlineno - prev = instr - return code.co_firstlineno - def do_enable(self, arg): """enable bpnumber [bpnumber ...] diff --git a/Lib/pickletools.py b/Lib/pickletools.py index 95a77aeb2afe2af..51ee4a7a2632ac0 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -1253,7 +1253,7 @@ def __init__(self, name, code, arg, stack_before=[], stack_after=[pyint], proto=2, - doc="""Long integer using found-byte length. + doc="""Long integer using four-byte length. A more efficient encoding of a Python long; the long4 encoding says it all."""), diff --git a/Lib/posixpath.py b/Lib/posixpath.py index e4f155e41a32216..33943b4403636a8 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -546,10 +546,11 @@ def relpath(path, start=None): def commonpath(paths): """Given a sequence of path names, returns the longest common sub-path.""" + paths = tuple(map(os.fspath, paths)) + if not paths: raise ValueError('commonpath() arg is an empty sequence') - paths = tuple(map(os.fspath, paths)) if isinstance(paths[0], bytes): sep = b'/' curdir = b'.' diff --git a/Lib/profile.py b/Lib/profile.py index 4b82523b03d64b0..f2f8c2f21333e0f 100755 --- a/Lib/profile.py +++ b/Lib/profile.py @@ -387,8 +387,9 @@ def simulate_cmd_complete(self): def print_stats(self, sort=-1): import pstats - pstats.Stats(self).strip_dirs().sort_stats(sort). \ - print_stats() + if not isinstance(sort, tuple): + sort = (sort,) + pstats.Stats(self).strip_dirs().sort_stats(*sort).print_stats() def dump_stats(self, file): with open(file, 'wb') as f: diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 96aa1dfc1aacf64..b0193b4a85164a7 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -127,9 +127,8 @@ def _finddoc(obj): cls = self.__class__ # Should be tested before isdatadescriptor(). elif isinstance(obj, property): - func = obj.fget - name = func.__name__ - cls = _findclass(func) + name = obj.__name__ + cls = _findclass(obj.fget) if cls is None or getattr(cls, name) is not obj: return None elif inspect.ismethoddescriptor(obj) or inspect.isdatadescriptor(obj): @@ -225,6 +224,19 @@ def classname(object, modname): name = object.__module__ + '.' + name return name +def parentname(object, modname): + """Get a name of the enclosing class (qualified it with a module name + if necessary) or module.""" + if '.' in object.__qualname__: + name = object.__qualname__.rpartition('.')[0] + if object.__module__ != modname: + return object.__module__ + '.' + name + else: + return name + else: + if object.__module__ != modname: + return object.__module__ + def isdata(object): """Check if an object is of a type that probably means it's data.""" return not (inspect.ismodule(object) or inspect.isclass(object) or @@ -319,13 +331,15 @@ def visiblename(name, all=None, obj=None): return not name.startswith('_') def classify_class_attrs(object): - """Wrap inspect.classify_class_attrs, with fixup for data descriptors.""" + """Wrap inspect.classify_class_attrs, with fixup for data descriptors and bound methods.""" results = [] for (name, kind, cls, value) in inspect.classify_class_attrs(object): if inspect.isdatadescriptor(value): kind = 'data descriptor' if isinstance(value, property) and value.fset is None: kind = 'readonly property' + elif kind == 'method' and _is_bound_method(value): + kind = 'static method' results.append((name, kind, cls, value)) return results @@ -537,7 +551,7 @@ def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')): '_thread', 'zipimport') or (file.startswith(basedir) and not file.startswith(os.path.join(basedir, 'site-packages')))) and - object.__name__ not in ('xml.etree', 'test.pydoc_mod')): + object.__name__ not in ('xml.etree', 'test.test_pydoc.pydoc_mod')): if docloc.startswith(("http://", "https://")): docloc = "{}/{}.html".format(docloc.rstrip("/"), object.__name__.lower()) else: @@ -681,6 +695,25 @@ def classlink(self, object, modname): module.__name__, name, classname(object, modname)) return classname(object, modname) + def parentlink(self, object, modname): + """Make a link for the enclosing class or module.""" + link = None + name, module = object.__name__, sys.modules.get(object.__module__) + if hasattr(module, name) and getattr(module, name) is object: + if '.' in object.__qualname__: + name = object.__qualname__.rpartition('.')[0] + if object.__module__ != modname: + link = '%s.html#%s' % (module.__name__, name) + else: + link = '#%s' % name + else: + if object.__module__ != modname: + link = '%s.html' % module.__name__ + if link: + return '<a href="%s">%s</a>' % (link, parentname(object, modname)) + else: + return parentname(object, modname) + def modulelink(self, object): """Make a link for a module.""" return '<a href="%s.html">%s</a>' % (object.__name__, object.__name__) @@ -822,9 +855,9 @@ def docmodule(self, object, name=None, mod=None, *ignored): cdict[key] = cdict[base] = modname + '.html#' + key funcs, fdict = [], {} for key, value in inspect.getmembers(object, inspect.isroutine): - # if __all__ exists, believe it. Otherwise use old heuristic. - if (all is not None or - inspect.isbuiltin(value) or inspect.getmodule(value) is object): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) fdict[key] = '#-' + key @@ -925,7 +958,7 @@ def spill(msg, attrs, predicate): push(self.docdata(value, name, mod)) else: push(self.document(value, name, mod, - funcs, classes, mdict, object)) + funcs, classes, mdict, object, homecls)) push('\n') return attrs @@ -1043,24 +1076,44 @@ def formatvalue(self, object): return self.grey('=' + self.repr(object)) def docroutine(self, object, name=None, mod=None, - funcs={}, classes={}, methods={}, cl=None): + funcs={}, classes={}, methods={}, cl=None, homecls=None): """Produce HTML documentation for a function or method object.""" realname = object.__name__ name = name or realname - anchor = (cl and cl.__name__ or '') + '-' + name + if homecls is None: + homecls = cl + anchor = ('' if cl is None else cl.__name__) + '-' + name note = '' - skipdocs = 0 + skipdocs = False + imfunc = None if _is_bound_method(object): - imclass = object.__self__.__class__ - if cl: - if imclass is not cl: - note = ' from ' + self.classlink(imclass, mod) + imself = object.__self__ + if imself is cl: + imfunc = getattr(object, '__func__', None) + elif inspect.isclass(imself): + note = ' class method of %s' % self.classlink(imself, mod) else: - if object.__self__ is not None: - note = ' method of %s instance' % self.classlink( - object.__self__.__class__, mod) - else: - note = ' unbound %s method' % self.classlink(imclass,mod) + note = ' method of %s instance' % self.classlink( + imself.__class__, mod) + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): + try: + objclass = object.__objclass__ + except AttributeError: + pass + else: + if cl is None: + note = ' unbound %s method' % self.classlink(objclass, mod) + elif objclass is not homecls: + note = ' from ' + self.classlink(objclass, mod) + else: + imfunc = object + if inspect.isfunction(imfunc) and homecls is not None and ( + imfunc.__module__ != homecls.__module__ or + imfunc.__qualname__ != homecls.__qualname__ + '.' + realname): + pname = self.parentlink(imfunc, mod) + if pname: + note = ' from %s' % pname if (inspect.iscoroutinefunction(object) or inspect.isasyncgenfunction(object)): @@ -1071,10 +1124,13 @@ def docroutine(self, object, name=None, mod=None, if name == realname: title = '<a name="%s"><strong>%s</strong></a>' % (anchor, realname) else: - if cl and inspect.getattr_static(cl, realname, []) is object: + if (cl is not None and + inspect.getattr_static(cl, realname, []) is object): reallink = '<a href="#%s">%s</a>' % ( cl.__name__ + '-' + realname, realname) - skipdocs = 1 + skipdocs = True + if note.startswith(' from '): + note = '' else: reallink = realname title = '<a name="%s"><strong>%s</strong></a> = %s' % ( @@ -1087,7 +1143,8 @@ def docroutine(self, object, name=None, mod=None, # XXX lambda's won't usually have func_annotations['return'] # since the syntax doesn't support but it is possible. # So removing parentheses isn't truly safe. - argspec = argspec[1:-1] # remove parentheses + if not object.__annotations__: + argspec = argspec[1:-1] # remove parentheses if not argspec: argspec = '(...)' @@ -1102,7 +1159,7 @@ def docroutine(self, object, name=None, mod=None, doc = doc and '<dd><span class="code">%s</span></dd>' % doc return '<dl><dt>%s</dt>%s</dl>\n' % (decl, doc) - def docdata(self, object, name=None, mod=None, cl=None): + def docdata(self, object, name=None, mod=None, cl=None, *ignored): """Produce html documentation for a data descriptor.""" results = [] push = results.append @@ -1213,7 +1270,7 @@ def formattree(self, tree, modname, parent=None, prefix=''): entry, modname, c, prefix + ' ') return result - def docmodule(self, object, name=None, mod=None): + def docmodule(self, object, name=None, mod=None, *ignored): """Produce text documentation for a given module object.""" name = object.__name__ # ignore the passed-in name synop, desc = splitdoc(getdoc(object)) @@ -1242,9 +1299,9 @@ def docmodule(self, object, name=None, mod=None): classes.append((key, value)) funcs = [] for key, value in inspect.getmembers(object, inspect.isroutine): - # if __all__ exists, believe it. Otherwise use old heuristic. - if (all is not None or - inspect.isbuiltin(value) or inspect.getmodule(value) is object): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) data = [] @@ -1392,7 +1449,7 @@ def spill(msg, attrs, predicate): push(self.docdata(value, name, mod)) else: push(self.document(value, - name, mod, object)) + name, mod, object, homecls)) return attrs def spilldescriptors(msg, attrs, predicate): @@ -1467,23 +1524,43 @@ def formatvalue(self, object): """Format an argument default value as text.""" return '=' + self.repr(object) - def docroutine(self, object, name=None, mod=None, cl=None): + def docroutine(self, object, name=None, mod=None, cl=None, homecls=None): """Produce text documentation for a function or method object.""" realname = object.__name__ name = name or realname + if homecls is None: + homecls = cl note = '' - skipdocs = 0 + skipdocs = False + imfunc = None if _is_bound_method(object): - imclass = object.__self__.__class__ - if cl: - if imclass is not cl: - note = ' from ' + classname(imclass, mod) + imself = object.__self__ + if imself is cl: + imfunc = getattr(object, '__func__', None) + elif inspect.isclass(imself): + note = ' class method of %s' % classname(imself, mod) else: - if object.__self__ is not None: - note = ' method of %s instance' % classname( - object.__self__.__class__, mod) - else: - note = ' unbound %s method' % classname(imclass,mod) + note = ' method of %s instance' % classname( + imself.__class__, mod) + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): + try: + objclass = object.__objclass__ + except AttributeError: + pass + else: + if cl is None: + note = ' unbound %s method' % classname(objclass, mod) + elif objclass is not homecls: + note = ' from ' + classname(objclass, mod) + else: + imfunc = object + if inspect.isfunction(imfunc) and homecls is not None and ( + imfunc.__module__ != homecls.__module__ or + imfunc.__qualname__ != homecls.__qualname__ + '.' + realname): + pname = parentname(imfunc, mod) + if pname: + note = ' from %s' % pname if (inspect.iscoroutinefunction(object) or inspect.isasyncgenfunction(object)): @@ -1494,8 +1571,11 @@ def docroutine(self, object, name=None, mod=None, cl=None): if name == realname: title = self.bold(realname) else: - if cl and inspect.getattr_static(cl, realname, []) is object: - skipdocs = 1 + if (cl is not None and + inspect.getattr_static(cl, realname, []) is object): + skipdocs = True + if note.startswith(' from '): + note = '' title = self.bold(name) + ' = ' + realname argspec = None @@ -1506,7 +1586,8 @@ def docroutine(self, object, name=None, mod=None, cl=None): # XXX lambda's won't usually have func_annotations['return'] # since the syntax doesn't support but it is possible. # So removing parentheses isn't truly safe. - argspec = argspec[1:-1] # remove parentheses + if not object.__annotations__: + argspec = argspec[1:-1] if not argspec: argspec = '(...)' decl = asyncqualifier + title + argspec + note @@ -1517,7 +1598,7 @@ def docroutine(self, object, name=None, mod=None, cl=None): doc = getdoc(object) or '' return decl + '\n' + (doc and self.indent(doc).rstrip() + '\n') - def docdata(self, object, name=None, mod=None, cl=None): + def docdata(self, object, name=None, mod=None, cl=None, *ignored): """Produce text documentation for a data descriptor.""" results = [] push = results.append @@ -1533,7 +1614,8 @@ def docdata(self, object, name=None, mod=None, cl=None): docproperty = docdata - def docother(self, object, name=None, mod=None, parent=None, maxlen=None, doc=None): + def docother(self, object, name=None, mod=None, parent=None, *ignored, + maxlen=None, doc=None): """Produce text documentation for a data object.""" repr = self.repr(object) if maxlen: diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index d453c1f038ef075..a49c38a51d39f20 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Wed Jan 17 13:09:41 2024 +# Autogenerated by Sphinx on Thu Feb 15 14:30:52 2024 # as part of the release process. topics = {'assert': 'The "assert" statement\n' '**********************\n' @@ -722,9 +722,9 @@ '\n' 'object.__dir__(self)\n' '\n' - ' Called when "dir()" is called on the object. A ' - 'sequence must be\n' - ' returned. "dir()" converts the returned sequence to a ' + ' Called when "dir()" is called on the object. An ' + 'iterable must be\n' + ' returned. "dir()" converts the returned iterable to a ' 'list and\n' ' sorts it.\n' '\n' @@ -751,8 +751,8 @@ 'returned.\n' '\n' 'The "__dir__" function should accept no arguments, and ' - 'return a\n' - 'sequence of strings that represents the names accessible ' + 'return an\n' + 'iterable of strings that represents the names accessible ' 'on module. If\n' 'present, this function overrides the standard "dir()" ' 'search on a\n' @@ -4921,7 +4921,7 @@ 'and continue running without the debugger using the "continue"\n' 'command.\n' '\n' - 'New in version 3.7: The built-in "breakpoint()", when called ' + 'Changed in version 3.7: The built-in "breakpoint()", when called ' 'with\n' 'defaults, can be used instead of "import pdb; pdb.set_trace()".\n' '\n' @@ -4965,11 +4965,11 @@ 'the\n' 'debugger upon program’s exit.\n' '\n' - 'New in version 3.2: "-c" option is introduced to execute ' + 'Changed in version 3.2: Added the "-c" option to execute ' 'commands as\n' - 'if given in a ".pdbrc" file, see Debugger Commands.\n' + 'if given in a ".pdbrc" file; see Debugger Commands.\n' '\n' - 'New in version 3.7: "-m" option is introduced to execute ' + 'Changed in version 3.7: Added the "-m" option to execute ' 'modules\n' 'similar to the way "python -m" does. As with a script, the ' 'debugger\n' @@ -5115,11 +5115,11 @@ '\n' ' Raises an auditing event "pdb.Pdb" with no arguments.\n' '\n' - ' New in version 3.1: The *skip* argument.\n' + ' Changed in version 3.1: Added the *skip* parameter.\n' '\n' - ' New in version 3.2: The *nosigint* argument. Previously, a ' - 'SIGINT\n' - ' handler was never set by Pdb.\n' + ' Changed in version 3.2: Added the *nosigint* parameter. ' + 'Previously,\n' + ' a SIGINT handler was never set by Pdb.\n' '\n' ' Changed in version 3.6: The *readrc* argument.\n' '\n' @@ -5466,7 +5466,7 @@ 'differs\n' ' from the current line.\n' '\n' - ' New in version 3.2: The ">>" marker.\n' + ' Changed in version 3.2: Added the ">>" marker.\n' '\n' 'll | longlist\n' '\n' @@ -5599,9 +5599,9 @@ '\n' ' New in version 3.2.\n' '\n' - ' New in version 3.13: "exit()" and "quit()" can be used to ' + ' Changed in version 3.13: "exit()" and "quit()" can be used to ' 'exit\n' - ' "interact" command.\n' + ' the "interact" command.\n' '\n' ' Changed in version 3.13: "interact" directs its output to ' 'the\n' @@ -6470,15 +6470,15 @@ 'originally\n' 'proposed by **PEP 448**.\n' '\n' - 'The trailing comma is required only to create a single tuple ' - '(a.k.a. a\n' - '*singleton*); it is optional in all other cases. A single ' - 'expression\n' - 'without a trailing comma doesn’t create a tuple, but rather ' - 'yields the\n' - 'value of that expression. (To create an empty tuple, use an ' - 'empty pair\n' - 'of parentheses: "()".)\n', + 'A trailing comma is required only to create a one-item tuple, ' + 'such as\n' + '"1,"; it is optional in all other cases. A single expression ' + 'without a\n' + 'trailing comma doesn’t create a tuple, but rather yields the ' + 'value of\n' + 'that expression. (To create an empty tuple, use an empty pair ' + 'of\n' + 'parentheses: "()".)\n', 'floating': 'Floating point literals\n' '***********************\n' '\n' @@ -10384,9 +10384,9 @@ '\n' 'object.__dir__(self)\n' '\n' - ' Called when "dir()" is called on the object. A sequence ' + ' Called when "dir()" is called on the object. An iterable ' 'must be\n' - ' returned. "dir()" converts the returned sequence to a ' + ' returned. "dir()" converts the returned iterable to a ' 'list and\n' ' sorts it.\n' '\n' @@ -10413,8 +10413,8 @@ 'returned.\n' '\n' 'The "__dir__" function should accept no arguments, and ' - 'return a\n' - 'sequence of strings that represents the names accessible on ' + 'return an\n' + 'iterable of strings that represents the names accessible on ' 'module. If\n' 'present, this function overrides the standard "dir()" search ' 'on a\n' @@ -14543,7 +14543,9 @@ 'name |\n' '+----------------------------------------------------+----------------------------------------------------+\n' '| codeobject.co_qualname | The fully ' - 'qualified function name |\n' + 'qualified function name New in version |\n' + '| | ' + '3.11. |\n' '+----------------------------------------------------+----------------------------------------------------+\n' '| codeobject.co_argcount | The total ' 'number of positional *parameters* |\n' @@ -15008,8 +15010,8 @@ 'around another object that alters the way in which that object is\n' 'retrieved from classes and class instances. The behaviour of class\n' 'method objects upon such retrieval is described above, under ' - '“User-\n' - 'defined methods”. Class method objects are created by the built-in\n' + '“instance\n' + 'methods”. Class method objects are created by the built-in\n' '"classmethod()" constructor.\n', 'typesfunctions': 'Functions\n' '*********\n' diff --git a/Lib/queue.py b/Lib/queue.py index 55f50088460f9e5..18fad8df08e4696 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -25,6 +25,10 @@ class Full(Exception): pass +class ShutDown(Exception): + '''Raised when put/get with shut-down queue.''' + + class Queue: '''Create a queue object with a given maximum size. @@ -54,6 +58,9 @@ def __init__(self, maxsize=0): self.all_tasks_done = threading.Condition(self.mutex) self.unfinished_tasks = 0 + # Queue shutdown state + self.is_shutdown = False + def task_done(self): '''Indicate that a formerly enqueued task is complete. @@ -65,6 +72,9 @@ def task_done(self): have been processed (meaning that a task_done() call was received for every item that had been put() into the queue). + shutdown(immediate=True) calls task_done() for each remaining item in + the queue. + Raises a ValueError if called more times than there were items placed in the queue. ''' @@ -129,8 +139,12 @@ def put(self, item, block=True, timeout=None): Otherwise ('block' is false), put an item on the queue if a free slot is immediately available, else raise the Full exception ('timeout' is ignored in that case). + + Raises ShutDown if the queue has been shut down. ''' with self.not_full: + if self.is_shutdown: + raise ShutDown if self.maxsize > 0: if not block: if self._qsize() >= self.maxsize: @@ -138,6 +152,8 @@ def put(self, item, block=True, timeout=None): elif timeout is None: while self._qsize() >= self.maxsize: self.not_full.wait() + if self.is_shutdown: + raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: @@ -147,6 +163,8 @@ def put(self, item, block=True, timeout=None): if remaining <= 0.0: raise Full self.not_full.wait(remaining) + if self.is_shutdown: + raise ShutDown self._put(item) self.unfinished_tasks += 1 self.not_empty.notify() @@ -161,14 +179,21 @@ def get(self, block=True, timeout=None): Otherwise ('block' is false), return an item if one is immediately available, else raise the Empty exception ('timeout' is ignored in that case). + + Raises ShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. ''' with self.not_empty: + if self.is_shutdown and not self._qsize(): + raise ShutDown if not block: if not self._qsize(): raise Empty elif timeout is None: while not self._qsize(): self.not_empty.wait() + if self.is_shutdown and not self._qsize(): + raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: @@ -178,6 +203,8 @@ def get(self, block=True, timeout=None): if remaining <= 0.0: raise Empty self.not_empty.wait(remaining) + if self.is_shutdown and not self._qsize(): + raise ShutDown item = self._get() self.not_full.notify() return item @@ -198,6 +225,27 @@ def get_nowait(self): ''' return self.get(block=False) + def shutdown(self, immediate=False): + '''Shut-down the queue, making queue gets and puts raise ShutDown. + + By default, gets will only raise once the queue is empty. Set + 'immediate' to True to make gets raise immediately instead. + + All blocked callers of put() will be unblocked, and also get() + and join() if 'immediate'. + ''' + with self.mutex: + self.is_shutdown = True + if immediate: + while self._qsize(): + self._get() + if self.unfinished_tasks > 0: + self.unfinished_tasks -= 1 + self.not_empty.notify_all() + # release all blocked threads in `join()` + self.all_tasks_done.notify_all() + self.not_full.notify_all() + # Override these methods to implement other queue organizations # (e.g. stack or priority queue). # These will only be called with appropriate locks held diff --git a/Lib/shutil.py b/Lib/shutil.py index acc9419be4dfca6..c19ea0607208afa 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -861,12 +861,12 @@ def move(src, dst, copy_function=copy2): similar to the Unix "mv" command. Return the file or directory's destination. - If the destination is a directory or a symlink to a directory, the source - is moved inside the directory. The destination path must not already - exist. + If dst is an existing directory or a symlink to a directory, then src is + moved inside that directory. The destination path in that directory must + not already exist. - If the destination already exists but is not a directory, it may be - overwritten depending on os.rename() semantics. + If dst already exists but is not a directory, it may be overwritten + depending on os.rename() semantics. If the destination is on our current filesystem, then rename() is used. Otherwise, src is copied to the destination and then removed. Symlinks are diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index 719dfc8947697d4..9dcce7dc76ced48 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -15,7 +15,7 @@ def _quote_value(value): return "'{0}'".format(value.replace("'", "''")) -def _iterdump(connection): +def _iterdump(connection, *, filter=None): """ Returns an iterator to the dump of the database in an SQL text format. @@ -32,15 +32,23 @@ def _iterdump(connection): yield('PRAGMA foreign_keys=OFF;') yield('BEGIN TRANSACTION;') + if filter: + # Return database objects which match the filter pattern. + filter_name_clause = 'AND "name" LIKE ?' + params = [filter] + else: + filter_name_clause = "" + params = [] # sqlite_master table contains the SQL CREATE statements for the database. - q = """ + q = f""" SELECT "name", "type", "sql" FROM "sqlite_master" WHERE "sql" NOT NULL AND "type" == 'table' + {filter_name_clause} ORDER BY "name" """ - schema_res = cu.execute(q) + schema_res = cu.execute(q, params) sqlite_sequence = [] for table_name, type, sql in schema_res.fetchall(): if table_name == 'sqlite_sequence': @@ -82,13 +90,14 @@ def _iterdump(connection): yield("{0};".format(row[0])) # Now when the type is 'index', 'trigger', or 'view' - q = """ + q = f""" SELECT "name", "type", "sql" FROM "sqlite_master" WHERE "sql" NOT NULL AND "type" IN ('index', 'trigger', 'view') + {filter_name_clause} """ - schema_res = cu.execute(q) + schema_res = cu.execute(q, params) for name, type, sql in schema_res.fetchall(): yield('{0};'.format(sql)) diff --git a/Lib/ssl.py b/Lib/ssl.py index 74a9d2d8fd4fb01..03d0121891ff4cb 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -994,71 +994,67 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, if context.check_hostname and not server_hostname: raise ValueError("check_hostname requires server_hostname") + sock_timeout = sock.gettimeout() kwargs = dict( family=sock.family, type=sock.type, proto=sock.proto, fileno=sock.fileno() ) self = cls.__new__(cls, **kwargs) super(SSLSocket, self).__init__(**kwargs) - sock_timeout = sock.gettimeout() sock.detach() - - self._context = context - self._session = session - self._closed = False - self._sslobj = None - self.server_side = server_side - self.server_hostname = context._encode_hostname(server_hostname) - self.do_handshake_on_connect = do_handshake_on_connect - self.suppress_ragged_eofs = suppress_ragged_eofs - - # See if we are connected + # Now SSLSocket is responsible for closing the file descriptor. try: - self.getpeername() - except OSError as e: - if e.errno != errno.ENOTCONN: - raise - connected = False - blocking = self.getblocking() - self.setblocking(False) + self._context = context + self._session = session + self._closed = False + self._sslobj = None + self.server_side = server_side + self.server_hostname = context._encode_hostname(server_hostname) + self.do_handshake_on_connect = do_handshake_on_connect + self.suppress_ragged_eofs = suppress_ragged_eofs + + # See if we are connected try: - # We are not connected so this is not supposed to block, but - # testing revealed otherwise on macOS and Windows so we do - # the non-blocking dance regardless. Our raise when any data - # is found means consuming the data is harmless. - notconn_pre_handshake_data = self.recv(1) + self.getpeername() except OSError as e: - # EINVAL occurs for recv(1) on non-connected on unix sockets. - if e.errno not in (errno.ENOTCONN, errno.EINVAL): + if e.errno != errno.ENOTCONN: raise - notconn_pre_handshake_data = b'' - self.setblocking(blocking) - if notconn_pre_handshake_data: - # This prevents pending data sent to the socket before it was - # closed from escaping to the caller who could otherwise - # presume it came through a successful TLS connection. - reason = "Closed before TLS handshake with data in recv buffer." - notconn_pre_handshake_data_error = SSLError(e.errno, reason) - # Add the SSLError attributes that _ssl.c always adds. - notconn_pre_handshake_data_error.reason = reason - notconn_pre_handshake_data_error.library = None - try: - self.close() - except OSError: - pass + connected = False + blocking = self.getblocking() + self.setblocking(False) try: - raise notconn_pre_handshake_data_error - finally: - # Explicitly break the reference cycle. - notconn_pre_handshake_data_error = None - else: - connected = True + # We are not connected so this is not supposed to block, but + # testing revealed otherwise on macOS and Windows so we do + # the non-blocking dance regardless. Our raise when any data + # is found means consuming the data is harmless. + notconn_pre_handshake_data = self.recv(1) + except OSError as e: + # EINVAL occurs for recv(1) on non-connected on unix sockets. + if e.errno not in (errno.ENOTCONN, errno.EINVAL): + raise + notconn_pre_handshake_data = b'' + self.setblocking(blocking) + if notconn_pre_handshake_data: + # This prevents pending data sent to the socket before it was + # closed from escaping to the caller who could otherwise + # presume it came through a successful TLS connection. + reason = "Closed before TLS handshake with data in recv buffer." + notconn_pre_handshake_data_error = SSLError(e.errno, reason) + # Add the SSLError attributes that _ssl.c always adds. + notconn_pre_handshake_data_error.reason = reason + notconn_pre_handshake_data_error.library = None + try: + raise notconn_pre_handshake_data_error + finally: + # Explicitly break the reference cycle. + notconn_pre_handshake_data_error = None + else: + connected = True - self.settimeout(sock_timeout) # Must come after setblocking() calls. - self._connected = connected - if connected: - # create the SSL object - try: + self.settimeout(sock_timeout) # Must come after setblocking() calls. + self._connected = connected + if connected: + # create the SSL object self._sslobj = self._context._wrap_socket( self, server_side, self.server_hostname, owner=self, session=self._session, @@ -1069,9 +1065,12 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, # non-blocking raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets") self.do_handshake() - except (OSError, ValueError): + except: + try: self.close() - raise + except OSError: + pass + raise return self @property diff --git a/Lib/statistics.py b/Lib/statistics.py index 83aaedb04515e02..7924123c05b8c37 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -112,6 +112,7 @@ 'fmean', 'geometric_mean', 'harmonic_mean', + 'kde', 'linear_regression', 'mean', 'median', @@ -137,7 +138,7 @@ from itertools import count, groupby, repeat from bisect import bisect_left, bisect_right from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum, sumprod -from math import isfinite, isinf +from math import isfinite, isinf, pi, cos, cosh from functools import reduce from operator import itemgetter from collections import Counter, namedtuple, defaultdict @@ -802,6 +803,171 @@ def multimode(data): return [value for value, count in counts.items() if count == maxcount] +def kde(data, h, kernel='normal'): + """Kernel Density Estimation: Create a continuous probability + density function from discrete samples. + + The basic idea is to smooth the data using a kernel function + to help draw inferences about a population from a sample. + + The degree of smoothing is controlled by the scaling parameter h + which is called the bandwidth. Smaller values emphasize local + features while larger values give smoother results. + + The kernel determines the relative weights of the sample data + points. Generally, the choice of kernel shape does not matter + as much as the more influential bandwidth smoothing parameter. + + Kernels that give some weight to every sample point: + + normal or gauss + logistic + sigmoid + + Kernels that only give weight to sample points within + the bandwidth: + + rectangular or uniform + triangular + parabolic or epanechnikov + quartic or biweight + triweight + cosine + + A StatisticsError will be raised if the data sequence is empty. + + Example + ------- + + Given a sample of six data points, construct a continuous + function that estimates the underlying probability density: + + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde(sample, h=1.5) + + Compute the area under the curve: + + >>> sum(f_hat(x) for x in range(-20, 20)) + 1.0 + + Plot the estimated probability density function at + evenly spaced points from -6 to 10: + + >>> for x in range(-6, 11): + ... density = f_hat(x) + ... plot = ' ' * int(density * 400) + 'x' + ... print(f'{x:2}: {density:.3f} {plot}') + ... + -6: 0.002 x + -5: 0.009 x + -4: 0.031 x + -3: 0.070 x + -2: 0.111 x + -1: 0.125 x + 0: 0.110 x + 1: 0.086 x + 2: 0.068 x + 3: 0.059 x + 4: 0.066 x + 5: 0.082 x + 6: 0.082 x + 7: 0.058 x + 8: 0.028 x + 9: 0.009 x + 10: 0.002 x + + References + ---------- + + Kernel density estimation and its application: + https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf + + Kernel functions in common use: + https://en.wikipedia.org/wiki/Kernel_(statistics)#kernel_functions_in_common_use + + Interactive graphical demonstration and exploration: + https://demonstrations.wolfram.com/KernelDensityEstimation/ + + """ + + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') + + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') + + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') + + match kernel: + + case 'normal' | 'gauss': + c = 1 / sqrt(2 * pi) + K = lambda t: c * exp(-1/2 * t * t) + support = None + + case 'logistic': + # 1.0 / (exp(t) + 2.0 + exp(-t)) + K = lambda t: 1/2 / (1.0 + cosh(t)) + support = None + + case 'sigmoid': + # (2/pi) / (exp(t) + exp(-t)) + c = 1 / pi + K = lambda t: c / cosh(t) + support = None + + case 'rectangular' | 'uniform': + K = lambda t: 1/2 + support = 1.0 + + case 'triangular': + K = lambda t: 1.0 - abs(t) + support = 1.0 + + case 'parabolic' | 'epanechnikov': + K = lambda t: 3/4 * (1.0 - t * t) + support = 1.0 + + case 'quartic' | 'biweight': + K = lambda t: 15/16 * (1.0 - t * t) ** 2 + support = 1.0 + + case 'triweight': + K = lambda t: 35/32 * (1.0 - t * t) ** 3 + support = 1.0 + + case 'cosine': + c1 = pi / 4 + c2 = pi / 2 + K = lambda t: c1 * cos(c2 * t) + support = 1.0 + + case _: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + + if support is None: + + def pdf(x): + return sum(K((x - x_i) / h) for x_i in data) / (n * h) + + else: + + sample = sorted(data) + bandwidth = h * support + + def pdf(x): + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum(K((x - x_i) / h) for x_i in supported) / (n * h) + + pdf.__doc__ = f'PDF estimate with {kernel=!r} and {h=!r}' + + return pdf + + # Notes on methods for computing quantiles # ---------------------------------------- # diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 20e0394507f5db1..f4dd0fdab4a3e4d 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2411,7 +2411,7 @@ def _extract_member(self, tarinfo, targetpath, set_attrs=True, if upperdirs and not os.path.exists(upperdirs): # Create directories that are not part of the archive with # default permissions. - os.makedirs(upperdirs) + os.makedirs(upperdirs, exist_ok=True) if tarinfo.islnk() or tarinfo.issym(): self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname)) @@ -2456,7 +2456,8 @@ def makedir(self, tarinfo, targetpath): # later in _extract_member(). os.mkdir(targetpath, 0o700) except FileExistsError: - pass + if not os.path.isdir(targetpath): + raise def makefile(self, tarinfo, targetpath): """Make a file called targetpath. diff --git a/Lib/test/.ruff.toml b/Lib/test/.ruff.toml index d6c1d8745036ece..1c9bac507209b12 100644 --- a/Lib/test/.ruff.toml +++ b/Lib/test/.ruff.toml @@ -1,7 +1,4 @@ fix = true -select = [ - "F811", # Redefinition of unused variable (useful for finding test methods with the same name) -] extend-exclude = [ # Excluded (run with the other AC files in its own separate ruff job in pre-commit) "test_clinic.py", @@ -21,3 +18,8 @@ extend-exclude = [ "test_pkg.py", "test_yield_from.py", ] + +[lint] +select = [ + "F811", # Redefinition of unused variable (useful for finding test methods with the same name) +] diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index c0d3ca50f17d69d..058537bab5af260 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3504,6 +3504,25 @@ def test_context(self): if self.TYPE == 'processes': self.assertRaises(OSError, l.accept) + def test_empty_authkey(self): + # bpo-43952: allow empty bytes as authkey + def handler(*args): + raise RuntimeError('Connection took too long...') + + def run(addr, authkey): + client = self.connection.Client(addr, authkey=authkey) + client.send(1729) + + key = b"" + + with self.connection.Listener(authkey=key) as listener: + threading.Thread(target=run, args=(listener.address, key)).start() + with listener.accept() as d: + self.assertEqual(d.recv(), 1729) + + if self.TYPE == 'processes': + self.assertRaises(OSError, listener.accept) + @unittest.skipUnless(util.abstract_sockets_supported, "test needs abstract socket support") def test_abstract_socket(self): @@ -3971,6 +3990,21 @@ def _new_shm_name(self, prefix): # test_multiprocessing_spawn, etc) in parallel. return prefix + str(os.getpid()) + def test_shared_memory_name_with_embedded_null(self): + name_tsmb = self._new_shm_name('test01_null') + sms = shared_memory.SharedMemory(name_tsmb, create=True, size=512) + self.addCleanup(sms.unlink) + with self.assertRaises(ValueError): + shared_memory.SharedMemory(name_tsmb + '\0a', create=False, size=512) + if shared_memory._USE_POSIX: + orig_name = sms._name + try: + sms._name = orig_name + '\0a' + with self.assertRaises(ValueError): + sms.unlink() + finally: + sms._name = orig_name + def test_shared_memory_basics(self): name_tsmb = self._new_shm_name('test01_tsmb') sms = shared_memory.SharedMemory(name_tsmb, create=True, size=512) @@ -4105,7 +4139,7 @@ def test_shared_memory_recreate(self): self.addCleanup(shm2.unlink) self.assertEqual(shm2._name, names[1]) - def test_invalid_shared_memory_cration(self): + def test_invalid_shared_memory_creation(self): # Test creating a shared memory segment with negative size with self.assertRaises(ValueError): sms_invalid = shared_memory.SharedMemory(create=True, size=-1) @@ -5609,8 +5643,9 @@ def create_and_register_resource(rtype): ''' for rtype in resource_tracker._CLEANUP_FUNCS: with self.subTest(rtype=rtype): - if rtype == "noop": + if rtype in ("noop", "dummy"): # Artefact resource type used by the resource_tracker + # or tests continue r, w = os.pipe() p = subprocess.Popen([sys.executable, @@ -5730,6 +5765,38 @@ def test_too_long_name_resource(self): with self.assertRaises(ValueError): resource_tracker.register(too_long_name_resource, rtype) + def _test_resource_tracker_leak_resources(self, cleanup): + # We use a separate instance for testing, since the main global + # _resource_tracker may be used to watch test infrastructure. + from multiprocessing.resource_tracker import ResourceTracker + tracker = ResourceTracker() + tracker.ensure_running() + self.assertTrue(tracker._check_alive()) + + self.assertIsNone(tracker._exitcode) + tracker.register('somename', 'dummy') + if cleanup: + tracker.unregister('somename', 'dummy') + expected_exit_code = 0 + else: + expected_exit_code = 1 + + self.assertTrue(tracker._check_alive()) + self.assertIsNone(tracker._exitcode) + tracker._stop() + self.assertEqual(tracker._exitcode, expected_exit_code) + + def test_resource_tracker_exit_code(self): + """ + Test the exit code of the resource tracker. + + If no leaked resources were found, exit code should be 0, otherwise 1 + """ + for cleanup in [True, False]: + with self.subTest(cleanup=cleanup): + self._test_resource_tracker_leak_resources( + cleanup=cleanup, + ) class TestSimpleQueue(unittest.TestCase): @@ -6113,6 +6180,24 @@ def test_spawn_sys_executable_none_allows_import(self): self.assertEqual(rc, 0) self.assertFalse(err, msg=err.decode('utf-8')) + def test_large_pool(self): + # + # gh-89240: Check that large pools are always okay + # + testfn = os_helper.TESTFN + self.addCleanup(os_helper.unlink, testfn) + with open(testfn, 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + import multiprocessing + def f(x): return x*x + if __name__ == '__main__': + with multiprocessing.Pool(200) as p: + print(sum(p.map(f, range(1000)))) + ''')) + rc, out, err = script_helper.assert_python_ok(testfn) + self.assertEqual("332833500", out.decode('utf-8').strip()) + self.assertFalse(err, msg=err.decode('utf-8')) + # # Mixins diff --git a/Lib/test/archiver_tests.py b/Lib/test/archiver_tests.py new file mode 100644 index 000000000000000..24745941b089239 --- /dev/null +++ b/Lib/test/archiver_tests.py @@ -0,0 +1,177 @@ +"""Tests common to tarfile and zipfile.""" + +import os +import sys + +from test.support import swap_attr +from test.support import os_helper + +class OverwriteTests: + + def setUp(self): + os.makedirs(self.testdir) + self.addCleanup(os_helper.rmtree, self.testdir) + + def create_file(self, path, content=b''): + with open(path, 'wb') as f: + f.write(content) + + def open(self, path): + raise NotImplementedError + + def extractall(self, ar): + raise NotImplementedError + + + def test_overwrite_file_as_file(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + def test_overwrite_dir_as_dir(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_overwrite_dir_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + self.assertTrue(os.path.isfile(os.path.join(target, 'file'))) + with open(os.path.join(target, 'file'), 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + def test_overwrite_dir_as_file(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_file) as ar: + with self.assertRaises(PermissionError if sys.platform == 'win32' + else IsADirectoryError): + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_overwrite_file_as_dir(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'content') + + def test_overwrite_file_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_implicit_dir) as ar: + with self.assertRaises(FileNotFoundError if sys.platform == 'win32' + else NotADirectoryError): + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'content') + + @os_helper.skip_unless_symlink + def test_overwrite_file_symlink_as_file(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + self.create_file(target2, b'content') + os.symlink('test2', target) + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isfile(target2)) + with open(target2, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_broken_file_symlink_as_file(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target) + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isfile(target2)) + with open(target2, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_dir_symlink_as_dir(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.mkdir(target2) + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isdir(target2)) + + @os_helper.skip_unless_symlink + def test_overwrite_dir_symlink_as_implicit_dir(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.mkdir(target2) + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isdir(target2)) + self.assertTrue(os.path.isfile(os.path.join(target2, 'file'))) + with open(os.path.join(target2, 'file'), 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_broken_dir_symlink_as_dir(self): + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertFalse(os.path.exists(target2)) + + @os_helper.skip_unless_symlink + def test_overwrite_broken_dir_symlink_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_implicit_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertFalse(os.path.exists(target2)) + + def test_concurrent_extract_dir(self): + target = os.path.join(self.testdir, 'test') + def concurrent_mkdir(*args, **kwargs): + orig_mkdir(*args, **kwargs) + orig_mkdir(*args, **kwargs) + with swap_attr(os, 'mkdir', concurrent_mkdir) as orig_mkdir: + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_concurrent_extract_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + def concurrent_mkdir(*args, **kwargs): + orig_mkdir(*args, **kwargs) + orig_mkdir(*args, **kwargs) + with swap_attr(os, 'mkdir', concurrent_mkdir) as orig_mkdir: + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + self.assertTrue(os.path.isfile(os.path.join(target, 'file'))) diff --git a/Lib/test/bisect_cmd.py b/Lib/test/bisect_cmd.py index 5cb804bd469dc35..aee2e8ac1208527 100755 --- a/Lib/test/bisect_cmd.py +++ b/Lib/test/bisect_cmd.py @@ -51,6 +51,7 @@ def python_cmd(): cmd = [sys.executable] cmd.extend(subprocess._args_from_interpreter_flags()) cmd.extend(subprocess._optim_args_from_interpreter_flags()) + cmd.extend(('-X', 'faulthandler')) return cmd @@ -77,9 +78,13 @@ def run_tests(args, tests, huntrleaks=None): write_tests(tmp, tests) cmd = python_cmd() - cmd.extend(['-m', 'test', '--matchfile', tmp]) + cmd.extend(['-u', '-m', 'test', '--matchfile', tmp]) cmd.extend(args.test_args) print("+ %s" % format_shell_args(cmd)) + + sys.stdout.flush() + sys.stderr.flush() + proc = subprocess.run(cmd) return proc.returncode finally: @@ -137,8 +142,8 @@ def main(): ntest = max(ntest // 2, 1) subtests = random.sample(tests, ntest) - print("[+] Iteration %s: run %s tests/%s" - % (iteration, len(subtests), len(tests))) + print(f"[+] Iteration {iteration}/{args.max_iter}: " + f"run {len(subtests)} tests/{len(tests)}") print() exitcode = run_tests(args, subtests) @@ -170,10 +175,10 @@ def main(): if len(tests) <= args.max_tests: print("Bisection completed in %s iterations and %s" % (iteration, datetime.timedelta(seconds=dt))) - sys.exit(1) else: print("Bisection failed after %s iterations and %s" % (iteration, datetime.timedelta(seconds=dt))) + sys.exit(1) if __name__ == "__main__": diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index b15aeb898d35a12..168f6f73f6186fa 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -4625,7 +4625,7 @@ Test_cls_no_params_impl(TestObj *self, PyTypeObject *cls); static PyObject * Test_cls_no_params(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "cls_no_params() takes no arguments"); return NULL; } @@ -4634,7 +4634,7 @@ Test_cls_no_params(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_s static PyObject * Test_cls_no_params_impl(TestObj *self, PyTypeObject *cls) -/*[clinic end generated code: output=cc8845f22cff3dcb input=e7e2e4e344e96a11]*/ +/*[clinic end generated code: output=4d68b4652c144af3 input=e7e2e4e344e96a11]*/ /*[clinic input] diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 53ad5e57ada0178..31fc383e29707a7 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1723,11 +1723,24 @@ def test_replace(self): def test_subclass_replace(self): class DateSubclass(self.theclass): - pass + def __new__(cls, *args, **kwargs): + result = self.theclass.__new__(cls, *args, **kwargs) + result.extra = 7 + return result dt = DateSubclass(2012, 1, 1) - self.assertIs(type(dt.replace(year=2013)), DateSubclass) - self.assertIs(type(copy.replace(dt, year=2013)), DateSubclass) + + test_cases = [ + ('self.replace', dt.replace(year=2013)), + ('copy.replace', copy.replace(dt, year=2013)), + ] + + for name, res in test_cases: + with self.subTest(name): + self.assertIs(type(res), DateSubclass) + self.assertEqual(res.year, 2013) + self.assertEqual(res.month, 1) + self.assertEqual(res.extra, 7) def test_subclass_date(self): @@ -3025,6 +3038,26 @@ def __new__(cls, *args, **kwargs): self.assertIsInstance(dt, DateTimeSubclass) self.assertEqual(dt.extra, 7) + def test_subclass_replace_fold(self): + class DateTimeSubclass(self.theclass): + pass + + dt = DateTimeSubclass(2012, 1, 1) + dt2 = DateTimeSubclass(2012, 1, 1, fold=1) + + test_cases = [ + ('self.replace', dt.replace(year=2013), 0), + ('self.replace', dt2.replace(year=2013), 1), + ('copy.replace', copy.replace(dt, year=2013), 0), + ('copy.replace', copy.replace(dt2, year=2013), 1), + ] + + for name, res, fold in test_cases: + with self.subTest(name, fold=fold): + self.assertIs(type(res), DateTimeSubclass) + self.assertEqual(res.year, 2013) + self.assertEqual(res.fold, fold) + def test_fromisoformat_datetime(self): # Test that isoformat() is reversible base_dates = [ @@ -3705,11 +3738,28 @@ def test_replace(self): def test_subclass_replace(self): class TimeSubclass(self.theclass): - pass + def __new__(cls, *args, **kwargs): + result = self.theclass.__new__(cls, *args, **kwargs) + result.extra = 7 + return result ctime = TimeSubclass(12, 30) - self.assertIs(type(ctime.replace(hour=10)), TimeSubclass) - self.assertIs(type(copy.replace(ctime, hour=10)), TimeSubclass) + ctime2 = TimeSubclass(12, 30, fold=1) + + test_cases = [ + ('self.replace', ctime.replace(hour=10), 0), + ('self.replace', ctime2.replace(hour=10), 1), + ('copy.replace', copy.replace(ctime, hour=10), 0), + ('copy.replace', copy.replace(ctime2, hour=10), 1), + ] + + for name, res, fold in test_cases: + with self.subTest(name, fold=fold): + self.assertIs(type(res), TimeSubclass) + self.assertEqual(res.hour, 10) + self.assertEqual(res.minute, 30) + self.assertEqual(res.extra, 7) + self.assertEqual(res.fold, fold) def test_subclass_time(self): @@ -5435,42 +5485,50 @@ def fromutc(self, dt): class Oddballs(unittest.TestCase): - def test_bug_1028306(self): + def test_date_datetime_comparison(self): + # bpo-1028306, bpo-5516 (gh-49766) # Trying to compare a date to a datetime should act like a mixed- # type comparison, despite that datetime is a subclass of date. as_date = date.today() as_datetime = datetime.combine(as_date, time()) - self.assertTrue(as_date != as_datetime) - self.assertTrue(as_datetime != as_date) - self.assertFalse(as_date == as_datetime) - self.assertFalse(as_datetime == as_date) - self.assertRaises(TypeError, lambda: as_date < as_datetime) - self.assertRaises(TypeError, lambda: as_datetime < as_date) - self.assertRaises(TypeError, lambda: as_date <= as_datetime) - self.assertRaises(TypeError, lambda: as_datetime <= as_date) - self.assertRaises(TypeError, lambda: as_date > as_datetime) - self.assertRaises(TypeError, lambda: as_datetime > as_date) - self.assertRaises(TypeError, lambda: as_date >= as_datetime) - self.assertRaises(TypeError, lambda: as_datetime >= as_date) - - # Nevertheless, comparison should work with the base-class (date) - # projection if use of a date method is forced. - self.assertEqual(as_date.__eq__(as_datetime), True) - different_day = (as_date.day + 1) % 20 + 1 - as_different = as_datetime.replace(day= different_day) - self.assertEqual(as_date.__eq__(as_different), False) + date_sc = SubclassDate(as_date.year, as_date.month, as_date.day) + datetime_sc = SubclassDatetime(as_date.year, as_date.month, + as_date.day, 0, 0, 0) + for d in (as_date, date_sc): + for dt in (as_datetime, datetime_sc): + for x, y in (d, dt), (dt, d): + self.assertTrue(x != y) + self.assertFalse(x == y) + self.assertRaises(TypeError, lambda: x < y) + self.assertRaises(TypeError, lambda: x <= y) + self.assertRaises(TypeError, lambda: x > y) + self.assertRaises(TypeError, lambda: x >= y) # And date should compare with other subclasses of date. If a # subclass wants to stop this, it's up to the subclass to do so. - date_sc = SubclassDate(as_date.year, as_date.month, as_date.day) - self.assertEqual(as_date, date_sc) - self.assertEqual(date_sc, as_date) - # Ditto for datetimes. - datetime_sc = SubclassDatetime(as_datetime.year, as_datetime.month, - as_date.day, 0, 0, 0) - self.assertEqual(as_datetime, datetime_sc) - self.assertEqual(datetime_sc, as_datetime) + for x, y in ((as_date, date_sc), + (date_sc, as_date), + (as_datetime, datetime_sc), + (datetime_sc, as_datetime)): + self.assertTrue(x == y) + self.assertFalse(x != y) + self.assertFalse(x < y) + self.assertFalse(x > y) + self.assertTrue(x <= y) + self.assertTrue(x >= y) + + # Nevertheless, comparison should work if other object is an instance + # of date or datetime class with overridden comparison operators. + # So special methods should return NotImplemented, as if + # date and datetime were independent classes. + for x, y in (as_date, as_datetime), (as_datetime, as_date): + self.assertEqual(x.__eq__(y), NotImplemented) + self.assertEqual(x.__ne__(y), NotImplemented) + self.assertEqual(x.__lt__(y), NotImplemented) + self.assertEqual(x.__gt__(y), NotImplemented) + self.assertEqual(x.__gt__(y), NotImplemented) + self.assertEqual(x.__ge__(y), NotImplemented) def test_extra_attributes(self): with self.assertWarns(DeprecationWarning): diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index 1eca123be0fecbf..65f54859e2a21d0 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -40,10 +40,12 @@ BaseException ├── ReferenceError ├── RuntimeError │ ├── NotImplementedError + │ ├── PythonFinalizationError │ └── RecursionError ├── StopAsyncIteration ├── StopIteration ├── SyntaxError + │ └── IncompleteInputError │ └── IndentationError │ └── TabError ├── SystemError diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 0053bce4292f64f..608b12bb6f2a381 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -347,6 +347,8 @@ def _create_parser(): help='override the working directory for the test run') group.add_argument('--cleanup', action='store_true', help='remove old test_python_* directories') + group.add_argument('--bisect', action='store_true', + help='if some tests fail, run test.bisect_cmd on them') group.add_argument('--dont-add-python-opts', dest='_add_python_opts', action='store_false', help="internal option, don't use it") diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py index ee890b5b1db4cdc..4ac95e23a56b8f8 100644 --- a/Lib/test/libregrtest/findtests.py +++ b/Lib/test/libregrtest/findtests.py @@ -23,6 +23,7 @@ "test_future_stmt", "test_gdb", "test_inspect", + "test_pydoc", "test_multiprocessing_fork", "test_multiprocessing_forkserver", "test_multiprocessing_spawn", diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 7ca1b1cb65ae40e..6fbe3d00981ddd2 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -7,8 +7,7 @@ import time import trace -from test import support -from test.support import os_helper, MS_WINDOWS +from test.support import os_helper, MS_WINDOWS, flush_std_streams from .cmdline import _parse_args, Namespace from .findtests import findtests, split_test_packages, list_cases @@ -73,6 +72,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.want_cleanup: bool = ns.cleanup self.want_rerun: bool = ns.rerun self.want_run_leaks: bool = ns.runleaks + self.want_bisect: bool = ns.bisect self.ci_mode: bool = (ns.fast_ci or ns.slow_ci) self.want_add_python_opts: bool = (_add_python_opts @@ -273,6 +273,55 @@ def rerun_failed_tests(self, runtests: RunTests): self.display_result(rerun_runtests) + def _run_bisect(self, runtests: RunTests, test: str, progress: str) -> bool: + print() + title = f"Bisect {test}" + if progress: + title = f"{title} ({progress})" + print(title) + print("#" * len(title)) + print() + + cmd = runtests.create_python_cmd() + cmd.extend([ + "-u", "-m", "test.bisect_cmd", + # Limit to 25 iterations (instead of 100) to not abuse CI resources + "--max-iter", "25", + "-v", + # runtests.match_tests is not used (yet) for bisect_cmd -i arg + ]) + cmd.extend(runtests.bisect_cmd_args()) + cmd.append(test) + print("+", shlex.join(cmd), flush=True) + + flush_std_streams() + + import subprocess + proc = subprocess.run(cmd, timeout=runtests.timeout) + exitcode = proc.returncode + + title = f"{title}: exit code {exitcode}" + print(title) + print("#" * len(title)) + print(flush=True) + + if exitcode: + print(f"Bisect failed with exit code {exitcode}") + return False + + return True + + def run_bisect(self, runtests: RunTests) -> None: + tests, _ = self.results.prepare_rerun(clear=False) + + for index, name in enumerate(tests, 1): + if len(tests) > 1: + progress = f"{index}/{len(tests)}" + else: + progress = "" + if not self._run_bisect(runtests, name, progress): + return + def display_result(self, runtests): # If running the test suite for PGO then no one cares about results. if runtests.pgo: @@ -335,10 +384,15 @@ def run_tests_sequentially(self, runtests) -> None: result = self.run_test(test_name, runtests, tracer) - # Unload the newly imported test modules (best effort finalization) + # Unload the newly imported test modules (best effort + # finalization). To work around gh-115490, don't unload + # test.support.interpreters and its submodules even if they + # weren't loaded before. + keep = "test.support.interpreters" new_modules = [module for module in sys.modules if module not in save_modules and - module.startswith(("test.", "test_"))] + module.startswith(("test.", "test_")) + and not module.startswith(keep)] for module in new_modules: sys.modules.pop(module, None) # Remove the attribute of the parent module. @@ -461,7 +515,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: setup_process() - if self.hunt_refleak and not self.num_workers: + if (runtests.hunt_refleak is not None) and (not self.num_workers): # gh-109739: WindowsLoadTracker thread interfers with refleak check use_load_tracker = False else: @@ -481,6 +535,9 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: if self.want_rerun and self.results.need_rerun(): self.rerun_failed_tests(runtests) + + if self.want_bisect and self.results.need_rerun(): + self.run_bisect(runtests) finally: if use_load_tracker: self.logger.stop_load_tracker() diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 7da16cf721f0973..f582c0d3e7ff134 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -88,9 +88,12 @@ def get_pooled_int(value): rc_before = alloc_before = fd_before = interned_before = 0 if not quiet: - print("beginning", repcount, "repetitions", file=sys.stderr) - print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, - flush=True) + print("beginning", repcount, "repetitions. Showing number of leaks " + "(. for 0 or less, X for 10 or more)", + file=sys.stderr) + numbers = ("1234567890"*(repcount//10 + 1))[:repcount] + numbers = numbers[:warmups] + ':' + numbers[warmups:] + print(numbers, file=sys.stderr, flush=True) results = None dash_R_cleanup(fs, ps, pic, zdc, abcs) @@ -116,13 +119,27 @@ def get_pooled_int(value): rc_after = gettotalrefcount() - interned_after * 2 fd_after = fd_count() - if not quiet: - print('.', end='', file=sys.stderr, flush=True) - rc_deltas[i] = get_pooled_int(rc_after - rc_before) alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before) fd_deltas[i] = get_pooled_int(fd_after - fd_before) + if not quiet: + # use max, not sum, so total_leaks is one of the pooled ints + total_leaks = max(rc_deltas[i], alloc_deltas[i], fd_deltas[i]) + if total_leaks <= 0: + symbol = '.' + elif total_leaks < 10: + symbol = ( + '.', '1', '2', '3', '4', '5', '6', '7', '8', '9', + )[total_leaks] + else: + symbol = 'X' + if i == warmups: + print(' ', end='', file=sys.stderr, flush=True) + print(symbol, end='', file=sys.stderr, flush=True) + del total_leaks + del symbol + alloc_before = alloc_after rc_before = rc_after fd_before = fd_after @@ -158,14 +175,20 @@ def check_fd_deltas(deltas): ]: # ignore warmup runs deltas = deltas[warmups:] - if checker(deltas): + failing = checker(deltas) + suspicious = any(deltas) + if failing or suspicious: msg = '%s leaked %s %s, sum=%s' % ( test_name, deltas, item_name, sum(deltas)) - print(msg, file=sys.stderr, flush=True) - with open(filename, "a", encoding="utf-8") as refrep: - print(msg, file=refrep) - refrep.flush() - failed = True + print(msg, end='', file=sys.stderr) + if failing: + print(file=sys.stderr, flush=True) + with open(filename, "a", encoding="utf-8") as refrep: + print(msg, file=refrep) + refrep.flush() + failed = True + else: + print(' (this is fine)', file=sys.stderr, flush=True) return (failed, results) @@ -201,8 +224,8 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): # Clear caches clear_caches() - # Clear type cache at the end: previous function calls can modify types - sys._clear_type_cache() + # Clear other caches last (previous function calls can re-populate them): + sys._clear_internal_caches() def warm_caches(): diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index a41ea8aba028c36..85c82052eae19b0 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -138,7 +138,7 @@ def get_coverage_results(self) -> trace.CoverageResults: def need_rerun(self): return bool(self.rerun_results) - def prepare_rerun(self) -> tuple[TestTuple, FilterDict]: + def prepare_rerun(self, *, clear: bool = True) -> tuple[TestTuple, FilterDict]: tests: TestList = [] match_tests_dict = {} for result in self.rerun_results: @@ -149,11 +149,12 @@ def prepare_rerun(self) -> tuple[TestTuple, FilterDict]: if match_tests: match_tests_dict[result.test_name] = match_tests - # Clear previously failed tests - self.rerun_bad.extend(self.bad) - self.bad.clear() - self.env_changed.clear() - self.rerun_results.clear() + if clear: + # Clear previously failed tests + self.rerun_bad.extend(self.bad) + self.bad.clear() + self.env_changed.clear() + self.rerun_results.clear() return (tuple(tests), match_tests_dict) diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index edd72276320e419..8e9779524c56a2f 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -2,7 +2,9 @@ import dataclasses import json import os +import shlex import subprocess +import sys from typing import Any from test import support @@ -67,6 +69,11 @@ class HuntRefleak: runs: int filename: StrPath + def bisect_cmd_args(self) -> list[str]: + # Ignore filename since it can contain colon (":"), + # and usually it's not used. Use the default filename. + return ["-R", f"{self.warmups}:{self.runs}:"] + @dataclasses.dataclass(slots=True, frozen=True) class RunTests: @@ -137,6 +144,49 @@ def json_file_use_stdout(self) -> bool: or support.is_wasi ) + def create_python_cmd(self) -> list[str]: + python_opts = support.args_from_interpreter_flags() + if self.python_cmd is not None: + executable = self.python_cmd + # Remove -E option, since --python=COMMAND can set PYTHON + # environment variables, such as PYTHONPATH, in the worker + # process. + python_opts = [opt for opt in python_opts if opt != "-E"] + else: + executable = (sys.executable,) + cmd = [*executable, *python_opts] + if '-u' not in python_opts: + cmd.append('-u') # Unbuffered stdout and stderr + if self.coverage: + cmd.append("-Xpresite=test.cov") + return cmd + + def bisect_cmd_args(self) -> list[str]: + args = [] + if self.fail_fast: + args.append("--failfast") + if self.fail_env_changed: + args.append("--fail-env-changed") + if self.timeout: + args.append(f"--timeout={self.timeout}") + if self.hunt_refleak is not None: + args.extend(self.hunt_refleak.bisect_cmd_args()) + if self.test_dir: + args.extend(("--testdir", self.test_dir)) + if self.memory_limit: + args.extend(("--memlimit", self.memory_limit)) + if self.gc_threshold: + args.append(f"--threshold={self.gc_threshold}") + if self.use_resources: + args.extend(("-u", ','.join(self.use_resources))) + if self.python_cmd: + cmd = shlex.join(self.python_cmd) + args.extend(("--python", cmd)) + if self.randomize: + args.append(f"--randomize") + args.append(f"--randseed={self.random_seed}") + return args + @dataclasses.dataclass(slots=True, frozen=True) class WorkerRunTests(RunTests): diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 7a6d33d4499943e..f8b8e45eca32768 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -3,7 +3,6 @@ import os from typing import Any, NoReturn -from test import support from test.support import os_helper, Py_DEBUG from .setup import setup_process, setup_test_dir @@ -19,23 +18,10 @@ def create_worker_process(runtests: WorkerRunTests, output_fd: int, tmp_dir: StrPath | None = None) -> subprocess.Popen: - python_cmd = runtests.python_cmd worker_json = runtests.as_json() - python_opts = support.args_from_interpreter_flags() - if python_cmd is not None: - executable = python_cmd - # Remove -E option, since --python=COMMAND can set PYTHON environment - # variables, such as PYTHONPATH, in the worker process. - python_opts = [opt for opt in python_opts if opt != "-E"] - else: - executable = (sys.executable,) - if runtests.coverage: - python_opts.append("-Xpresite=test.cov") - cmd = [*executable, *python_opts, - '-u', # Unbuffered stdout and stderr - '-m', 'test.libregrtest.worker', - worker_json] + cmd = runtests.create_python_cmd() + cmd.extend(['-m', 'test.libregrtest.worker', worker_json]) env = dict(os.environ) if tmp_dir is not None: diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index d9ab21d4941cdb9..26118e14bb97e08 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -562,3 +562,8 @@ def test_exhausted_iterator(self): self.assertEqual(list(exhit), []) self.assertEqual(list(empit), [9]) self.assertEqual(a, self.type2test([1, 2, 3, 9])) + + # gh-115733: Crash when iterating over exhausted iterator + exhit = iter(self.type2test([1, 2, 3])) + for _ in exhit: + next(exhit, 1) diff --git a/Lib/test/sortperf.py b/Lib/test/sortperf.py deleted file mode 100644 index 14a9d827ed57c56..000000000000000 --- a/Lib/test/sortperf.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Sort performance test. - -See main() for command line syntax. -See tabulate() for output format. - -""" - -import sys -import time -import random -import marshal -import tempfile -import os - -td = tempfile.gettempdir() - -def randfloats(n): - """Return a list of n random floats in [0, 1).""" - # Generating floats is expensive, so this writes them out to a file in - # a temp directory. If the file already exists, it just reads them - # back in and shuffles them a bit. - fn = os.path.join(td, "rr%06d" % n) - try: - fp = open(fn, "rb") - except OSError: - r = random.random - result = [r() for i in range(n)] - try: - try: - fp = open(fn, "wb") - marshal.dump(result, fp) - fp.close() - fp = None - finally: - if fp: - try: - os.unlink(fn) - except OSError: - pass - except OSError as msg: - print("can't write", fn, ":", msg) - else: - result = marshal.load(fp) - fp.close() - # Shuffle it a bit... - for i in range(10): - i = random.randrange(n) - temp = result[:i] - del result[:i] - temp.reverse() - result.extend(temp) - del temp - assert len(result) == n - return result - -def flush(): - sys.stdout.flush() - -def doit(L): - t0 = time.perf_counter() - L.sort() - t1 = time.perf_counter() - print("%6.2f" % (t1-t0), end=' ') - flush() - -def tabulate(r): - r"""Tabulate sort speed for lists of various sizes. - - The sizes are 2**i for i in r (the argument, a list). - - The output displays i, 2**i, and the time to sort arrays of 2**i - floating point numbers with the following properties: - - *sort: random data - \sort: descending data - /sort: ascending data - 3sort: ascending, then 3 random exchanges - +sort: ascending, then 10 random at the end - %sort: ascending, then randomly replace 1% of the elements w/ random values - ~sort: many duplicates - =sort: all equal - !sort: worst case scenario - - """ - cases = tuple([ch + "sort" for ch in r"*\/3+%~=!"]) - fmt = ("%2s %7s" + " %6s"*len(cases)) - print(fmt % (("i", "2**i") + cases)) - for i in r: - n = 1 << i - L = randfloats(n) - print("%2d %7d" % (i, n), end=' ') - flush() - doit(L) # *sort - L.reverse() - doit(L) # \sort - doit(L) # /sort - - # Do 3 random exchanges. - for dummy in range(3): - i1 = random.randrange(n) - i2 = random.randrange(n) - L[i1], L[i2] = L[i2], L[i1] - doit(L) # 3sort - - # Replace the last 10 with random floats. - if n >= 10: - L[-10:] = [random.random() for dummy in range(10)] - doit(L) # +sort - - # Replace 1% of the elements at random. - for dummy in range(n // 100): - L[random.randrange(n)] = random.random() - doit(L) # %sort - - # Arrange for lots of duplicates. - if n > 4: - del L[4:] - L = L * (n // 4) - # Force the elements to be distinct objects, else timings can be - # artificially low. - L = list(map(lambda x: --x, L)) - doit(L) # ~sort - del L - - # All equal. Again, force the elements to be distinct objects. - L = list(map(abs, [-0.5] * n)) - doit(L) # =sort - del L - - # This one looks like [3, 2, 1, 0, 0, 1, 2, 3]. It was a bad case - # for an older implementation of quicksort, which used the median - # of the first, last and middle elements as the pivot. - half = n // 2 - L = list(range(half - 1, -1, -1)) - L.extend(range(half)) - # Force to float, so that the timings are comparable. This is - # significantly faster if we leave them as ints. - L = list(map(float, L)) - doit(L) # !sort - print() - -def main(): - """Main program when invoked as a script. - - One argument: tabulate a single row. - Two arguments: tabulate a range (inclusive). - Extra arguments are used to seed the random generator. - - """ - # default range (inclusive) - k1 = 15 - k2 = 20 - if sys.argv[1:]: - # one argument: single point - k1 = k2 = int(sys.argv[1]) - if sys.argv[2:]: - # two arguments: specify range - k2 = int(sys.argv[2]) - if sys.argv[3:]: - # derive random seed from remaining arguments - x = 1 - for a in sys.argv[3:]: - x = 69069 * x + hash(a) - random.seed(x) - r = range(k1, k2+1) # include the end point - tabulate(r) - -if __name__ == '__main__': - main() diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index f2e6af078a5f291..401b2ce1fe213cb 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -43,7 +43,7 @@ "requires_limited_api", "requires_specialization", # sys "MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi", - "check_impl_detail", "unix_shell", "setswitchinterval", + "is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval", # os "get_pagesize", # network @@ -522,7 +522,7 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): is_android = hasattr(sys, 'getandroidapilevel') -if sys.platform not in ('win32', 'vxworks'): +if sys.platform not in {"win32", "vxworks", "ios", "tvos", "watchos"}: unix_shell = '/system/bin/sh' if is_android else '/bin/sh' else: unix_shell = None @@ -532,19 +532,35 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): is_emscripten = sys.platform == "emscripten" is_wasi = sys.platform == "wasi" -has_fork_support = hasattr(os, "fork") and not is_emscripten and not is_wasi +# Apple mobile platforms (iOS/tvOS/watchOS) are POSIX-like but do not +# have subprocess or fork support. +is_apple_mobile = sys.platform in {"ios", "tvos", "watchos"} +is_apple = is_apple_mobile or sys.platform == "darwin" + +has_fork_support = hasattr(os, "fork") and not ( + is_emscripten + or is_wasi + or is_apple_mobile +) def requires_fork(): return unittest.skipUnless(has_fork_support, "requires working os.fork()") -has_subprocess_support = not is_emscripten and not is_wasi +has_subprocess_support = not ( + is_emscripten + or is_wasi + or is_apple_mobile +) def requires_subprocess(): """Used for subprocess, os.spawn calls, fd inheritance""" return unittest.skipUnless(has_subprocess_support, "requires subprocess support") # Emscripten's socket emulation and WASI sockets have limitations. -has_socket_support = not is_emscripten and not is_wasi +has_socket_support = not ( + is_emscripten + or is_wasi +) def requires_working_socket(*, module=False): """Skip tests or modules that require working sockets @@ -1711,19 +1727,22 @@ def _check_tracemalloc(): def check_free_after_iterating(test, iter, cls, args=()): - class A(cls): - def __del__(self): - nonlocal done - done = True - try: - next(it) - except StopIteration: - pass - done = False - it = iter(A(*args)) - # Issue 26494: Shouldn't crash - test.assertRaises(StopIteration, next, it) + def wrapper(): + class A(cls): + def __del__(self): + nonlocal done + done = True + try: + next(it) + except StopIteration: + pass + + it = iter(A(*args)) + # Issue 26494: Shouldn't crash + test.assertRaises(StopIteration, next, it) + + wrapper() # The sequence should be deallocated just after the end of iterating gc_collect() test.assertTrue(done) @@ -2362,6 +2381,46 @@ def sleeping_retry(timeout, err_msg=None, /, delay = min(delay * 2, max_delay) +class CPUStopwatch: + """Context manager to roughly time a CPU-bound operation. + + Disables GC. Uses CPU time if it can (i.e. excludes sleeps & time of + other processes). + + N.B.: + - This *includes* time spent in other threads. + - Some systems only have a coarse resolution; check + stopwatch.clock_info.rseolution if. + + Usage: + + with ProcessStopwatch() as stopwatch: + ... + elapsed = stopwatch.seconds + resolution = stopwatch.clock_info.resolution + """ + def __enter__(self): + get_time = time.process_time + clock_info = time.get_clock_info('process_time') + if get_time() <= 0: # some platforms like WASM lack process_time() + get_time = time.monotonic + clock_info = time.get_clock_info('monotonic') + self.context = disable_gc() + self.context.__enter__() + self.get_time = get_time + self.clock_info = clock_info + self.start_time = get_time() + return self + + def __exit__(self, *exc): + try: + end_time = self.get_time() + finally: + result = self.context.__exit__(*exc) + self.seconds = end_time - self.start_time + return result + + @contextlib.contextmanager def adjust_int_max_str_digits(max_digits): """Temporarily change the integer string conversion length limit.""" diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index 15a908e9663593f..d02ffbae1113c0a 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -6,7 +6,7 @@ # aliases: from _xxsubinterpreters import ( - InterpreterError, InterpreterNotFoundError, + InterpreterError, InterpreterNotFoundError, NotShareableError, is_shareable, ) @@ -14,7 +14,8 @@ __all__ = [ 'get_current', 'get_main', 'create', 'list_all', 'is_shareable', 'Interpreter', - 'InterpreterError', 'InterpreterNotFoundError', 'ExecFailure', + 'InterpreterError', 'InterpreterNotFoundError', 'ExecutionFailed', + 'NotShareableError', 'create_queue', 'Queue', 'QueueEmpty', 'QueueFull', ] @@ -42,7 +43,11 @@ def __getattr__(name): {formatted} """.strip() -class ExecFailure(RuntimeError): +class ExecutionFailed(RuntimeError): + """An unhandled exception happened during execution. + + This is raised from Interpreter.exec() and Interpreter.call(). + """ def __init__(self, excinfo): msg = excinfo.formatted @@ -157,7 +162,7 @@ def prepare_main(self, ns=None, /, **kwargs): ns = dict(ns, **kwargs) if ns is not None else kwargs _interpreters.set___main___attrs(self._id, ns) - def exec_sync(self, code, /): + def exec(self, code, /): """Run the given source code in the interpreter. This is essentially the same as calling the builtin "exec" @@ -166,10 +171,10 @@ def exec_sync(self, code, /): There is no return value. - If the code raises an unhandled exception then an ExecFailure - is raised, which summarizes the unhandled exception. The actual - exception is discarded because objects cannot be shared between - interpreters. + If the code raises an unhandled exception then an ExecutionFailed + exception is raised, which summarizes the unhandled exception. + The actual exception is discarded because objects cannot be + shared between interpreters. This blocks the current Python thread until done. During that time, the previous interpreter is allowed to run @@ -177,11 +182,35 @@ def exec_sync(self, code, /): """ excinfo = _interpreters.exec(self._id, code) if excinfo is not None: - raise ExecFailure(excinfo) + raise ExecutionFailed(excinfo) + + def call(self, callable, /): + """Call the object in the interpreter with given args/kwargs. + + Only functions that take no arguments and have no closure + are supported. - def run(self, code, /): + The return value is discarded. + + If the callable raises an exception then the error display + (including full traceback) is send back between the interpreters + and an ExecutionFailed exception is raised, much like what + happens with Interpreter.exec(). + """ + # XXX Support args and kwargs. + # XXX Support arbitrary callables. + # XXX Support returning the return value (e.g. via pickle). + excinfo = _interpreters.call(self._id, callable) + if excinfo is not None: + raise ExecutionFailed(excinfo) + + def call_in_thread(self, callable, /): + """Return a new thread that calls the object in the interpreter. + + The return value and any raised exception are discarded. + """ def task(): - self.exec_sync(code) + self.call(callable) t = threading.Thread(target=task) t.start() return t diff --git a/Lib/test/support/interpreters/queues.py b/Lib/test/support/interpreters/queues.py index aead0c40ca9667e..2cc616be337a501 100644 --- a/Lib/test/support/interpreters/queues.py +++ b/Lib/test/support/interpreters/queues.py @@ -1,5 +1,6 @@ """Cross-interpreter Queues High Level Module.""" +import pickle import queue import time import weakref @@ -31,20 +32,26 @@ class QueueFull(_queues.QueueFull, queue.Full): """ -def create(maxsize=0): +_SHARED_ONLY = 0 +_PICKLED = 1 + +def create(maxsize=0, *, syncobj=False): """Return a new cross-interpreter queue. The queue may be used to pass data safely between interpreters. + + "syncobj" sets the default for Queue.put() + and Queue.put_nowait(). """ - qid = _queues.create(maxsize) - return Queue(qid) + fmt = _SHARED_ONLY if syncobj else _PICKLED + qid = _queues.create(maxsize, fmt) + return Queue(qid, _fmt=fmt) def list_all(): """Return a list of all open queues.""" - return [Queue(qid) - for qid in _queues.list_all()] - + return [Queue(qid, _fmt=fmt) + for qid, fmt in _queues.list_all()] _known_queues = weakref.WeakValueDictionary() @@ -52,17 +59,20 @@ def list_all(): class Queue: """A cross-interpreter queue.""" - def __new__(cls, id, /): + def __new__(cls, id, /, *, _fmt=None): # There is only one instance for any given ID. if isinstance(id, int): id = int(id) else: raise TypeError(f'id must be an int, got {id!r}') + if _fmt is None: + _fmt = _queues.get_default_fmt(id) try: self = _known_queues[id] except KeyError: self = super().__new__(cls) self._id = id + self._fmt = _fmt _known_queues[id] = self _queues.bind(id) return self @@ -105,20 +115,50 @@ def qsize(self): return _queues.get_count(self._id) def put(self, obj, timeout=None, *, + syncobj=None, _delay=10 / 1000, # 10 milliseconds ): """Add the object to the queue. This blocks while the queue is full. + + If "syncobj" is None (the default) then it uses the + queue's default, set with create_queue().. + + If "syncobj" is false then all objects are supported, + at the expense of worse performance. + + If "syncobj" is true then the object must be "shareable". + Examples of "shareable" objects include the builtin singletons, + str, and memoryview. One benefit is that such objects are + passed through the queue efficiently. + + The key difference, though, is conceptual: the corresponding + object returned from Queue.get() will be strictly equivalent + to the given obj. In other words, the two objects will be + effectively indistinguishable from each other, even if the + object is mutable. The received object may actually be the + same object, or a copy (immutable values only), or a proxy. + Regardless, the received object should be treated as though + the original has been shared directly, whether or not it + actually is. That's a slightly different and stronger promise + than just (initial) equality, which is all "syncobj=False" + can promise. """ + if syncobj is None: + fmt = self._fmt + else: + fmt = _SHARED_ONLY if syncobj else _PICKLED if timeout is not None: timeout = int(timeout) if timeout < 0: raise ValueError(f'timeout value must be non-negative') end = time.time() + timeout + if fmt is _PICKLED: + obj = pickle.dumps(obj) while True: try: - _queues.put(self._id, obj) + _queues.put(self._id, obj, fmt) except _queues.QueueFull as exc: if timeout is not None and time.time() >= end: exc.__class__ = QueueFull @@ -127,9 +167,15 @@ def put(self, obj, timeout=None, *, else: break - def put_nowait(self, obj): + def put_nowait(self, obj, *, syncobj=None): + if syncobj is None: + fmt = self._fmt + else: + fmt = _SHARED_ONLY if syncobj else _PICKLED + if fmt is _PICKLED: + obj = pickle.dumps(obj) try: - return _queues.put(self._id, obj) + _queues.put(self._id, obj, fmt) except _queues.QueueFull as exc: exc.__class__ = QueueFull raise # re-raise @@ -148,12 +194,18 @@ def get(self, timeout=None, *, end = time.time() + timeout while True: try: - return _queues.get(self._id) + obj, fmt = _queues.get(self._id) except _queues.QueueEmpty as exc: if timeout is not None and time.time() >= end: exc.__class__ = QueueEmpty raise # re-raise time.sleep(_delay) + else: + break + if fmt == _PICKLED: + obj = pickle.loads(obj) + else: + assert fmt == _SHARED_ONLY return obj def get_nowait(self): diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 20f38fd36a8876e..22787e32b5f3abe 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -22,8 +22,8 @@ # TESTFN_UNICODE is a non-ascii filename TESTFN_UNICODE = TESTFN_ASCII + "-\xe0\xf2\u0258\u0141\u011f" -if sys.platform == 'darwin': - # In Mac OS X's VFS API file names are, by definition, canonically +if support.is_apple: + # On Apple's VFS API file names are, by definition, canonically # decomposed Unicode, encoded using UTF-8. See QA1173: # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html import unicodedata @@ -48,8 +48,8 @@ 'encoding (%s). Unicode filename tests may not be effective' % (TESTFN_UNENCODABLE, sys.getfilesystemencoding())) TESTFN_UNENCODABLE = None -# macOS and Emscripten deny unencodable filenames (invalid utf-8) -elif sys.platform not in {'darwin', 'emscripten', 'wasi'}: +# Apple and Emscripten deny unencodable filenames (invalid utf-8) +elif not support.is_apple and sys.platform not in {"emscripten", "wasi"}: try: # ascii and utf-8 cannot encode the byte 0xff b'\xff'.decode(sys.getfilesystemencoding()) diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index c87cde4b3d1fab1..19dcbb207e914af 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -5,11 +5,6 @@ import sys import types -try: - import _multiprocessing -except ModuleNotFoundError: - _multiprocessing = None - if support.check_sanitizer(address=True, memory=True): SKIP_MODULES = frozenset(( @@ -36,17 +31,6 @@ class FailedImport(RuntimeError): class AllTest(unittest.TestCase): - def setUp(self): - # concurrent.futures uses a __getattr__ hook. Its __all__ triggers - # import of a submodule, which fails when _multiprocessing is not - # available. - if _multiprocessing is None: - sys.modules["_multiprocessing"] = types.ModuleType("_multiprocessing") - - def tearDown(self): - if _multiprocessing is None: - sys.modules.pop("_multiprocessing") - def check_all(self, modname): names = {} with warnings_helper.check_warnings( diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 7c1f5d36999a3d5..617b1721f3dbb1c 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -2274,6 +2274,34 @@ def test_parse_known_args(self): (NS(foo=False, bar=0.5, w=7, x='b'), ['-W', '-X', 'Y', 'Z']), ) + def test_parse_known_args_with_single_dash_option(self): + parser = ErrorRaisingArgumentParser() + parser.add_argument('-k', '--known', action='count', default=0) + parser.add_argument('-n', '--new', action='count', default=0) + self.assertEqual(parser.parse_known_args(['-k', '-u']), + (NS(known=1, new=0), ['-u'])) + self.assertEqual(parser.parse_known_args(['-u', '-k']), + (NS(known=1, new=0), ['-u'])) + self.assertEqual(parser.parse_known_args(['-ku']), + (NS(known=1, new=0), ['-u'])) + self.assertArgumentParserError(parser.parse_known_args, ['-k=u']) + self.assertEqual(parser.parse_known_args(['-uk']), + (NS(known=0, new=0), ['-uk'])) + self.assertEqual(parser.parse_known_args(['-u=k']), + (NS(known=0, new=0), ['-u=k'])) + self.assertEqual(parser.parse_known_args(['-kunknown']), + (NS(known=1, new=0), ['-unknown'])) + self.assertArgumentParserError(parser.parse_known_args, ['-k=unknown']) + self.assertEqual(parser.parse_known_args(['-ku=nknown']), + (NS(known=1, new=0), ['-u=nknown'])) + self.assertEqual(parser.parse_known_args(['-knew']), + (NS(known=1, new=1), ['-ew'])) + self.assertArgumentParserError(parser.parse_known_args, ['-kn=ew']) + self.assertArgumentParserError(parser.parse_known_args, ['-k-new']) + self.assertArgumentParserError(parser.parse_known_args, ['-kn-ew']) + self.assertEqual(parser.parse_known_args(['-kne-w']), + (NS(known=1, new=1), ['-e-w'])) + def test_dest(self): parser = ErrorRaisingArgumentParser() parser.add_argument('--foo', action='store_true') @@ -2836,6 +2864,27 @@ def test_help(self): ''' self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + def test_help_subparser_all_mutually_exclusive_group_members_suppressed(self): + self.maxDiff = None + parser = ErrorRaisingArgumentParser(prog='PROG') + commands = parser.add_subparsers(title="commands", dest="command") + cmd_foo = commands.add_parser("foo") + group = cmd_foo.add_mutually_exclusive_group() + group.add_argument('--verbose', action='store_true', help=argparse.SUPPRESS) + group.add_argument('--quiet', action='store_true', help=argparse.SUPPRESS) + longopt = '--' + 'long'*32 + longmeta = 'LONG'*32 + cmd_foo.add_argument(longopt) + expected = f'''\ + usage: PROG foo [-h] + [{longopt} {longmeta}] + + options: + -h, --help show this help message and exit + {longopt} {longmeta} + ''' + self.assertEqual(cmd_foo.format_help(), textwrap.dedent(expected)) + def test_empty_group(self): # See issue 26952 parser = argparse.ArgumentParser() @@ -3922,7 +3971,7 @@ class TestHelpUsageWithParentheses(HelpTestCase): options: -h, --help show this help message and exit - -p {1 (option A), 2 (option B)}, --optional {1 (option A), 2 (option B)} + -p, --optional {1 (option A), 2 (option B)} ''' version = '' @@ -4405,8 +4454,8 @@ class TestHelpAlternatePrefixChars(HelpTestCase): help = usage + '''\ options: - ^^foo foo help - ;b BAR, ;;bar BAR bar help + ^^foo foo help + ;b, ;;bar BAR bar help ''' version = '' @@ -5099,7 +5148,8 @@ def test_optional(self): string = ( "Action(option_strings=['--foo', '-a', '-b'], dest='b', " "nargs='+', const=None, default=42, type='int', " - "choices=[1, 2, 3], required=False, help='HELP', metavar='METAVAR')") + "choices=[1, 2, 3], required=False, help='HELP', " + "metavar='METAVAR', deprecated=False)") self.assertStringEqual(option, string) def test_argument(self): @@ -5116,7 +5166,8 @@ def test_argument(self): string = ( "Action(option_strings=[], dest='x', nargs='?', " "const=None, default=2.5, type=%r, choices=[0.5, 1.5, 2.5], " - "required=True, help='H HH H', metavar='MV MV MV')" % float) + "required=True, help='H HH H', metavar='MV MV MV', " + "deprecated=False)" % float) self.assertStringEqual(argument, string) def test_namespace(self): @@ -5308,6 +5359,139 @@ def spam(string_to_convert): args = parser.parse_args('--foo spam!'.split()) self.assertEqual(NS(foo='foo_converted'), args) + +# ============================================== +# Check that deprecated arguments output warning +# ============================================== + +class TestDeprecatedArguments(TestCase): + + def test_deprecated_option(self): + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--foo', deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', 'spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['-f', 'spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '-f' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', 'spam', '-f', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertRegex(stderr, "warning: option '-f' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 2) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', 'spam', '--foo', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + def test_deprecated_boolean_option(self): + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--foo', action=argparse.BooleanOptionalAction, deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args(['--foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['-f']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '-f' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['--no-foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--no-foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', '--no-foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertRegex(stderr, "warning: option '--no-foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 2) + + def test_deprecated_arguments(self): + parser = argparse.ArgumentParser() + parser.add_argument('foo', nargs='?', deprecated=True) + parser.add_argument('bar', nargs='?', deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args([]) + stderr = stderr.getvalue() + self.assertEqual(stderr.count('is deprecated'), 0) + + with captured_stderr() as stderr: + parser.parse_args(['spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['spam', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertRegex(stderr, "warning: argument 'bar' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 2) + + def test_deprecated_varargument(self): + parser = argparse.ArgumentParser() + parser.add_argument('foo', nargs='*', deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args([]) + stderr = stderr.getvalue() + self.assertEqual(stderr.count('is deprecated'), 0) + + with captured_stderr() as stderr: + parser.parse_args(['spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['spam', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + def test_deprecated_subparser(self): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + subparsers.add_parser('foo', aliases=['baz'], deprecated=True) + subparsers.add_parser('bar') + + with captured_stderr() as stderr: + parser.parse_args(['bar']) + stderr = stderr.getvalue() + self.assertEqual(stderr.count('is deprecated'), 0) + + with captured_stderr() as stderr: + parser.parse_args(['foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: command 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['baz']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: command 'baz' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + # ================================================================== # Check semantics regarding the default argument and type conversion # ================================================================== @@ -5405,6 +5589,22 @@ def test_zero_or_more_optional(self): args = parser.parse_args([]) self.assertEqual(NS(x=[]), args) + def test_double_dash(self): + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--foo', nargs='*') + parser.add_argument('bar', nargs='*') + + args = parser.parse_args(['--foo=--']) + self.assertEqual(NS(foo=['--'], bar=[]), args) + args = parser.parse_args(['--foo', '--']) + self.assertEqual(NS(foo=[], bar=[]), args) + args = parser.parse_args(['-f--']) + self.assertEqual(NS(foo=['--'], bar=[]), args) + args = parser.parse_args(['-f', '--']) + self.assertEqual(NS(foo=[], bar=[]), args) + args = parser.parse_args(['--foo', 'a', 'b', '--', 'c', 'd']) + self.assertEqual(NS(foo=['a', 'b'], bar=['c', 'd']), args) + # =========================== # parse_intermixed_args tests diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index a219fa365e7f202..95383be9659eb9e 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1014,6 +1014,29 @@ def test_pop(self): array.array(self.typecode, self.example[3:]+self.example[:-1]) ) + def test_clear(self): + a = array.array(self.typecode, self.example) + with self.assertRaises(TypeError): + a.clear(42) + a.clear() + self.assertEqual(len(a), 0) + self.assertEqual(a.typecode, self.typecode) + + a = array.array(self.typecode) + a.clear() + self.assertEqual(len(a), 0) + self.assertEqual(a.typecode, self.typecode) + + a = array.array(self.typecode, self.example) + a.clear() + a.append(self.example[2]) + a.append(self.example[3]) + self.assertEqual(a, array.array(self.typecode, self.example[2:4])) + + with memoryview(a): + with self.assertRaises(BufferError): + a.clear() + def test_reverse(self): a = array.array(self.typecode, self.example) self.assertRaises(TypeError, a.reverse, 42) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 3789ac22e3899cf..7cecf319e3638f0 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -525,17 +525,38 @@ def test_field_attr_existence(self): if name == 'Index': continue if self._is_ast_node(name, item): - x = item() + x = self._construct_ast_class(item) if isinstance(x, ast.AST): self.assertIs(type(x._fields), tuple) + def _construct_ast_class(self, cls): + kwargs = {} + for name, typ in cls.__annotations__.items(): + if typ is str: + kwargs[name] = 'capybara' + elif typ is int: + kwargs[name] = 42 + elif typ is object: + kwargs[name] = b'capybara' + elif isinstance(typ, type) and issubclass(typ, ast.AST): + kwargs[name] = self._construct_ast_class(typ) + return cls(**kwargs) + def test_arguments(self): x = ast.arguments() self.assertEqual(x._fields, ('posonlyargs', 'args', 'vararg', 'kwonlyargs', 'kw_defaults', 'kwarg', 'defaults')) - - with self.assertRaises(AttributeError): - x.args + self.assertEqual(x.__annotations__, { + 'posonlyargs': list[ast.arg], + 'args': list[ast.arg], + 'vararg': ast.arg | None, + 'kwonlyargs': list[ast.arg], + 'kw_defaults': list[ast.expr], + 'kwarg': ast.arg | None, + 'defaults': list[ast.expr], + }) + + self.assertEqual(x.args, []) self.assertIsNone(x.vararg) x = ast.arguments(*range(1, 8)) @@ -551,7 +572,7 @@ def test_field_attr_writable_deprecated(self): self.assertEqual(x._fields, 666) def test_field_attr_writable(self): - x = ast.Constant() + x = ast.Constant(1) # We can assign to _fields x._fields = 666 self.assertEqual(x._fields, 666) @@ -611,15 +632,22 @@ def test_classattrs_deprecated(self): self.assertEqual([str(w.message) for w in wlog], [ 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + "Constant.__init__ missing 1 required positional argument: 'value'. This will become " + 'an error in Python 3.15.', 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + "Constant.__init__ missing 1 required positional argument: 'value'. This will become " + 'an error in Python 3.15.', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + "Constant.__init__ got an unexpected keyword argument 'foo'. Support for " + 'arbitrary keyword arguments is deprecated and will be removed in Python ' + '3.15.', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', @@ -636,7 +664,8 @@ def test_classattrs_deprecated(self): ]) def test_classattrs(self): - x = ast.Constant() + with self.assertWarns(DeprecationWarning): + x = ast.Constant() self.assertEqual(x._fields, ('value', 'kind')) with self.assertRaises(AttributeError): @@ -651,7 +680,7 @@ def test_classattrs(self): with self.assertRaises(AttributeError): x.foobar - x = ast.Constant(lineno=2) + x = ast.Constant(lineno=2, value=3) self.assertEqual(x.lineno, 2) x = ast.Constant(42, lineno=0) @@ -662,8 +691,9 @@ def test_classattrs(self): self.assertRaises(TypeError, ast.Constant, 1, None, 2) self.assertRaises(TypeError, ast.Constant, 1, None, 2, lineno=0) - # Arbitrary keyword arguments are supported - self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') + # Arbitrary keyword arguments are supported (but deprecated) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"): ast.Constant(1, value=2) @@ -815,11 +845,11 @@ def test_isinstance(self): assertBytesDeprecated(self.assertNotIsInstance, Constant('42'), Bytes) assertNameConstantDeprecated(self.assertNotIsInstance, Constant(42), NameConstant) assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis) - assertNumDeprecated(self.assertNotIsInstance, Constant(), Num) - assertStrDeprecated(self.assertNotIsInstance, Constant(), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant(), Bytes) - assertNameConstantDeprecated(self.assertNotIsInstance, Constant(), NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(), Ellipsis) + assertNumDeprecated(self.assertNotIsInstance, Constant(None), Num) + assertStrDeprecated(self.assertNotIsInstance, Constant(None), Str) + assertBytesDeprecated(self.assertNotIsInstance, Constant(None), Bytes) + assertNameConstantDeprecated(self.assertNotIsInstance, Constant(1), NameConstant) + assertEllipsisDeprecated(self.assertNotIsInstance, Constant(None), Ellipsis) class S(str): pass with assertStrDeprecated(): @@ -888,8 +918,9 @@ def test_module(self): self.assertEqual(x.body, body) def test_nodeclasses(self): - # Zero arguments constructor explicitly allowed - x = ast.BinOp() + # Zero arguments constructor explicitly allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.BinOp() self.assertEqual(x._fields, ('left', 'op', 'right')) # Random attribute allowed too @@ -927,8 +958,9 @@ def test_nodeclasses(self): self.assertEqual(x.right, 3) self.assertEqual(x.lineno, 0) - # Random kwargs also allowed - x = ast.BinOp(1, 2, 3, foobarbaz=42) + # Random kwargs also allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.BinOp(1, 2, 3, foobarbaz=42) self.assertEqual(x.foobarbaz, 42) def test_no_fields(self): @@ -941,8 +973,9 @@ def test_pickling(self): for protocol in range(pickle.HIGHEST_PROTOCOL + 1): for ast in (compile(i, "?", "exec", 0x400) for i in exec_tests): - ast2 = pickle.loads(pickle.dumps(ast, protocol)) - self.assertEqual(to_tuple(ast2), to_tuple(ast)) + with self.subTest(ast=ast, protocol=protocol): + ast2 = pickle.loads(pickle.dumps(ast, protocol)) + self.assertEqual(to_tuple(ast2), to_tuple(ast)) def test_invalid_sum(self): pos = dict(lineno=2, col_offset=3) @@ -1045,19 +1078,15 @@ def test_positional_only_feature_version(self): with self.assertRaises(SyntaxError): ast.parse('lambda x=1, /: ...', feature_version=(3, 7)) - def test_parenthesized_with_feature_version(self): - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 10)) - # While advertised as a feature in Python 3.10, this was allowed starting 3.9 - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 9)) - with self.assertRaises(SyntaxError): - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 8)) - ast.parse('with CtxManager() as example: ...', feature_version=(3, 8)) - def test_assignment_expression_feature_version(self): ast.parse('(x := 0)', feature_version=(3, 8)) with self.assertRaises(SyntaxError): ast.parse('(x := 0)', feature_version=(3, 7)) + def test_conditional_context_managers_parse_with_low_feature_version(self): + # regression test for gh-115881 + ast.parse('with (x() if y else z()): ...', feature_version=(3, 8)) + def test_exception_groups_feature_version(self): code = dedent(''' try: ... @@ -1314,8 +1343,9 @@ def test_copy_location(self): 'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, ' 'col_offset=0, end_lineno=1, end_col_offset=5))' ) - src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1) - new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None)) + func = ast.Name('spam', ast.Load()) + src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1, func=func) + new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None, func=func)) self.assertIsNone(new.end_lineno) self.assertIsNone(new.end_col_offset) self.assertEqual(new.lineno, 1) @@ -1574,15 +1604,15 @@ def test_level_as_none(self): self.assertIn('sleep', ns) def test_recursion_direct(self): - e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) + e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) e.operand = e with self.assertRaises(RecursionError): with support.infinite_recursion(): compile(ast.Expression(e), "<test>", "eval") def test_recursion_indirect(self): - e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) - f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) + e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) + f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) e.operand = f f.operand = e with self.assertRaises(RecursionError): @@ -2870,6 +2900,23 @@ def visit_Call(self, node: ast.Call): self.assertASTTransformation(PrintToLog, code, expected) +class ASTConstructorTests(unittest.TestCase): + """Test the autogenerated constructors for AST nodes.""" + + def test_FunctionDef(self): + args = ast.arguments() + self.assertEqual(args.args, []) + self.assertEqual(args.posonlyargs, []) + with self.assertWarnsRegex(DeprecationWarning, + r"FunctionDef\.__init__ missing 1 required positional argument: 'name'"): + node = ast.FunctionDef(args=args) + self.assertFalse(hasattr(node, "name")) + self.assertEqual(node.decorator_list, []) + node = ast.FunctionDef(name='foo', args=args) + self.assertEqual(node.name, 'foo') + self.assertEqual(node.decorator_list, []) + + @support.cpython_only class ModuleStateTests(unittest.TestCase): # bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state. diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index c2080977e9d5877..4cd872d3a5b2d89 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -231,6 +231,22 @@ def test_set_default_executor_error(self): self.assertIsNone(self.loop._default_executor) + def test_shutdown_default_executor_timeout(self): + class DummyExecutor(concurrent.futures.ThreadPoolExecutor): + def shutdown(self, wait=True, *, cancel_futures=False): + if wait: + time.sleep(0.1) + + self.loop._process_events = mock.Mock() + self.loop._write_to_self = mock.Mock() + executor = DummyExecutor() + self.loop.set_default_executor(executor) + + with self.assertWarnsRegex(RuntimeWarning, + "The executor did not finishing joining"): + self.loop.run_until_complete( + self.loop.shutdown_default_executor(timeout=0.01)) + def test_call_soon(self): def cb(): pass @@ -1232,7 +1248,7 @@ def test_create_datagram_endpoint_wrong_sock(self): with sock: coro = self.loop.create_datagram_endpoint(MyProto, sock=sock) with self.assertRaisesRegex(ValueError, - 'A UDP Socket was expected'): + 'A datagram socket was expected'): self.loop.run_until_complete(coro) def test_create_connection_no_host_port_sock(self): diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index b25c0975736e20e..5b9c871e1d1b5a0 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1815,6 +1815,7 @@ def check_killed(self, returncode): else: self.assertEqual(-signal.SIGKILL, returncode) + @support.requires_subprocess() def test_subprocess_exec(self): prog = os.path.join(os.path.dirname(__file__), 'echo.py') @@ -1836,6 +1837,7 @@ def test_subprocess_exec(self): self.check_killed(proto.returncode) self.assertEqual(b'Python The Winner', proto.data[1]) + @support.requires_subprocess() def test_subprocess_interactive(self): prog = os.path.join(os.path.dirname(__file__), 'echo.py') @@ -1863,6 +1865,7 @@ def test_subprocess_interactive(self): self.loop.run_until_complete(proto.completed) self.check_killed(proto.returncode) + @support.requires_subprocess() def test_subprocess_shell(self): connect = self.loop.subprocess_shell( functools.partial(MySubprocessProtocol, self.loop), @@ -1879,6 +1882,7 @@ def test_subprocess_shell(self): self.assertEqual(proto.data[2], b'') transp.close() + @support.requires_subprocess() def test_subprocess_exitcode(self): connect = self.loop.subprocess_shell( functools.partial(MySubprocessProtocol, self.loop), @@ -1890,6 +1894,7 @@ def test_subprocess_exitcode(self): self.assertEqual(7, proto.returncode) transp.close() + @support.requires_subprocess() def test_subprocess_close_after_finish(self): connect = self.loop.subprocess_shell( functools.partial(MySubprocessProtocol, self.loop), @@ -1904,6 +1909,7 @@ def test_subprocess_close_after_finish(self): self.assertEqual(7, proto.returncode) self.assertIsNone(transp.close()) + @support.requires_subprocess() def test_subprocess_kill(self): prog = os.path.join(os.path.dirname(__file__), 'echo.py') @@ -1920,6 +1926,7 @@ def test_subprocess_kill(self): self.check_killed(proto.returncode) transp.close() + @support.requires_subprocess() def test_subprocess_terminate(self): prog = os.path.join(os.path.dirname(__file__), 'echo.py') @@ -1937,6 +1944,7 @@ def test_subprocess_terminate(self): transp.close() @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP") + @support.requires_subprocess() def test_subprocess_send_signal(self): # bpo-31034: Make sure that we get the default signal handler (killing # the process). The parent process may have decided to ignore SIGHUP, @@ -1961,6 +1969,7 @@ def test_subprocess_send_signal(self): finally: signal.signal(signal.SIGHUP, old_handler) + @support.requires_subprocess() def test_subprocess_stderr(self): prog = os.path.join(os.path.dirname(__file__), 'echo2.py') @@ -1982,6 +1991,7 @@ def test_subprocess_stderr(self): self.assertTrue(proto.data[2].startswith(b'ERR:test'), proto.data[2]) self.assertEqual(0, proto.returncode) + @support.requires_subprocess() def test_subprocess_stderr_redirect_to_stdout(self): prog = os.path.join(os.path.dirname(__file__), 'echo2.py') @@ -2007,6 +2017,7 @@ def test_subprocess_stderr_redirect_to_stdout(self): transp.close() self.assertEqual(0, proto.returncode) + @support.requires_subprocess() def test_subprocess_close_client_stream(self): prog = os.path.join(os.path.dirname(__file__), 'echo3.py') @@ -2041,6 +2052,7 @@ def test_subprocess_close_client_stream(self): self.loop.run_until_complete(proto.completed) self.check_killed(proto.returncode) + @support.requires_subprocess() def test_subprocess_wait_no_same_group(self): # start the new process in a new session connect = self.loop.subprocess_shell( @@ -2053,6 +2065,7 @@ def test_subprocess_wait_no_same_group(self): self.assertEqual(7, proto.returncode) transp.close() + @support.requires_subprocess() def test_subprocess_exec_invalid_args(self): async def connect(**kwds): await self.loop.subprocess_exec( @@ -2066,6 +2079,7 @@ async def connect(**kwds): with self.assertRaises(ValueError): self.loop.run_until_complete(connect(shell=True)) + @support.requires_subprocess() def test_subprocess_shell_invalid_args(self): async def connect(cmd=None, **kwds): @@ -2236,7 +2250,7 @@ def test_handle_repr(self): h = asyncio.Handle(noop, (1, 2), self.loop) filename, lineno = test_utils.get_function_source(noop) self.assertEqual(repr(h), - '<Handle noop(1, 2) at %s:%s>' + '<Handle noop() at %s:%s>' % (filename, lineno)) # cancelled handle @@ -2254,14 +2268,14 @@ def test_handle_repr(self): # partial function cb = functools.partial(noop, 1, 2) h = asyncio.Handle(cb, (3,), self.loop) - regex = (r'^<Handle noop\(1, 2\)\(3\) at %s:%s>$' + regex = (r'^<Handle noop\(\)\(\) at %s:%s>$' % (re.escape(filename), lineno)) self.assertRegex(repr(h), regex) # partial function with keyword args cb = functools.partial(noop, x=1) h = asyncio.Handle(cb, (2, 3), self.loop) - regex = (r'^<Handle noop\(x=1\)\(2, 3\) at %s:%s>$' + regex = (r'^<Handle noop\(\)\(\) at %s:%s>$' % (re.escape(filename), lineno)) self.assertRegex(repr(h), regex) @@ -2302,6 +2316,24 @@ def test_handle_repr_debug(self): '<Handle cancelled noop(1, 2) at %s:%s created at %s:%s>' % (filename, lineno, create_filename, create_lineno)) + # partial function + cb = functools.partial(noop, 1, 2) + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.Handle(cb, (3,), self.loop) + regex = (r'^<Handle noop\(1, 2\)\(3\) at %s:%s created at %s:%s>$' + % (re.escape(filename), lineno, + re.escape(create_filename), create_lineno)) + self.assertRegex(repr(h), regex) + + # partial function with keyword args + cb = functools.partial(noop, x=1) + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.Handle(cb, (2, 3), self.loop) + regex = (r'^<Handle noop\(x=1\)\(2, 3\) at %s:%s created at %s:%s>$' + % (re.escape(filename), lineno, + re.escape(create_filename), create_lineno)) + self.assertRegex(repr(h), regex) + def test_handle_source_traceback(self): loop = asyncio.get_event_loop_policy().new_event_loop() loop.set_debug(True) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 9029efd2355b46f..a0884bffe6b0de1 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -816,6 +816,98 @@ async def func(): # originally raised. self.assertIs(err.exception, raised) + async def test_cancelled_wakeup(self): + # Test that a task cancelled at the "same" time as it is woken + # up as part of a Condition.notify() does not result in a lost wakeup. + # This test simulates a cancel while the target task is awaiting initial + # wakeup on the wakeup queue. + condition = asyncio.Condition() + state = 0 + async def consumer(): + nonlocal state + async with condition: + while True: + await condition.wait_for(lambda: state != 0) + if state < 0: + return + state -= 1 + + # create two consumers + c = [asyncio.create_task(consumer()) for _ in range(2)] + # wait for them to settle + await asyncio.sleep(0) + async with condition: + # produce one item and wake up one + state += 1 + condition.notify(1) + + # Cancel it while it is awaiting to be run. + # This cancellation could come from the outside + c[0].cancel() + + # now wait for the item to be consumed + # if it doesn't means that our "notify" didn"t take hold. + # because it raced with a cancel() + try: + async with asyncio.timeout(0.01): + await condition.wait_for(lambda: state == 0) + except TimeoutError: + pass + self.assertEqual(state, 0) + + # clean up + state = -1 + condition.notify_all() + await c[1] + + async def test_cancelled_wakeup_relock(self): + # Test that a task cancelled at the "same" time as it is woken + # up as part of a Condition.notify() does not result in a lost wakeup. + # This test simulates a cancel while the target task is acquiring the lock + # again. + condition = asyncio.Condition() + state = 0 + async def consumer(): + nonlocal state + async with condition: + while True: + await condition.wait_for(lambda: state != 0) + if state < 0: + return + state -= 1 + + # create two consumers + c = [asyncio.create_task(consumer()) for _ in range(2)] + # wait for them to settle + await asyncio.sleep(0) + async with condition: + # produce one item and wake up one + state += 1 + condition.notify(1) + + # now we sleep for a bit. This allows the target task to wake up and + # settle on re-aquiring the lock + await asyncio.sleep(0) + + # Cancel it while awaiting the lock + # This cancel could come the outside. + c[0].cancel() + + # now wait for the item to be consumed + # if it doesn't means that our "notify" didn"t take hold. + # because it raced with a cancel() + try: + async with asyncio.timeout(0.01): + await condition.wait_for(lambda: state == 0) + except TimeoutError: + pass + self.assertEqual(state, 0) + + # clean up + state = -1 + condition.notify_all() + await c[1] + class SemaphoreTests(unittest.IsolatedAsyncioTestCase): def test_initial_value_zero(self): diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py index c42856e578b8cc8..fcaa2f6ade2b767 100644 --- a/Lib/test/test_asyncio/test_proactor_events.py +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -585,11 +585,10 @@ def test_sendto_memoryview(self): def test_sendto_no_data(self): transport = self.datagram_transport() - transport._buffer.append((b'data', ('0.0.0.0', 12345))) - transport.sendto(b'', ()) - self.assertFalse(self.sock.sendto.called) - self.assertEqual( - [(b'data', ('0.0.0.0', 12345))], list(transport._buffer)) + transport.sendto(b'', ('0.0.0.0', 1234)) + self.assertTrue(self.proactor.sendto.called) + self.proactor.sendto.assert_called_with( + self.sock, b'', addr=('0.0.0.0', 1234)) def test_sendto_buffer(self): transport = self.datagram_transport() @@ -628,6 +627,19 @@ def test_sendto_buffer_memoryview(self): list(transport._buffer)) self.assertIsInstance(transport._buffer[1][0], bytes) + def test_sendto_buffer_nodata(self): + data2 = b'' + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport._write_fut = object() + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.proactor.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + @mock.patch('asyncio.proactor_events.logger') def test_sendto_exception(self, m_log): data = b'data' diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index c22b780b5edcb83..aaeda33dd0c677f 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -1280,11 +1280,10 @@ def test_sendto_memoryview(self): def test_sendto_no_data(self): transport = self.datagram_transport() - transport._buffer.append((b'data', ('0.0.0.0', 12345))) - transport.sendto(b'', ()) - self.assertFalse(self.sock.sendto.called) + transport.sendto(b'', ('0.0.0.0', 1234)) + self.assertTrue(self.sock.sendto.called) self.assertEqual( - [(b'data', ('0.0.0.0', 12345))], list(transport._buffer)) + self.sock.sendto.call_args[0], (b'', ('0.0.0.0', 1234))) def test_sendto_buffer(self): transport = self.datagram_transport() @@ -1320,6 +1319,18 @@ def test_sendto_buffer_memoryview(self): list(transport._buffer)) self.assertIsInstance(transport._buffer[1][0], bytes) + def test_sendto_buffer_nodata(self): + data2 = b'' + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.sock.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + def test_sendto_tryagain(self): data = b'data' diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 3c8cc5f3649180d..bf123ebf9bd158f 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -10,7 +10,6 @@ import unittest from unittest import mock import warnings -from test.support import socket_helper try: import ssl except ImportError: @@ -18,6 +17,7 @@ import asyncio from test.test_asyncio import utils as test_utils +from test.support import requires_subprocess, socket_helper def tearDownModule(): @@ -770,6 +770,7 @@ async def client(addr): self.assertEqual(msg2, b"hello world 2!\n") @unittest.skipIf(sys.platform == 'win32', "Don't have pipes") + @requires_subprocess() def test_read_all_from_pipe_reader(self): # See asyncio issue 168. This test is derived from the example # subprocess_attach_read_pipe.py, but we configure the @@ -1129,6 +1130,31 @@ async def inner(httpd): self.assertEqual(messages, []) + def test_unclosed_server_resource_warnings(self): + async def inner(rd, wr): + fut.set_result(True) + with self.assertWarns(ResourceWarning) as cm: + del wr + gc.collect() + self.assertEqual(len(cm.warnings), 1) + self.assertTrue(str(cm.warnings[0].message).startswith("unclosed <StreamWriter")) + + async def outer(): + srv = await asyncio.start_server(inner, socket_helper.HOSTv4, 0) + async with srv: + addr = srv.sockets[0].getsockname() + with socket.create_connection(addr): + # Give the loop some time to notice the connection + await fut + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + fut = self.loop.create_future() + self.loop.run_until_complete(outer()) + + self.assertEqual(messages, []) + def _basetest_unhandled_exceptions(self, handle_echo): port = socket_helper.find_unused_port() diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 808b21c66175511..f50a9ebc031ba8a 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -47,6 +47,7 @@ def _start(self, *args, **kwargs): self._proc.pid = -1 +@support.requires_subprocess() class SubprocessTransportTests(test_utils.TestCase): def setUp(self): super().setUp() @@ -110,6 +111,7 @@ def test_subprocess_repr(self): transport.close() +@support.requires_subprocess() class SubprocessMixin: def test_stdin_stdout(self): diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index d2c8cba6acfa31c..59ef9f5f58cabc3 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1874,7 +1874,7 @@ async def runner(): wsock.close() -@unittest.skipUnless(hasattr(os, 'fork'), 'requires os.fork()') +@support.requires_fork() class TestFork(unittest.IsolatedAsyncioTestCase): async def test_fork_not_share_event_loop(self): diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index f6171d3ed4efd73..409c8c109e885f6 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -545,6 +545,40 @@ def test_b85encode(self): self.check_other_types(base64.b85encode, b"www.python.org", b'cXxL#aCvlSZ*DGca%T') + def test_z85encode(self): + eq = self.assertEqual + + tests = { + b'': b'', + b'www.python.org': b'CxXl-AcVLsz/dgCA+t', + bytes(range(255)): b"""009c61o!#m2NH?C3>iWS5d]J*6CRx17-skh9337x""" + b"""ar.{NbQB=+c[cR@eg&FcfFLssg=mfIi5%2YjuU>)kTv.7l}6Nnnj=AD""" + b"""oIFnTp/ga?r8($2sxO*itWpVyu$0IOwmYv=xLzi%y&a6dAb/]tBAI+J""" + b"""CZjQZE0{D[FpSr8GOteoH(41EJe-<UKDCY&L:dM3N3<zjOsMmzPRn9P""" + b"""Q[%@^ShV!$TGwUeU^7HuW6^uKXvGh.YUh4]Z})[9-kP:p:JqPF+*1CV""" + b"""^9Zp<!yAd4/Xb0k*$*&A&nJXQ<MkK!>&}x#)cTlf[Bu8v].4}L}1:^-""" + b"""@qDP""", + b"""abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ""" + b"""0123456789!@#0^&*();:<>,. []{}""": + b"""vpA.SwObN*x>?B1zeKohADlbxB-}$ND3R+ylQTvjm[uizoh55PpF:[^""" + b"""q=D:$s6eQefFLssg=mfIi5@cEbqrBJdKV-ciY]OSe*aw7DWL""", + b'no padding..': b'zF{UpvpS[.zF7NO', + b'zero compression\x00\x00\x00\x00': b'Ds.bnay/tbAb]JhB7]Mg00000', + b'zero compression\x00\x00\x00': b'Ds.bnay/tbAb]JhB7]Mg0000', + b"""Boundary:\x00\x00\x00\x00""": b"""lt}0:wmoI7iSGcW00""", + b'Space compr: ': b'q/DePwGUG3ze:IRarR^H', + b'\xff': b'@@', + b'\xff'*2: b'%nJ', + b'\xff'*3: b'%nS9', + b'\xff'*4: b'%nSc0', + } + + for data, res in tests.items(): + eq(base64.z85encode(data), res) + + self.check_other_types(base64.z85encode, b"www.python.org", + b'CxXl-AcVLsz/dgCA+t') + def test_a85decode(self): eq = self.assertEqual @@ -626,6 +660,41 @@ def test_b85decode(self): self.check_other_types(base64.b85decode, b'cXxL#aCvlSZ*DGca%T', b"www.python.org") + def test_z85decode(self): + eq = self.assertEqual + + tests = { + b'': b'', + b'CxXl-AcVLsz/dgCA+t': b'www.python.org', + b"""009c61o!#m2NH?C3>iWS5d]J*6CRx17-skh9337x""" + b"""ar.{NbQB=+c[cR@eg&FcfFLssg=mfIi5%2YjuU>)kTv.7l}6Nnnj=AD""" + b"""oIFnTp/ga?r8($2sxO*itWpVyu$0IOwmYv=xLzi%y&a6dAb/]tBAI+J""" + b"""CZjQZE0{D[FpSr8GOteoH(41EJe-<UKDCY&L:dM3N3<zjOsMmzPRn9P""" + b"""Q[%@^ShV!$TGwUeU^7HuW6^uKXvGh.YUh4]Z})[9-kP:p:JqPF+*1CV""" + b"""^9Zp<!yAd4/Xb0k*$*&A&nJXQ<MkK!>&}x#)cTlf[Bu8v].4}L}1:^-""" + b"""@qDP""": bytes(range(255)), + b"""vpA.SwObN*x>?B1zeKohADlbxB-}$ND3R+ylQTvjm[uizoh55PpF:[^""" + b"""q=D:$s6eQefFLssg=mfIi5@cEbqrBJdKV-ciY]OSe*aw7DWL""": + b"""abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ""" + b"""0123456789!@#0^&*();:<>,. []{}""", + b'zF{UpvpS[.zF7NO': b'no padding..', + b'Ds.bnay/tbAb]JhB7]Mg00000': b'zero compression\x00\x00\x00\x00', + b'Ds.bnay/tbAb]JhB7]Mg0000': b'zero compression\x00\x00\x00', + b"""lt}0:wmoI7iSGcW00""": b"""Boundary:\x00\x00\x00\x00""", + b'q/DePwGUG3ze:IRarR^H': b'Space compr: ', + b'@@': b'\xff', + b'%nJ': b'\xff'*2, + b'%nS9': b'\xff'*3, + b'%nSc0': b'\xff'*4, + } + + for data, res in tests.items(): + eq(base64.z85decode(data), res) + eq(base64.z85decode(data.decode("ascii")), res) + + self.check_other_types(base64.z85decode, b'CxXl-AcVLsz/dgCA+t', + b'www.python.org') + def test_a85_padding(self): eq = self.assertEqual @@ -707,6 +776,21 @@ def test_b85decode_errors(self): self.assertRaises(ValueError, base64.b85decode, b'|NsC') self.assertRaises(ValueError, base64.b85decode, b'|NsC1') + def test_z85decode_errors(self): + illegal = list(range(33)) + \ + list(b'"\',;_`|\\~') + \ + list(range(128, 256)) + for c in illegal: + with self.assertRaises(ValueError, msg=bytes([c])): + base64.z85decode(b'0000' + bytes([c])) + + # b'\xff\xff\xff\xff' encodes to b'%nSc0', the following will overflow: + self.assertRaises(ValueError, base64.z85decode, b'%') + self.assertRaises(ValueError, base64.z85decode, b'%n') + self.assertRaises(ValueError, base64.z85decode, b'%nS') + self.assertRaises(ValueError, base64.z85decode, b'%nSc') + self.assertRaises(ValueError, base64.z85decode, b'%nSc1') + def test_decode_nonascii_str(self): decode_funcs = (base64.b64decode, base64.standard_b64decode, @@ -714,7 +798,8 @@ def test_decode_nonascii_str(self): base64.b32decode, base64.b16decode, base64.b85decode, - base64.a85decode) + base64.a85decode, + base64.z85decode) for f in decode_funcs: self.assertRaises(ValueError, f, 'with non-ascii \xcb') diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 72a06d6af450e32..5b1b95b9c82064c 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4585,6 +4585,33 @@ def test_c_buffer(self): buf.__release_buffer__(mv) self.assertEqual(buf.references, 0) + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_buffer_invalid_flags(self): + buf = _testcapi.testBuf() + self.assertRaises(SystemError, buf.__buffer__, PyBUF_READ) + self.assertRaises(SystemError, buf.__buffer__, PyBUF_WRITE) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_fill_buffer_invalid_flags(self): + # PyBuffer_FillInfo + source = b"abc" + self.assertRaises(SystemError, _testcapi.buffer_fill_info, + source, 0, PyBUF_READ) + self.assertRaises(SystemError, _testcapi.buffer_fill_info, + source, 0, PyBUF_WRITE) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_fill_buffer_readonly_and_writable(self): + source = b"abc" + with _testcapi.buffer_fill_info(source, 1, PyBUF_SIMPLE) as m: + self.assertEqual(bytes(m), b"abc") + self.assertTrue(m.readonly) + with _testcapi.buffer_fill_info(source, 0, PyBUF_WRITABLE) as m: + self.assertEqual(bytes(m), b"abc") + self.assertFalse(m.readonly) + self.assertRaises(BufferError, _testcapi.buffer_fill_info, + source, 1, PyBUF_WRITABLE) + def test_inheritance(self): class A(bytearray): def __buffer__(self, flags): diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index fcddd147bac63ef..9a0bf524e3943fd 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -308,14 +308,13 @@ class C3(C2): pass self.assertTrue(callable(c3)) def test_chr(self): + self.assertEqual(chr(0), '\0') self.assertEqual(chr(32), ' ') self.assertEqual(chr(65), 'A') self.assertEqual(chr(97), 'a') self.assertEqual(chr(0xff), '\xff') - self.assertRaises(ValueError, chr, 1<<24) - self.assertEqual(chr(sys.maxunicode), - str('\\U0010ffff'.encode("ascii"), 'unicode-escape')) self.assertRaises(TypeError, chr) + self.assertRaises(TypeError, chr, 65.0) self.assertEqual(chr(0x0000FFFF), "\U0000FFFF") self.assertEqual(chr(0x00010000), "\U00010000") self.assertEqual(chr(0x00010001), "\U00010001") @@ -327,7 +326,11 @@ def test_chr(self): self.assertEqual(chr(0x0010FFFF), "\U0010FFFF") self.assertRaises(ValueError, chr, -1) self.assertRaises(ValueError, chr, 0x00110000) - self.assertRaises((OverflowError, ValueError), chr, 2**32) + self.assertRaises(ValueError, chr, 1<<24) + self.assertRaises(ValueError, chr, 2**32-1) + self.assertRaises(ValueError, chr, -2**32) + self.assertRaises(ValueError, chr, 2**1000) + self.assertRaises(ValueError, chr, -2**1000) def test_cmp(self): self.assertTrue(not hasattr(builtins, "cmp")) @@ -611,6 +614,14 @@ def __dir__(self): self.assertIsInstance(res, list) self.assertTrue(res == ["a", "b", "c"]) + # dir(obj__dir__iterable) + class Foo(object): + def __dir__(self): + return {"b", "c", "a"} + res = dir(Foo()) + self.assertIsInstance(res, list) + self.assertEqual(sorted(res), ["a", "b", "c"]) + # dir(obj__dir__not_sequence) class Foo(object): def __dir__(self): diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index a3804a945f2e3a4..71bb1e75c6affd7 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1599,6 +1599,13 @@ def test_extend(self): a = bytearray(b'') a.extend([Indexable(ord('a'))]) self.assertEqual(a, b'a') + a = bytearray(b'abc') + self.assertRaisesRegex(TypeError, # Override for string. + "expected iterable of integers; got: 'str'", + a.extend, 'def') + self.assertRaisesRegex(TypeError, # But not for others. + "can't extend bytearray with float", + a.extend, 1.0) def test_remove(self): b = bytearray(b'hello') diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 1f0b9adc3698b4a..772f0eacce28f52 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -3,19 +3,19 @@ import array import unittest +import io from io import BytesIO, DEFAULT_BUFFER_SIZE import os import pickle import glob import tempfile -import pathlib import random import shutil import subprocess import threading from test.support import import_helper from test.support import threading_helper -from test.support.os_helper import unlink +from test.support.os_helper import unlink, FakePath import _compression import sys @@ -537,12 +537,136 @@ def testMultiStreamOrdering(self): with BZ2File(self.filename) as bz2f: self.assertEqual(bz2f.read(), data1 + data2) + def testOpenFilename(self): + with BZ2File(self.filename, "wb") as f: + f.write(b'content') + self.assertIsInstance(f.fileno(), int) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with BZ2File(self.filename, "ab") as f: + f.write(b'appendix') + self.assertIsInstance(f.fileno(), int) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with BZ2File(self.filename, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertIsInstance(f.fileno(), int) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.fileno() + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def testOpenFileWithName(self): + with open(self.filename, 'wb') as raw: + with BZ2File(raw, 'wb') as f: + f.write(b'content') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with open(self.filename, 'ab') as raw: + with BZ2File(raw, 'ab') as f: + f.write(b'appendix') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with open(self.filename, 'rb') as raw: + with BZ2File(raw, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.fileno() + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def testOpenFileWithoutName(self): + bio = BytesIO() + with BZ2File(bio, 'wb') as f: + f.write(b'content') + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertRaises(ValueError, f.fileno) + + with BZ2File(bio, 'ab') as f: + f.write(b'appendix') + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertRaises(ValueError, f.fileno) + + bio.seek(0) + with BZ2File(bio, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertRaises(io.UnsupportedOperation, f.fileno) + with self.assertRaises(ValueError): + f.fileno() + + def testOpenFileWithIntName(self): + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with open(fd, 'wb') as raw: + with BZ2File(raw, 'wb') as f: + f.write(b'content') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertRaises(ValueError, f.fileno) + + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_APPEND) + with open(fd, 'ab') as raw: + with BZ2File(raw, 'ab') as f: + f.write(b'appendix') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertRaises(ValueError, f.fileno) + + fd = os.open(self.filename, os.O_RDONLY) + with open(fd, 'rb') as raw: + with BZ2File(raw, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.fileno(), raw.fileno()) + with self.assertRaises(ValueError): + f.fileno() + def testOpenBytesFilename(self): str_filename = self.filename - try: - bytes_filename = str_filename.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(str_filename) with BZ2File(bytes_filename, "wb") as f: f.write(self.DATA) with BZ2File(bytes_filename, "rb") as f: @@ -552,7 +676,7 @@ def testOpenBytesFilename(self): self.assertEqual(f.read(), self.DATA) def testOpenPathLikeFilename(self): - filename = pathlib.Path(self.filename) + filename = FakePath(self.filename) with BZ2File(filename, "wb") as f: f.write(self.DATA) with BZ2File(filename, "rb") as f: diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 3c8fc35e3c116d0..2a6a5d287b04ee9 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -155,7 +155,7 @@ def test_varargs16_kw(self): min, 0, default=1, key=2, foo=3) def test_varargs17_kw(self): - msg = r"'foo' is an invalid keyword argument for print\(\)$" + msg = r"print\(\) got an unexpected keyword argument 'foo'$" self.assertRaisesRegex(TypeError, msg, print, 0, sep=1, end=2, file=3, flush=4, foo=5) @@ -928,7 +928,7 @@ def check_suggestion_includes(self, message): self.assertIn(f"Did you mean '{message}'?", str(cm.exception)) @contextlib.contextmanager - def check_suggestion_not_pressent(self): + def check_suggestion_not_present(self): with self.assertRaises(TypeError) as cm: yield self.assertNotIn("Did you mean", str(cm.exception)) @@ -946,7 +946,7 @@ def foo(blech=None, /, aaa=None, *args, late1=None): for keyword, suggestion in cases: with self.subTest(keyword): - ctx = self.check_suggestion_includes(suggestion) if suggestion else self.check_suggestion_not_pressent() + ctx = self.check_suggestion_includes(suggestion) if suggestion else self.check_suggestion_not_present() with ctx: foo(**{keyword:None}) @@ -987,6 +987,32 @@ def case_change_over_substitution(BLuch=None, Luch = None, fluch = None): with self.check_suggestion_includes(suggestion): func(bluch=None) + def test_unexpected_keyword_suggestion_via_getargs(self): + with self.check_suggestion_includes("maxsplit"): + "foo".split(maxsplt=1) + + self.assertRaisesRegex( + TypeError, r"split\(\) got an unexpected keyword argument 'blech'$", + "foo".split, blech=1 + ) + with self.check_suggestion_not_present(): + "foo".split(blech=1) + with self.check_suggestion_not_present(): + "foo".split(more_noise=1, maxsplt=1) + + # Also test the vgetargskeywords path + with self.check_suggestion_includes("name"): + ImportError(namez="oops") + + self.assertRaisesRegex( + TypeError, r"ImportError\(\) got an unexpected keyword argument 'blech'$", + ImportError, blech=1 + ) + with self.check_suggestion_not_present(): + ImportError(blech=1) + with self.check_suggestion_not_present(): + ImportError(blech=1, namez="oops") + @cpython_only class TestRecursion(unittest.TestCase): diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py index 57a7238588eae09..cca6145bc90c047 100644 --- a/Lib/test/test_capi/test_dict.py +++ b/Lib/test/test_capi/test_dict.py @@ -339,6 +339,28 @@ def test_dict_setdefault(self): # CRASHES setdefault({}, 'a', NULL) # CRASHES setdefault(NULL, 'a', 5) + def test_dict_setdefaultref(self): + setdefault = _testcapi.dict_setdefaultref + dct = {} + self.assertEqual(setdefault(dct, 'a', 5), 5) + self.assertEqual(dct, {'a': 5}) + self.assertEqual(setdefault(dct, 'a', 8), 5) + self.assertEqual(dct, {'a': 5}) + + dct2 = DictSubclass() + self.assertEqual(setdefault(dct2, 'a', 5), 5) + self.assertEqual(dct2, {'a': 5}) + self.assertEqual(setdefault(dct2, 'a', 8), 5) + self.assertEqual(dct2, {'a': 5}) + + self.assertRaises(TypeError, setdefault, {}, [], 5) # unhashable + self.assertRaises(SystemError, setdefault, UserDict(), 'a', 5) + self.assertRaises(SystemError, setdefault, [1], 0, 5) + self.assertRaises(SystemError, setdefault, 42, 'a', 5) + # CRASHES setdefault({}, NULL, 5) + # CRASHES setdefault({}, 'a', NULL) + # CRASHES setdefault(NULL, 'a', 5) + def test_mapping_keys_valuesitems(self): class BadMapping(dict): def keys(self): diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 9b6aef27625ad0e..12039803ba543eb 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -667,7 +667,7 @@ def test_invalid_keyword(self): try: getargs_keywords((1,2),3,arg5=10,arg666=666) except TypeError as err: - self.assertEqual(str(err), "'arg666' is an invalid keyword argument for this function") + self.assertEqual(str(err), "this function got an unexpected keyword argument 'arg666'") else: self.fail('TypeError should have been raised') @@ -675,7 +675,7 @@ def test_surrogate_keyword(self): try: getargs_keywords((1,2), 3, (4,(5,6)), (7,8,9), **{'\uDC80': 10}) except TypeError as err: - self.assertEqual(str(err), "'\udc80' is an invalid keyword argument for this function") + self.assertEqual(str(err), "this function got an unexpected keyword argument '\udc80'") else: self.fail('TypeError should have been raised') @@ -742,12 +742,12 @@ def test_too_many_args(self): def test_invalid_keyword(self): # extraneous keyword arg with self.assertRaisesRegex(TypeError, - "'monster' is an invalid keyword argument for this function"): + "this function got an unexpected keyword argument 'monster'"): getargs_keyword_only(1, 2, monster=666) def test_surrogate_keyword(self): with self.assertRaisesRegex(TypeError, - "'\udc80' is an invalid keyword argument for this function"): + "this function got an unexpected keyword argument '\udc80'"): getargs_keyword_only(1, 2, **{'\uDC80': 10}) def test_weird_str_subclass(self): @@ -761,7 +761,7 @@ def __hash__(self): "invalid keyword argument for this function"): getargs_keyword_only(1, 2, **{BadStr("keyword_only"): 3}) with self.assertRaisesRegex(TypeError, - "invalid keyword argument for this function"): + "this function got an unexpected keyword argument"): getargs_keyword_only(1, 2, **{BadStr("monster"): 666}) def test_weird_str_subclass2(self): @@ -774,7 +774,7 @@ def __hash__(self): "invalid keyword argument for this function"): getargs_keyword_only(1, 2, **{BadStr("keyword_only"): 3}) with self.assertRaisesRegex(TypeError, - "invalid keyword argument for this function"): + "this function got an unexpected keyword argument"): getargs_keyword_only(1, 2, **{BadStr("monster"): 666}) @@ -807,7 +807,7 @@ def test_required_args(self): def test_empty_keyword(self): with self.assertRaisesRegex(TypeError, - "'' is an invalid keyword argument for this function"): + "this function got an unexpected keyword argument ''"): self.getargs(1, 2, **{'': 666}) @@ -1204,7 +1204,7 @@ def test_basic(self): "function missing required argument 'a'"): parse((), {}, 'O', ['a']) with self.assertRaisesRegex(TypeError, - "'b' is an invalid keyword argument"): + "this function got an unexpected keyword argument 'b'"): parse((), {'b': 1}, '|O', ['a']) with self.assertRaisesRegex(TypeError, fr"argument for function given by name \('a'\) " @@ -1278,10 +1278,10 @@ def test_nonascii_keywords(self): fr"and position \(1\)"): parse((1,), {name: 2}, 'O|O', [name, 'b']) with self.assertRaisesRegex(TypeError, - f"'{name}' is an invalid keyword argument"): + f"this function got an unexpected keyword argument '{name}'"): parse((), {name: 1}, '|O', ['b']) with self.assertRaisesRegex(TypeError, - "'b' is an invalid keyword argument"): + "this function got an unexpected keyword argument 'b'"): parse((), {'b': 1}, '|O', [name]) invalid = name.encode() + (name.encode()[:-1] or b'\x80') @@ -1301,17 +1301,17 @@ def test_nonascii_keywords(self): for name2 in ('b', 'ë', 'ĉ', 'Ɐ', '𐀁'): with self.subTest(name2=name2): with self.assertRaisesRegex(TypeError, - f"'{name2}' is an invalid keyword argument"): + f"this function got an unexpected keyword argument '{name2}'"): parse((), {name2: 1}, '|O', [name]) name2 = name.encode().decode('latin1') if name2 != name: with self.assertRaisesRegex(TypeError, - f"'{name2}' is an invalid keyword argument"): + f"this function got an unexpected keyword argument '{name2}'"): parse((), {name2: 1}, '|O', [name]) name3 = name + '3' with self.assertRaisesRegex(TypeError, - f"'{name2}' is an invalid keyword argument"): + f"this function got an unexpected keyword argument '{name2}'"): parse((), {name2: 1, name3: 2}, '|OO', [name, name3]) def test_nested_tuple(self): diff --git a/Lib/test/test_capi/test_list.py b/Lib/test/test_capi/test_list.py index eb03d51d3def378..dceb4fce3c077bf 100644 --- a/Lib/test/test_capi/test_list.py +++ b/Lib/test/test_capi/test_list.py @@ -82,10 +82,8 @@ def test_list_get_size(self): # CRASHES size(UserList()) # CRASHES size(NULL) - - def test_list_getitem(self): - # Test PyList_GetItem() - getitem = _testcapi.list_getitem + def check_list_get_item(self, getitem, exctype): + # Common test cases for PyList_GetItem() and PyList_GetItemRef() lst = [1, 2, 3] self.assertEqual(getitem(lst, 0), 1) self.assertEqual(getitem(lst, 2), 3) @@ -93,12 +91,19 @@ def test_list_getitem(self): self.assertRaises(IndexError, getitem, lst, -1) self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MIN) self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MAX) - self.assertRaises(SystemError, getitem, 42, 1) - self.assertRaises(SystemError, getitem, (1, 2, 3), 1) - self.assertRaises(SystemError, getitem, {1: 2}, 1) - + self.assertRaises(exctype, getitem, 42, 1) + self.assertRaises(exctype, getitem, (1, 2, 3), 1) + self.assertRaises(exctype, getitem, {1: 2}, 1) # CRASHES getitem(NULL, 1) + def test_list_getitem(self): + # Test PyList_GetItem() + self.check_list_get_item(_testcapi.list_getitem, SystemError) + + def test_list_get_item_ref(self): + # Test PyList_GetItemRef() + self.check_list_get_item(_testcapi.list_get_item_ref, TypeError) + def test_list_get_item(self): # Test PyList_GET_ITEM() get_item = _testcapi.list_get_item @@ -112,7 +117,6 @@ def test_list_get_item(self): # CRASHES get_item(21, 2) # CRASHES get_item(NULL, 1) - def test_list_setitem(self): # Test PyList_SetItem() setitem = _testcapi.list_setitem diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 8e3ef25d1ff86fc..9f5ee507a8eb85e 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -1,5 +1,6 @@ import unittest import sys +import test.support as support from test.support import import_helper @@ -423,6 +424,170 @@ def test_long_asvoidptr(self): self.assertRaises(OverflowError, asvoidptr, -2**1000) # CRASHES asvoidptr(NULL) + def test_long_asnativebytes(self): + import math + from _testcapi import ( + pylong_asnativebytes as asnativebytes, + SIZE_MAX, + ) + + # Abbreviate sizeof(Py_ssize_t) to SZ because we use it a lot + SZ = int(math.ceil(math.log(SIZE_MAX + 1) / math.log(2)) / 8) + MAX_SSIZE = 2 ** (SZ * 8 - 1) - 1 + MAX_USIZE = 2 ** (SZ * 8) - 1 + if support.verbose: + print(f"SIZEOF_SIZE={SZ}\n{MAX_SSIZE=:016X}\n{MAX_USIZE=:016X}") + + # These tests check that the requested buffer size is correct. + # This matches our current implementation: We only specify that the + # return value is a size *sufficient* to hold the result when queried + # using n_bytes=0. If our implementation changes, feel free to update + # the expectations here -- or loosen them to be range checks. + # (i.e. 0 *could* be stored in 1 byte and 512 in 2) + for v, expect in [ + (0, SZ), + (512, SZ), + (-512, SZ), + (MAX_SSIZE, SZ), + (MAX_USIZE, SZ + 1), + (-MAX_SSIZE, SZ), + (-MAX_USIZE, SZ + 1), + (2**255-1, 32), + (-(2**255-1), 32), + (2**256-1, 33), + (-(2**256-1), 33), + ]: + with self.subTest(f"sizeof-{v:X}"): + buffer = bytearray(b"\x5a") + self.assertEqual(expect, asnativebytes(v, buffer, 0, -1), + "PyLong_AsNativeBytes(v, <unknown>, 0, -1)") + self.assertEqual(buffer, b"\x5a", + "buffer overwritten when it should not have been") + # Also check via the __index__ path + self.assertEqual(expect, asnativebytes(Index(v), buffer, 0, -1), + "PyLong_AsNativeBytes(Index(v), <unknown>, 0, -1)") + self.assertEqual(buffer, b"\x5a", + "buffer overwritten when it should not have been") + + # Test that we populate n=2 bytes but do not overwrite more. + buffer = bytearray(b"\x99"*3) + self.assertEqual(2, asnativebytes(4, buffer, 2, 0), # BE + "PyLong_AsNativeBytes(v, <3 byte buffer>, 2, 0) // BE") + self.assertEqual(buffer, b"\x00\x04\x99") + self.assertEqual(2, asnativebytes(4, buffer, 2, 1), # LE + "PyLong_AsNativeBytes(v, <3 byte buffer>, 2, 1) // LE") + self.assertEqual(buffer, b"\x04\x00\x99") + + # We request as many bytes as `expect_be` contains, and always check + # the result (both big and little endian). We check the return value + # independently, since the buffer should always be filled correctly even + # if we need more bytes + for v, expect_be, expect_n in [ + (0, b'\x00', 1), + (0, b'\x00' * 2, 2), + (0, b'\x00' * 8, min(8, SZ)), + (1, b'\x01', 1), + (1, b'\x00' * 10 + b'\x01', min(11, SZ)), + (42, b'\x2a', 1), + (42, b'\x00' * 10 + b'\x2a', min(11, SZ)), + (-1, b'\xff', 1), + (-1, b'\xff' * 10, min(11, SZ)), + (-42, b'\xd6', 1), + (-42, b'\xff' * 10 + b'\xd6', min(11, SZ)), + # Extracts 255 into a single byte, but requests sizeof(Py_ssize_t) + (255, b'\xff', SZ), + (255, b'\x00\xff', 2), + (256, b'\x01\x00', 2), + # Extracts successfully (unsigned), but requests 9 bytes + (2**63, b'\x80' + b'\x00' * 7, 9), + # "Extracts", but requests 9 bytes + (-2**63, b'\x80' + b'\x00' * 7, 9), + (2**63, b'\x00\x80' + b'\x00' * 7, 9), + (-2**63, b'\xff\x80' + b'\x00' * 7, 9), + + (2**255-1, b'\x7f' + b'\xff' * 31, 32), + (-(2**255-1), b'\x80' + b'\x00' * 30 + b'\x01', 32), + # Request extra bytes, but result says we only needed 32 + (-(2**255-1), b'\xff\x80' + b'\x00' * 30 + b'\x01', 32), + (-(2**255-1), b'\xff\xff\x80' + b'\x00' * 30 + b'\x01', 32), + + # Extracting 256 bits of integer will request 33 bytes, but still + # copy as many bits as possible into the buffer. So we *can* copy + # into a 32-byte buffer, though negative number may be unrecoverable + (2**256-1, b'\xff' * 32, 33), + (2**256-1, b'\x00' + b'\xff' * 32, 33), + (-(2**256-1), b'\x00' * 31 + b'\x01', 33), + (-(2**256-1), b'\xff' + b'\x00' * 31 + b'\x01', 33), + (-(2**256-1), b'\xff\xff' + b'\x00' * 31 + b'\x01', 33), + + # The classic "Windows HRESULT as negative number" case + # HRESULT hr; + # PyLong_CopyBits(<-2147467259>, &hr, sizeof(HRESULT)) + # assert(hr == E_FAIL) + (-2147467259, b'\x80\x00\x40\x05', 4), + ]: + with self.subTest(f"{v:X}-{len(expect_be)}bytes"): + n = len(expect_be) + # Fill the buffer with dummy data to ensure all bytes + # are overwritten. + buffer = bytearray(b"\xa5"*n) + expect_le = expect_be[::-1] + + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 0), + f"PyLong_AsNativeBytes(v, buffer, {n}, <big>)") + self.assertEqual(expect_be, buffer[:n], "<big>") + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 1), + f"PyLong_AsNativeBytes(v, buffer, {n}, <little>)") + self.assertEqual(expect_le, buffer[:n], "<little>") + + # Check a few error conditions. These are validated in code, but are + # unspecified in docs, so if we make changes to the implementation, it's + # fine to just update these tests rather than preserve the behaviour. + with self.assertRaises(SystemError): + asnativebytes(1, buffer, 0, 2) + with self.assertRaises(TypeError): + asnativebytes('not a number', buffer, 0, -1) + + def test_long_fromnativebytes(self): + import math + from _testcapi import ( + pylong_fromnativebytes as fromnativebytes, + SIZE_MAX, + ) + + # Abbreviate sizeof(Py_ssize_t) to SZ because we use it a lot + SZ = int(math.ceil(math.log(SIZE_MAX + 1) / math.log(2)) / 8) + MAX_SSIZE = 2 ** (SZ * 8 - 1) - 1 + MAX_USIZE = 2 ** (SZ * 8) - 1 + + for v_be, expect_s, expect_u in [ + (b'\x00', 0, 0), + (b'\x01', 1, 1), + (b'\xff', -1, 255), + (b'\x00\xff', 255, 255), + (b'\xff\xff', -1, 65535), + ]: + with self.subTest(f"{expect_s}-{expect_u:X}-{len(v_be)}bytes"): + n = len(v_be) + v_le = v_be[::-1] + + self.assertEqual(expect_s, fromnativebytes(v_be, n, 0, 1), + f"PyLong_FromNativeBytes(buffer, {n}, <big>)") + self.assertEqual(expect_s, fromnativebytes(v_le, n, 1, 1), + f"PyLong_FromNativeBytes(buffer, {n}, <little>)") + self.assertEqual(expect_u, fromnativebytes(v_be, n, 0, 0), + f"PyLong_FromUnsignedNativeBytes(buffer, {n}, <big>)") + self.assertEqual(expect_u, fromnativebytes(v_le, n, 1, 0), + f"PyLong_FromUnsignedNativeBytes(buffer, {n}, <little>)") + + # Check native endian when the result would be the same either + # way and we can test it. + if v_be == v_le: + self.assertEqual(expect_s, fromnativebytes(v_be, n, -1, 1), + f"PyLong_FromNativeBytes(buffer, {n}, <native>)") + self.assertEqual(expect_u, fromnativebytes(v_be, n, -1, 0), + f"PyLong_FromUnsignedNativeBytes(buffer, {n}, <native>)") + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 5c8c0596610303b..a43726f05a448d3 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1,10 +1,15 @@ import contextlib import opcode +import sys import textwrap import unittest +import gc +import os import _testinternalcapi +from test.support import script_helper + @contextlib.contextmanager def temporary_optimizer(opt): @@ -28,16 +33,16 @@ def clear_executors(func): class TestOptimizerAPI(unittest.TestCase): - def test_get_counter_optimizer_dealloc(self): + def test_new_counter_optimizer_dealloc(self): # See gh-108727 def f(): - _testinternalcapi.get_counter_optimizer() + _testinternalcapi.new_counter_optimizer() f() def test_get_set_optimizer(self): old = _testinternalcapi.get_optimizer() - opt = _testinternalcapi.get_counter_optimizer() + opt = _testinternalcapi.new_counter_optimizer() try: _testinternalcapi.set_optimizer(opt) self.assertEqual(_testinternalcapi.get_optimizer(), opt) @@ -58,7 +63,7 @@ def loop(): loop = ns['loop'] for repeat in range(5): - opt = _testinternalcapi.get_counter_optimizer() + opt = _testinternalcapi.new_counter_optimizer() with temporary_optimizer(opt): self.assertEqual(opt.get_count(), 0) with clear_executors(loop): @@ -86,7 +91,7 @@ def long_loop(): """), ns, ns) long_loop = ns['long_loop'] - opt = _testinternalcapi.get_counter_optimizer() + opt = _testinternalcapi.new_counter_optimizer() with temporary_optimizer(opt): self.assertEqual(opt.get_count(), 0) long_loop() @@ -98,7 +103,7 @@ def testfunc(x): while i < x: i += 1 - opt = _testinternalcapi.get_counter_optimizer() + opt = _testinternalcapi.new_counter_optimizer() with temporary_optimizer(opt): testfunc(1000) code, replace_code = testfunc.__code__, testfunc.__code__.replace() @@ -119,11 +124,20 @@ def get_first_executor(func): return None +def iter_opnames(ex): + for item in ex: + yield item[0] + + +def get_opnames(ex): + return set(iter_opnames(ex)) + + class TestExecutorInvalidation(unittest.TestCase): def setUp(self): self.old = _testinternalcapi.get_optimizer() - self.opt = _testinternalcapi.get_counter_optimizer() + self.opt = _testinternalcapi.new_counter_optimizer() _testinternalcapi.set_optimizer(self.opt) def tearDown(self): @@ -172,7 +186,7 @@ def f(): pass """), ns, ns) f = ns['f'] - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): f() exe = get_first_executor(f) @@ -181,6 +195,23 @@ def f(): _testinternalcapi.invalidate_executors(f.__code__) self.assertFalse(exe.is_valid()) + def test_sys__clear_internal_caches(self): + def f(): + for _ in range(1000): + pass + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + f() + exe = get_first_executor(f) + self.assertIsNotNone(exe) + self.assertTrue(exe.is_valid()) + sys._clear_internal_caches() + self.assertFalse(exe.is_valid()) + exe = get_first_executor(f) + self.assertIsNone(exe) + + +@unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") class TestUops(unittest.TestCase): def test_basic_loop(self): @@ -189,15 +220,15 @@ def testfunc(x): while i < x: i += 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(1000) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_SET_IP", uops) - self.assertIn("_LOAD_FAST", uops) + self.assertIn("_LOAD_FAST_0", uops) def test_extended_arg(self): "Check EXTENDED_ARG handling in superblock creation" @@ -236,7 +267,7 @@ def many_vars(): """), ns, ns) many_vars = ns["many_vars"] - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): ex = get_first_executor(many_vars) self.assertIsNone(ex) @@ -244,7 +275,8 @@ def many_vars(): ex = get_first_executor(many_vars) self.assertIsNotNone(ex) - self.assertIn(("_LOAD_FAST", 259, 0), list(ex)) + self.assertTrue(any((opcode, oparg, operand) == ("_LOAD_FAST", 259, 0) + for opcode, oparg, _, operand in list(ex))) def test_unspecialized_unpack(self): # An example of an unspecialized opcode @@ -258,14 +290,14 @@ def testfunc(x): while i < x: i += 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_UNPACK_SEQUENCE", uops) def test_pop_jump_if_false(self): @@ -274,13 +306,13 @@ def testfunc(n): while i < n: i += 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_IS_TRUE_POP", uops) def test_pop_jump_if_none(self): @@ -289,13 +321,13 @@ def testfunc(a): if x is None: x = 0 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(range(20)) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_IS_NOT_NONE_POP", uops) def test_pop_jump_if_not_none(self): @@ -305,13 +337,13 @@ def testfunc(a): if x is not None: x = 0 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(range(20)) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_IS_NONE_POP", uops) def test_pop_jump_if_true(self): @@ -320,13 +352,13 @@ def testfunc(n): while not i >= n: i += 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_IS_FALSE_POP", uops) def test_jump_backward(self): @@ -335,13 +367,13 @@ def testfunc(n): while i < n: i += 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_JUMP_TO_TOP", uops) def test_jump_forward(self): @@ -355,13 +387,13 @@ def testfunc(n): a += 1 return a - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) # Since there is no JUMP_FORWARD instruction, # look for indirect evidence: the += operator self.assertIn("_BINARY_OP_ADD_INT", uops) @@ -373,7 +405,7 @@ def testfunc(n): total += i return total - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): total = testfunc(20) self.assertEqual(total, 190) @@ -382,7 +414,7 @@ def testfunc(n): self.assertIsNotNone(ex) # for i, (opname, oparg) in enumerate(ex): # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_NOT_EXHAUSTED_RANGE", uops) # Verification that the jump goes past END_FOR # is done by manual inspection of the output @@ -394,7 +426,7 @@ def testfunc(a): total += i return total - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): a = list(range(20)) total = testfunc(a) @@ -404,7 +436,7 @@ def testfunc(a): self.assertIsNotNone(ex) # for i, (opname, oparg) in enumerate(ex): # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_NOT_EXHAUSTED_LIST", uops) # Verification that the jump goes past END_FOR # is done by manual inspection of the output @@ -416,7 +448,7 @@ def testfunc(a): total += i return total - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): a = tuple(range(20)) total = testfunc(a) @@ -426,7 +458,7 @@ def testfunc(a): self.assertIsNotNone(ex) # for i, (opname, oparg) in enumerate(ex): # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_NOT_EXHAUSTED_TUPLE", uops) # Verification that the jump goes past END_FOR # is done by manual inspection of the output @@ -436,7 +468,7 @@ def testfunc(it): for x in it: pass - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): a = [1, 2, 3] it = iter(a) @@ -452,13 +484,13 @@ def dummy(x): for i in range(n): dummy(i) - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_PUSH_FRAME", uops) self.assertIn("_BINARY_OP_ADD_INT", uops) @@ -470,13 +502,13 @@ def testfunc(n): else: i = 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_IS_FALSE_POP", uops) def test_for_iter_tier_two(self): @@ -498,7 +530,7 @@ def testfunc(n, m): x += 1000*i + j return x - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): x = testfunc(10, 10) @@ -506,7 +538,7 @@ def testfunc(n, m): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_FOR_ITER_TIER_TWO", uops) def test_confidence_score(self): @@ -527,18 +559,358 @@ def testfunc(n): bits += 1 return bits - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): x = testfunc(20) self.assertEqual(x, 40) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - ops = [opname for opname, _, _ in ex] - count = ops.count("_GUARD_IS_TRUE_POP") - # Because Each 'if' halves the score, the second branch is - # too much already. - self.assertEqual(count, 1) + ops = list(iter_opnames(ex)) + #Since branch is 50/50 the trace could go either way. + count = ops.count("_GUARD_IS_TRUE_POP") + ops.count("_GUARD_IS_FALSE_POP") + self.assertLessEqual(count, 2) + + +@unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") +class TestUopsOptimization(unittest.TestCase): + + def _run_with_optimizer(self, testfunc, arg): + res = None + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + res = testfunc(arg) + + ex = get_first_executor(testfunc) + return res, ex + + + def test_int_type_propagation(self): + def testfunc(loops): + num = 0 + for i in range(loops): + x = num + num + a = x + 1 + num += 1 + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertIsNotNone(ex) + self.assertEqual(res, 63) + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertGreaterEqual(len(binop_count), 3) + self.assertLessEqual(len(guard_both_int_count), 1) + + def test_int_type_propagation_through_frame(self): + def double(x): + return x + x + def testfunc(loops): + num = 0 + for i in range(loops): + x = num + num + a = double(x) + num += 1 + return a + + opt = _testinternalcapi.new_uop_optimizer() + res = None + with temporary_optimizer(opt): + res = testfunc(32) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + self.assertEqual(res, 124) + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertGreaterEqual(len(binop_count), 3) + self.assertLessEqual(len(guard_both_int_count), 1) + + def test_int_type_propagation_from_frame(self): + def double(x): + return x + x + def testfunc(loops): + num = 0 + for i in range(loops): + a = double(num) + x = a + a + num += 1 + return x + + opt = _testinternalcapi.new_uop_optimizer() + res = None + with temporary_optimizer(opt): + res = testfunc(32) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + self.assertEqual(res, 124) + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertGreaterEqual(len(binop_count), 3) + self.assertLessEqual(len(guard_both_int_count), 1) + + def test_int_impure_region(self): + def testfunc(loops): + num = 0 + while num < loops: + x = num + num + y = 1 + x // 2 + a = x + y + num += 1 + return a + + res, ex = self._run_with_optimizer(testfunc, 64) + self.assertIsNotNone(ex) + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + self.assertGreaterEqual(len(binop_count), 3) + + def test_call_py_exact_args(self): + def testfunc(n): + def dummy(x): + return x+1 + for i in range(n): + dummy(i) + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_PUSH_FRAME", uops) + self.assertIn("_BINARY_OP_ADD_INT", uops) + self.assertNotIn("_CHECK_PEP_523", uops) + + def test_int_type_propagate_through_range(self): + def testfunc(n): + + for i in range(n): + x = i + i + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 62) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_BOTH_INT", uops) + + def test_int_value_numbering(self): + def testfunc(n): + + y = 1 + for i in range(n): + x = y + z = x + a = z + b = a + res = x + z + a + b + return res + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 4) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_BOTH_INT", uops) + guard_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertEqual(len(guard_count), 1) + + def test_comprehension(self): + def testfunc(n): + for _ in range(n): + return [i for i in range(n)] + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, list(range(32))) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_BINARY_OP_ADD_INT", uops) + + def test_call_py_exact_args_disappearing(self): + def dummy(x): + return x+1 + + def testfunc(n): + for i in range(n): + dummy(i) + + opt = _testinternalcapi.new_uop_optimizer() + # Trigger specialization + testfunc(8) + with temporary_optimizer(opt): + del dummy + gc.collect() + + def dummy(x): + return x + 2 + testfunc(32) + + ex = get_first_executor(testfunc) + # Honestly as long as it doesn't crash it's fine. + # Whether we get an executor or not is non-deterministic, + # because it's decided by when the function is freed. + # This test is a little implementation specific. + + def test_promote_globals_to_constants(self): + + result = script_helper.run_python_until_end('-c', textwrap.dedent(""" + import _testinternalcapi + import opcode + + def get_first_executor(func): + code = func.__code__ + co_code = code.co_code + JUMP_BACKWARD = opcode.opmap["JUMP_BACKWARD"] + for i in range(0, len(co_code), 2): + if co_code[i] == JUMP_BACKWARD: + try: + return _testinternalcapi.get_executor(code, i) + except ValueError: + pass + return None + + def get_opnames(ex): + return {item[0] for item in ex} + + def testfunc(n): + for i in range(n): + x = range(i) + return x + + opt = _testinternalcapi.new_uop_optimizer() + _testinternalcapi.set_optimizer(opt) + testfunc(64) + + ex = get_first_executor(testfunc) + assert ex is not None + uops = get_opnames(ex) + assert "_LOAD_GLOBAL_BUILTINS" not in uops + assert "_LOAD_CONST_INLINE_BORROW_WITH_NULL" in uops + """)) + self.assertEqual(result[0].rc, 0, result) + + def test_float_add_constant_propagation(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + a = a + 0.1 + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertAlmostEqual(res, 4.2) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_both_float_count), 1) + # TODO gh-115506: this assertion may change after propagating constants. + # We'll also need to verify that propagation actually occurs. + self.assertIn("_BINARY_OP_ADD_FLOAT", uops) + + def test_float_subtract_constant_propagation(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + a = a - 0.1 + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertAlmostEqual(res, -2.2) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_both_float_count), 1) + # TODO gh-115506: this assertion may change after propagating constants. + # We'll also need to verify that propagation actually occurs. + self.assertIn("_BINARY_OP_SUBTRACT_FLOAT", uops) + + def test_float_multiply_constant_propagation(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + a = a * 2.0 + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertAlmostEqual(res, 2 ** 32) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_both_float_count), 1) + # TODO gh-115506: this assertion may change after propagating constants. + # We'll also need to verify that propagation actually occurs. + self.assertIn("_BINARY_OP_MULTIPLY_FLOAT", uops) + + def test_compare_op_type_propagation_float(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + x = a == a + x = a == a + x = a == a + x = a == a + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertTrue(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_both_float_count), 1) + self.assertIn("_COMPARE_OP_FLOAT", uops) + + def test_compare_op_type_propagation_int(self): + def testfunc(n): + a = 1 + for _ in range(n): + x = a == a + x = a == a + x = a == a + x = a == a + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertTrue(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertLessEqual(len(guard_both_float_count), 1) + self.assertIn("_COMPARE_OP_INT", uops) + + def test_compare_op_type_propagation_unicode(self): + def testfunc(n): + a = "" + for _ in range(n): + x = a == a + x = a == a + x = a == a + x = a == a + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertTrue(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_UNICODE"] + self.assertLessEqual(len(guard_both_float_count), 1) + self.assertIn("_COMPARE_OP_STR", uops) + + def test_type_inconsistency(self): + ns = {} + exec(textwrap.dedent(""" + def testfunc(n): + for i in range(n): + x = _test_global + _test_global + """), globals(), ns) + testfunc = ns['testfunc'] + # Must be a real global else it won't be optimized to _LOAD_CONST_INLINE + global _test_global + _test_global = 0 + _, ex = self._run_with_optimizer(testfunc, 16) + self.assertIsNone(ex) + _test_global = 1.2 + _, ex = self._run_with_optimizer(testfunc, 16) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_BOTH_INT", uops) + self.assertIn("_BINARY_OP_ADD_INT", uops) if __name__ == "__main__": diff --git a/Lib/test/test_capi/test_structmembers.py b/Lib/test/test_capi/test_structmembers.py index 2cf46b203478dc2..08ca1f828529cf2 100644 --- a/Lib/test/test_capi/test_structmembers.py +++ b/Lib/test/test_capi/test_structmembers.py @@ -14,6 +14,13 @@ PY_SSIZE_T_MAX, PY_SSIZE_T_MIN, ) + +class Index: + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + # There are two classes: one using <structmember.h> and another using # `Py_`-prefixed API. They should behave the same in Python @@ -38,75 +45,99 @@ class ReadWriteTests: def setUp(self): self.ts = _make_test_object(self.cls) + def _test_write(self, name, value, expected=None): + if expected is None: + expected = value + ts = self.ts + setattr(ts, name, value) + self.assertEqual(getattr(ts, name), expected) + + def _test_warn(self, name, value, expected=None): + ts = self.ts + self.assertWarns(RuntimeWarning, setattr, ts, name, value) + if expected is not None: + self.assertEqual(getattr(ts, name), expected) + + def _test_overflow(self, name, value): + ts = self.ts + self.assertRaises(OverflowError, setattr, ts, name, value) + + def _test_int_range(self, name, minval, maxval, *, hardlimit=None, + indexlimit=None): + if hardlimit is None: + hardlimit = (minval, maxval) + ts = self.ts + self._test_write(name, minval) + self._test_write(name, maxval) + hardminval, hardmaxval = hardlimit + self._test_overflow(name, hardminval-1) + self._test_overflow(name, hardmaxval+1) + self._test_overflow(name, 2**1000) + self._test_overflow(name, -2**1000) + if hardminval < minval: + self._test_warn(name, hardminval) + self._test_warn(name, minval-1, maxval) + if maxval < hardmaxval: + self._test_warn(name, maxval+1, minval) + self._test_warn(name, hardmaxval) + + if indexlimit is False: + self.assertRaises(TypeError, setattr, ts, name, Index(minval)) + self.assertRaises(TypeError, setattr, ts, name, Index(maxval)) + else: + self._test_write(name, Index(minval), minval) + self._test_write(name, Index(maxval), maxval) + self._test_overflow(name, Index(hardminval-1)) + self._test_overflow(name, Index(hardmaxval+1)) + self._test_overflow(name, Index(2**1000)) + self._test_overflow(name, Index(-2**1000)) + if hardminval < minval: + self._test_warn(name, Index(hardminval)) + self._test_warn(name, Index(minval-1), maxval) + if maxval < hardmaxval: + self._test_warn(name, Index(maxval+1), minval) + self._test_warn(name, Index(hardmaxval)) + def test_bool(self): ts = self.ts ts.T_BOOL = True - self.assertEqual(ts.T_BOOL, True) + self.assertIs(ts.T_BOOL, True) ts.T_BOOL = False - self.assertEqual(ts.T_BOOL, False) + self.assertIs(ts.T_BOOL, False) self.assertRaises(TypeError, setattr, ts, 'T_BOOL', 1) + self.assertRaises(TypeError, setattr, ts, 'T_BOOL', 0) + self.assertRaises(TypeError, setattr, ts, 'T_BOOL', None) def test_byte(self): - ts = self.ts - ts.T_BYTE = CHAR_MAX - self.assertEqual(ts.T_BYTE, CHAR_MAX) - ts.T_BYTE = CHAR_MIN - self.assertEqual(ts.T_BYTE, CHAR_MIN) - ts.T_UBYTE = UCHAR_MAX - self.assertEqual(ts.T_UBYTE, UCHAR_MAX) + self._test_int_range('T_BYTE', CHAR_MIN, CHAR_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) + self._test_int_range('T_UBYTE', 0, UCHAR_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) def test_short(self): - ts = self.ts - ts.T_SHORT = SHRT_MAX - self.assertEqual(ts.T_SHORT, SHRT_MAX) - ts.T_SHORT = SHRT_MIN - self.assertEqual(ts.T_SHORT, SHRT_MIN) - ts.T_USHORT = USHRT_MAX - self.assertEqual(ts.T_USHORT, USHRT_MAX) + self._test_int_range('T_SHORT', SHRT_MIN, SHRT_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) + self._test_int_range('T_USHORT', 0, USHRT_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) def test_int(self): - ts = self.ts - ts.T_INT = INT_MAX - self.assertEqual(ts.T_INT, INT_MAX) - ts.T_INT = INT_MIN - self.assertEqual(ts.T_INT, INT_MIN) - ts.T_UINT = UINT_MAX - self.assertEqual(ts.T_UINT, UINT_MAX) + self._test_int_range('T_INT', INT_MIN, INT_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) + self._test_int_range('T_UINT', 0, UINT_MAX, + hardlimit=(LONG_MIN, ULONG_MAX)) def test_long(self): - ts = self.ts - ts.T_LONG = LONG_MAX - self.assertEqual(ts.T_LONG, LONG_MAX) - ts.T_LONG = LONG_MIN - self.assertEqual(ts.T_LONG, LONG_MIN) - ts.T_ULONG = ULONG_MAX - self.assertEqual(ts.T_ULONG, ULONG_MAX) + self._test_int_range('T_LONG', LONG_MIN, LONG_MAX) + self._test_int_range('T_ULONG', 0, ULONG_MAX, + hardlimit=(LONG_MIN, ULONG_MAX)) def test_py_ssize_t(self): - ts = self.ts - ts.T_PYSSIZET = PY_SSIZE_T_MAX - self.assertEqual(ts.T_PYSSIZET, PY_SSIZE_T_MAX) - ts.T_PYSSIZET = PY_SSIZE_T_MIN - self.assertEqual(ts.T_PYSSIZET, PY_SSIZE_T_MIN) + self._test_int_range('T_PYSSIZET', PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, indexlimit=False) def test_longlong(self): - ts = self.ts - if not hasattr(ts, "T_LONGLONG"): - self.skipTest("long long not present") - - ts.T_LONGLONG = LLONG_MAX - self.assertEqual(ts.T_LONGLONG, LLONG_MAX) - ts.T_LONGLONG = LLONG_MIN - self.assertEqual(ts.T_LONGLONG, LLONG_MIN) - - ts.T_ULONGLONG = ULLONG_MAX - self.assertEqual(ts.T_ULONGLONG, ULLONG_MAX) - - ## make sure these will accept a plain int as well as a long - ts.T_LONGLONG = 3 - self.assertEqual(ts.T_LONGLONG, 3) - ts.T_ULONGLONG = 4 - self.assertEqual(ts.T_ULONGLONG, 4) + self._test_int_range('T_LONGLONG', LLONG_MIN, LLONG_MAX) + self._test_int_range('T_ULONGLONG', 0, ULLONG_MAX, + hardlimit=(LONG_MIN, ULLONG_MAX)) def test_bad_assignments(self): ts = self.ts @@ -116,10 +147,9 @@ def test_bad_assignments(self): 'T_SHORT', 'T_USHORT', 'T_INT', 'T_UINT', 'T_LONG', 'T_ULONG', + 'T_LONGLONG', 'T_ULONGLONG', 'T_PYSSIZET' ] - if hasattr(ts, 'T_LONGLONG'): - integer_attributes.extend(['T_LONGLONG', 'T_ULONGLONG']) # issue8014: this produced 'bad argument to internal function' # internal error @@ -139,46 +169,6 @@ class ReadWriteTests_OldAPI(ReadWriteTests, unittest.TestCase): class ReadWriteTests_NewAPI(ReadWriteTests, unittest.TestCase): cls = _test_structmembersType_NewAPI -class TestWarnings: - def setUp(self): - self.ts = _make_test_object(self.cls) - - def test_byte_max(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_BYTE = CHAR_MAX+1 - - def test_byte_min(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_BYTE = CHAR_MIN-1 - - def test_ubyte_max(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_UBYTE = UCHAR_MAX+1 - - def test_short_max(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_SHORT = SHRT_MAX+1 - - def test_short_min(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_SHORT = SHRT_MIN-1 - - def test_ushort_max(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_USHORT = USHRT_MAX+1 - -class TestWarnings_OldAPI(TestWarnings, unittest.TestCase): - cls = _test_structmembersType_OldAPI - -class TestWarnings_NewAPI(TestWarnings, unittest.TestCase): - cls = _test_structmembersType_NewAPI - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_time.py b/Lib/test/test_capi/test_time.py new file mode 100644 index 000000000000000..10b7fbf2c372a3b --- /dev/null +++ b/Lib/test/test_capi/test_time.py @@ -0,0 +1,71 @@ +import time +import unittest +from test.support import import_helper +_testcapi = import_helper.import_module('_testcapi') + + +PyTime_MIN = _testcapi.PyTime_MIN +PyTime_MAX = _testcapi.PyTime_MAX +SEC_TO_NS = 10 ** 9 +DAY_TO_SEC = (24 * 60 * 60) +# Worst clock resolution: maximum delta between two clock reads. +CLOCK_RES = 0.050 + + +class CAPITest(unittest.TestCase): + def test_min_max(self): + # PyTime_t is just int64_t + self.assertEqual(PyTime_MIN, -2**63) + self.assertEqual(PyTime_MAX, 2**63 - 1) + + def check_clock(self, c_func, py_func): + t1 = c_func() + t2 = py_func() + self.assertAlmostEqual(t1, t2, delta=CLOCK_RES) + + def test_assecondsdouble(self): + # Test PyTime_AsSecondsDouble() + def ns_to_sec(ns): + if abs(ns) % SEC_TO_NS == 0: + return float(ns // SEC_TO_NS) + else: + return float(ns) / SEC_TO_NS + + seconds = ( + 0, + 1, + DAY_TO_SEC, + 365 * DAY_TO_SEC, + ) + values = { + PyTime_MIN, + PyTime_MIN + 1, + PyTime_MAX - 1, + PyTime_MAX, + } + for second in seconds: + ns = second * SEC_TO_NS + values.add(ns) + # test nanosecond before/after to test rounding + values.add(ns - 1) + values.add(ns + 1) + for ns in list(values): + if (-ns) > PyTime_MAX: + continue + values.add(-ns) + for ns in sorted(values): + with self.subTest(ns=ns): + self.assertEqual(_testcapi.PyTime_AsSecondsDouble(ns), + ns_to_sec(ns)) + + def test_monotonic(self): + # Test PyTime_Monotonic() + self.check_clock(_testcapi.PyTime_Monotonic, time.monotonic) + + def test_perf_counter(self): + # Test PyTime_PerfCounter() + self.check_clock(_testcapi.PyTime_PerfCounter, time.perf_counter) + + def test_time(self): + # Test PyTime_time() + self.check_clock(_testcapi.PyTime_Time, time.time) diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 5981712c80c3a96..ae062b1bda26b7c 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -151,8 +151,8 @@ def test_watch_out_of_range_watcher_id(self): def test_watch_unassigned_watcher_id(self): d = {} - with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 1"): - self.watch(1, d) + with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 3"): + self.watch(3, d) def test_unwatch_non_dict(self): with self.watcher() as wid: @@ -168,8 +168,8 @@ def test_unwatch_out_of_range_watcher_id(self): def test_unwatch_unassigned_watcher_id(self): d = {} - with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 1"): - self.unwatch(1, d) + with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 3"): + self.unwatch(3, d) def test_clear_out_of_range_watcher_id(self): with self.assertRaisesRegex(ValueError, r"Invalid dict watcher ID -1"): @@ -178,8 +178,8 @@ def test_clear_out_of_range_watcher_id(self): self.clear_watcher(8) # DICT_MAX_WATCHERS = 8 def test_clear_unassigned_watcher_id(self): - with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 1"): - self.clear_watcher(1) + with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 3"): + self.clear_watcher(3) class TestTypeWatchers(unittest.TestCase): diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 1531aad4f1f779d..d59271435e9eb05 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -771,6 +771,22 @@ def add_one_level(): with self.assertRaises(RecursionError): add_one_level() + def testMetaclassCallOptimization(self): + calls = 0 + + class TypeMetaclass(type): + def __call__(cls, *args, **kwargs): + nonlocal calls + calls += 1 + return type.__call__(cls, *args, **kwargs) + + class Type(metaclass=TypeMetaclass): + def __init__(self, obj): + self._obj = obj + + for i in range(100): + Type(i) + self.assertEqual(calls, 100) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 7323bdd801f4be0..f5e9b11ad1cc8ac 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -711,7 +711,7 @@ def fn(): class ClinicLinearFormatTest(TestCase): def _test(self, input, output, **kwargs): - computed = clinic.linear_format(input, **kwargs) + computed = libclinic.linear_format(input, **kwargs) self.assertEqual(output, computed) def test_empty_strings(self): @@ -761,6 +761,19 @@ def test_multiline_substitution(self): def """, name='bingle\nbungle\n') + def test_text_before_block_marker(self): + regex = re.escape("found before '{marker}'") + with self.assertRaisesRegex(clinic.ClinicError, regex): + libclinic.linear_format("no text before marker for you! {marker}", + marker="not allowed!") + + def test_text_after_block_marker(self): + regex = re.escape("found after '{marker}'") + with self.assertRaisesRegex(clinic.ClinicError, regex): + libclinic.linear_format("{marker} no text after marker for you!", + marker="not allowed!") + + class InertParser: def __init__(self, clinic): pass @@ -1900,7 +1913,7 @@ def test_parameters_no_more_than_one_vararg(self): *vararg1: object *vararg2: object """ - self.expect_failure(block, err, lineno=0) + self.expect_failure(block, err, lineno=3) def test_function_not_at_column_0(self): function = self.parse_function(""" @@ -2286,10 +2299,10 @@ def test_non_ascii_character_in_docstring(self): self.parse(block) # The line numbers are off; this is a known limitation. expected = dedent("""\ - Warning in file 'clinic_tests' on line 0: + Warning: Non-ascii characters are not allowed in docstrings: 'á' - Warning in file 'clinic_tests' on line 0: + Warning: Non-ascii characters are not allowed in docstrings: 'ü', 'á', 'ß' """) @@ -2390,7 +2403,7 @@ def test_state_func_docstring_no_summary(self): docstring1 docstring2 """ - self.expect_failure(block, err, lineno=0) + self.expect_failure(block, err, lineno=3) def test_state_func_docstring_only_one_param_template(self): err = "You may not specify {parameters} more than once in a docstring!" @@ -2404,7 +2417,7 @@ def test_state_func_docstring_only_one_param_template(self): these are the params again: {parameters} """ - self.expect_failure(block, err, lineno=0) + self.expect_failure(block, err, lineno=7) class ClinicExternalTest(TestCase): @@ -3288,6 +3301,26 @@ def test_cloned_func_with_converter_exception_message(self): func = getattr(ac_tester, name) self.assertEqual(func(), name) + def test_meth_method_no_params(self): + obj = ac_tester.TestClass() + meth = obj.meth_method_no_params + check = partial(self.assertRaisesRegex, TypeError, "no arguments") + check(meth, 1) + check(meth, a=1) + + def test_meth_method_no_params_capi(self): + from _testcapi import pyobject_vectorcall + obj = ac_tester.TestClass() + meth = obj.meth_method_no_params + pyobject_vectorcall(meth, None, None) + pyobject_vectorcall(meth, (), None) + pyobject_vectorcall(meth, (), ()) + pyobject_vectorcall(meth, None, ()) + + check = partial(self.assertRaisesRegex, TypeError, "no arguments") + check(pyobject_vectorcall, meth, (1,), None) + check(pyobject_vectorcall, meth, (1,), ("a",)) + def test_depr_star_new(self): cls = ac_tester.DeprStarNew cls() diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 48754d5a63da3b1..3a5a8abf81e43d4 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -14,8 +14,7 @@ import textwrap from test import support -from test.support import import_helper -from test.support import os_helper +from test.support import import_helper, is_apple, os_helper from test.support.script_helper import ( make_pkg, make_script, make_zip_pkg, make_zip_script, assert_python_ok, assert_python_failure, spawn_python, kill_python) @@ -557,12 +556,17 @@ def test_pep_409_verbiage(self): self.assertTrue(text[3].startswith('NameError')) def test_non_ascii(self): - # Mac OS X denies the creation of a file with an invalid UTF-8 name. + # Apple platforms deny the creation of a file with an invalid UTF-8 name. # Windows allows creating a name with an arbitrary bytes name, but # Python cannot a undecodable bytes argument to a subprocess. - # WASI does not permit invalid UTF-8 names. - if (os_helper.TESTFN_UNDECODABLE - and sys.platform not in ('win32', 'darwin', 'emscripten', 'wasi')): + # Emscripten/WASI does not permit invalid UTF-8 names. + if ( + os_helper.TESTFN_UNDECODABLE + and sys.platform not in { + "win32", "emscripten", "wasi" + } + and not is_apple + ): name = os.fsdecode(os_helper.TESTFN_UNDECODABLE) elif os_helper.TESTFN_NONASCII: name = os_helper.TESTFN_NONASCII diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index d8fb826edeb6816..46bebfc7af675b1 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -865,6 +865,7 @@ def __init__(self, f, test): self.test = test def run(self): del self.f + gc_collect() self.test.assertEqual(LAST_FREED, 500) SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500)) diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 747c0f9683c19c7..259778a5cade98f 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -160,6 +160,7 @@ def setUp(self): self.console = code.InteractiveConsole(local_exit=True) self.mock_sys() + @unittest.skipIf(sys.flags.no_site, "exit() isn't defined unless there's a site module") def test_exit(self): # default exit message self.infunc.side_effect = ["exit()"] diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 7e6f811e17cfa2d..1fb492ecebd6687 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1,5 +1,6 @@ """Unit tests for collections.py.""" +import array import collections import copy import doctest @@ -1972,6 +1973,7 @@ def test_MutableSequence(self): for sample in [list, bytearray, deque]: self.assertIsInstance(sample(), MutableSequence) self.assertTrue(issubclass(sample, MutableSequence)) + self.assertTrue(issubclass(array.array, MutableSequence)) self.assertFalse(issubclass(str, MutableSequence)) self.validate_abstract_methods(MutableSequence, '__contains__', '__iter__', '__len__', '__getitem__', '__setitem__', '__delitem__', 'insert') diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 3b1ceceaa6305f7..0126780982059a3 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1,4 +1,6 @@ +import contextlib import dis +import io import math import os import unittest @@ -747,7 +749,7 @@ def f(): return "unused" self.assertEqual(f.__code__.co_consts, - ("docstring", "used")) + (f.__doc__, "used")) @support.cpython_only def test_remove_unused_consts_no_docstring(self): @@ -792,7 +794,7 @@ def test_strip_unused_None(self): def f1(): "docstring" return 42 - self.assertEqual(f1.__code__.co_consts, ("docstring", 42)) + self.assertEqual(f1.__code__.co_consts, (f1.__doc__, 42)) # This is a regression test for a CPython specific peephole optimizer # implementation bug present in a few releases. It's assertion verifies @@ -812,6 +814,30 @@ def unused_code_at_end(): 'RETURN_CONST', list(dis.get_instructions(unused_code_at_end))[-1].opname) + @support.cpython_only + def test_docstring_omitted(self): + # See gh-115347 + src = textwrap.dedent(""" + def f(): + "docstring1" + def h(): + "docstring2" + return 42 + + class C: + "docstring3" + pass + + return h + """) + for opt in [-1, 0, 1, 2]: + with self.subTest(opt=opt): + code = compile(src, "<test>", "exec", optimize=opt) + output = io.StringIO() + with contextlib.redirect_stdout(output): + dis.dis(code) + self.assertNotIn('NOP' , output.getvalue()) + def test_dont_merge_constants(self): # Issue #25843: compile() must not merge constants which are equal # but have a different type. @@ -1021,6 +1047,8 @@ def no_code2(): for func in (no_code1, no_code2): with self.subTest(func=func): + if func is no_code1 and no_code1.__doc__ is None: + continue code = func.__code__ [(start, end, line)] = code.co_lines() self.assertEqual(start, 0) @@ -1104,6 +1132,17 @@ async def test(aseq): code_lines = self.get_code_lines(test.__code__) self.assertEqual(expected_lines, code_lines) + def check_line_numbers(self, code, opnames=None): + # Check that all instructions whose op matches opnames + # have a line number. opnames can be a single name, or + # a sequence of names. If it is None, match all ops. + + if isinstance(opnames, str): + opnames = (opnames, ) + for inst in dis.Bytecode(code): + if opnames and inst.opname in opnames: + self.assertIsNotNone(inst.positions.lineno) + def test_line_number_synthetic_jump_multiple_predecessors(self): def f(): for x in it: @@ -1113,25 +1152,52 @@ def f(): except OSError: pass - # Ensure that all JUMP_BACKWARDs have line number - code = f.__code__ - for inst in dis.Bytecode(code): - if inst.opname == 'JUMP_BACKWARD': - self.assertIsNotNone(inst.positions.lineno) + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') - def test_lineno_of_backward_jump(self): + def test_line_number_synthetic_jump_multiple_predecessors_nested(self): + def f(): + for x in it: + try: + X = 3 + except OSError: + try: + if C3: + X = 4 + except OSError: + pass + return 42 + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_line_number_synthetic_jump_multiple_predecessors_more_nested(self): + def f(): + for x in it: + try: + X = 3 + except OSError: + try: + if C3: + if C4: + X = 4 + except OSError: + try: + if C3: + if C4: + X = 5 + except OSError: + pass + return 42 + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_lineno_of_backward_jump_conditional_in_loop(self): # Issue gh-107901 def f(): for i in x: if y: pass - linenos = list(inst.positions.lineno - for inst in dis.get_instructions(f.__code__) - if inst.opname == 'JUMP_BACKWARD') - - self.assertTrue(len(linenos) > 0) - self.assertTrue(all(l is not None for l in linenos)) + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') def test_big_dict_literal(self): # The compiler has a flushing point in "compiler_dict" that calls compiles @@ -1460,6 +1526,7 @@ def test_multiline_boolean_expression(self): self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_TRUE', line=4, end_line=4, column=8, end_column=13, occurrence=2) + @unittest.skipIf(sys.flags.optimize, "Assertions are disabled in optimized mode") def test_multiline_assert(self): snippet = textwrap.dedent("""\ assert (a > 0 and diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index 5696433e529d0a2..ab9f04dd63af202 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -1,3 +1,6 @@ +import dis +import io +import textwrap import types from test.support.bytecode_helper import AssemblerTestCase @@ -22,11 +25,13 @@ def complete_metadata(self, metadata, filename="myfile.py"): metadata.setdefault('filename', filename) return metadata - def assemble_test(self, insts, metadata, expected): + def insts_to_code_object(self, insts, metadata): metadata = self.complete_metadata(metadata) insts = self.complete_insts_info(insts) + return self.get_code_object(metadata['filename'], insts, metadata) - co = self.get_code_object(metadata['filename'], insts, metadata) + def assemble_test(self, insts, metadata, expected): + co = self.insts_to_code_object(insts, metadata) self.assertIsInstance(co, types.CodeType) expected_metadata = {} @@ -108,3 +113,35 @@ def inner(): expected = {(0,): 0, (1,): 1, (2,): 0, (120,): 0, (121,): 1} self.assemble_test(instructions, metadata, expected) + + + def test_exception_table(self): + metadata = { + 'filename' : 'exc.py', + 'name' : 'exc', + 'consts' : {2 : 0}, + } + + # code for "try: pass\n except: pass" + insts = [ + ('RESUME', 0), + ('SETUP_FINALLY', 3), + ('RETURN_CONST', 0), + ('SETUP_CLEANUP', 8), + ('PUSH_EXC_INFO', 0), + ('POP_TOP', 0), + ('POP_EXCEPT', 0), + ('RETURN_CONST', 0), + ('COPY', 3), + ('POP_EXCEPT', 0), + ('RERAISE', 1), + ] + co = self.insts_to_code_object(insts, metadata) + output = io.StringIO() + dis.dis(co, file=output) + exc_table = textwrap.dedent(""" + ExceptionTable: + L1 to L2 -> L2 [0] + L2 to L3 -> L3 [1] lasti + """) + self.assertTrue(output.getvalue().endswith(exc_table)) diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index dbeadd9ca47c63d..166294a40c1cb7b 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -8,7 +8,7 @@ class IsolatedCodeGenTests(CodegenTestCase): def codegen_test(self, snippet, expected_insts): import ast - a = ast.parse(snippet, "my_file.py", "exec"); + a = ast.parse(snippet, "my_file.py", "exec") insts = self.generate_code(a) self.assertInstructionsMatch(insts, expected_insts) @@ -54,3 +54,8 @@ def test_for_loop(self): ('RETURN_VALUE', None), ] self.codegen_test(snippet, expected) + + def test_syntax_error__return_not_in_function(self): + snippet = "return 42" + with self.assertRaisesRegex(SyntaxError, "'return' outside function"): + self.codegen_test(snippet, None) diff --git a/Lib/test/test_concurrent_futures/__init__.py b/Lib/test/test_concurrent_futures/__init__.py index 430fa93aa456a2d..b38bd38d338f0ee 100644 --- a/Lib/test/test_concurrent_futures/__init__.py +++ b/Lib/test/test_concurrent_futures/__init__.py @@ -1,10 +1,12 @@ import os.path import unittest from test import support -from test.support import import_helper +from test.support import threading_helper + + +# Adjust if we ever have a platform with processes but not threads. +threading_helper.requires_working_threading(module=True) -# Skip tests if _multiprocessing wasn't built. -import_helper.import_module('_multiprocessing') if support.check_sanitizer(address=True, memory=True): # gh-90791: Skip the test because it is too slow when Python is built diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 1e7d4344740943d..6a79fe69ec37cf4 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -1,8 +1,10 @@ import threading import time +import unittest import weakref from concurrent import futures from test import support +from test.support import Py_GIL_DISABLED def mul(x, y): @@ -83,10 +85,21 @@ def test_no_stale_references(self): my_object_collected = threading.Event() my_object_callback = weakref.ref( my_object, lambda obj: my_object_collected.set()) - # Deliberately discarding the future. - self.executor.submit(my_object.my_method) + fut = self.executor.submit(my_object.my_method) del my_object + if Py_GIL_DISABLED: + # Due to biased reference counting, my_object might only be + # deallocated while the thread that created it runs -- if the + # thread is paused waiting on an event, it may not merge the + # refcount of the queued object. For that reason, we wait for the + # task to finish (so that it's no longer referenced) and force a + # GC to ensure that it is collected. + fut.result() # Wait for the task to finish. + support.gc_collect() + else: + del fut # Deliberately discard the future. + collected = my_object_collected.wait(timeout=support.SHORT_TIMEOUT) self.assertTrue(collected, "Stale reference not collected within timeout.") diff --git a/Lib/test/test_concurrent_futures/test_init.py b/Lib/test/test_concurrent_futures/test_init.py index ce01e0ff0f287a0..113a4d1c54be03b 100644 --- a/Lib/test/test_concurrent_futures/test_init.py +++ b/Lib/test/test_concurrent_futures/test_init.py @@ -3,7 +3,10 @@ import queue import time import unittest +import sys from concurrent.futures._base import BrokenExecutor +from concurrent.futures.process import _check_system_limits + from logging.handlers import QueueHandler from test import support @@ -109,6 +112,36 @@ def _assert_logged(self, msg): create_executor_tests(globals(), FailingInitializerMixin) +@unittest.skipIf(sys.platform == "win32", "Resource Tracker doesn't run on Windows") +class FailingInitializerResourcesTest(unittest.TestCase): + """ + Source: https://github.com/python/cpython/issues/104090 + """ + + def _test(self, test_class): + try: + _check_system_limits() + except NotImplementedError: + self.skipTest("ProcessPoolExecutor unavailable on this system") + + runner = unittest.TextTestRunner() + runner.run(test_class('test_initializer')) + + # GH-104090: + # Stop resource tracker manually now, so we can verify there are not leaked resources by checking + # the process exit code + from multiprocessing.resource_tracker import _resource_tracker + _resource_tracker._stop() + + self.assertEqual(_resource_tracker._exitcode, 0) + + def test_spawn(self): + self._test(ProcessPoolSpawnFailingInitializerTest) + + def test_forkserver(self): + self._test(ProcessPoolForkserverFailingInitializerTest) + + def setUpModule(): setup_module() diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 3e61b0c9387c6fa..7fc59a05f3deac0 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -98,6 +98,7 @@ def test_ressources_gced_in_workers(self): # explicitly destroy the object to ensure that EventfulGCObj.__del__() # is called while manager is still running. + support.gc_collect() obj = None support.gc_collect() diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py index dc48bec796b87fe..3e855031913042f 100644 --- a/Lib/test/test_concurrent_futures/util.py +++ b/Lib/test/test_concurrent_futures/util.py @@ -136,6 +136,12 @@ def strip_mixin(name): def setup_module(): - unittest.addModuleCleanup(multiprocessing.util._cleanup_tests) + try: + _check_system_limits() + except NotImplementedError: + pass + else: + unittest.addModuleCleanup(multiprocessing.util._cleanup_tests) + thread_info = threading_helper.threading_setup() unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 69fef5945ae66f5..d74ab7e016f78c0 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -64,8 +64,7 @@ def _test_arg_valid(self, ctor, arg): ctor(arg, delimiter='\t', skipinitialspace=True) ctor(arg, escapechar='\t', skipinitialspace=True) ctor(arg, quotechar='\t', skipinitialspace=True) - self.assertRaises(ValueError, ctor, arg, - delimiter=' ', skipinitialspace=True) + ctor(arg, delimiter=' ', skipinitialspace=True) self.assertRaises(ValueError, ctor, arg, escapechar=' ', skipinitialspace=True) self.assertRaises(ValueError, ctor, arg, @@ -192,9 +191,6 @@ def _write_error_test(self, exc, fields, **kwargs): def test_write_arg_valid(self): self._write_error_test(csv.Error, None) - self._write_test((), '') - self._write_test([None], '""') - self._write_error_test(csv.Error, [None], quoting = csv.QUOTE_NONE) # Check that exceptions are passed up the chain self._write_error_test(OSError, BadIterable()) class BadList: @@ -208,7 +204,6 @@ class BadItem: def __str__(self): raise OSError self._write_error_test(OSError, [BadItem()]) - def test_write_bigfield(self): # This exercises the buffer realloc functionality bigstring = 'X' * 50000 @@ -270,9 +265,11 @@ def test_write_lineterminator(self): writer = csv.writer(sio, lineterminator=lineterminator) writer.writerow(['a', 'b']) writer.writerow([1, 2]) + writer.writerow(['\r', '\n']) self.assertEqual(sio.getvalue(), f'a,b{lineterminator}' - f'1,2{lineterminator}') + f'1,2{lineterminator}' + f'"\r","\n"{lineterminator}') def test_write_iterable(self): self._write_test(iter(['a', 1, 'p,q']), 'a,1,"p,q"') @@ -315,6 +312,49 @@ def test_writerows_with_none(self): fileobj.seek(0) self.assertEqual(fileobj.read(), 'a\r\n""\r\n') + + def test_write_empty_fields(self): + self._write_test((), '') + self._write_test([''], '""') + self._write_error_test(csv.Error, [''], quoting=csv.QUOTE_NONE) + self._write_test([''], '""', quoting=csv.QUOTE_STRINGS) + self._write_test([''], '""', quoting=csv.QUOTE_NOTNULL) + self._write_test([None], '""') + self._write_error_test(csv.Error, [None], quoting=csv.QUOTE_NONE) + self._write_error_test(csv.Error, [None], quoting=csv.QUOTE_STRINGS) + self._write_error_test(csv.Error, [None], quoting=csv.QUOTE_NOTNULL) + self._write_test(['', ''], ',') + self._write_test([None, None], ',') + + def test_write_empty_fields_space_delimiter(self): + self._write_test([''], '""', delimiter=' ', skipinitialspace=False) + self._write_test([''], '""', delimiter=' ', skipinitialspace=True) + self._write_test([None], '""', delimiter=' ', skipinitialspace=False) + self._write_test([None], '""', delimiter=' ', skipinitialspace=True) + + self._write_test(['', ''], ' ', delimiter=' ', skipinitialspace=False) + self._write_test(['', ''], '"" ""', delimiter=' ', skipinitialspace=True) + self._write_test([None, None], ' ', delimiter=' ', skipinitialspace=False) + self._write_test([None, None], '"" ""', delimiter=' ', skipinitialspace=True) + + self._write_test(['', ''], ' ', delimiter=' ', skipinitialspace=False, + quoting=csv.QUOTE_NONE) + self._write_error_test(csv.Error, ['', ''], + delimiter=' ', skipinitialspace=True, + quoting=csv.QUOTE_NONE) + for quoting in csv.QUOTE_STRINGS, csv.QUOTE_NOTNULL: + self._write_test(['', ''], '"" ""', delimiter=' ', skipinitialspace=False, + quoting=quoting) + self._write_test(['', ''], '"" ""', delimiter=' ', skipinitialspace=True, + quoting=quoting) + + for quoting in csv.QUOTE_NONE, csv.QUOTE_STRINGS, csv.QUOTE_NOTNULL: + self._write_test([None, None], ' ', delimiter=' ', skipinitialspace=False, + quoting=quoting) + self._write_error_test(csv.Error, [None, None], + delimiter=' ', skipinitialspace=True, + quoting=quoting) + def test_writerows_errors(self): with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj) @@ -392,10 +432,26 @@ def test_read_quoting(self): # will this fail where locale uses comma for decimals? self._read_test([',3,"5",7.3, 9'], [['', 3, '5', 7.3, 9]], quoting=csv.QUOTE_NONNUMERIC) + self._read_test([',3,"5",7.3, 9'], [[None, '3', '5', '7.3', ' 9']], + quoting=csv.QUOTE_NOTNULL) + self._read_test([',3,"5",7.3, 9'], [[None, 3, '5', 7.3, 9]], + quoting=csv.QUOTE_STRINGS) + + self._read_test([',,"",'], [['', '', '', '']]) + self._read_test([',,"",'], [['', '', '', '']], + quoting=csv.QUOTE_NONNUMERIC) + self._read_test([',,"",'], [[None, None, '', None]], + quoting=csv.QUOTE_NOTNULL) + self._read_test([',,"",'], [[None, None, '', None]], + quoting=csv.QUOTE_STRINGS) + self._read_test(['"a\nb", 7'], [['a\nb', ' 7']]) self.assertRaises(ValueError, self._read_test, ['abc,3'], [[]], quoting=csv.QUOTE_NONNUMERIC) + self.assertRaises(ValueError, self._read_test, + ['abc,3'], [[]], + quoting=csv.QUOTE_STRINGS) self._read_test(['1,@,3,@,5'], [['1', ',3,', '5']], quotechar='@') self._read_test(['1,\0,3,\0,5'], [['1', ',3,', '5']], quotechar='\0') @@ -403,6 +459,23 @@ def test_read_skipinitialspace(self): self._read_test(['no space, space, spaces,\ttab'], [['no space', 'space', 'spaces', '\ttab']], skipinitialspace=True) + self._read_test([' , , '], + [['', '', '']], + skipinitialspace=True) + self._read_test([' , , '], + [[None, None, None]], + skipinitialspace=True, quoting=csv.QUOTE_NOTNULL) + self._read_test([' , , '], + [[None, None, None]], + skipinitialspace=True, quoting=csv.QUOTE_STRINGS) + + def test_read_space_delimiter(self): + self._read_test(['a b', ' a ', ' ', ''], + [['a', '', '', 'b'], ['', '', 'a', '', ''], ['', '', ''], []], + delimiter=' ', skipinitialspace=False) + self._read_test(['a b', ' a ', ' ', ''], + [['a', 'b'], ['a', ''], [''], []], + delimiter=' ', skipinitialspace=True) def test_read_bigfield(self): # This exercises the buffer realloc functionality and field size @@ -436,22 +509,44 @@ def test_read_linenum(self): self.assertEqual(r.line_num, 3) def test_roundtrip_quoteed_newlines(self): - with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: - writer = csv.writer(fileobj) - rows = [['a\nb','b'],['c','x\r\nd']] - writer.writerows(rows) - fileobj.seek(0) - for i, row in enumerate(csv.reader(fileobj)): - self.assertEqual(row, rows[i]) + rows = [ + ['\na', 'b\nc', 'd\n'], + ['\re', 'f\rg', 'h\r'], + ['\r\ni', 'j\r\nk', 'l\r\n'], + ['\n\rm', 'n\n\ro', 'p\n\r'], + ['\r\rq', 'r\r\rs', 't\r\r'], + ['\n\nu', 'v\n\nw', 'x\n\n'], + ] + for lineterminator in '\r\n', '\n', '\r': + with self.subTest(lineterminator=lineterminator): + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: + writer = csv.writer(fileobj, lineterminator=lineterminator) + writer.writerows(rows) + fileobj.seek(0) + for i, row in enumerate(csv.reader(fileobj)): + self.assertEqual(row, rows[i]) def test_roundtrip_escaped_unquoted_newlines(self): - with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: - writer = csv.writer(fileobj,quoting=csv.QUOTE_NONE,escapechar="\\") - rows = [['a\nb','b'],['c','x\r\nd']] - writer.writerows(rows) - fileobj.seek(0) - for i, row in enumerate(csv.reader(fileobj,quoting=csv.QUOTE_NONE,escapechar="\\")): - self.assertEqual(row,rows[i]) + rows = [ + ['\na', 'b\nc', 'd\n'], + ['\re', 'f\rg', 'h\r'], + ['\r\ni', 'j\r\nk', 'l\r\n'], + ['\n\rm', 'n\n\ro', 'p\n\r'], + ['\r\rq', 'r\r\rs', 't\r\r'], + ['\n\nu', 'v\n\nw', 'x\n\n'], + ] + for lineterminator in '\r\n', '\n', '\r': + with self.subTest(lineterminator=lineterminator): + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: + writer = csv.writer(fileobj, lineterminator=lineterminator, + quoting=csv.QUOTE_NONE, escapechar="\\") + writer.writerows(rows) + fileobj.seek(0) + for i, row in enumerate(csv.reader(fileobj, + quoting=csv.QUOTE_NONE, + escapechar="\\")): + self.assertEqual(row, rows[i]) + class TestDialectRegistry(unittest.TestCase): def test_registry_badargs(self): @@ -530,10 +625,10 @@ class space(csv.excel): escapechar = "\\" with TemporaryFile("w+", encoding="utf-8") as fileobj: - fileobj.write("abc def\nc1ccccc1 benzene\n") + fileobj.write("abc def\nc1ccccc1 benzene\n") fileobj.seek(0) reader = csv.reader(fileobj, dialect=space()) - self.assertEqual(next(reader), ["abc", "def"]) + self.assertEqual(next(reader), ["abc", "", "", "def"]) self.assertEqual(next(reader), ["c1ccccc1", "benzene"]) def compare_dialect_123(self, expected, *writeargs, **kwwriteargs): @@ -1139,8 +1234,9 @@ class mydialect(csv.Dialect): self.assertRaises(csv.Error, create_invalid, field_name, 5) self.assertRaises(ValueError, create_invalid, field_name, "\n") self.assertRaises(ValueError, create_invalid, field_name, "\r") - self.assertRaises(ValueError, create_invalid, field_name, " ", - skipinitialspace=True) + if field_name != "delimiter": + self.assertRaises(ValueError, create_invalid, field_name, " ", + skipinitialspace=True) class TestSniffer(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_aligned_structures.py b/Lib/test/test_ctypes/test_aligned_structures.py new file mode 100644 index 000000000000000..a208fb9a00966ad --- /dev/null +++ b/Lib/test/test_ctypes/test_aligned_structures.py @@ -0,0 +1,286 @@ +from ctypes import ( + c_char, c_uint32, c_uint16, c_ubyte, c_byte, alignment, sizeof, + BigEndianStructure, LittleEndianStructure, + BigEndianUnion, LittleEndianUnion, +) +import struct +import unittest + + +class TestAlignedStructures(unittest.TestCase): + def test_aligned_string(self): + for base, e in ( + (LittleEndianStructure, "<"), + (BigEndianStructure, ">"), + ): + data = bytearray(struct.pack(f"{e}i12x16s", 7, b"hello world!")) + class Aligned(base): + _align_ = 16 + _fields_ = [ + ('value', c_char * 12) + ] + + class Main(base): + _fields_ = [ + ('first', c_uint32), + ('string', Aligned), + ] + + main = Main.from_buffer(data) + self.assertEqual(main.first, 7) + self.assertEqual(main.string.value, b'hello world!') + self.assertEqual(bytes(main.string), b'hello world!\0\0\0\0') + self.assertEqual(Main.string.offset, 16) + self.assertEqual(Main.string.size, 16) + self.assertEqual(alignment(main.string), 16) + self.assertEqual(alignment(main), 16) + + def test_aligned_structures(self): + for base, data in ( + (LittleEndianStructure, bytearray(b"\1\0\0\0\1\0\0\0\7\0\0\0")), + (BigEndianStructure, bytearray(b"\1\0\0\0\1\0\0\0\7\0\0\0")), + ): + class SomeBools(base): + _align_ = 4 + _fields_ = [ + ("bool1", c_ubyte), + ("bool2", c_ubyte), + ] + class Main(base): + _fields_ = [ + ("x", c_ubyte), + ("y", SomeBools), + ("z", c_ubyte), + ] + + main = Main.from_buffer(data) + self.assertEqual(alignment(SomeBools), 4) + self.assertEqual(alignment(main), 4) + self.assertEqual(alignment(main.y), 4) + self.assertEqual(Main.x.size, 1) + self.assertEqual(Main.y.offset, 4) + self.assertEqual(Main.y.size, 4) + self.assertEqual(main.y.bool1, True) + self.assertEqual(main.y.bool2, False) + self.assertEqual(Main.z.offset, 8) + self.assertEqual(main.z, 7) + + def test_oversized_structure(self): + data = bytearray(b"\0" * 8) + for base in (LittleEndianStructure, BigEndianStructure): + class SomeBoolsTooBig(base): + _align_ = 8 + _fields_ = [ + ("bool1", c_ubyte), + ("bool2", c_ubyte), + ("bool3", c_ubyte), + ] + class Main(base): + _fields_ = [ + ("y", SomeBoolsTooBig), + ("z", c_uint32), + ] + with self.assertRaises(ValueError) as ctx: + Main.from_buffer(data) + self.assertEqual( + ctx.exception.args[0], + 'Buffer size too small (4 instead of at least 8 bytes)' + ) + + def test_aligned_subclasses(self): + for base, e in ( + (LittleEndianStructure, "<"), + (BigEndianStructure, ">"), + ): + data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) + class UnalignedSub(base): + x: c_uint32 + _fields_ = [ + ("x", c_uint32), + ] + + class AlignedStruct(UnalignedSub): + _align_ = 8 + _fields_ = [ + ("y", c_uint32), + ] + + class Main(base): + _fields_ = [ + ("a", c_uint32), + ("b", AlignedStruct) + ] + + main = Main.from_buffer(data) + self.assertEqual(alignment(main.b), 8) + self.assertEqual(alignment(main), 8) + self.assertEqual(sizeof(main.b), 8) + self.assertEqual(sizeof(main), 16) + self.assertEqual(main.a, 1) + self.assertEqual(main.b.x, 3) + self.assertEqual(main.b.y, 4) + self.assertEqual(Main.b.offset, 8) + self.assertEqual(Main.b.size, 8) + + def test_aligned_union(self): + for sbase, ubase, e in ( + (LittleEndianStructure, LittleEndianUnion, "<"), + (BigEndianStructure, BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) + class AlignedUnion(ubase): + _align_ = 8 + _fields_ = [ + ("a", c_uint32), + ("b", c_ubyte * 7), + ] + + class Main(sbase): + _fields_ = [ + ("first", c_uint32), + ("union", AlignedUnion), + ] + + main = Main.from_buffer(data) + self.assertEqual(main.first, 1) + self.assertEqual(main.union.a, 3) + self.assertEqual(bytes(main.union.b), data[8:-1]) + self.assertEqual(Main.union.offset, 8) + self.assertEqual(Main.union.size, 8) + self.assertEqual(alignment(main.union), 8) + self.assertEqual(alignment(main), 8) + + def test_aligned_struct_in_union(self): + for sbase, ubase, e in ( + (LittleEndianStructure, LittleEndianUnion, "<"), + (BigEndianStructure, BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) + class Sub(sbase): + _align_ = 8 + _fields_ = [ + ("x", c_uint32), + ("y", c_uint32), + ] + + class MainUnion(ubase): + _fields_ = [ + ("a", c_uint32), + ("b", Sub), + ] + + class Main(sbase): + _fields_ = [ + ("first", c_uint32), + ("union", MainUnion), + ] + + main = Main.from_buffer(data) + self.assertEqual(Main.first.size, 4) + self.assertEqual(alignment(main.union), 8) + self.assertEqual(alignment(main), 8) + self.assertEqual(Main.union.offset, 8) + self.assertEqual(Main.union.size, 8) + self.assertEqual(main.first, 1) + self.assertEqual(main.union.a, 3) + self.assertEqual(main.union.b.x, 3) + self.assertEqual(main.union.b.y, 4) + + def test_smaller_aligned_subclassed_union(self): + for sbase, ubase, e in ( + (LittleEndianStructure, LittleEndianUnion, "<"), + (BigEndianStructure, BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}H2xI", 1, 0xD60102D7)) + class SubUnion(ubase): + _align_ = 2 + _fields_ = [ + ("unsigned", c_ubyte), + ("signed", c_byte), + ] + + class MainUnion(SubUnion): + _fields_ = [ + ("num", c_uint32) + ] + + class Main(sbase): + _fields_ = [ + ("first", c_uint16), + ("union", MainUnion), + ] + + main = Main.from_buffer(data) + self.assertEqual(main.union.num, 0xD60102D7) + self.assertEqual(main.union.unsigned, data[4]) + self.assertEqual(main.union.signed, data[4] - 256) + self.assertEqual(alignment(main), 4) + self.assertEqual(alignment(main.union), 4) + self.assertEqual(Main.union.offset, 4) + self.assertEqual(Main.union.size, 4) + self.assertEqual(Main.first.size, 2) + + def test_larger_aligned_subclassed_union(self): + for ubase, e in ( + (LittleEndianUnion, "<"), + (BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}I4x", 0xD60102D6)) + class SubUnion(ubase): + _align_ = 8 + _fields_ = [ + ("unsigned", c_ubyte), + ("signed", c_byte), + ] + + class Main(SubUnion): + _fields_ = [ + ("num", c_uint32) + ] + + main = Main.from_buffer(data) + self.assertEqual(alignment(main), 8) + self.assertEqual(sizeof(main), 8) + self.assertEqual(main.num, 0xD60102D6) + self.assertEqual(main.unsigned, 0xD6) + self.assertEqual(main.signed, -42) + + def test_aligned_packed_structures(self): + for sbase, e in ( + (LittleEndianStructure, "<"), + (BigEndianStructure, ">"), + ): + data = bytearray(struct.pack(f"{e}B2H4xB", 1, 2, 3, 4)) + + class Inner(sbase): + _align_ = 8 + _fields_ = [ + ("x", c_uint16), + ("y", c_uint16), + ] + + class Main(sbase): + _pack_ = 1 + _fields_ = [ + ("a", c_ubyte), + ("b", Inner), + ("c", c_ubyte), + ] + + main = Main.from_buffer(data) + self.assertEqual(sizeof(main), 10) + self.assertEqual(Main.b.offset, 1) + # Alignment == 8 because _pack_ wins out. + self.assertEqual(alignment(main.b), 8) + # Size is still 8 though since inside this Structure, it will have + # effect. + self.assertEqual(sizeof(main.b), 8) + self.assertEqual(Main.c.offset, 9) + self.assertEqual(main.a, 1) + self.assertEqual(main.b.x, 2) + self.assertEqual(main.b.y, 3) + self.assertEqual(main.c, 4) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_ctypes/test_as_parameter.py b/Lib/test/test_ctypes/test_as_parameter.py index a1a8745e737fa25..ca75e748256083b 100644 --- a/Lib/test/test_ctypes/test_as_parameter.py +++ b/Lib/test/test_ctypes/test_as_parameter.py @@ -221,5 +221,16 @@ class AsParamPropertyWrapperTestCase(BasicWrapTestCase): wrap = AsParamPropertyWrapper +class AsParamNestedWrapperTestCase(BasicWrapTestCase): + """Test that _as_parameter_ is evaluated recursively. + + The _as_parameter_ attribute can be another object which + defines its own _as_parameter_ attribute. + """ + + def wrap(self, param): + return AsParamWrapper(AsParamWrapper(AsParamWrapper(param))) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_callbacks.py b/Lib/test/test_ctypes/test_callbacks.py index 19f4158c0ac8461..64f92ffdca6a3f8 100644 --- a/Lib/test/test_ctypes/test_callbacks.py +++ b/Lib/test/test_ctypes/test_callbacks.py @@ -148,9 +148,10 @@ def callback(a, b): print(f"a={a}, b={b}, c={c}") return c dll = cdll[_ctypes_test.__file__] - # With no fix for i38748, the next line will raise OSError and cause the test to fail. - self.assertEqual(dll._test_i38748_runCallback(callback, 5, 10), 15) - + with support.captured_stdout() as out: + # With no fix for i38748, the next line will raise OSError and cause the test to fail. + self.assertEqual(dll._test_i38748_runCallback(callback, 5, 10), 15) + self.assertEqual(out.getvalue(), "a=5, b=10, c=15\n") if hasattr(ctypes, 'WINFUNCTYPE'): class StdcallCallbacks(Callbacks): diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 272d427875ae407..ede74b0dd15ccfc 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -22,6 +22,8 @@ import typing # Needed for the string "typing.ClassVar[int]" to work as an annotation. import dataclasses # Needed for the string "dataclasses.InitVar[int]" to work as an annotation. +from test import support + # Just any custom exception we can catch. class CustomError(Exception): pass @@ -2216,6 +2218,7 @@ def assertDocStrEqual(self, a, b): # whitespace stripped. self.assertEqual(a.replace(' ', ''), b.replace(' ', '')) + @support.requires_docstrings def test_existing_docstring_not_overridden(self): @dataclass class C: diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index e3924d8ec8b5c1d..4be7c5649da68aa 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -6,6 +6,13 @@ from test.support import import_helper from test.support import os_helper + +try: + from dbm import sqlite3 as dbm_sqlite3 +except ImportError: + dbm_sqlite3 = None + + try: from dbm import ndbm except ImportError: @@ -213,6 +220,27 @@ def test_whichdb_ndbm(self): for path in fnames: self.assertIsNone(self.dbm.whichdb(path)) + @unittest.skipUnless(dbm_sqlite3, reason='Test requires dbm.sqlite3') + def test_whichdb_sqlite3(self): + # Databases created by dbm.sqlite3 are detected correctly. + with dbm_sqlite3.open(_fname, "c") as db: + db["key"] = "value" + self.assertEqual(self.dbm.whichdb(_fname), "dbm.sqlite3") + + @unittest.skipUnless(dbm_sqlite3, reason='Test requires dbm.sqlite3') + def test_whichdb_sqlite3_existing_db(self): + # Existing sqlite3 databases are detected correctly. + sqlite3 = import_helper.import_module("sqlite3") + try: + # Create an empty database. + with sqlite3.connect(_fname) as cx: + cx.execute("CREATE TABLE dummy(database)") + cx.commit() + finally: + cx.close() + self.assertEqual(self.dbm.whichdb(_fname), "dbm.sqlite3") + + def setUp(self): self.addCleanup(cleaunup_test_dir) setup_test_dir() diff --git a/Lib/test/test_dbm_dumb.py b/Lib/test/test_dbm_dumb.py index a481175b3bfdbde..672f9092207cf62 100644 --- a/Lib/test/test_dbm_dumb.py +++ b/Lib/test/test_dbm_dumb.py @@ -246,9 +246,27 @@ def test_missing_data(self): _delete_files() with self.assertRaises(FileNotFoundError): dumbdbm.open(_fname, value) + self.assertFalse(os.path.exists(_fname + '.dat')) self.assertFalse(os.path.exists(_fname + '.dir')) self.assertFalse(os.path.exists(_fname + '.bak')) + for value in ('c', 'n'): + _delete_files() + with dumbdbm.open(_fname, value) as f: + self.assertTrue(os.path.exists(_fname + '.dat')) + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.bak')) + + for value in ('c', 'n'): + _delete_files() + with dumbdbm.open(_fname, value) as f: + f['key'] = 'value' + self.assertTrue(os.path.exists(_fname + '.dat')) + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertTrue(os.path.exists(_fname + '.bak')) + def test_missing_index(self): with dumbdbm.open(_fname, 'n') as f: pass @@ -259,6 +277,60 @@ def test_missing_index(self): self.assertFalse(os.path.exists(_fname + '.dir')) self.assertFalse(os.path.exists(_fname + '.bak')) + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + f['key'] = 'value' + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertTrue(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + os.unlink(_fname + '.bak') + + def test_sync_empty_unmodified(self): + with dumbdbm.open(_fname, 'n') as f: + pass + os.unlink(_fname + '.dir') + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + f.sync() + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + f.sync() + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + + def test_sync_nonempty_unmodified(self): + with dumbdbm.open(_fname, 'n') as f: + pass + os.unlink(_fname + '.dir') + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + f['key'] = 'value' + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + f.sync() + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertTrue(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + os.unlink(_fname + '.bak') + f.sync() + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + def test_invalid_flag(self): for flag in ('x', 'rf', None): with self.assertRaisesRegex(ValueError, diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py new file mode 100644 index 000000000000000..7a49fd2f924f8d3 --- /dev/null +++ b/Lib/test/test_dbm_sqlite3.py @@ -0,0 +1,311 @@ +import sys +import test.support +import unittest +from contextlib import closing +from functools import partial +from pathlib import Path +from test.support import cpython_only, import_helper, os_helper + +dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") +# N.B. The test will fail on some platforms without sqlite3 +# if the sqlite3 import is above the import of dbm.sqlite3. +# This is deliberate: if the import helper managed to import dbm.sqlite3, +# we must inevitably be able to import sqlite3. Else, we have a problem. +import sqlite3 +from dbm.sqlite3 import _normalize_uri + + +class _SQLiteDbmTests(unittest.TestCase): + + def setUp(self): + self.filename = os_helper.TESTFN + db = dbm_sqlite3.open(self.filename, "c") + db.close() + + def tearDown(self): + for suffix in "", "-wal", "-shm": + os_helper.unlink(self.filename + suffix) + + +class URI(unittest.TestCase): + + def test_uri_substitutions(self): + dataset = ( + ("/absolute/////b/c", "/absolute/b/c"), + ("PRE#MID##END", "PRE%23MID%23%23END"), + ("%#?%%#", "%25%23%3F%25%25%23"), + ) + for path, normalized in dataset: + with self.subTest(path=path, normalized=normalized): + self.assertTrue(_normalize_uri(path).endswith(normalized)) + + @unittest.skipUnless(sys.platform == "win32", "requires Windows") + def test_uri_windows(self): + dataset = ( + # Relative subdir. + (r"2018\January.xlsx", + "2018/January.xlsx"), + # Absolute with drive letter. + (r"C:\Projects\apilibrary\apilibrary.sln", + "/C:/Projects/apilibrary/apilibrary.sln"), + # Relative with drive letter. + (r"C:Projects\apilibrary\apilibrary.sln", + "/C:Projects/apilibrary/apilibrary.sln"), + ) + for path, normalized in dataset: + with self.subTest(path=path, normalized=normalized): + if not Path(path).is_absolute(): + self.skipTest(f"skipping relative path: {path!r}") + self.assertTrue(_normalize_uri(path).endswith(normalized)) + + +class ReadOnly(_SQLiteDbmTests): + + def setUp(self): + super().setUp() + with dbm_sqlite3.open(self.filename, "w") as db: + db[b"key1"] = "value1" + db[b"key2"] = "value2" + self.db = dbm_sqlite3.open(self.filename, "r") + + def tearDown(self): + self.db.close() + super().tearDown() + + def test_readonly_read(self): + self.assertEqual(self.db[b"key1"], b"value1") + self.assertEqual(self.db[b"key2"], b"value2") + + def test_readonly_write(self): + with self.assertRaises(dbm_sqlite3.error): + self.db[b"new"] = "value" + + def test_readonly_delete(self): + with self.assertRaises(dbm_sqlite3.error): + del self.db[b"key1"] + + def test_readonly_keys(self): + self.assertEqual(self.db.keys(), [b"key1", b"key2"]) + + def test_readonly_iter(self): + self.assertEqual([k for k in self.db], [b"key1", b"key2"]) + + +class ReadWrite(_SQLiteDbmTests): + + def setUp(self): + super().setUp() + self.db = dbm_sqlite3.open(self.filename, "w") + + def tearDown(self): + self.db.close() + super().tearDown() + + def db_content(self): + with closing(sqlite3.connect(self.filename)) as cx: + keys = [r[0] for r in cx.execute("SELECT key FROM Dict")] + vals = [r[0] for r in cx.execute("SELECT value FROM Dict")] + return keys, vals + + def test_readwrite_unique_key(self): + self.db["key"] = "value" + self.db["key"] = "other" + keys, vals = self.db_content() + self.assertEqual(keys, [b"key"]) + self.assertEqual(vals, [b"other"]) + + def test_readwrite_delete(self): + self.db["key"] = "value" + self.db["new"] = "other" + + del self.db[b"new"] + keys, vals = self.db_content() + self.assertEqual(keys, [b"key"]) + self.assertEqual(vals, [b"value"]) + + del self.db[b"key"] + keys, vals = self.db_content() + self.assertEqual(keys, []) + self.assertEqual(vals, []) + + def test_readwrite_null_key(self): + with self.assertRaises(dbm_sqlite3.error): + self.db[None] = "value" + + def test_readwrite_null_value(self): + with self.assertRaises(dbm_sqlite3.error): + self.db[b"key"] = None + + +class Misuse(_SQLiteDbmTests): + + def setUp(self): + super().setUp() + self.db = dbm_sqlite3.open(self.filename, "w") + + def tearDown(self): + self.db.close() + super().tearDown() + + def test_misuse_double_create(self): + self.db["key"] = "value" + with dbm_sqlite3.open(self.filename, "c") as db: + self.assertEqual(db[b"key"], b"value") + + def test_misuse_double_close(self): + self.db.close() + + def test_misuse_invalid_flag(self): + regex = "must be.*'r'.*'w'.*'c'.*'n', not 'invalid'" + with self.assertRaisesRegex(ValueError, regex): + dbm_sqlite3.open(self.filename, flag="invalid") + + def test_misuse_double_delete(self): + self.db["key"] = "value" + del self.db[b"key"] + with self.assertRaises(KeyError): + del self.db[b"key"] + + def test_misuse_invalid_key(self): + with self.assertRaises(KeyError): + self.db[b"key"] + + def test_misuse_iter_close1(self): + self.db["1"] = 1 + it = iter(self.db) + self.db.close() + with self.assertRaises(dbm_sqlite3.error): + next(it) + + def test_misuse_iter_close2(self): + self.db["1"] = 1 + self.db["2"] = 2 + it = iter(self.db) + next(it) + self.db.close() + with self.assertRaises(dbm_sqlite3.error): + next(it) + + def test_misuse_use_after_close(self): + self.db.close() + with self.assertRaises(dbm_sqlite3.error): + self.db[b"read"] + with self.assertRaises(dbm_sqlite3.error): + self.db[b"write"] = "value" + with self.assertRaises(dbm_sqlite3.error): + del self.db[b"del"] + with self.assertRaises(dbm_sqlite3.error): + len(self.db) + with self.assertRaises(dbm_sqlite3.error): + self.db.keys() + + def test_misuse_reinit(self): + with self.assertRaises(dbm_sqlite3.error): + self.db.__init__("new.db", flag="n", mode=0o666) + + def test_misuse_empty_filename(self): + for flag in "r", "w", "c", "n": + with self.assertRaises(dbm_sqlite3.error): + db = dbm_sqlite3.open("", flag="c") + + +class DataTypes(_SQLiteDbmTests): + + dataset = ( + # (raw, coerced) + (42, b"42"), + (3.14, b"3.14"), + ("string", b"string"), + (b"bytes", b"bytes"), + ) + + def setUp(self): + super().setUp() + self.db = dbm_sqlite3.open(self.filename, "w") + + def tearDown(self): + self.db.close() + super().tearDown() + + def test_datatypes_values(self): + for raw, coerced in self.dataset: + with self.subTest(raw=raw, coerced=coerced): + self.db["key"] = raw + self.assertEqual(self.db[b"key"], coerced) + + def test_datatypes_keys(self): + for raw, coerced in self.dataset: + with self.subTest(raw=raw, coerced=coerced): + self.db[raw] = "value" + self.assertEqual(self.db[coerced], b"value") + # Raw keys are silently coerced to bytes. + self.assertEqual(self.db[raw], b"value") + del self.db[raw] + + def test_datatypes_replace_coerced(self): + self.db["10"] = "value" + self.db[b"10"] = "value" + self.db[10] = "value" + self.assertEqual(self.db.keys(), [b"10"]) + + +class CorruptDatabase(_SQLiteDbmTests): + """Verify that database exceptions are raised as dbm.sqlite3.error.""" + + def setUp(self): + super().setUp() + with closing(sqlite3.connect(self.filename)) as cx: + with cx: + cx.execute("DROP TABLE IF EXISTS Dict") + cx.execute("CREATE TABLE Dict (invalid_schema)") + + def check(self, flag, fn, should_succeed=False): + with closing(dbm_sqlite3.open(self.filename, flag)) as db: + with self.assertRaises(dbm_sqlite3.error): + fn(db) + + @staticmethod + def read(db): + return db["key"] + + @staticmethod + def write(db): + db["key"] = "value" + + @staticmethod + def iter(db): + next(iter(db)) + + @staticmethod + def keys(db): + db.keys() + + @staticmethod + def del_(db): + del db["key"] + + @staticmethod + def len_(db): + len(db) + + def test_corrupt_readwrite(self): + for flag in "r", "w", "c": + with self.subTest(flag=flag): + check = partial(self.check, flag=flag) + check(fn=self.read) + check(fn=self.write) + check(fn=self.iter) + check(fn=self.keys) + check(fn=self.del_) + check(fn=self.len_) + + def test_corrupt_force_new(self): + with closing(dbm_sqlite3.open(self.filename, "n")) as db: + db["foo"] = "write" + _ = db[b"foo"] + next(iter(db)) + del db[b"foo"] + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 7a5fe62b4673720..f23ea8af0c8772f 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -41,6 +41,7 @@ darwin_malloc_err_warning, is_emscripten) from test.support.import_helper import import_fresh_module from test.support import threading_helper +from test.support import warnings_helper import random import inspect import threading @@ -1109,6 +1110,13 @@ def test_formatting(self): ('z>z6.1f', '-0.', 'zzz0.0'), ('x>z6.1f', '-0.', 'xxx0.0'), ('🖤>z6.1f', '-0.', '🖤🖤🖤0.0'), # multi-byte fill char + ('\x00>z6.1f', '-0.', '\x00\x00\x000.0'), # null fill char + + # issue 114563 ('z' format on F type in cdecimal) + ('z3,.10F', '-6.24E-323', '0.0000000000'), + + # issue 91060 ('#' format in cdecimal) + ('#', '0', '0.'), # issue 6850 ('a=-7.0', '0.12345', 'aaaa0.1'), @@ -1237,7 +1245,14 @@ def test_deprecated_N_format(self): else: self.assertRaises(ValueError, format, h, 'N') self.assertRaises(ValueError, format, h, '010.3N') - + with warnings_helper.check_no_warnings(self): + self.assertEqual(format(h, 'N>10.3'), 'NN6.63E-34') + self.assertEqual(format(h, 'N>10.3n'), 'NN6.63e-34') + self.assertEqual(format(h, 'N>10.3e'), 'N6.626e-34') + self.assertEqual(format(h, 'N>10.3f'), 'NNNNN0.000') + self.assertRaises(ValueError, format, h, '>Nf') + self.assertRaises(ValueError, format, h, '10Nf') + self.assertRaises(ValueError, format, h, 'Nx') @run_with_locale('LC_ALL', 'ps_AF') def test_wide_char_separator_decimal_point(self): @@ -5718,6 +5733,21 @@ def test_c_signaldict_segfault(self): with self.assertRaisesRegex(ValueError, err_msg): sd.copy() + def test_format_fallback_capitals(self): + # Fallback to _pydecimal formatting (triggered by `#` format which + # is unsupported by mpdecimal) should honor the current context. + x = C.Decimal('6.09e+23') + self.assertEqual(format(x, '#'), '6.09E+23') + with C.localcontext(capitals=0): + self.assertEqual(format(x, '#'), '6.09e+23') + + def test_format_fallback_rounding(self): + y = C.Decimal('6.09') + self.assertEqual(format(y, '#.1f'), '6.1') + with C.localcontext(rounding=C.ROUND_DOWN): + self.assertEqual(format(y, '#.1f'), '6.0') + + @requires_docstrings @requires_cdecimal class SignatureTest(unittest.TestCase): diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index ae1dfacd7262e40..4679f297fd7f4a1 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -166,7 +166,7 @@ def test_contains(self): with self.assertRaises(RuntimeError): n in d - def test_contains_count_stop_crashes(self): + def test_contains_count_index_stop_crashes(self): class A: def __eq__(self, other): d.clear() @@ -178,6 +178,10 @@ def __eq__(self, other): with self.assertRaises(RuntimeError): _ = d.count(3) + d = deque([A()]) + with self.assertRaises(RuntimeError): + d.index(0) + def test_extend(self): d = deque('a') self.assertRaises(TypeError, d.extend, 1) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index fd0af9b30a0a71a..5404d8d3b99d5d4 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -1594,7 +1594,11 @@ def f(cls, arg): cm = classmethod(f) cm_dict = {'__annotations__': {}, - '__doc__': "f docstring", + '__doc__': ( + "f docstring" + if support.HAVE_DOCSTRINGS + else None + ), '__module__': __name__, '__name__': 'f', '__qualname__': f.__qualname__} @@ -4734,6 +4738,20 @@ class X(object): with self.assertRaises(AttributeError): del X.__abstractmethods__ + def test_gh55664(self): + # gh-55664: issue a warning when the + # __dict__ of a class contains non-string keys + with self.assertWarnsRegex(RuntimeWarning, 'MyClass'): + MyClass = type('MyClass', (), {1: 2}) + + class meta(type): + def __new__(mcls, name, bases, ns): + ns[1] = 2 + return super().__new__(mcls, name, bases, ns) + + with self.assertWarnsRegex(RuntimeWarning, 'MyClass'): + MyClass = meta('MyClass', (), {}) + def test_proxy_call(self): class FakeStr: __class__ = str @@ -5151,7 +5169,8 @@ class Base2(object): mykey = 'from Base2' mykey2 = 'from Base2' - X = type('X', (Base,), {MyKey(): 5}) + with self.assertWarnsRegex(RuntimeWarning, 'X'): + X = type('X', (Base,), {MyKey(): 5}) # mykey is read from Base self.assertEqual(X.mykey, 'from Base') # mykey2 is read from Base2 because MyKey.__eq__ has set __bases__ diff --git a/Lib/test/test_descrtut.py b/Lib/test/test_descrtut.py index 13e3ea41bdb76c5..f097c4e7300baac 100644 --- a/Lib/test/test_descrtut.py +++ b/Lib/test/test_descrtut.py @@ -39,16 +39,16 @@ def merge(self, other): Here's the new type at work: >>> print(defaultdict) # show our type - <class 'test.test_descrtut.defaultdict'> + <class '%(modname)s.defaultdict'> >>> print(type(defaultdict)) # its metatype <class 'type'> >>> a = defaultdict(default=0.0) # create an instance >>> print(a) # show the instance {} >>> print(type(a)) # show its type - <class 'test.test_descrtut.defaultdict'> + <class '%(modname)s.defaultdict'> >>> print(a.__class__) # show its class - <class 'test.test_descrtut.defaultdict'> + <class '%(modname)s.defaultdict'> >>> print(type(a) is a.__class__) # its type is its class True >>> a[1] = 3.25 # modify the instance @@ -99,7 +99,7 @@ def merge(self, other): >>> print(sortdict(a.__dict__)) {'default': -1000, 'x1': 100, 'x2': 200} >>> -""" +""" % {'modname': __name__} class defaultdict2(dict): __slots__ = ['default'] @@ -264,19 +264,19 @@ def merge(self, other): ... print("classmethod", cls, y) >>> C.foo(1) - classmethod <class 'test.test_descrtut.C'> 1 + classmethod <class '%(modname)s.C'> 1 >>> c = C() >>> c.foo(1) - classmethod <class 'test.test_descrtut.C'> 1 + classmethod <class '%(modname)s.C'> 1 >>> class D(C): ... pass >>> D.foo(1) - classmethod <class 'test.test_descrtut.D'> 1 + classmethod <class '%(modname)s.D'> 1 >>> d = D() >>> d.foo(1) - classmethod <class 'test.test_descrtut.D'> 1 + classmethod <class '%(modname)s.D'> 1 This prints "classmethod __main__.D 1" both times; in other words, the class passed as the first argument of foo() is the class involved in the @@ -292,18 +292,18 @@ class passed as the first argument of foo() is the class involved in the >>> E.foo(1) E.foo() called - classmethod <class 'test.test_descrtut.C'> 1 + classmethod <class '%(modname)s.C'> 1 >>> e = E() >>> e.foo(1) E.foo() called - classmethod <class 'test.test_descrtut.C'> 1 + classmethod <class '%(modname)s.C'> 1 In this example, the call to C.foo() from E.foo() will see class C as its first argument, not class E. This is to be expected, since the call specifies the class C. But it stresses the difference between these class methods and methods defined in metaclasses (where an upcall to a metamethod would pass the target class as an explicit first argument). -""" +""" % {'modname': __name__} test_5 = """ diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index a5917da346dded5..a93cb509b651c51 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1986,6 +1986,22 @@ def f(opcode, oparg, offset, *init_args): self.assertEqual(f(opcode.opmap["BINARY_OP"], 3, *args), (3, '<<')) self.assertEqual(f(opcode.opmap["CALL_INTRINSIC_1"], 2, *args), (2, 'INTRINSIC_IMPORT_STAR')) + def test_custom_arg_resolver(self): + class MyArgResolver(dis.ArgResolver): + def offset_from_jump_arg(self, op, arg, offset): + return arg + 1 + + def get_label_for_offset(self, offset): + return 2 * offset + + def f(opcode, oparg, offset, *init_args): + arg_resolver = MyArgResolver(*init_args) + return arg_resolver.get_argval_argrepr(opcode, oparg, offset) + offset = 42 + self.assertEqual(f(opcode.opmap["JUMP_BACKWARD"], 1, offset), (2, 'to L4')) + self.assertEqual(f(opcode.opmap["SETUP_FINALLY"], 2, offset), (3, 'to L6')) + + def get_instructions(self, code): return dis._get_instructions_bytes(code) diff --git a/Lib/test/test_doctest/decorator_mod.py b/Lib/test/test_doctest/decorator_mod.py new file mode 100644 index 000000000000000..9f106888411202b --- /dev/null +++ b/Lib/test/test_doctest/decorator_mod.py @@ -0,0 +1,10 @@ +# This module is used in `doctest_lineno.py`. +import functools + + +def decorator(f): + @functools.wraps(f) + def inner(): + return f() + + return inner diff --git a/Lib/test/test_doctest/doctest_lineno.py b/Lib/test/test_doctest/doctest_lineno.py index 677c569cf710ebd..0dbcd9a11eaba2f 100644 --- a/Lib/test/test_doctest/doctest_lineno.py +++ b/Lib/test/test_doctest/doctest_lineno.py @@ -67,3 +67,12 @@ def property_with_doctest(self): # https://github.com/python/cpython/issues/99433 str_wrapper = object().__str__ + + +# https://github.com/python/cpython/issues/115392 +from test.test_doctest.decorator_mod import decorator + +@decorator +@decorator +def func_with_docstring_wrapped(): + """Some unrelated info.""" diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index 7015255db1f7f0e..43be200b983227b 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -685,6 +685,7 @@ def basics(): r""" None test.test_doctest.doctest_lineno.MethodWrapper.method_without_docstring 61 test.test_doctest.doctest_lineno.MethodWrapper.property_with_doctest 4 test.test_doctest.doctest_lineno.func_with_docstring + 77 test.test_doctest.doctest_lineno.func_with_docstring_wrapped 12 test.test_doctest.doctest_lineno.func_with_doctest None test.test_doctest.doctest_lineno.func_without_docstring diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index bdb0e55f21069f7..f7e80749c456f8f 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -2985,6 +2985,11 @@ def test_address_list_with_unicode_names_in_quotes(self): '=?utf-8?q?H=C3=BCbsch?= Kaktus <beautiful@example.com>,\n' ' =?utf-8?q?bei=C3=9Ft_bei=C3=9Ft?= <biter@example.com>\n') + def test_address_list_with_list_separator_after_fold(self): + to = '0123456789' * 8 + '@foo, ä <foo@bar>' + self._test(parser.get_address_list(to)[0], + '0123456789' * 8 + '@foo,\n =?utf-8?q?=C3=A4?= <foo@bar>\n') + # XXX Need tests with comments on various sides of a unicode token, # and with unicode tokens in the comments. Spaces inside the quotes # currently don't do the right thing. diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index 69ab2a4feaa9389..a4b36a90d8815ea 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -6,6 +6,8 @@ import test.support import unittest import unittest.mock +from importlib.resources.abc import Traversable +from pathlib import Path import ensurepip import ensurepip._uninstall @@ -20,41 +22,35 @@ def test_version(self): # Test version() with tempfile.TemporaryDirectory() as tmpdir: self.touch(tmpdir, "pip-1.2.3b1-py2.py3-none-any.whl") - with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), - unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)): + with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', Path(tmpdir)): self.assertEqual(ensurepip.version(), '1.2.3b1') - def test_get_packages_no_dir(self): - # Test _get_packages() without a wheel package directory - with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), - unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None)): - packages = ensurepip._get_packages() - - # when bundled wheel packages are used, we get _PIP_VERSION + def test_version_no_dir(self): + # Test version() without a wheel package directory + with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None): + # when the bundled pip wheel is used, we get _PIP_VERSION self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version()) - # use bundled wheel packages - self.assertIsNotNone(packages['pip'].wheel_name) + def test_selected_wheel_path_no_dir(self): + pip_filename = f'pip-{ensurepip._PIP_VERSION}-py3-none-any.whl' + with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None): + with ensurepip._get_pip_whl_path_ctx() as bundled_wheel_path: + self.assertEqual(pip_filename, bundled_wheel_path.name) - def test_get_packages_with_dir(self): - # Test _get_packages() with a wheel package directory + def test_selected_wheel_path_with_dir(self): + # Test _get_pip_whl_path_ctx() with a wheel package directory pip_filename = "pip-20.2.2-py2.py3-none-any.whl" with tempfile.TemporaryDirectory() as tmpdir: self.touch(tmpdir, pip_filename) - # not used, make sure that it's ignored + # not used, make sure that they're ignored + self.touch(tmpdir, "pip-1.2.3-py2.py3-none-any.whl") self.touch(tmpdir, "wheel-0.34.2-py2.py3-none-any.whl") + self.touch(tmpdir, "pip-script.py") - with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), - unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)): - packages = ensurepip._get_packages() - - self.assertEqual(packages['pip'].version, '20.2.2') - self.assertEqual(packages['pip'].wheel_path, - os.path.join(tmpdir, pip_filename)) - - # wheel package is ignored - self.assertEqual(sorted(packages), ['pip']) + with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', Path(tmpdir)): + with ensurepip._get_pip_whl_path_ctx() as bundled_wheel_path: + self.assertEqual(pip_filename, bundled_wheel_path.name) class EnsurepipMixin: @@ -69,7 +65,7 @@ def setUp(self): real_devnull = os.devnull os_patch = unittest.mock.patch("ensurepip.os") patched_os = os_patch.start() - # But expose os.listdir() used by _find_packages() + # But expose os.listdir() used by _find_wheel_pkg_dir_pip() patched_os.listdir = os.listdir self.addCleanup(os_patch.stop) patched_os.devnull = real_devnull diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index d045739efa46b8b..27f8bbaf952afca 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -447,7 +447,7 @@ def spam(cls): def test_bad_new_super(self): with self.assertRaisesRegex( TypeError, - 'has no members defined', + 'do not use .super...__new__;', ): class BadSuper(self.enum_type): def __new__(cls, value): @@ -1048,6 +1048,22 @@ class TestPlainEnumFunction(_EnumTests, _PlainOutputTests, unittest.TestCase): class TestPlainFlagClass(_EnumTests, _PlainOutputTests, _FlagTests, unittest.TestCase): enum_type = Flag + def test_none_member(self): + class FlagWithNoneMember(Flag): + A = 1 + E = None + + self.assertEqual(FlagWithNoneMember.A.value, 1) + self.assertIs(FlagWithNoneMember.E.value, None) + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with |"): + FlagWithNoneMember.A | FlagWithNoneMember.E + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with &"): + FlagWithNoneMember.E & FlagWithNoneMember.A + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with \^"): + FlagWithNoneMember.A ^ FlagWithNoneMember.E + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be inverted"): + ~FlagWithNoneMember.E + class TestPlainFlagFunction(_EnumTests, _PlainOutputTests, _FlagTests, unittest.TestCase): enum_type = Flag @@ -2344,6 +2360,40 @@ class SomeTuple(tuple, Enum): globals()['SomeTuple'] = SomeTuple test_pickle_dump_load(self.assertIs, SomeTuple.first) + def test_tuple_subclass_with_auto_1(self): + from collections import namedtuple + T = namedtuple('T', 'index desc') + class SomeEnum(T, Enum): + __qualname__ = 'SomeEnum' # needed for pickle protocol 4 + first = auto(), 'for the money' + second = auto(), 'for the show' + third = auto(), 'for the music' + self.assertIs(type(SomeEnum.first), SomeEnum) + self.assertEqual(SomeEnum.third.value, (3, 'for the music')) + self.assertIsInstance(SomeEnum.third.value, T) + self.assertEqual(SomeEnum.first.index, 1) + self.assertEqual(SomeEnum.second.desc, 'for the show') + globals()['SomeEnum'] = SomeEnum + globals()['T'] = T + test_pickle_dump_load(self.assertIs, SomeEnum.first) + + def test_tuple_subclass_with_auto_2(self): + from collections import namedtuple + T = namedtuple('T', 'index desc') + class SomeEnum(Enum): + __qualname__ = 'SomeEnum' # needed for pickle protocol 4 + first = T(auto(), 'for the money') + second = T(auto(), 'for the show') + third = T(auto(), 'for the music') + self.assertIs(type(SomeEnum.first), SomeEnum) + self.assertEqual(SomeEnum.third.value, (3, 'for the music')) + self.assertIsInstance(SomeEnum.third.value, T) + self.assertEqual(SomeEnum.first.value.index, 1) + self.assertEqual(SomeEnum.second.value.desc, 'for the show') + globals()['SomeEnum'] = SomeEnum + globals()['T'] = T + test_pickle_dump_load(self.assertIs, SomeEnum.first) + def test_duplicate_values_give_unique_enum_items(self): class AutoNumber(Enum): first = () @@ -3359,6 +3409,17 @@ def __new__(cls, int_value, *value_aliases): self.assertIs(Types(2), Types.NetList) self.assertIs(Types('nl'), Types.NetList) + def test_no_members(self): + with self.assertRaisesRegex( + TypeError, + 'has no members', + ): + Enum(7) + with self.assertRaisesRegex( + TypeError, + 'has no members', + ): + Flag(7) class TestOrder(unittest.TestCase): "test usage of the `_order_` attribute" @@ -4817,22 +4878,22 @@ class Color(enum.Enum) | The value of the Enum member. | | ---------------------------------------------------------------------- - | Methods inherited from enum.EnumType: + | Static methods inherited from enum.EnumType: | - | __contains__(value) from enum.EnumType + | __contains__(value) | Return True if `value` is in `cls`. | | `value` is in `cls` if: | 1) `value` is a member of `cls`, or | 2) `value` is the value of one of the `cls`'s members. | - | __getitem__(name) from enum.EnumType + | __getitem__(name) | Return the member matching `name`. | - | __iter__() from enum.EnumType + | __iter__() | Return members in definition order. | - | __len__() from enum.EnumType + | __len__() | Return the number of members (no aliases) | | ---------------------------------------------------------------------- @@ -4857,11 +4918,11 @@ class Color(enum.Enum) | | Data and other attributes defined here: | - | YELLOW = <Color.YELLOW: 3> + | CYAN = <Color.CYAN: 1> | | MAGENTA = <Color.MAGENTA: 2> | - | CYAN = <Color.CYAN: 1> + | YELLOW = <Color.YELLOW: 3> | | ---------------------------------------------------------------------- | Data descriptors inherited from enum.Enum: @@ -4871,7 +4932,18 @@ class Color(enum.Enum) | value | | ---------------------------------------------------------------------- - | Data descriptors inherited from enum.EnumType: + | Static methods inherited from enum.EnumType: + | + | __contains__(value) + | + | __getitem__(name) + | + | __iter__() + | + | __len__() + | + | ---------------------------------------------------------------------- + | Readonly properties inherited from enum.EnumType: | | __members__""" diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index c57488e44aecc64..c5eff8ad8ccca17 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -301,6 +301,7 @@ def baz(): { 6 0="""''', 5, 13) + check('b"fooжжж"'.encode(), 1, 1, 1, 10) # Errors thrown by symtable.c check('x = [(yield i) for i in range(3)]', 1, 7) @@ -1917,7 +1918,7 @@ def test_attributes(self): self.assertEqual(exc.name, 'somename') self.assertEqual(exc.path, 'somepath') - msg = "'invalid' is an invalid keyword argument for ImportError" + msg = r"ImportError\(\) got an unexpected keyword argument 'invalid'" with self.assertRaisesRegex(TypeError, msg): ImportError('test', invalid='keyword') diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py new file mode 100644 index 000000000000000..86c07de507e39cb --- /dev/null +++ b/Lib/test/test_external_inspection.py @@ -0,0 +1,84 @@ +import unittest +import os +import textwrap +import importlib +import sys +from test.support import os_helper, SHORT_TIMEOUT +from test.support.script_helper import make_script + +import subprocess + +PROCESS_VM_READV_SUPPORTED = False + +try: + from _testexternalinspection import PROCESS_VM_READV_SUPPORTED + from _testexternalinspection import get_stack_trace +except ImportError: + unittest.skip("Test only runs when _testexternalinspection is available") + +def _make_test_script(script_dir, script_basename, source): + to_return = make_script(script_dir, script_basename, source) + importlib.invalidate_caches() + return to_return + +class TestGetStackTrace(unittest.TestCase): + + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + def test_remote_stack_trace(self): + # Spawn a process with some realistic Python code + script = textwrap.dedent("""\ + import time, sys, os + def bar(): + for x in range(100): + if x == 50: + baz() + def baz(): + foo() + + def foo(): + fifo = sys.argv[1] + with open(sys.argv[1], "w") as fifo: + fifo.write("ready") + time.sleep(1000) + + bar() + """) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + fifo = f"{work_dir}/the_fifo" + os.mkfifo(fifo) + script_name = _make_test_script(script_dir, 'script', script) + try: + p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + with open(fifo, "r") as fifo_file: + response = fifo_file.read() + self.assertEqual(response, "ready") + stack_trace = get_stack_trace(p.pid) + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + finally: + os.remove(fifo) + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + + expected_stack_trace = [ + 'foo', + 'baz', + 'bar', + '<module>' + ] + self.assertEqual(stack_trace, expected_stack_trace) + + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + def test_self_trace(self): + stack_trace = get_stack_trace(os.getpid()) + self.assertEqual(stack_trace[0], "test_self_trace") + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 203dd6fe57dcd99..6d734d052454d34 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -6,7 +6,9 @@ import struct import sys import unittest -from test.support import verbose, cpython_only, get_pagesize +from test.support import ( + cpython_only, get_pagesize, is_apple, requires_subprocess, verbose +) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink @@ -56,8 +58,10 @@ def get_lockdata(): else: start_len = "qq" - if (sys.platform.startswith(('netbsd', 'freebsd', 'openbsd')) - or sys.platform == 'darwin'): + if ( + sys.platform.startswith(('netbsd', 'freebsd', 'openbsd')) + or is_apple + ): if struct.calcsize('l') == 8: off_t = 'l' pid_t = 'i' @@ -157,6 +161,7 @@ def test_flock(self): self.assertRaises(TypeError, fcntl.flock, 'spam', fcntl.LOCK_SH) @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError") + @requires_subprocess() def test_lockf_exclusive(self): self.f = open(TESTFN, 'wb+') cmd = fcntl.LOCK_EX | fcntl.LOCK_NB @@ -169,6 +174,7 @@ def test_lockf_exclusive(self): self.assertEqual(p.exitcode, 0) @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError") + @requires_subprocess() def test_lockf_share(self): self.f = open(TESTFN, 'wb+') cmd = fcntl.LOCK_SH | fcntl.LOCK_NB diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index 06d9b454add34cd..06d5a8abf320835 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -484,6 +484,14 @@ def testInvalidFd(self): import msvcrt self.assertRaises(OSError, msvcrt.get_osfhandle, make_bad_fd()) + def testBooleanFd(self): + for fd in False, True: + with self.assertWarnsRegex(RuntimeWarning, + 'bool is used as a file descriptor') as cm: + f = self.FileIO(fd, closefd=False) + f.close() + self.assertEqual(cm.filename, __file__) + def testBadModeArgument(self): # verify that we get a sensible error message for bad mode argument bad_mode = "qwerty" diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index af3cb214ab0ac11..b45bd098a366845 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -1314,6 +1314,33 @@ def test_float_format_testfile(self): self.assertEqual(float(format(f, fmt2)), float(rhs)) self.assertEqual(float(format(-f, fmt2)), float('-' + rhs)) + def test_complex_handling(self): + # See issue gh-102840 for more details. + + a = F(1, 2) + b = 1j + message = "unsupported operand type(s) for %s: '%s' and '%s'" + # test forward + self.assertRaisesMessage(TypeError, + message % ("%", "Fraction", "complex"), + operator.mod, a, b) + self.assertRaisesMessage(TypeError, + message % ("//", "Fraction", "complex"), + operator.floordiv, a, b) + self.assertRaisesMessage(TypeError, + message % ("divmod()", "Fraction", "complex"), + divmod, a, b) + # test reverse + self.assertRaisesMessage(TypeError, + message % ("%", "complex", "Fraction"), + operator.mod, b, a) + self.assertRaisesMessage(TypeError, + message % ("//", "complex", "Fraction"), + operator.floordiv, b, a) + self.assertRaisesMessage(TypeError, + message % ("divmod()", "complex", "Fraction"), + divmod, b, a) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 7f17666a8d9697b..f8812c281c2deba 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -13,7 +13,7 @@ _testcapi = None from test import support -from test.support import threading_helper +from test.support import threading_helper, Py_GIL_DISABLED from test.support.script_helper import assert_python_ok @@ -55,6 +55,27 @@ class C: # The reference was released by .clear() self.assertIs(None, wr()) + def test_clear_locals_after_f_locals_access(self): + # see gh-113939 + class C: + pass + + wr = None + def inner(): + nonlocal wr + c = C() + wr = weakref.ref(c) + 1/0 + + try: + inner() + except ZeroDivisionError as exc: + support.gc_collect() + self.assertIsNotNone(wr()) + exc.__traceback__.tb_next.tb_frame.clear() + support.gc_collect() + self.assertIsNone(wr()) + def test_clear_does_not_clear_specials(self): class C: pass @@ -273,6 +294,7 @@ def gen(): assert_python_ok("-c", code) @support.cpython_only + @unittest.skipIf(Py_GIL_DISABLED, "test requires precise GC scheduling") def test_sneaky_frame_object(self): def trace(frame, event, arg): @@ -309,6 +331,7 @@ def f(): # on the *very next* allocation: gc.collect() gc.set_threshold(1, 0, 0) + sys._clear_internal_caches() # Okay, so here's the nightmare scenario: # - We're tracing the resumption of a generator, which creates a new # frame object. diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 2f191ea7a44c161..bed0e6d973b8da3 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -18,6 +18,7 @@ from unittest import TestCase, skipUnless from test import support +from test.support import requires_subprocess from test.support import threading_helper from test.support import socket_helper from test.support import warnings_helper @@ -542,8 +543,8 @@ def test_set_pasv(self): self.assertFalse(self.client.passiveserver) def test_voidcmd(self): - self.client.voidcmd('echo 200') - self.client.voidcmd('echo 299') + self.assertEqual(self.client.voidcmd('echo 200'), '200') + self.assertEqual(self.client.voidcmd('echo 299'), '299') self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 199') self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 300') @@ -900,6 +901,7 @@ def retr(): @skipUnless(ssl, "SSL not available") +@requires_subprocess() class TestTLS_FTPClassMixin(TestFTPClass): """Repeat TestFTPClass tests starting the TLS layer for both control and data connections first. @@ -916,6 +918,7 @@ def setUp(self, encoding=DEFAULT_ENCODING): @skipUnless(ssl, "SSL not available") +@requires_subprocess() class TestTLS_FTPClass(TestCase): """Specific TLS_FTP class tests.""" diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 7c66b906d308baf..4eb322644fc541b 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -31,10 +31,6 @@ decimal = import_helper.import_fresh_module('decimal', fresh=['_decimal']) -_partial_types = [py_functools.partial] -if c_functools: - _partial_types.append(c_functools.partial) - @contextlib.contextmanager def replaced_module(name, replacement): @@ -207,10 +203,7 @@ def test_repr(self): kwargs = {'a': object(), 'b': object()} kwargs_reprs = ['a={a!r}, b={b!r}'.format_map(kwargs), 'b={b!r}, a={a!r}'.format_map(kwargs)] - if self.partial in _partial_types: - name = 'functools.partial' - else: - name = self.partial.__name__ + name = f"{self.partial.__module__}.{self.partial.__qualname__}" f = self.partial(capture) self.assertEqual(f'{name}({capture!r})', repr(f)) @@ -229,10 +222,7 @@ def test_repr(self): for kwargs_repr in kwargs_reprs]) def test_recursive_repr(self): - if self.partial in _partial_types: - name = 'functools.partial' - else: - name = self.partial.__name__ + name = f"{self.partial.__module__}.{self.partial.__qualname__}" f = self.partial(capture) f.__setstate__((f, (), {}, {})) @@ -2867,11 +2857,26 @@ def _(arg: typing.Union[int, typing.Iterable[str]]): def test_invalid_positional_argument(self): @functools.singledispatch - def f(*args): + def f(*args, **kwargs): pass msg = 'f requires at least 1 positional argument' with self.assertRaisesRegex(TypeError, msg): f() + msg = 'f requires at least 1 positional argument' + with self.assertRaisesRegex(TypeError, msg): + f(a=1) + + def test_invalid_positional_argument_singledispatchmethod(self): + class A: + @functools.singledispatchmethod + def t(self, *args, **kwargs): + pass + msg = 't requires at least 1 positional argument' + with self.assertRaisesRegex(TypeError, msg): + A().t() + msg = 't requires at least 1 positional argument' + with self.assertRaisesRegex(TypeError, msg): + A().t(a=1) def test_union(self): @functools.singledispatch diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index b01f344cb14a1a7..dd09643788d62fb 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -363,6 +363,7 @@ def __del__(self): # To minimize variations, though, we first store the get_count() results # and check them at the end. @refcount_test + @unittest.skipIf(Py_GIL_DISABLED, 'needs precise allocation counts') def test_get_count(self): gc.collect() a, b, c = gc.get_count() diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index ca1228ee7008a9a..6fcb5d58dd7f348 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -33,6 +33,7 @@ def skip_if_different_mount_drives(): import parser from stack import Stack import tier1_generator + import optimizer_generator def handle_stderr(): @@ -793,5 +794,168 @@ def test_annotated_op(self): self.run_cases_test(input, output) + def test_deopt_and_exit(self): + input = """ + pure op(OP, (arg1 -- out)) { + DEOPT_IF(1); + EXIT_IF(1); + } + """ + output = "" + with self.assertRaises(Exception): + self.run_cases_test(input, output) + +class TestGeneratedAbstractCases(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.maxDiff = None + + self.temp_dir = tempfile.gettempdir() + self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") + self.temp_input2_filename = os.path.join(self.temp_dir, "input2.txt") + self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") + + def tearDown(self) -> None: + for filename in [ + self.temp_input_filename, + self.temp_input2_filename, + self.temp_output_filename, + ]: + try: + os.remove(filename) + except: + pass + super().tearDown() + + def run_cases_test(self, input: str, input2: str, expected: str): + with open(self.temp_input_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with open(self.temp_input2_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input2) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with handle_stderr(): + optimizer_generator.generate_tier2_abstract_from_files( + [self.temp_input_filename, self.temp_input2_filename], + self.temp_output_filename + ) + + with open(self.temp_output_filename) as temp_output: + lines = temp_output.readlines() + while lines and lines[0].startswith(("// ", "#", " #", "\n")): + lines.pop(0) + while lines and lines[-1].startswith(("#", "\n")): + lines.pop(-1) + actual = "".join(lines) + self.assertEqual(actual.strip(), expected.strip()) + + def test_overridden_abstract(self): + input = """ + pure op(OP, (--)) { + spam(); + } + """ + input2 = """ + pure op(OP, (--)) { + eggs(); + } + """ + output = """ + case OP: { + eggs(); + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_overridden_abstract_args(self): + input = """ + pure op(OP, (arg1 -- out)) { + spam(); + } + op(OP2, (arg1 -- out)) { + eggs(); + } + """ + input2 = """ + op(OP, (arg1 -- out)) { + eggs(); + } + """ + output = """ + case OP: { + _Py_UopsSymbol *arg1; + _Py_UopsSymbol *out; + arg1 = stack_pointer[-1]; + eggs(); + stack_pointer[-1] = out; + break; + } + + case OP2: { + _Py_UopsSymbol *out; + out = sym_new_unknown(ctx); + if (out == NULL) goto out_of_space; + stack_pointer[-1] = out; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_no_overridden_case(self): + input = """ + pure op(OP, (arg1 -- out)) { + spam(); + } + + pure op(OP2, (arg1 -- out)) { + } + + """ + input2 = """ + pure op(OP2, (arg1 -- out)) { + } + """ + output = """ + case OP: { + _Py_UopsSymbol *out; + out = sym_new_unknown(ctx); + if (out == NULL) goto out_of_space; + stack_pointer[-1] = out; + break; + } + + case OP2: { + _Py_UopsSymbol *arg1; + _Py_UopsSymbol *out; + arg1 = stack_pointer[-1]; + stack_pointer[-1] = out; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_missing_override_failure(self): + input = """ + pure op(OP, (arg1 -- out)) { + spam(); + } + """ + input2 = """ + pure op(OTHER, (arg1 -- out)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(AssertionError, "All abstract uops"): + self.run_cases_test(input, input2, output) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 4f311c2d498e9f4..f407ee3caf154c2 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -7,9 +7,9 @@ import sys import unittest import warnings -from test.support import is_emscripten -from test.support import os_helper -from test.support import warnings_helper +from test.support import ( + is_apple, is_emscripten, os_helper, warnings_helper +) from test.support.script_helper import assert_python_ok from test.support.os_helper import FakePath @@ -165,6 +165,12 @@ def test_exists_fd(self): os.close(w) self.assertFalse(self.pathmodule.exists(r)) + def test_exists_bool(self): + for fd in False, True: + with self.assertWarnsRegex(RuntimeWarning, + 'bool is used as a file descriptor'): + self.pathmodule.exists(fd) + def test_isdir(self): filename = os_helper.TESTFN bfilename = os.fsencode(filename) @@ -483,12 +489,16 @@ def test_abspath_issue3426(self): self.assertIsInstance(abspath(path), str) def test_nonascii_abspath(self): - if (os_helper.TESTFN_UNDECODABLE - # macOS and Emscripten deny the creation of a directory with an - # invalid UTF-8 name. Windows allows creating a directory with an - # arbitrary bytes name, but fails to enter this directory - # (when the bytes name is used). - and sys.platform not in ('win32', 'darwin', 'emscripten', 'wasi')): + if ( + os_helper.TESTFN_UNDECODABLE + # Apple platforms and Emscripten/WASI deny the creation of a + # directory with an invalid UTF-8 name. Windows allows creating a + # directory with an arbitrary bytes name, but fails to enter this + # directory (when the bytes name is used). + and sys.platform not in { + "win32", "emscripten", "wasi" + } and not is_apple + ): name = os_helper.TESTFN_UNDECODABLE elif os_helper.TESTFN_NONASCII: name = os_helper.TESTFN_NONASCII diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index aa5fac8eca13542..8b2ea8f89f5dafb 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -333,6 +333,17 @@ def test_recursive_glob(self): eq(glob.glob('**', recursive=True, include_hidden=True), [join(*i) for i in full+rec]) + def test_glob_non_directory(self): + eq = self.assertSequencesEqual_noorder + eq(self.rglob('EF'), self.joins(('EF',))) + eq(self.rglob('EF', ''), []) + eq(self.rglob('EF', '*'), []) + eq(self.rglob('EF', '**'), []) + eq(self.rglob('nonexistent'), []) + eq(self.rglob('nonexistent', ''), []) + eq(self.rglob('nonexistent', '*'), []) + eq(self.rglob('nonexistent', '**'), []) + def test_glob_many_open_files(self): depth = 30 base = os.path.join(self.tempdir, 'deep') diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index 128f933787a3f67..d220c7d06e50c97 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -5,7 +5,6 @@ import functools import io import os -import pathlib import struct import sys import unittest @@ -79,16 +78,18 @@ def test_write(self): f.close() def test_write_read_with_pathlike_file(self): - filename = pathlib.Path(self.filename) + filename = os_helper.FakePath(self.filename) with gzip.GzipFile(filename, 'w') as f: f.write(data1 * 50) self.assertIsInstance(f.name, str) + self.assertEqual(f.name, self.filename) with gzip.GzipFile(filename, 'a') as f: f.write(data1) with gzip.GzipFile(filename) as f: d = f.read() self.assertEqual(d, data1 * 51) self.assertIsInstance(f.name, str) + self.assertEqual(f.name, self.filename) # The following test_write_xy methods test that write accepts # the corresponding bytes-like object type as input @@ -472,13 +473,118 @@ def test_textio_readlines(self): with io.TextIOWrapper(f, encoding="ascii") as t: self.assertEqual(t.readlines(), lines) + def test_fileobj_with_name(self): + with open(self.filename, "xb") as raw: + with gzip.GzipFile(fileobj=raw, mode="x") as f: + f.write(b'one') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + with open(self.filename, "wb") as raw: + with gzip.GzipFile(fileobj=raw, mode="w") as f: + f.write(b'two') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + with open(self.filename, "ab") as raw: + with gzip.GzipFile(fileobj=raw, mode="a") as f: + f.write(b'three') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + with open(self.filename, "rb") as raw: + with gzip.GzipFile(fileobj=raw, mode="r") as f: + self.assertEqual(f.read(), b'twothree') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + def test_fileobj_from_fdopen(self): # Issue #13781: Opening a GzipFile for writing fails when using a # fileobj created with os.fdopen(). - fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT) - with os.fdopen(fd, "wb") as f: - with gzip.GzipFile(fileobj=f, mode="w") as g: - pass + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + with os.fdopen(fd, "xb") as raw: + with gzip.GzipFile(fileobj=raw, mode="x") as f: + f.write(b'one') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.closed, True) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with os.fdopen(fd, "wb") as raw: + with gzip.GzipFile(fileobj=raw, mode="w") as f: + f.write(b'two') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_APPEND) + with os.fdopen(fd, "ab") as raw: + with gzip.GzipFile(fileobj=raw, mode="a") as f: + f.write(b'three') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + + fd = os.open(self.filename, os.O_RDONLY) + with os.fdopen(fd, "rb") as raw: + with gzip.GzipFile(fileobj=raw, mode="r") as f: + self.assertEqual(f.read(), b'twothree') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) def test_fileobj_mode(self): gzip.GzipFile(self.filename, "wb").close() @@ -508,17 +614,69 @@ def test_fileobj_mode(self): def test_bytes_filename(self): str_filename = self.filename - try: - bytes_filename = str_filename.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(str_filename) with gzip.GzipFile(bytes_filename, "wb") as f: f.write(data1 * 50) + self.assertEqual(f.name, bytes_filename) with gzip.GzipFile(bytes_filename, "rb") as f: self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, bytes_filename) # Sanity check that we are actually operating on the right file. with gzip.GzipFile(str_filename, "rb") as f: self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, str_filename) + + def test_fileobj_without_name(self): + bio = io.BytesIO() + with gzip.GzipFile(fileobj=bio, mode='wb') as f: + f.write(data1 * 50) + self.assertEqual(f.name, '') + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + bio.seek(0) + with gzip.GzipFile(fileobj=bio, mode='rb') as f: + self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, '') + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + + def test_fileobj_and_filename(self): + filename2 = self.filename + 'new' + with (open(self.filename, 'wb') as fileobj, + gzip.GzipFile(fileobj=fileobj, filename=filename2, mode='wb') as f): + f.write(data1 * 50) + self.assertEqual(f.name, filename2) + with (open(self.filename, 'rb') as fileobj, + gzip.GzipFile(fileobj=fileobj, filename=filename2, mode='rb') as f): + self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, filename2) + # Sanity check that we are actually operating on the right file. + with gzip.GzipFile(self.filename, 'rb') as f: + self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, self.filename) def test_decompress_limited(self): """Decompressed data buffering should be limited""" @@ -707,13 +865,16 @@ def test_binary_modes(self): self.assertEqual(file_data, uncompressed) def test_pathlike_file(self): - filename = pathlib.Path(self.filename) + filename = os_helper.FakePath(self.filename) with gzip.open(filename, "wb") as f: f.write(data1 * 50) + self.assertEqual(f.name, self.filename) with gzip.open(filename, "ab") as f: f.write(data1) + self.assertEqual(f.name, self.filename) with gzip.open(filename) as f: self.assertEqual(f.read(), data1 * 51) + self.assertEqual(f.name, self.filename) def test_implicit_binary_modes(self): # Test implicit binary modes (no "b" or "t" in mode string). diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index a39a2c45ebc2e28..1502fba9f3e8b8b 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -479,6 +479,14 @@ def test_exercise_all_methods(self): self.fail("Exception raised during normal usage of HMAC class.") +class UpdateTestCase(unittest.TestCase): + @hashlib_helper.requires_hashdigest('sha256') + def test_with_str_update(self): + with self.assertRaises(TypeError): + h = hmac.new(b"key", digestmod='sha256') + h.update("invalid update") + + class CopyTestCase(unittest.TestCase): @hashlib_helper.requires_hashdigest('sha256') diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 089bf5be40a0e27..6e63a8872d9c6e1 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -2408,6 +2408,22 @@ def test_connect_put_request(self): self.assertIn(b'PUT / HTTP/1.1\r\nHost: %(host)s\r\n' % d, self.conn.sock.data) + def test_connect_put_request_ipv6(self): + self.conn.set_tunnel('[1:2:3::4]', 1234) + self.conn.request('PUT', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertIn(b'CONNECT [1:2:3::4]:1234', self.conn.sock.data) + self.assertIn(b'Host: [1:2:3::4]:1234', self.conn.sock.data) + + def test_connect_put_request_ipv6_port(self): + self.conn.set_tunnel('[1:2:3::4]:1234') + self.conn.request('PUT', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertIn(b'CONNECT [1:2:3::4]:1234', self.conn.sock.data) + self.assertIn(b'Host: [1:2:3::4]:1234', self.conn.sock.data) + def test_tunnel_debuglog(self): expected_header = 'X-Dummy: 1' response_text = 'HTTP/1.0 200 OK\r\n{}\r\n\r\n'.format(expected_header) diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 9fa6ecf9c08e279..d762ec6102ab8aa 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -30,8 +30,9 @@ import unittest from test import support -from test.support import os_helper -from test.support import threading_helper +from test.support import ( + is_apple, os_helper, requires_subprocess, threading_helper +) support.requires_working_socket(module=True) @@ -410,8 +411,8 @@ def close_conn(): reader.close() return body - @unittest.skipIf(sys.platform == 'darwin', - 'undecodable name cannot always be decoded on macOS') + @unittest.skipIf(is_apple, + 'undecodable name cannot always be decoded on Apple platforms') @unittest.skipIf(sys.platform == 'win32', 'undecodable name cannot be decoded on win32') @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, @@ -422,11 +423,11 @@ def test_undecodable_filename(self): with open(os.path.join(self.tempdir, filename), 'wb') as f: f.write(os_helper.TESTFN_UNDECODABLE) response = self.request(self.base_url + '/') - if sys.platform == 'darwin': - # On Mac OS the HFS+ filesystem replaces bytes that aren't valid - # UTF-8 into a percent-encoded value. + if is_apple: + # On Apple platforms the HFS+ filesystem replaces bytes that + # aren't valid UTF-8 into a percent-encoded value. for name in os.listdir(self.tempdir): - if name != 'test': # Ignore a filename created in setUp(). + if name != 'test': # Ignore a filename created in setUp(). filename = name break body = self.check_status_and_reason(response, HTTPStatus.OK) @@ -697,6 +698,7 @@ def test_html_escape_filename(self): @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, "This test can't be run reliably as root (issue #13308).") +@requires_subprocess() class CGIHTTPServerTestCase(BaseTestCase): class request_handler(NoLogRequestHandler, CGIHTTPRequestHandler): _test_case_self = None # populated by each setUp() method call. diff --git a/Lib/test/test_importlib/test_lazy.py b/Lib/test/test_importlib/test_lazy.py index cc993f333e355af..38ab21907b58d9d 100644 --- a/Lib/test/test_importlib/test_lazy.py +++ b/Lib/test/test_importlib/test_lazy.py @@ -2,9 +2,12 @@ from importlib import abc from importlib import util import sys +import time +import threading import types import unittest +from test.support import threading_helper from test.test_importlib import util as test_util @@ -40,6 +43,7 @@ class TestingImporter(abc.MetaPathFinder, abc.Loader): module_name = 'lazy_loader_test' mutated_name = 'changed' loaded = None + load_count = 0 source_code = 'attr = 42; __name__ = {!r}'.format(mutated_name) def find_spec(self, name, path, target=None): @@ -48,8 +52,10 @@ def find_spec(self, name, path, target=None): return util.spec_from_loader(name, util.LazyLoader(self)) def exec_module(self, module): + time.sleep(0.01) # Simulate a slow load. exec(self.source_code, module.__dict__) self.loaded = module + self.load_count += 1 class LazyLoaderTests(unittest.TestCase): @@ -59,8 +65,9 @@ def test_init(self): # Classes that don't define exec_module() trigger TypeError. util.LazyLoader(object) - def new_module(self, source_code=None): - loader = TestingImporter() + def new_module(self, source_code=None, loader=None): + if loader is None: + loader = TestingImporter() if source_code is not None: loader.source_code = source_code spec = util.spec_from_loader(TestingImporter.module_name, @@ -140,6 +147,37 @@ def test_module_already_in_sys(self): # Force the load; just care that no exception is raised. module.__name__ + @threading_helper.requires_working_threading() + def test_module_load_race(self): + with test_util.uncache(TestingImporter.module_name): + loader = TestingImporter() + module = self.new_module(loader=loader) + self.assertEqual(loader.load_count, 0) + + class RaisingThread(threading.Thread): + exc = None + def run(self): + try: + super().run() + except Exception as exc: + self.exc = exc + + def access_module(): + return module.attr + + threads = [] + for _ in range(2): + threads.append(thread := RaisingThread(target=access_module)) + thread.start() + + # Races could cause errors + for thread in threads: + thread.join() + self.assertIsNone(thread.exc) + + # Or multiple load attempts + self.assertEqual(loader.load_count, 1) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_inspect/inspect_fodder.py b/Lib/test/test_inspect/inspect_fodder.py index 60ba7aa78394e87..febd54c86fe1d19 100644 --- a/Lib/test/test_inspect/inspect_fodder.py +++ b/Lib/test/test_inspect/inspect_fodder.py @@ -68,9 +68,9 @@ class FesteringGob(MalodorousPervert, ParrotDroppings): def abuse(self, a, b, c): pass - @property - def contradiction(self): + def _getter(self): pass + contradiction = property(_getter) async def lobbest(grenade): pass diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 4611f62b293ff97..9dc37852ec205a6 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -206,12 +206,33 @@ def test_iscoroutine(self): gen_coro = gen_coroutine_function_example(1) coro = coroutine_function_example(1) + class PMClass: + async_generator_partialmethod_example = functools.partialmethod( + async_generator_function_example) + coroutine_partialmethod_example = functools.partialmethod( + coroutine_function_example) + gen_coroutine_partialmethod_example = functools.partialmethod( + gen_coroutine_function_example) + + # partialmethods on the class, bound to an instance + pm_instance = PMClass() + async_gen_coro_pmi = pm_instance.async_generator_partialmethod_example + gen_coro_pmi = pm_instance.gen_coroutine_partialmethod_example + coro_pmi = pm_instance.coroutine_partialmethod_example + + # partialmethods on the class, unbound but accessed via the class + async_gen_coro_pmc = PMClass.async_generator_partialmethod_example + gen_coro_pmc = PMClass.gen_coroutine_partialmethod_example + coro_pmc = PMClass.coroutine_partialmethod_example + self.assertFalse( inspect.iscoroutinefunction(gen_coroutine_function_example)) self.assertFalse( inspect.iscoroutinefunction( functools.partial(functools.partial( gen_coroutine_function_example)))) + self.assertFalse(inspect.iscoroutinefunction(gen_coro_pmi)) + self.assertFalse(inspect.iscoroutinefunction(gen_coro_pmc)) self.assertFalse(inspect.iscoroutine(gen_coro)) self.assertTrue( @@ -220,6 +241,8 @@ def test_iscoroutine(self): inspect.isgeneratorfunction( functools.partial(functools.partial( gen_coroutine_function_example)))) + self.assertTrue(inspect.isgeneratorfunction(gen_coro_pmi)) + self.assertTrue(inspect.isgeneratorfunction(gen_coro_pmc)) self.assertTrue(inspect.isgenerator(gen_coro)) async def _fn3(): @@ -285,6 +308,8 @@ def do_something_static(): inspect.iscoroutinefunction( functools.partial(functools.partial( coroutine_function_example)))) + self.assertTrue(inspect.iscoroutinefunction(coro_pmi)) + self.assertTrue(inspect.iscoroutinefunction(coro_pmc)) self.assertTrue(inspect.iscoroutine(coro)) self.assertFalse( @@ -297,6 +322,8 @@ def do_something_static(): inspect.isgeneratorfunction( functools.partial(functools.partial( coroutine_function_example)))) + self.assertFalse(inspect.isgeneratorfunction(coro_pmi)) + self.assertFalse(inspect.isgeneratorfunction(coro_pmc)) self.assertFalse(inspect.isgenerator(coro)) self.assertFalse( @@ -311,6 +338,8 @@ def do_something_static(): inspect.isasyncgenfunction( functools.partial(functools.partial( async_generator_function_example)))) + self.assertTrue(inspect.isasyncgenfunction(async_gen_coro_pmi)) + self.assertTrue(inspect.isasyncgenfunction(async_gen_coro_pmc)) self.assertTrue(inspect.isasyncgen(async_gen_coro)) coro.close(); gen_coro.close(); # silence warnings @@ -3108,6 +3137,10 @@ def m1d(*args, **kwargs): int)) def test_signature_on_classmethod(self): + self.assertEqual(self.signature(classmethod), + ((('function', ..., ..., "positional_only"),), + ...)) + class Test: @classmethod def foo(cls, arg1, *, arg2=1): @@ -3126,6 +3159,10 @@ def foo(cls, arg1, *, arg2=1): ...)) def test_signature_on_staticmethod(self): + self.assertEqual(self.signature(staticmethod), + ((('function', ..., ..., "positional_only"),), + ...)) + class Test: @staticmethod def foo(cls, *, arg): @@ -3389,7 +3426,7 @@ def test(self: 'anno', x): def test_signature_on_fake_partialmethod(self): def foo(a): pass - foo._partialmethod = 'spam' + foo.__partialmethod__ = 'spam' self.assertEqual(str(inspect.signature(foo)), '(a)') def test_signature_on_decorated(self): @@ -3649,16 +3686,20 @@ class Bar(Spam, Foo): ((('a', ..., ..., "positional_or_keyword"),), ...)) - class Wrapped: - pass - Wrapped.__wrapped__ = lambda a: None - self.assertEqual(self.signature(Wrapped), + def test_signature_on_wrapper(self): + class Wrapper: + def __call__(self, b): + pass + wrapper = Wrapper() + wrapper.__wrapped__ = lambda a: None + self.assertEqual(self.signature(wrapper), ((('a', ..., ..., "positional_or_keyword"),), ...)) # wrapper loop: - Wrapped.__wrapped__ = Wrapped + wrapper = Wrapper() + wrapper.__wrapped__ = wrapper with self.assertRaisesRegex(ValueError, 'wrapper loop'): - self.signature(Wrapped) + self.signature(wrapper) def test_signature_on_lambdas(self): self.assertEqual(self.signature((lambda a=10: a)), @@ -4970,6 +5011,14 @@ def test_recursion_limit(self): with self.assertRaisesRegex(ValueError, 'wrapper loop'): inspect.unwrap(obj) + def test_wrapped_descriptor(self): + self.assertIs(inspect.unwrap(NTimesUnwrappable), NTimesUnwrappable) + self.assertIs(inspect.unwrap(staticmethod), staticmethod) + self.assertIs(inspect.unwrap(classmethod), classmethod) + self.assertIs(inspect.unwrap(staticmethod(classmethod)), classmethod) + self.assertIs(inspect.unwrap(classmethod(staticmethod)), staticmethod) + + class TestMain(unittest.TestCase): def test_only_source(self): module = importlib.import_module('unittest') diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 0bf55facad9fedb..47fc50a0e203497 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -664,84 +664,78 @@ def test_denial_of_service_prevented_int_to_str(self): """Regression test: ensure we fail before performing O(N**2) work.""" maxdigits = sys.get_int_max_str_digits() assert maxdigits < 50_000, maxdigits # A test prerequisite. - get_time = time.process_time - if get_time() <= 0: # some platforms like WASM lack process_time() - get_time = time.monotonic huge_int = int(f'0x{"c"*65_000}', base=16) # 78268 decimal digits. digits = 78_268 - with support.adjust_int_max_str_digits(digits): - start = get_time() + with ( + support.adjust_int_max_str_digits(digits), + support.CPUStopwatch() as sw_convert): huge_decimal = str(huge_int) - seconds_to_convert = get_time() - start self.assertEqual(len(huge_decimal), digits) # Ensuring that we chose a slow enough conversion to measure. # It takes 0.1 seconds on a Zen based cloud VM in an opt build. # Some OSes have a low res 1/64s timer, skip if hard to measure. - if seconds_to_convert < 1/64: + if sw_convert.seconds < sw_convert.clock_info.resolution * 2: raise unittest.SkipTest('"slow" conversion took only ' - f'{seconds_to_convert} seconds.') + f'{sw_convert.seconds} seconds.') # We test with the limit almost at the size needed to check performance. # The performant limit check is slightly fuzzy, give it a some room. with support.adjust_int_max_str_digits(int(.995 * digits)): - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_huge): str(huge_int) - seconds_to_fail_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLessEqual(seconds_to_fail_huge, seconds_to_convert/2) + self.assertLessEqual(sw_fail_huge.seconds, sw_convert.seconds/2) # Now we test that a conversion that would take 30x as long also fails # in a similarly fast fashion. extra_huge_int = int(f'0x{"c"*500_000}', base=16) # 602060 digits. - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_extra_huge): # If not limited, 8 seconds said Zen based cloud VM. str(extra_huge_int) - seconds_to_fail_extra_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLess(seconds_to_fail_extra_huge, seconds_to_convert/2) + self.assertLess(sw_fail_extra_huge.seconds, sw_convert.seconds/2) def test_denial_of_service_prevented_str_to_int(self): """Regression test: ensure we fail before performing O(N**2) work.""" maxdigits = sys.get_int_max_str_digits() assert maxdigits < 100_000, maxdigits # A test prerequisite. - get_time = time.process_time - if get_time() <= 0: # some platforms like WASM lack process_time() - get_time = time.monotonic digits = 133700 huge = '8'*digits - with support.adjust_int_max_str_digits(digits): - start = get_time() + with ( + support.adjust_int_max_str_digits(digits), + support.CPUStopwatch() as sw_convert): int(huge) - seconds_to_convert = get_time() - start # Ensuring that we chose a slow enough conversion to measure. # It takes 0.1 seconds on a Zen based cloud VM in an opt build. # Some OSes have a low res 1/64s timer, skip if hard to measure. - if seconds_to_convert < 1/64: + if sw_convert.seconds < sw_convert.clock_info.resolution * 2: raise unittest.SkipTest('"slow" conversion took only ' - f'{seconds_to_convert} seconds.') + f'{sw_convert.seconds} seconds.') with support.adjust_int_max_str_digits(digits - 1): - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_huge): int(huge) - seconds_to_fail_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLessEqual(seconds_to_fail_huge, seconds_to_convert/2) + self.assertLessEqual(sw_fail_huge.seconds, sw_convert.seconds/2) # Now we test that a conversion that would take 30x as long also fails # in a similarly fast fashion. extra_huge = '7'*1_200_000 - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_extra_huge): # If not limited, 8 seconds in the Zen based cloud VM. int(extra_huge) - seconds_to_fail_extra_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLessEqual(seconds_to_fail_extra_huge, seconds_to_convert/2) + self.assertLessEqual(sw_fail_extra_huge.seconds, sw_convert.seconds/2) def test_power_of_two_bases_unlimited(self): """The limit does not apply to power of 2 bases.""" diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index aefd326977095f9..363143fa810f35d 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -280,7 +280,7 @@ def test_subinterpreter(self): def test_finished(self): r, w = self.pipe() interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os os.write({w}, b'x') """) @@ -312,7 +312,7 @@ def test_with_only_background_threads(self): FINISHED = b'F' interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os import threading @@ -326,7 +326,7 @@ def task(): self.assertFalse(interp.is_running()) os.write(w_thread, DONE) - interp.exec_sync('t.join()') + interp.exec('t.join()') self.assertEqual(os.read(r_interp, 1), FINISHED) @@ -393,7 +393,7 @@ def test_from_sibling(self): interp2 = interpreters.create() self.assertEqual(set(interpreters.list_all()), {main, interp1, interp2}) - interp1.exec_sync(dedent(f""" + interp1.exec(dedent(f""" from test.support import interpreters interp2 = interpreters.Interpreter({interp2.id}) interp2.close() @@ -427,7 +427,7 @@ def test_subthreads_still_running(self): FINISHED = b'F' interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os import threading import time @@ -503,27 +503,27 @@ def test_not_shareable(self): interp.prepare_main(spam={'spam': 'eggs', 'foo': 'bar'}) # Make sure neither was actually bound. - with self.assertRaises(interpreters.ExecFailure): - interp.exec_sync('print(foo)') - with self.assertRaises(interpreters.ExecFailure): - interp.exec_sync('print(spam)') + with self.assertRaises(interpreters.ExecutionFailed): + interp.exec('print(foo)') + with self.assertRaises(interpreters.ExecutionFailed): + interp.exec('print(spam)') -class TestInterpreterExecSync(TestBase): +class TestInterpreterExec(TestBase): def test_success(self): interp = interpreters.create() script, file = _captured_script('print("it worked!", end="")') with file: - interp.exec_sync(script) + interp.exec(script) out = file.read() self.assertEqual(out, 'it worked!') def test_failure(self): interp = interpreters.create() - with self.assertRaises(interpreters.ExecFailure): - interp.exec_sync('raise Exception') + with self.assertRaises(interpreters.ExecutionFailed): + interp.exec('raise Exception') def test_display_preserved_exception(self): tempdir = self.temp_dir() @@ -542,21 +542,21 @@ def script(): spam.eggs() interp = interpreters.create() - interp.exec_sync(script) + interp.exec(script) """) stdout, stderr = self.assert_python_failure(scriptfile) self.maxDiff = None - interpmod_line, = (l for l in stderr.splitlines() if ' exec_sync' in l) - # File "{interpreters.__file__}", line 179, in exec_sync + interpmod_line, = (l for l in stderr.splitlines() if ' exec' in l) + # File "{interpreters.__file__}", line 179, in exec self.assertEqual(stderr, dedent(f"""\ Traceback (most recent call last): File "{scriptfile}", line 9, in <module> - interp.exec_sync(script) - ~~~~~~~~~~~~~~~~^^^^^^^^ + interp.exec(script) + ~~~~~~~~~~~^^^^^^^^ {interpmod_line.strip()} - raise ExecFailure(excinfo) - test.support.interpreters.ExecFailure: RuntimeError: uh-oh! + raise ExecutionFailed(excinfo) + test.support.interpreters.ExecutionFailed: RuntimeError: uh-oh! Uncaught in the interpreter: @@ -578,7 +578,7 @@ def test_in_thread(self): script, file = _captured_script('print("it worked!", end="")') with file: def f(): - interp.exec_sync(script) + interp.exec(script) t = threading.Thread(target=f) t.start() @@ -604,7 +604,7 @@ def test_fork(self): with open('{file.name}', 'w', encoding='utf-8') as out: out.write('{expected}') """) - interp.exec_sync(script) + interp.exec(script) file.seek(0) content = file.read() @@ -615,17 +615,17 @@ def test_already_running(self): interp = interpreters.create() with _running(interp): with self.assertRaises(RuntimeError): - interp.exec_sync('print("spam")') + interp.exec('print("spam")') def test_bad_script(self): interp = interpreters.create() with self.assertRaises(TypeError): - interp.exec_sync(10) + interp.exec(10) def test_bytes_for_script(self): interp = interpreters.create() with self.assertRaises(TypeError): - interp.exec_sync(b'print("spam")') + interp.exec(b'print("spam")') def test_with_background_threads_still_running(self): r_interp, w_interp = self.pipe() @@ -636,7 +636,7 @@ def test_with_background_threads_still_running(self): FINISHED = b'F' interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os import threading @@ -648,46 +648,229 @@ def task(): t.start() os.write({w_interp}, {RAN!r}) """) - interp.exec_sync(f"""if True: + interp.exec(f"""if True: os.write({w_interp}, {RAN!r}) """) os.write(w_thread, DONE) - interp.exec_sync('t.join()') + interp.exec('t.join()') self.assertEqual(os.read(r_interp, 1), RAN) self.assertEqual(os.read(r_interp, 1), RAN) self.assertEqual(os.read(r_interp, 1), FINISHED) # test_xxsubinterpreters covers the remaining - # Interpreter.exec_sync() behavior. + # Interpreter.exec() behavior. -class TestInterpreterRun(TestBase): - - def test_success(self): - interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: - t = interp.run(script) +def call_func_noop(): + pass + + +def call_func_return_shareable(): + return (1, None) + + +def call_func_return_not_shareable(): + return [1, 2, 3] + + +def call_func_failure(): + raise Exception('spam!') + + +def call_func_ident(value): + return value + + +def get_call_func_closure(value): + def call_func_closure(): + return value + return call_func_closure + + +class Spam: + + @staticmethod + def noop(): + pass + + @classmethod + def from_values(cls, *values): + return cls(values) + + def __init__(self, value): + self.value = value + + def __call__(self, *args, **kwargs): + return (self.value, args, kwargs) + + def __eq__(self, other): + if not isinstance(other, Spam): + return NotImplemented + return self.value == other.value + + def run(self, *args, **kwargs): + return (self.value, args, kwargs) + + +def call_func_complex(op, /, value=None, *args, exc=None, **kwargs): + if exc is not None: + raise exc + if op == '': + raise ValueError('missing op') + elif op == 'ident': + if args or kwargs: + raise Exception((args, kwargs)) + return value + elif op == 'full-ident': + return (value, args, kwargs) + elif op == 'globals': + if value is not None or args or kwargs: + raise Exception((value, args, kwargs)) + return __name__ + elif op == 'interpid': + if value is not None or args or kwargs: + raise Exception((value, args, kwargs)) + return interpreters.get_current().id + elif op == 'closure': + if args or kwargs: + raise Exception((args, kwargs)) + return get_call_func_closure(value) + elif op == 'custom': + if args or kwargs: + raise Exception((args, kwargs)) + return Spam(value) + elif op == 'custom-inner': + if args or kwargs: + raise Exception((args, kwargs)) + class Eggs(Spam): + pass + return Eggs(value) + elif not isinstance(op, str): + raise TypeError(op) + else: + raise NotImplementedError(op) + + +class TestInterpreterCall(TestBase): + + # signature + # - blank + # - args + # - kwargs + # - args, kwargs + # return + # - nothing (None) + # - simple + # - closure + # - custom + # ops: + # - do nothing + # - fail + # - echo + # - do complex, relative to interpreter + # scope + # - global func + # - local closure + # - returned closure + # - callable type instance + # - type + # - classmethod + # - staticmethod + # - instance method + # exception + # - builtin + # - custom + # - preserves info (e.g. SyntaxError) + # - matching error display + + def test_call(self): + interp = interpreters.create() + + for i, (callable, args, kwargs) in enumerate([ + (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_not_shareable, (), {}), + (Spam.noop, (), {}), + ]): + with self.subTest(f'success case #{i+1}'): + res = interp.call(callable) + self.assertIs(res, None) + + for i, (callable, args, kwargs) in enumerate([ + (call_func_ident, ('spamspamspam',), {}), + (get_call_func_closure, (42,), {}), + (get_call_func_closure(42), (), {}), + (Spam.from_values, (), {}), + (Spam.from_values, (1, 2, 3), {}), + (Spam, ('???'), {}), + (Spam(101), (), {}), + (Spam(10101).run, (), {}), + (call_func_complex, ('ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), + (call_func_complex, ('globals',), {}), + (call_func_complex, ('interpid',), {}), + (call_func_complex, ('closure',), {'value': '~~~'}), + (call_func_complex, ('custom', 'spam!'), {}), + (call_func_complex, ('custom-inner', 'eggs!'), {}), + (call_func_complex, ('???',), {'exc': ValueError('spam')}), + ]): + with self.subTest(f'invalid case #{i+1}'): + with self.assertRaises(Exception): + if args or kwargs: + raise Exception((args, kwargs)) + interp.call(callable) + + with self.assertRaises(interpreters.ExecutionFailed): + interp.call(call_func_failure) + + def test_call_in_thread(self): + interp = interpreters.create() + + for i, (callable, args, kwargs) in enumerate([ + (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_not_shareable, (), {}), + (Spam.noop, (), {}), + ]): + with self.subTest(f'success case #{i+1}'): + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(callable) + t.join() + self.assertIsNone(ctx.caught) + + for i, (callable, args, kwargs) in enumerate([ + (call_func_ident, ('spamspamspam',), {}), + (get_call_func_closure, (42,), {}), + (get_call_func_closure(42), (), {}), + (Spam.from_values, (), {}), + (Spam.from_values, (1, 2, 3), {}), + (Spam, ('???'), {}), + (Spam(101), (), {}), + (Spam(10101).run, (), {}), + (call_func_complex, ('ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), + (call_func_complex, ('globals',), {}), + (call_func_complex, ('interpid',), {}), + (call_func_complex, ('closure',), {'value': '~~~'}), + (call_func_complex, ('custom', 'spam!'), {}), + (call_func_complex, ('custom-inner', 'eggs!'), {}), + (call_func_complex, ('???',), {'exc': ValueError('spam')}), + ]): + with self.subTest(f'invalid case #{i+1}'): + if args or kwargs: + continue + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(callable) + t.join() + self.assertIsNotNone(ctx.caught) + + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(call_func_failure) t.join() - out = file.read() - - self.assertEqual(out, 'it worked!') - - def test_failure(self): - caught = False - def excepthook(args): - nonlocal caught - caught = True - threading.excepthook = excepthook - try: - interp = interpreters.create() - t = interp.run('raise Exception') - t.join() - - self.assertTrue(caught) - except BaseException: - threading.excepthook = threading.__excepthook__ + self.assertIsNotNone(ctx.caught) class TestIsShareable(TestBase): diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index 3c3e18832d41683..07e503837bcf75b 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -120,7 +120,7 @@ def test_send_recv_main(self): def test_send_recv_same_interpreter(self): interp = interpreters.create() - interp.exec_sync(dedent(""" + interp.exec(dedent(""" from test.support.interpreters import channels r, s = channels.create() orig = b'spam' @@ -193,7 +193,7 @@ def test_send_recv_nowait_main_with_default(self): def test_send_recv_nowait_same_interpreter(self): interp = interpreters.create() - interp.exec_sync(dedent(""" + interp.exec(dedent(""" from test.support.interpreters import channels r, s = channels.create() orig = b'spam' diff --git a/Lib/test/test_interpreters/test_lifecycle.py b/Lib/test/test_interpreters/test_lifecycle.py index c2917d839904f91..67b6f439c3191fc 100644 --- a/Lib/test/test_interpreters/test_lifecycle.py +++ b/Lib/test/test_interpreters/test_lifecycle.py @@ -124,7 +124,7 @@ def test_sys_path_0(self): orig = sys.path[0] interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import json import sys print(json.dumps({{ diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 2a8ca99c1f6e3f3..65b5435fb00b04b 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -51,20 +51,20 @@ def test_shareable(self): queue1 = queues.create() interp = interpreters.create() - interp.exec_sync(dedent(f""" + interp.exec(dedent(f""" from test.support.interpreters import queues queue1 = queues.Queue({queue1.id}) """)); with self.subTest('same interpreter'): queue2 = queues.create() - queue1.put(queue2) + queue1.put(queue2, syncobj=True) queue3 = queue1.get() self.assertIs(queue3, queue2) with self.subTest('from current interpreter'): queue4 = queues.create() - queue1.put(queue4) + queue1.put(queue4, syncobj=True) out = _run_output(interp, dedent(""" queue4 = queue1.get() print(queue4.id) @@ -75,7 +75,7 @@ def test_shareable(self): with self.subTest('from subinterpreter'): out = _run_output(interp, dedent(""" queue5 = queues.create() - queue1.put(queue5) + queue1.put(queue5, syncobj=True) print(queue5.id) """)) qid = int(out) @@ -118,7 +118,7 @@ class TestQueueOps(TestBase): def test_empty(self): queue = queues.create() before = queue.empty() - queue.put(None) + queue.put(None, syncobj=True) during = queue.empty() queue.get() after = queue.empty() @@ -133,7 +133,7 @@ def test_full(self): queue = queues.create(3) for _ in range(3): actual.append(queue.full()) - queue.put(None) + queue.put(None, syncobj=True) actual.append(queue.full()) for _ in range(3): queue.get() @@ -147,16 +147,16 @@ def test_qsize(self): queue = queues.create() for _ in range(3): actual.append(queue.qsize()) - queue.put(None) + queue.put(None, syncobj=True) actual.append(queue.qsize()) queue.get() actual.append(queue.qsize()) - queue.put(None) + queue.put(None, syncobj=True) actual.append(queue.qsize()) for _ in range(3): queue.get() actual.append(queue.qsize()) - queue.put(None) + queue.put(None, syncobj=True) actual.append(queue.qsize()) queue.get() actual.append(queue.qsize()) @@ -165,30 +165,81 @@ def test_qsize(self): def test_put_get_main(self): expected = list(range(20)) - queue = queues.create() - for i in range(20): - queue.put(i) - actual = [queue.get() for _ in range(20)] + for syncobj in (True, False): + kwds = dict(syncobj=syncobj) + with self.subTest(f'syncobj={syncobj}'): + queue = queues.create() + for i in range(20): + queue.put(i, **kwds) + actual = [queue.get() for _ in range(20)] - self.assertEqual(actual, expected) + self.assertEqual(actual, expected) def test_put_timeout(self): - queue = queues.create(2) - queue.put(None) - queue.put(None) - with self.assertRaises(queues.QueueFull): - queue.put(None, timeout=0.1) - queue.get() - queue.put(None) + for syncobj in (True, False): + kwds = dict(syncobj=syncobj) + with self.subTest(f'syncobj={syncobj}'): + queue = queues.create(2) + queue.put(None, **kwds) + queue.put(None, **kwds) + with self.assertRaises(queues.QueueFull): + queue.put(None, timeout=0.1, **kwds) + queue.get() + queue.put(None, **kwds) def test_put_nowait(self): - queue = queues.create(2) - queue.put_nowait(None) - queue.put_nowait(None) - with self.assertRaises(queues.QueueFull): - queue.put_nowait(None) - queue.get() - queue.put_nowait(None) + for syncobj in (True, False): + kwds = dict(syncobj=syncobj) + with self.subTest(f'syncobj={syncobj}'): + queue = queues.create(2) + queue.put_nowait(None, **kwds) + queue.put_nowait(None, **kwds) + with self.assertRaises(queues.QueueFull): + queue.put_nowait(None, **kwds) + queue.get() + queue.put_nowait(None, **kwds) + + def test_put_syncobj(self): + for obj in [ + None, + True, + 10, + 'spam', + b'spam', + (0, 'a'), + ]: + with self.subTest(repr(obj)): + queue = queues.create() + queue.put(obj, syncobj=True) + obj2 = queue.get() + self.assertEqual(obj2, obj) + + for obj in [ + [1, 2, 3], + {'a': 13, 'b': 17}, + ]: + with self.subTest(repr(obj)): + queue = queues.create() + with self.assertRaises(interpreters.NotShareableError): + queue.put(obj, syncobj=True) + + def test_put_not_syncobj(self): + for obj in [ + None, + True, + 10, + 'spam', + b'spam', + (0, 'a'), + # not shareable + [1, 2, 3], + {'a': 13, 'b': 17}, + ]: + with self.subTest(repr(obj)): + queue = queues.create() + queue.put(obj, syncobj=False) + obj2 = queue.get() + self.assertEqual(obj2, obj) def test_get_timeout(self): queue = queues.create() @@ -200,13 +251,41 @@ def test_get_nowait(self): with self.assertRaises(queues.QueueEmpty): queue.get_nowait() + def test_put_get_default_syncobj(self): + expected = list(range(20)) + queue = queues.create(syncobj=True) + for i in range(20): + queue.put(i) + actual = [queue.get() for _ in range(20)] + + self.assertEqual(actual, expected) + + obj = [1, 2, 3] # lists are not shareable + with self.assertRaises(interpreters.NotShareableError): + queue.put(obj) + + def test_put_get_default_not_syncobj(self): + expected = list(range(20)) + queue = queues.create(syncobj=False) + for i in range(20): + queue.put(i) + actual = [queue.get() for _ in range(20)] + + self.assertEqual(actual, expected) + + obj = [1, 2, 3] # lists are not shareable + queue.put(obj) + obj2 = queue.get() + self.assertEqual(obj, obj2) + self.assertIsNot(obj, obj2) + def test_put_get_same_interpreter(self): interp = interpreters.create() - interp.exec_sync(dedent(""" + interp.exec(dedent(""" from test.support.interpreters import queues queue = queues.create() orig = b'spam' - queue.put(orig) + queue.put(orig, syncobj=True) obj = queue.get() assert obj == orig, 'expected: obj == orig' assert obj is not orig, 'expected: obj is not orig' @@ -219,7 +298,7 @@ def test_put_get_different_interpreters(self): self.assertEqual(len(queues.list_all()), 2) obj1 = b'spam' - queue1.put(obj1) + queue1.put(obj1, syncobj=True) out = _run_output( interp, @@ -236,7 +315,7 @@ def test_put_get_different_interpreters(self): obj2 = b'eggs' print(id(obj2)) assert queue2.qsize() == 0, 'expected: queue2.qsize() == 0' - queue2.put(obj2) + queue2.put(obj2, syncobj=True) assert queue2.qsize() == 1, 'expected: queue2.qsize() == 1' """)) self.assertEqual(len(queues.list_all()), 2) @@ -258,8 +337,8 @@ def test_put_cleared_with_subinterpreter(self): queue = queues.Queue({queue.id}) obj1 = b'spam' obj2 = b'eggs' - queue.put(obj1) - queue.put(obj2) + queue.put(obj1, syncobj=True) + queue.put(obj2, syncobj=True) """)) self.assertEqual(queue.qsize(), 2) @@ -281,12 +360,12 @@ def f(): break except queues.QueueEmpty: continue - queue2.put(obj) + queue2.put(obj, syncobj=True) t = threading.Thread(target=f) t.start() orig = b'spam' - queue1.put(orig) + queue1.put(orig, syncobj=True) obj = queue2.get() t.join() diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 3a37ed09dd89434..973d05d4f96dcb8 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -4,8 +4,9 @@ import subprocess import sys import tempfile -import threading from textwrap import dedent +import threading +import types import unittest from test import support @@ -41,7 +42,7 @@ def _run_output(interp, request, init=None): with rpipe: if init: interp.prepare_main(init) - interp.exec_sync(script) + interp.exec(script) return rpipe.read() @@ -49,7 +50,7 @@ def _run_output(interp, request, init=None): def _running(interp): r, w = os.pipe() def run(): - interp.exec_sync(dedent(f""" + interp.exec(dedent(f""" # wait for "signal" with open({r}) as rpipe: rpipe.read() @@ -84,6 +85,18 @@ def temp_dir(self): self.addCleanup(lambda: os_helper.rmtree(tempdir)) return tempdir + @contextlib.contextmanager + def captured_thread_exception(self): + ctx = types.SimpleNamespace(caught=None) + def excepthook(args): + ctx.caught = args + orig_excepthook = threading.excepthook + threading.excepthook = excepthook + try: + yield ctx + finally: + threading.excepthook = orig_excepthook + def make_script(self, filename, dirname=None, text=None): if text: text = dedent(text) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 9e28b936e00bd57..5491c0575dbd3f9 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -39,11 +39,9 @@ from test import support from test.support.script_helper import ( assert_python_ok, assert_python_failure, run_python_until_end) -from test.support import import_helper -from test.support import os_helper -from test.support import threading_helper -from test.support import warnings_helper -from test.support import skip_if_sanitizer +from test.support import ( + import_helper, is_apple, os_helper, skip_if_sanitizer, threading_helper, warnings_helper +) from test.support.os_helper import FakePath import codecs @@ -259,6 +257,27 @@ class PyMockUnseekableIO(MockUnseekableIO, pyio.BytesIO): UnsupportedOperation = pyio.UnsupportedOperation +class MockCharPseudoDevFileIO(MockFileIO): + # GH-95782 + # ftruncate() does not work on these special files (and CPython then raises + # appropriate exceptions), so truncate() does not have to be accounted for + # here. + def __init__(self, data): + super().__init__(data) + + def seek(self, *args): + return 0 + + def tell(self, *args): + return 0 + +class CMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, io.BytesIO): + pass + +class PyMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, pyio.BytesIO): + pass + + class MockNonBlockWriterIO: def __init__(self): @@ -606,10 +625,10 @@ def test_raw_bytes_io(self): self.read_ops(f, True) def test_large_file_ops(self): - # On Windows and Mac OSX this test consumes large resources; It takes - # a long time to build the >2 GiB file and takes >2 GiB of disk space - # therefore the resource must be enabled to run this test. - if sys.platform[:3] == 'win' or sys.platform == 'darwin': + # On Windows and Apple platforms this test consumes large resources; It + # takes a long time to build the >2 GiB file and takes >2 GiB of disk + # space therefore the resource must be enabled to run this test. + if sys.platform[:3] == 'win' or is_apple: support.requires( 'largefile', 'test requires %s bytes and a long time to run' % self.LARGE) @@ -1650,6 +1669,30 @@ def test_truncate_on_read_only(self): self.assertRaises(self.UnsupportedOperation, bufio.truncate) self.assertRaises(self.UnsupportedOperation, bufio.truncate, 0) + def test_tell_character_device_file(self): + # GH-95782 + # For the (former) bug in BufferedIO to manifest, the wrapped IO obj + # must be able to produce at least 2 bytes. + raw = self.MockCharPseudoDevFileIO(b"12") + buf = self.tp(raw) + self.assertEqual(buf.tell(), 0) + self.assertEqual(buf.read(1), b"1") + self.assertEqual(buf.tell(), 0) + + def test_seek_character_device_file(self): + raw = self.MockCharPseudoDevFileIO(b"12") + buf = self.tp(raw) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + self.assertEqual(buf.seek(1, io.SEEK_SET), 0) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + self.assertEqual(buf.read(1), b"1") + + # In the C implementation, tell() sets the BufferedIO's abs_pos to 0, + # which means that the next seek() could return a negative offset if it + # does not sanity-check: + self.assertEqual(buf.tell(), 0) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + class CBufferedReaderTest(BufferedReaderTest, SizeofTest): tp = io.BufferedReader @@ -2499,6 +2542,28 @@ def test_interleaved_read_write(self): f.flush() self.assertEqual(raw.getvalue(), b'a2c') + def test_read1_after_write(self): + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + self.assertEqual(f.read1(1), b'b') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + self.assertEqual(f.read1(), b'bcd') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + # XXX: read(100) returns different numbers of bytes + # in Python and C implementations. + self.assertEqual(f.read1(100)[:3], b'bcd') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + def test_interleaved_readline_write(self): with self.BytesIO(b'ab\ncdef\ng\n') as raw: with self.tp(raw) as f: @@ -4860,7 +4925,7 @@ def load_tests(loader, tests, pattern): # classes in the __dict__ of each test. mocks = (MockRawIO, MisbehavedRawIO, MockFileIO, CloseFailureIO, MockNonBlockWriterIO, MockUnseekableIO, MockRawIOWithoutRead, - SlowFlushRawIO) + SlowFlushRawIO, MockCharPseudoDevFileIO) all_members = io.__all__ c_io_ns = {name : getattr(io, name) for name in all_members} py_io_ns = {name : getattr(pyio, name) for name in all_members} diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 30aedb0db3bb3db..9606d5beab71cb0 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -302,7 +302,7 @@ def __eq__(self, other): # listiter_reduce_general self.assertEqual( run("reversed", orig["reversed"](list(range(8)))), - (iter, ([],)) + (reversed, ([],)) ) for case in types: diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 72dd40136cfdb24..e42df3d9496bc82 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -5,6 +5,7 @@ import os.path import tempfile import tokenize +from importlib.machinery import ModuleSpec from test import support from test.support import os_helper @@ -97,6 +98,16 @@ class BadUnicode_WithDeclaration(GetLineTestsBadData, unittest.TestCase): file_byte_string = b'# coding=utf-8\n\x80abc' +class FakeLoader: + def get_source(self, fullname): + return f'source for {fullname}' + + +class NoSourceLoader: + def get_source(self, fullname): + return None + + class LineCacheTests(unittest.TestCase): def test_getline(self): @@ -238,6 +249,33 @@ def raise_memoryerror(*args, **kwargs): self.assertEqual(lines3, []) self.assertEqual(linecache.getlines(FILENAME), lines) + def test_loader(self): + filename = 'scheme://path' + + for loader in (None, object(), NoSourceLoader()): + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': loader} + self.assertEqual(linecache.getlines(filename, module_globals), []) + + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader()} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + + for spec in (None, object(), ModuleSpec('', FakeLoader())): + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader(), + '__spec__': spec} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + + linecache.clearcache() + spec = ModuleSpec('x.y.z', FakeLoader()) + module_globals = {'__name__': 'a.b.c', '__loader__': spec.loader, + '__spec__': spec} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for x.y.z\n']) + class LineCacheInvalidationTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index f95a78aff0c7118..2868dd01545b95f 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -156,6 +156,18 @@ def method(self): self.assertEqual(C.y, [4, 4, 4, 4, 4]) self.assertIs(C().method(), C) + def test_references_super(self): + code = """ + res = [super for x in [1]] + """ + self._check_in_scopes(code, outputs={"res": [super]}) + + def test_references___class__(self): + code = """ + res = [__class__ for x in [1]] + """ + self._check_in_scopes(code, raises=NameError) + def test_inner_cell_shadows_outer(self): code = """ items = [(lambda: i) for i in range(5)] diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 908e242b85f5e7a..d87142698efa8b3 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -4089,6 +4089,7 @@ def test_queue_listener(self): self.que_logger.critical(self.next_message()) finally: listener.stop() + listener.stop() # gh-114706 - ensure no crash if called again self.assertTrue(handler.matches(levelno=logging.WARNING, message='1')) self.assertTrue(handler.matches(levelno=logging.ERROR, message='2')) self.assertTrue(handler.matches(levelno=logging.CRITICAL, message='3')) @@ -5477,6 +5478,7 @@ def test_critical(self): self.assertEqual(record.levelno, logging.CRITICAL) self.assertEqual(record.msg, msg) self.assertEqual(record.args, (self.recording,)) + self.assertEqual(record.funcName, 'test_critical') def test_is_enabled_for(self): old_disable = self.adapter.logger.manager.disable @@ -5495,15 +5497,9 @@ def test_has_handlers(self): self.assertFalse(self.adapter.hasHandlers()) def test_nested(self): - class Adapter(logging.LoggerAdapter): - prefix = 'Adapter' - - def process(self, msg, kwargs): - return f"{self.prefix} {msg}", kwargs - msg = 'Adapters can be nested, yo.' - adapter = Adapter(logger=self.logger, extra=None) - adapter_adapter = Adapter(logger=adapter, extra=None) + adapter = PrefixAdapter(logger=self.logger, extra=None) + adapter_adapter = PrefixAdapter(logger=adapter, extra=None) adapter_adapter.prefix = 'AdapterAdapter' self.assertEqual(repr(adapter), repr(adapter_adapter)) adapter_adapter.log(logging.CRITICAL, msg, self.recording) @@ -5512,6 +5508,7 @@ def process(self, msg, kwargs): self.assertEqual(record.levelno, logging.CRITICAL) self.assertEqual(record.msg, f"Adapter AdapterAdapter {msg}") self.assertEqual(record.args, (self.recording,)) + self.assertEqual(record.funcName, 'test_nested') orig_manager = adapter_adapter.manager self.assertIs(adapter.manager, orig_manager) self.assertIs(self.logger.manager, orig_manager) @@ -5527,6 +5524,61 @@ def process(self, msg, kwargs): self.assertIs(adapter.manager, orig_manager) self.assertIs(self.logger.manager, orig_manager) + def test_styled_adapter(self): + # Test an example from the Cookbook. + records = self.recording.records + adapter = StyleAdapter(self.logger) + adapter.warning('Hello, {}!', 'world') + self.assertEqual(str(records[-1].msg), 'Hello, world!') + self.assertEqual(records[-1].funcName, 'test_styled_adapter') + adapter.log(logging.WARNING, 'Goodbye {}.', 'world') + self.assertEqual(str(records[-1].msg), 'Goodbye world.') + self.assertEqual(records[-1].funcName, 'test_styled_adapter') + + def test_nested_styled_adapter(self): + records = self.recording.records + adapter = PrefixAdapter(self.logger) + adapter.prefix = '{}' + adapter2 = StyleAdapter(adapter) + adapter2.warning('Hello, {}!', 'world') + self.assertEqual(str(records[-1].msg), '{} Hello, world!') + self.assertEqual(records[-1].funcName, 'test_nested_styled_adapter') + adapter2.log(logging.WARNING, 'Goodbye {}.', 'world') + self.assertEqual(str(records[-1].msg), '{} Goodbye world.') + self.assertEqual(records[-1].funcName, 'test_nested_styled_adapter') + + def test_find_caller_with_stacklevel(self): + the_level = 1 + trigger = self.adapter.warning + + def innermost(): + trigger('test', stacklevel=the_level) + + def inner(): + innermost() + + def outer(): + inner() + + records = self.recording.records + outer() + self.assertEqual(records[-1].funcName, 'innermost') + lineno = records[-1].lineno + the_level += 1 + outer() + self.assertEqual(records[-1].funcName, 'inner') + self.assertGreater(records[-1].lineno, lineno) + lineno = records[-1].lineno + the_level += 1 + outer() + self.assertEqual(records[-1].funcName, 'outer') + self.assertGreater(records[-1].lineno, lineno) + lineno = records[-1].lineno + the_level += 1 + outer() + self.assertEqual(records[-1].funcName, 'test_find_caller_with_stacklevel') + self.assertGreater(records[-1].lineno, lineno) + def test_extra_in_records(self): self.adapter = logging.LoggerAdapter(logger=self.logger, extra={'foo': '1'}) @@ -5568,6 +5620,30 @@ def test_extra_merged_log_call_has_precedence(self): self.assertEqual(record.foo, '2') +class PrefixAdapter(logging.LoggerAdapter): + prefix = 'Adapter' + + def process(self, msg, kwargs): + return f"{self.prefix} {msg}", kwargs + + +class Message: + def __init__(self, fmt, args): + self.fmt = fmt + self.args = args + + def __str__(self): + return self.fmt.format(*self.args) + + +class StyleAdapter(logging.LoggerAdapter): + def log(self, level, msg, /, *args, stacklevel=1, **kwargs): + if self.isEnabledFor(level): + msg, kwargs = self.process(msg, kwargs) + self.logger.log(level, Message(msg, args), **kwargs, + stacklevel=stacklevel+1) + + class LoggerTest(BaseTest, AssertErrorMessage): def setUp(self): @@ -6117,6 +6193,43 @@ def test_compute_files_to_delete(self): self.assertTrue(fn.startswith(prefix + '.') and fn[len(prefix) + 2].isdigit()) + def test_compute_files_to_delete_same_filename_different_extensions(self): + # See GH-93205 for background + wd = pathlib.Path(tempfile.mkdtemp(prefix='test_logging_')) + self.addCleanup(shutil.rmtree, wd) + times = [] + dt = datetime.datetime.now() + n_files = 10 + for _ in range(n_files): + times.append(dt.strftime('%Y-%m-%d_%H-%M-%S')) + dt += datetime.timedelta(seconds=5) + prefixes = ('a.log', 'a.log.b') + files = [] + rotators = [] + for i, prefix in enumerate(prefixes): + backupCount = i+1 + rotator = logging.handlers.TimedRotatingFileHandler(wd / prefix, when='s', + interval=5, + backupCount=backupCount, + delay=True) + rotators.append(rotator) + for t in times: + files.append('%s.%s' % (prefix, t)) + # Create empty files + for f in files: + (wd / f).touch() + # Now the checks that only the correct files are offered up for deletion + for i, prefix in enumerate(prefixes): + backupCount = i+1 + rotator = rotators[i] + candidates = rotator.getFilesToDelete() + self.assertEqual(len(candidates), n_files - backupCount) + matcher = re.compile(r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(\.\w+)?$") + for c in candidates: + d, fn = os.path.split(c) + self.assertTrue(fn.startswith(prefix)) + suffix = fn[(len(prefix)+1):] + self.assertRegex(suffix, matcher) def secs(**kw): return datetime.timedelta(**kw) // datetime.timedelta(seconds=1) diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index 65e6488c5d7b106..db290e139327e02 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -2,7 +2,6 @@ import array from io import BytesIO, UnsupportedOperation, DEFAULT_BUFFER_SIZE import os -import pathlib import pickle import random import sys @@ -12,7 +11,7 @@ from test.support import _4G, bigmemtest from test.support.import_helper import import_module from test.support.os_helper import ( - TESTFN, unlink + TESTFN, unlink, FakePath ) lzma = import_module("lzma") @@ -548,7 +547,7 @@ def test_init(self): pass def test_init_with_PathLike_filename(self): - filename = pathlib.Path(TESTFN) + filename = FakePath(TESTFN) with TempFile(filename, COMPRESSED_XZ): with LZMAFile(filename) as f: self.assertEqual(f.read(), INPUT) @@ -585,11 +584,10 @@ def test_init_with_x_mode(self): self.addCleanup(unlink, TESTFN) for mode in ("x", "xb"): unlink(TESTFN) - with LZMAFile(TESTFN, mode): + with LZMAFile(TESTFN, mode) as f: pass with self.assertRaises(FileExistsError): - with LZMAFile(TESTFN, mode): - pass + LZMAFile(TESTFN, mode) def test_init_bad_mode(self): with self.assertRaises(ValueError): @@ -867,17 +865,59 @@ def test_read_from_file(self): with LZMAFile(TESTFN) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + self.assertIsInstance(f.fileno(), int) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) def test_read_from_file_with_bytes_filename(self): - try: - bytes_filename = TESTFN.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(TESTFN) with TempFile(TESTFN, COMPRESSED_XZ): with LZMAFile(bytes_filename) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + def test_read_from_fileobj(self): + with TempFile(TESTFN, COMPRESSED_XZ): + with open(TESTFN, 'rb') as raw: + with LZMAFile(raw) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def test_read_from_fileobj_with_int_name(self): + with TempFile(TESTFN, COMPRESSED_XZ): + fd = os.open(TESTFN, os.O_RDONLY) + with open(fd, 'rb') as raw: + with LZMAFile(raw) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + def test_read_incomplete(self): with LZMAFile(BytesIO(COMPRESSED_XZ[:128])) as f: self.assertRaises(EOFError, f.read) @@ -1051,6 +1091,17 @@ def test_write_to_file(self): try: with LZMAFile(TESTFN, "w") as f: f.write(INPUT) + self.assertIsInstance(f.fileno(), int) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + expected = lzma.compress(INPUT) with open(TESTFN, "rb") as f: self.assertEqual(f.read(), expected) @@ -1058,10 +1109,7 @@ def test_write_to_file(self): unlink(TESTFN) def test_write_to_file_with_bytes_filename(self): - try: - bytes_filename = TESTFN.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(TESTFN) try: with LZMAFile(bytes_filename, "w") as f: f.write(INPUT) @@ -1071,6 +1119,51 @@ def test_write_to_file_with_bytes_filename(self): finally: unlink(TESTFN) + def test_write_to_fileobj(self): + try: + with open(TESTFN, "wb") as raw: + with LZMAFile(raw, "w") as f: + f.write(INPUT) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + + def test_write_to_fileobj_with_int_name(self): + try: + fd = os.open(TESTFN, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with open(fd, 'wb') as raw: + with LZMAFile(raw, "w") as f: + f.write(INPUT) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + def test_write_append_to_file(self): part1 = INPUT[:1024] part2 = INPUT[1024:1536] @@ -1276,7 +1369,7 @@ def test_filename(self): self.assertEqual(f.read(), INPUT * 2) def test_with_pathlike_filename(self): - filename = pathlib.Path(TESTFN) + filename = FakePath(TESTFN) with TempFile(filename): with lzma.open(filename, "wb") as f: f.write(INPUT) diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index d84faad0eb34069..d4628f91daf7e85 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -10,6 +10,7 @@ import tempfile from test import support from test.support import os_helper +from test.support import refleak_helper from test.support import socket_helper import unittest import textwrap @@ -1127,12 +1128,14 @@ def test_add_from_string(self): # Add a string starting with 'From ' to the mailbox key = self._box.add('From foo@bar blah\nFrom: foo\n\n0\n') self.assertEqual(self._box[key].get_from(), 'foo@bar blah') + self.assertEqual(self._box[key].get_unixfrom(), 'From foo@bar blah') self.assertEqual(self._box[key].get_payload(), '0\n') def test_add_from_bytes(self): # Add a byte string starting with 'From ' to the mailbox key = self._box.add(b'From foo@bar blah\nFrom: foo\n\n0\n') self.assertEqual(self._box[key].get_from(), 'foo@bar blah') + self.assertEqual(self._box[key].get_unixfrom(), 'From foo@bar blah') self.assertEqual(self._box[key].get_payload(), '0\n') def test_add_mbox_or_mmdf_message(self): @@ -1667,18 +1670,23 @@ def test_initialize_with_unixfrom(self): msg = mailbox.Message(_sample_message) msg.set_unixfrom('From foo@bar blah') msg = mailbox.mboxMessage(msg) - self.assertEqual(msg.get_from(), 'foo@bar blah', msg.get_from()) + self.assertEqual(msg.get_from(), 'foo@bar blah') + self.assertEqual(msg.get_unixfrom(), 'From foo@bar blah') def test_from(self): # Get and set "From " line msg = mailbox.mboxMessage(_sample_message) self._check_from(msg) + self.assertIsNone(msg.get_unixfrom()) msg.set_from('foo bar') self.assertEqual(msg.get_from(), 'foo bar') + self.assertIsNone(msg.get_unixfrom()) msg.set_from('foo@bar', True) self._check_from(msg, 'foo@bar') + self.assertIsNone(msg.get_unixfrom()) msg.set_from('blah@temp', time.localtime()) self._check_from(msg, 'blah@temp') + self.assertIsNone(msg.get_unixfrom()) def test_flags(self): # Use get_flags(), set_flags(), add_flag(), remove_flag() @@ -1866,6 +1874,7 @@ def test_maildir_to_mboxmmdf(self): self.assertEqual(msg.get_flags(), result) self.assertEqual(msg.get_from(), 'MAILER-DAEMON %s' % time.asctime(time.gmtime(0.0))) + self.assertIsNone(msg.get_unixfrom()) msg_maildir.set_subdir('cur') self.assertEqual(class_(msg_maildir).get_flags(), 'RODFA') @@ -1914,10 +1923,12 @@ def test_mboxmmdf_to_mboxmmdf(self): msg_mboxMMDF = class_(_sample_message) msg_mboxMMDF.set_flags('RODFA') msg_mboxMMDF.set_from('foo@bar') + self.assertIsNone(msg_mboxMMDF.get_unixfrom()) for class2_ in (mailbox.mboxMessage, mailbox.MMDFMessage): msg2 = class2_(msg_mboxMMDF) self.assertEqual(msg2.get_flags(), 'RODFA') self.assertEqual(msg2.get_from(), 'foo@bar') + self.assertIsNone(msg2.get_unixfrom()) def test_mboxmmdf_to_mh(self): # Convert mboxMessage and MMDFMessage to MHMessage @@ -2433,6 +2444,9 @@ def test__all__(self): def tearDownModule(): support.reap_children() + # reap_children may have re-populated caches: + if refleak_helper.hunting_for_refleaks(): + sys._clear_internal_caches() if __name__ == '__main__': diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 6e17e010e7f355d..615568e6af21028 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -1,5 +1,5 @@ from test import support -from test.support import os_helper, requires_debug_ranges +from test.support import is_apple_mobile, os_helper, requires_debug_ranges from test.support.script_helper import assert_python_ok import array import io @@ -286,7 +286,7 @@ def test_recursion_limit(self): #if os.name == 'nt' and support.Py_DEBUG: if os.name == 'nt': MAX_MARSHAL_STACK_DEPTH = 1000 - elif sys.platform == 'wasi': + elif sys.platform == 'wasi' or is_apple_mobile: MAX_MARSHAL_STACK_DEPTH = 1500 else: MAX_MARSHAL_STACK_DEPTH = 2000 diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index d64aee71fc48b13..01bba0ac2eed5a4 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -96,14 +96,12 @@ def test_non_standard_types(self): # First try strict eq(self.db.guess_type('foo.xul', strict=True), (None, None)) eq(self.db.guess_extension('image/jpg', strict=True), None) - eq(self.db.guess_extension('image/webp', strict=True), None) # And then non-strict eq(self.db.guess_type('foo.xul', strict=False), ('text/xul', None)) eq(self.db.guess_type('foo.XUL', strict=False), ('text/xul', None)) eq(self.db.guess_type('foo.invalid', strict=False), (None, None)) eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') - eq(self.db.guess_extension('image/webp', strict=False), '.webp') def test_filename_with_url_delimiters(self): # bpo-38449: URL delimiters cases should be handled also. @@ -183,6 +181,7 @@ def check_extensions(): self.assertEqual(mimetypes.guess_extension('application/xml'), '.xsl') self.assertEqual(mimetypes.guess_extension('audio/mpeg'), '.mp3') self.assertEqual(mimetypes.guess_extension('image/avif'), '.avif') + self.assertEqual(mimetypes.guess_extension('image/webp'), '.webp') self.assertEqual(mimetypes.guess_extension('image/jpeg'), '.jpg') self.assertEqual(mimetypes.guess_extension('image/tiff'), '.tiff') self.assertEqual(mimetypes.guess_extension('message/rfc822'), '.eml') diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index b89621e08577be0..ac759757d24659c 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -1,5 +1,5 @@ from test.support import ( - requires, _2G, _4G, gc_collect, cpython_only, is_emscripten + requires, _2G, _4G, gc_collect, cpython_only, is_emscripten, is_apple, ) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink @@ -1067,7 +1067,7 @@ def tearDown(self): unlink(TESTFN) def _make_test_file(self, num_zeroes, tail): - if sys.platform[:3] == 'win' or sys.platform == 'darwin': + if sys.platform[:3] == 'win' or is_apple: requires('largefile', 'test requires %s bytes and a long time to run' % str(0x180000000)) f = open(TESTFN, 'w+b') diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index a64d1ed79decd83..2fd822036bcff5e 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -35,6 +35,9 @@ def g1(): TEST_TOOL2 = 3 TEST_TOOL3 = 4 +def nth_line(func, offset): + return func.__code__.co_firstlineno + offset + class MonitoringBasicTest(unittest.TestCase): def test_has_objects(self): @@ -529,8 +532,8 @@ def test_lines_single(self): f1() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - start = LineMonitoringTest.test_lines_single.__code__.co_firstlineno - self.assertEqual(events, [start+7, 16, start+8]) + start = nth_line(LineMonitoringTest.test_lines_single, 0) + self.assertEqual(events, [start+7, nth_line(f1, 1), start+8]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) @@ -547,8 +550,13 @@ def test_lines_loop(self): floop() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - start = LineMonitoringTest.test_lines_loop.__code__.co_firstlineno - self.assertEqual(events, [start+7, 23, 24, 23, 24, 23, start+8]) + start = nth_line(LineMonitoringTest.test_lines_loop, 0) + floop_1 = nth_line(floop, 1) + floop_2 = nth_line(floop, 2) + self.assertEqual( + events, + [start+7, floop_1, floop_2, floop_1, floop_2, floop_1, start+8] + ) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) @@ -569,8 +577,8 @@ def test_lines_two(self): sys.monitoring.set_events(TEST_TOOL, 0); sys.monitoring.set_events(TEST_TOOL2, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) - start = LineMonitoringTest.test_lines_two.__code__.co_firstlineno - expected = [start+10, 16, start+11] + start = nth_line(LineMonitoringTest.test_lines_two, 0) + expected = [start+10, nth_line(f1, 1), start+11] self.assertEqual(events, expected) self.assertEqual(events2, expected) finally: @@ -1466,9 +1474,8 @@ def func(): ('branch', 'func', 4, 4), ('line', 'func', 5), ('line', 'meth', 1), - ('jump', 'func', 5, 5), - ('jump', 'func', 5, '[offset=114]'), - ('branch', 'func', '[offset=120]', '[offset=124]'), + ('jump', 'func', 5, '[offset=118]'), + ('branch', 'func', '[offset=122]', '[offset=126]'), ('line', 'get_events', 11)]) self.check_events(func, recorders = FLOW_AND_LINE_RECORDERS, expected = [ @@ -1482,9 +1489,8 @@ def func(): ('line', 'func', 5), ('line', 'meth', 1), ('return', 'meth', None), - ('jump', 'func', 5, 5), - ('jump', 'func', 5, '[offset=114]'), - ('branch', 'func', '[offset=120]', '[offset=124]'), + ('jump', 'func', 5, '[offset=118]'), + ('branch', 'func', '[offset=122]', '[offset=126]'), ('return', 'func', None), ('line', 'get_events', 11)]) @@ -1801,7 +1807,7 @@ class TestOptimizer(MonitoringTestBase, unittest.TestCase): def setUp(self): import _testinternalcapi self.old_opt = _testinternalcapi.get_optimizer() - opt = _testinternalcapi.get_counter_optimizer() + opt = _testinternalcapi.new_counter_optimizer() _testinternalcapi.set_optimizer(opt) super(TestOptimizer, self).setUp() diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 7b2fa844827ae9b..f2017bdffcf9680 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -298,6 +298,16 @@ def test_named_expression_invalid_set_comprehension_iterable_expression(self): with self.assertRaisesRegex(SyntaxError, msg): exec(f"lambda: {code}", {}) # Function scope + def test_named_expression_invalid_mangled_class_variables(self): + code = """class Foo: + def bar(self): + [[(__x:=2) for _ in range(2)] for __x in range(2)] + """ + + with self.assertRaisesRegex(SyntaxError, + "assignment expression cannot rebind comprehension iteration variable '__x'"): + exec(code, {}, {}) + class NamedExpressionAssignmentTest(unittest.TestCase): @@ -674,6 +684,18 @@ def test_named_expression_scope_in_genexp(self): for idx, elem in enumerate(genexp): self.assertEqual(elem, b[idx] + a) + def test_named_expression_scope_mangled_names(self): + class Foo: + def f(self_): + global __x1 + __x1 = 0 + [_Foo__x1 := 1 for a in [2]] + self.assertEqual(__x1, 1) + [__x1 := 2 for a in [3]] + self.assertEqual(__x1, 2) + + Foo().f() + self.assertEqual(_Foo__x1, 2) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_optimizer.py b/Lib/test/test_optimizer.py index b56bf3cfd9560ea..899a45073173347 100644 --- a/Lib/test/test_optimizer.py +++ b/Lib/test/test_optimizer.py @@ -1,9 +1,15 @@ -import _testinternalcapi import unittest import types +from test.support import import_helper + + +_testinternalcapi = import_helper.import_module("_testinternalcapi") class TestRareEventCounters(unittest.TestCase): + def setUp(self): + _testinternalcapi.reset_rare_event_counters() + def test_set_class(self): class A: pass @@ -71,5 +77,12 @@ def func(x=0): _testinternalcapi.get_rare_event_counters()["func_modification"] ) + +class TestOptimizerSymbols(unittest.TestCase): + + def test_optimizer_symbols(self): + _testinternalcapi.uop_symbols_test() + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index e6f0dfde8cb4aea..2372ac4c21efd9f 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2195,12 +2195,15 @@ def test_chmod(self): class TestInvalidFD(unittest.TestCase): singles = ["fchdir", "dup", "fdatasync", "fstat", "fstatvfs", "fsync", "tcgetpgrp", "ttyname"] + singles_fildes = {"fchdir", "fdatasync", "fsync"} #singles.append("close") #We omit close because it doesn't raise an exception on some platforms def get_single(f): def helper(self): if hasattr(os, f): self.check(getattr(os, f)) + if f in self.singles_fildes: + self.check_bool(getattr(os, f)) return helper for f in singles: locals()["test_"+f] = get_single(f) @@ -2214,8 +2217,16 @@ def check(self, f, *args, **kwargs): self.fail("%r didn't raise an OSError with a bad file descriptor" % f) + def check_bool(self, f, *args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + for fd in False, True: + with self.assertRaises(RuntimeWarning): + f(fd, *args, **kwargs) + def test_fdopen(self): self.check(os.fdopen, encoding="utf-8") + self.check_bool(os.fdopen, encoding="utf-8") @unittest.skipUnless(hasattr(os, 'isatty'), 'test needs os.isatty()') def test_isatty(self): @@ -2277,11 +2288,14 @@ def test_fchown(self): def test_fpathconf(self): self.check(os.pathconf, "PC_NAME_MAX") self.check(os.fpathconf, "PC_NAME_MAX") + self.check_bool(os.pathconf, "PC_NAME_MAX") + self.check_bool(os.fpathconf, "PC_NAME_MAX") @unittest.skipUnless(hasattr(os, 'ftruncate'), 'test needs os.ftruncate()') def test_ftruncate(self): self.check(os.truncate, 0) self.check(os.ftruncate, 0) + self.check_bool(os.truncate, 0) @unittest.skipUnless(hasattr(os, 'lseek'), 'test needs os.lseek()') def test_lseek(self): @@ -3492,22 +3506,22 @@ class ProgramPriorityTests(unittest.TestCase): """Tests for os.getpriority() and os.setpriority().""" def test_set_get_priority(self): - base = os.getpriority(os.PRIO_PROCESS, os.getpid()) - os.setpriority(os.PRIO_PROCESS, os.getpid(), base + 1) - try: - new_prio = os.getpriority(os.PRIO_PROCESS, os.getpid()) - if base >= 19 and new_prio <= 19: - raise unittest.SkipTest("unable to reliably test setpriority " - "at current nice level of %s" % base) - else: - self.assertEqual(new_prio, base + 1) - finally: - try: - os.setpriority(os.PRIO_PROCESS, os.getpid(), base) - except OSError as err: - if err.errno != errno.EACCES: - raise + code = f"""if 1: + import os + os.setpriority(os.PRIO_PROCESS, os.getpid(), {base} + 1) + print(os.getpriority(os.PRIO_PROCESS, os.getpid())) + """ + + # Subprocess inherits the current process' priority. + _, out, _ = assert_python_ok("-c", code) + new_prio = int(out) + # nice value cap is 19 for linux and 20 for FreeBSD + if base >= 19 and new_prio <= base: + raise unittest.SkipTest("unable to reliably test setpriority " + "at current nice level of %s" % base) + else: + self.assertEqual(new_prio, base + 1) @unittest.skipUnless(hasattr(os, 'sendfile'), "test needs os.sendfile()") @@ -3848,6 +3862,7 @@ def test_does_not_crash(self): self.assertGreaterEqual(size.columns, 0) self.assertGreaterEqual(size.lines, 0) + @support.requires_subprocess() def test_stty_match(self): """Check if stty returns the same results @@ -4536,15 +4551,49 @@ def test_dup2(self): self.assertEqual(os.dup2(fd, fd3, inheritable=False), fd3) self.assertFalse(os.get_inheritable(fd3)) - @unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") - def test_openpty(self): - master_fd, slave_fd = os.openpty() - self.addCleanup(os.close, master_fd) - self.addCleanup(os.close, slave_fd) - self.assertEqual(os.get_inheritable(master_fd), False) - self.assertEqual(os.get_inheritable(slave_fd), False) +@unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") +class PseudoterminalTests(unittest.TestCase): + def open_pty(self): + """Open a pty fd-pair, and schedule cleanup for it""" + main_fd, second_fd = os.openpty() + self.addCleanup(os.close, main_fd) + self.addCleanup(os.close, second_fd) + return main_fd, second_fd - @unittest.skipUnless(hasattr(os, 'spawnl'), "need os.openpty()") + def test_openpty(self): + main_fd, second_fd = self.open_pty() + self.assertEqual(os.get_inheritable(main_fd), False) + self.assertEqual(os.get_inheritable(second_fd), False) + + @unittest.skipUnless(hasattr(os, 'ptsname'), "need os.ptsname()") + @unittest.skipUnless(hasattr(os, 'O_RDWR'), "need os.O_RDWR") + @unittest.skipUnless(hasattr(os, 'O_NOCTTY'), "need os.O_NOCTTY") + def test_open_via_ptsname(self): + main_fd, second_fd = self.open_pty() + second_path = os.ptsname(main_fd) + reopened_second_fd = os.open(second_path, os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, reopened_second_fd) + os.write(reopened_second_fd, b'foo') + self.assertEqual(os.read(main_fd, 3), b'foo') + + @unittest.skipUnless(hasattr(os, 'posix_openpt'), "need os.posix_openpt()") + @unittest.skipUnless(hasattr(os, 'grantpt'), "need os.grantpt()") + @unittest.skipUnless(hasattr(os, 'unlockpt'), "need os.unlockpt()") + @unittest.skipUnless(hasattr(os, 'ptsname'), "need os.ptsname()") + @unittest.skipUnless(hasattr(os, 'O_RDWR'), "need os.O_RDWR") + @unittest.skipUnless(hasattr(os, 'O_NOCTTY'), "need os.O_NOCTTY") + def test_posix_pty_functions(self): + mother_fd = os.posix_openpt(os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, mother_fd) + os.grantpt(mother_fd) + os.unlockpt(mother_fd) + son_path = os.ptsname(mother_fd) + son_fd = os.open(son_path, os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, son_fd) + self.assertEqual(os.ptsname(mother_fd), os.ttyname(son_fd)) + + @unittest.skipUnless(hasattr(os, 'spawnl'), "need os.spawnl()") + @support.requires_subprocess() def test_pipe_spawnl(self): # gh-77046: On Windows, os.pipe() file descriptors must be created with # _O_NOINHERIT to make them non-inheritable. UCRT has no public API to diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 5ce3b605c58e637..c0dcf314da4bfc3 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -189,7 +189,7 @@ def test_fspath_common(self): self._check_str(p.__fspath__(), ('a/b',)) self._check_str(os.fspath(p), ('a/b',)) - def test_bytes(self): + def test_bytes_exc_message(self): P = self.cls message = (r"argument should be a str or an os\.PathLike object " r"where __fspath__ returns a str, not 'bytes'") @@ -199,22 +199,6 @@ def test_bytes(self): P(b'a', 'b') with self.assertRaisesRegex(TypeError, message): P('a', b'b') - with self.assertRaises(TypeError): - P('a').joinpath(b'b') - with self.assertRaises(TypeError): - P('a') / b'b' - with self.assertRaises(TypeError): - b'a' / P('b') - with self.assertRaises(TypeError): - P('a').match(b'b') - with self.assertRaises(TypeError): - P('a').relative_to(b'b') - with self.assertRaises(TypeError): - P('a').with_name(b'b') - with self.assertRaises(TypeError): - P('a').with_stem(b'b') - with self.assertRaises(TypeError): - P('a').with_suffix(b'b') def test_as_bytes_common(self): sep = os.fsencode(self.sep) @@ -327,13 +311,6 @@ def test_with_stem_empty(self): self.assertRaises(ValueError, P('a/b').with_stem, '') self.assertRaises(ValueError, P('a/b').with_stem, '.') - def test_with_suffix_empty(self): - # Path doesn't have a "filename" component. - P = self.cls - self.assertRaises(ValueError, P('').with_suffix, '.gz') - self.assertRaises(ValueError, P('.').with_suffix, '.gz') - self.assertRaises(ValueError, P('/').with_suffix, '.gz') - def test_relative_to_several_args(self): P = self.cls p = P('a/b') @@ -1265,18 +1242,6 @@ def test_glob_above_recursion_limit(self): with set_recursion_limit(recursion_limit): list(base.glob('**/')) - def test_glob_recursive_no_trailing_slash(self): - P = self.cls - p = P(self.base) - with self.assertWarns(FutureWarning): - p.glob('**') - with self.assertWarns(FutureWarning): - p.glob('*/**') - with self.assertWarns(FutureWarning): - p.rglob('**') - with self.assertWarns(FutureWarning): - p.rglob('*/**') - def test_glob_pathlike(self): P = self.cls p = P(self.base) @@ -1285,6 +1250,19 @@ def test_glob_pathlike(self): self.assertEqual(expect, set(p.glob(P(pattern)))) self.assertEqual(expect, set(p.glob(FakePath(pattern)))) + @needs_symlinks + def test_glob_dot(self): + P = self.cls + with os_helper.change_cwd(P(self.base, "dirC")): + self.assertEqual( + set(P('.').glob('*')), {P("fileC"), P("novel.txt"), P("dirD")}) + self.assertEqual( + set(P('.').glob('**')), {P("fileC"), P("novel.txt"), P("dirD"), P("dirD/fileD"), P(".")}) + self.assertEqual( + set(P('.').glob('**/*')), {P("fileC"), P("novel.txt"), P("dirD"), P("dirD/fileD")}) + self.assertEqual( + set(P('.').glob('**/*/*')), {P("dirD/fileD")}) + def test_rglob_pathlike(self): P = self.cls p = P(self.base, "dirC") diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index ab989cb5503f99d..5bfb76f85c79094 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -155,6 +155,31 @@ def test_constructor_common(self): P('a/b/c') P('/a/b/c') + def test_bytes(self): + P = self.cls + with self.assertRaises(TypeError): + P(b'a') + with self.assertRaises(TypeError): + P(b'a', 'b') + with self.assertRaises(TypeError): + P('a', b'b') + with self.assertRaises(TypeError): + P('a').joinpath(b'b') + with self.assertRaises(TypeError): + P('a') / b'b' + with self.assertRaises(TypeError): + b'a' / P('b') + with self.assertRaises(TypeError): + P('a').match(b'b') + with self.assertRaises(TypeError): + P('a').relative_to(b'b') + with self.assertRaises(TypeError): + P('a').with_name(b'b') + with self.assertRaises(TypeError): + P('a').with_stem(b'b') + with self.assertRaises(TypeError): + P('a').with_suffix(b'b') + def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object # from a str subclass instance, and it then gets converted to @@ -932,6 +957,8 @@ def test_with_stem_empty(self): self.assertEqual(P('/').with_stem('d'), P('/d')) self.assertEqual(P('a/b').with_stem(''), P('a/')) self.assertEqual(P('a/b').with_stem('.'), P('a/.')) + self.assertRaises(ValueError, P('foo.gz').with_stem, '') + self.assertRaises(ValueError, P('/a/b/foo.gz').with_stem, '') def test_with_stem_seps(self): P = self.cls @@ -977,9 +1004,8 @@ def test_with_suffix_windows(self): def test_with_suffix_empty(self): P = self.cls # Path doesn't have a "filename" component. - self.assertEqual(P('').with_suffix('.gz'), P('.gz')) - self.assertEqual(P('.').with_suffix('.gz'), P('..gz')) - self.assertEqual(P('/').with_suffix('.gz'), P('/.gz')) + self.assertRaises(ValueError, P('').with_suffix, '.gz') + self.assertRaises(ValueError, P('/').with_suffix, '.gz') def test_with_suffix_seps(self): P = self.cls @@ -1766,16 +1792,26 @@ def _check(path, glob, expected): _check(p, "*/fileB", ["dirB/fileB", "linkB/fileB"]) _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/..", "dirB/linkD/.."]) + _check(p, "dir*/**", [ + "dirA/", "dirA/linkC", "dirA/linkC/fileB", "dirA/linkC/linkD", "dirA/linkC/linkD/fileB", + "dirB/", "dirB/fileB", "dirB/linkD", "dirB/linkD/fileB", + "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", + "dirE/"]) _check(p, "dir*/**/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", "dirC/", "dirC/dirD/", "dirE/"]) _check(p, "dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", "dirB/linkD/..", "dirA/linkC/linkD/..", "dirC/..", "dirC/dirD/..", "dirE/.."]) + _check(p, "dir*/*/**", [ + "dirA/linkC/", "dirA/linkC/linkD", "dirA/linkC/fileB", "dirA/linkC/linkD/fileB", + "dirB/linkD/", "dirB/linkD/fileB", + "dirC/dirD/", "dirC/dirD/fileD"]) _check(p, "dir*/*/**/", ["dirA/linkC/", "dirA/linkC/linkD/", "dirB/linkD/", "dirC/dirD/"]) _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirA/linkC/linkD/..", "dirB/linkD/..", "dirC/dirD/.."]) _check(p, "dir*/**/fileC", ["dirC/fileC"]) _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) _check(p, "*/dirD/**/", ["dirC/dirD/"]) @needs_symlinks @@ -1792,12 +1828,20 @@ def _check(path, glob, expected): _check(p, "*/fileB", ["dirB/fileB"]) _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/"]) _check(p, "dir*/*/..", ["dirC/dirD/.."]) + _check(p, "dir*/**", [ + "dirA/", "dirA/linkC", + "dirB/", "dirB/fileB", "dirB/linkD", + "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", + "dirE/"]) _check(p, "dir*/**/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) _check(p, "dir*/**/..", ["dirA/..", "dirB/..", "dirC/..", "dirC/dirD/..", "dirE/.."]) + _check(p, "dir*/*/**", ["dirC/dirD/", "dirC/dirD/fileD"]) _check(p, "dir*/*/**/", ["dirC/dirD/"]) _check(p, "dir*/*/**/..", ["dirC/dirD/.."]) _check(p, "dir*/**/fileC", ["dirC/fileC"]) + _check(p, "dir*/*/../dirD/**", ["dirC/dirD/../dirD/", "dirC/dirD/../dirD/fileD"]) _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) _check(p, "*/dirD/**/", ["dirC/dirD/"]) def test_rglob_common(self): @@ -1834,10 +1878,13 @@ def _check(glob, expected): "dirC/dirD", "dirC/dirD/fileD"]) _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p.rglob("dir*/**"), ["dirC/dirD/", "dirC/dirD/fileD"]) _check(p.rglob("dir*/**/"), ["dirC/dirD/"]) _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) _check(p.rglob("*/"), ["dirC/dirD/"]) _check(p.rglob(""), ["dirC/", "dirC/dirD/"]) + _check(p.rglob("**"), [ + "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt"]) _check(p.rglob("**/"), ["dirC/", "dirC/dirD/"]) # gh-91616, a re module regression _check(p.rglob("*.txt"), ["dirC/novel.txt"]) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index c64df62c761471d..2b0795cdad707e1 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -779,58 +779,62 @@ def test_pdb_where_command(): (Pdb) continue """ -def test_pdb_interact_command(): - """Test interact command - >>> g = 0 - >>> dict_g = {} +# skip this test if sys.flags.no_site = True; +# exit() isn't defined unless there's a site module. +if not sys.flags.no_site: + def test_pdb_interact_command(): + """Test interact command - >>> def test_function(): - ... x = 1 - ... lst_local = [] - ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + >>> g = 0 + >>> dict_g = {} - >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE - ... 'interact', - ... 'x', - ... 'g', - ... 'x = 2', - ... 'g = 3', - ... 'dict_g["a"] = True', - ... 'lst_local.append(x)', - ... 'exit()', - ... 'p x', - ... 'p g', - ... 'p dict_g', - ... 'p lst_local', - ... 'continue', - ... ]): - ... test_function() - --Return-- - > <doctest test.test_pdb.test_pdb_interact_command[2]>(4)test_function()->None - -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - (Pdb) interact - *pdb interact start* - ... x - 1 - ... g - 0 - ... x = 2 - ... g = 3 - ... dict_g["a"] = True - ... lst_local.append(x) - ... exit() - *exit from pdb interact command* - (Pdb) p x - 1 - (Pdb) p g - 0 - (Pdb) p dict_g - {'a': True} - (Pdb) p lst_local - [2] - (Pdb) continue - """ + >>> def test_function(): + ... x = 1 + ... lst_local = [] + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + + >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'interact', + ... 'x', + ... 'g', + ... 'x = 2', + ... 'g = 3', + ... 'dict_g["a"] = True', + ... 'lst_local.append(x)', + ... 'exit()', + ... 'p x', + ... 'p g', + ... 'p dict_g', + ... 'p lst_local', + ... 'continue', + ... ]): + ... test_function() + --Return-- + > <doctest test.test_pdb.test_pdb_interact_command[2]>(4)test_function()->None + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) interact + *pdb interact start* + ... x + 1 + ... g + 0 + ... x = 2 + ... g = 3 + ... dict_g["a"] = True + ... lst_local.append(x) + ... exit() + *exit from pdb interact command* + (Pdb) p x + 1 + (Pdb) p g + 0 + (Pdb) p dict_g + {'a': True} + (Pdb) p lst_local + [2] + (Pdb) continue + """ def test_convenience_variables(): """Test convenience variables @@ -2661,7 +2665,7 @@ def quux(): pass """.encode(), 'bœr', - ('bœr', 4), + ('bœr', 5), ) def test_find_function_found_with_encoding_cookie(self): @@ -2678,7 +2682,7 @@ def quux(): pass """.encode('iso-8859-15'), 'bœr', - ('bœr', 5), + ('bœr', 6), ) def test_find_function_found_with_bom(self): @@ -2688,9 +2692,34 @@ def bœr(): pass """.encode(), 'bœr', - ('bœr', 1), + ('bœr', 2), ) + def test_find_function_first_executable_line(self): + code = textwrap.dedent("""\ + def foo(): pass + + def bar(): + pass # line 4 + + def baz(): + # comment + pass # line 8 + + def mul(): + # code on multiple lines + code = compile( # line 12 + 'def f()', + '<string>', + 'exec', + ) + """).encode() + + self._assert_find_function(code, 'foo', ('foo', 1)) + self._assert_find_function(code, 'bar', ('bar', 4)) + self._assert_find_function(code, 'baz', ('baz', 8)) + self._assert_find_function(code, 'mul', ('mul', 12)) + def test_issue7964(self): # open the file as binary so we can force \r\n newline with open(os_helper.TESTFN, 'wb') as f: diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 2ea186c85c8823e..dffedd0b1fc4767 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -1065,6 +1065,22 @@ def test_conditional_jump_backward_const_condition(self): ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + def test_except_handler_label(self): + insts = [ + ('SETUP_FINALLY', handler := self.Label(), 10), + ('POP_BLOCK', 0, -1), + ('RETURN_CONST', 1, 11), + handler, + ('RETURN_CONST', 2, 12), + ] + expected_insts = [ + ('SETUP_FINALLY', handler := self.Label(), 10), + ('RETURN_CONST', 1, 11), + handler, + ('RETURN_CONST', 2, 12), + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + def test_no_unsafe_static_swap(self): # We can't change order of two stores to the same location insts = [ diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index b2245ddf72f7085..19f977971570b76 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -564,10 +564,12 @@ def test_exceptions(self): if exc in (BlockingIOError, ResourceWarning, StopAsyncIteration, + PythonFinalizationError, RecursionError, EncodingWarning, BaseExceptionGroup, - ExceptionGroup): + ExceptionGroup, + IncompleteInputError): continue if exc is not OSError and issubclass(exc, OSError): self.assertEqual(reverse_mapping('builtins', name), diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 216973350319fe6..648e18d0150ef08 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -472,7 +472,8 @@ def test_macos(self): 'root:xnu-4570.71.2~1/RELEASE_X86_64'), 'x86_64', 'i386') arch = ('64bit', '') - with mock.patch.object(platform, 'uname', return_value=uname), \ + with mock.patch.object(sys, "platform", "darwin"), \ + mock.patch.object(platform, 'uname', return_value=uname), \ mock.patch.object(platform, 'architecture', return_value=arch): for mac_ver, expected_terse, expected in [ # darwin: mac_ver() returns empty strings diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 9c382ace806e0f8..a45f620e18dc1d0 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1,7 +1,7 @@ "Test posix functions" from test import support -from test.support import import_helper +from test.support import is_apple from test.support import os_helper from test.support import warnings_helper from test.support.script_helper import assert_python_ok @@ -781,9 +781,10 @@ def check_stat(uid, gid): check_stat(uid, gid) self.assertRaises(OSError, chown_func, first_param, 0, -1) check_stat(uid, gid) - if 0 not in os.getgroups(): - self.assertRaises(OSError, chown_func, first_param, -1, 0) - check_stat(uid, gid) + if hasattr(os, 'getgroups'): + if 0 not in os.getgroups(): + self.assertRaises(OSError, chown_func, first_param, -1, 0) + check_stat(uid, gid) # test illegal types for t in str, float: self.assertRaises(TypeError, chown_func, first_param, t(uid), gid) @@ -1256,8 +1257,8 @@ def test_sched_priority(self): self.assertIsInstance(lo, int) self.assertIsInstance(hi, int) self.assertGreaterEqual(hi, lo) - # OSX evidently just returns 15 without checking the argument. - if sys.platform != "darwin": + # Apple plaforms return 15 without checking the argument. + if not is_apple: self.assertRaises(OSError, posix.sched_get_priority_min, -23) self.assertRaises(OSError, posix.sched_get_priority_max, -23) @@ -1513,6 +1514,13 @@ def test_stat_dir_fd(self): self.assertRaises(OverflowError, posix.stat, name, dir_fd=10**20) + for fd in False, True: + with self.assertWarnsRegex(RuntimeWarning, + 'bool is used as a file descriptor') as cm: + with self.assertRaises(OSError): + posix.stat('nonexisting', dir_fd=fd) + self.assertEqual(cm.filename, __file__) + @unittest.skipUnless(os.utime in os.supports_dir_fd, "test needs dir_fd support in os.utime()") def test_utime_dir_fd(self): with self.prepare_file() as (dir_fd, name, fullname): @@ -2028,11 +2036,13 @@ def test_dup2(self): @unittest.skipUnless(hasattr(os, 'posix_spawn'), "test needs os.posix_spawn") +@support.requires_subprocess() class TestPosixSpawn(unittest.TestCase, _PosixSpawnMixin): spawn_func = getattr(posix, 'posix_spawn', None) @unittest.skipUnless(hasattr(os, 'posix_spawnp'), "test needs os.posix_spawnp") +@support.requires_subprocess() class TestPosixSpawnP(unittest.TestCase, _PosixSpawnMixin): spawn_func = getattr(posix, 'posix_spawnp', None) diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 86ce1b1d41ba61a..cbb7c4c52d96979 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -703,7 +703,9 @@ def check_error(exc, paths): self.assertRaises(exc, posixpath.commonpath, [os.fsencode(p) for p in paths]) + self.assertRaises(TypeError, posixpath.commonpath, None) self.assertRaises(ValueError, posixpath.commonpath, []) + self.assertRaises(ValueError, posixpath.commonpath, iter([])) check_error(ValueError, ['/usr', 'usr']) check_error(ValueError, ['usr', '/usr']) diff --git a/Lib/test/test_profile.py b/Lib/test/test_profile.py index a1dfc9abbb8ef75..0f16b92334999ce 100644 --- a/Lib/test/test_profile.py +++ b/Lib/test/test_profile.py @@ -7,7 +7,7 @@ from difflib import unified_diff from io import StringIO from test.support.os_helper import TESTFN, unlink, temp_dir, change_cwd -from contextlib import contextmanager +from contextlib import contextmanager, redirect_stdout import profile from test.profilee import testfunc, timer @@ -92,6 +92,11 @@ def test_run(self): self.profilermodule.run("int('1')", filename=TESTFN) self.assertTrue(os.path.exists(TESTFN)) + def test_run_with_sort_by_values(self): + with redirect_stdout(StringIO()) as f: + self.profilermodule.run("int('1')", sort=('tottime', 'stdname')) + self.assertIn("Ordered by: internal time, standard name", f.getvalue()) + def test_runctx(self): with silent(): self.profilermodule.runctx("testfunc()", globals(), locals()) diff --git a/Lib/test/test_property.py b/Lib/test/test_property.py index c12c908d2ee32d0..408e64f53142db1 100644 --- a/Lib/test/test_property.py +++ b/Lib/test/test_property.py @@ -183,6 +183,77 @@ def test_refleaks_in___init__(self): fake_prop.__init__('fget', 'fset', 'fdel', 'doc') self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + @support.refcount_test + def test_gh_115618(self): + # Py_XDECREF() was improperly called for None argument + # in property methods. + gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') + prop = property() + refs_before = gettotalrefcount() + for i in range(100): + prop = prop.getter(None) + self.assertIsNone(prop.fget) + for i in range(100): + prop = prop.setter(None) + self.assertIsNone(prop.fset) + for i in range(100): + prop = prop.deleter(None) + self.assertIsNone(prop.fdel) + self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + + def test_property_name(self): + def getter(self): + return 42 + + def setter(self, value): + pass + + class A: + @property + def foo(self): + return 1 + + @foo.setter + def oof(self, value): + pass + + bar = property(getter) + baz = property(None, setter) + + self.assertEqual(A.foo.__name__, 'foo') + self.assertEqual(A.oof.__name__, 'oof') + self.assertEqual(A.bar.__name__, 'bar') + self.assertEqual(A.baz.__name__, 'baz') + + A.quux = property(getter) + self.assertEqual(A.quux.__name__, 'getter') + A.quux.__name__ = 'myquux' + self.assertEqual(A.quux.__name__, 'myquux') + self.assertEqual(A.bar.__name__, 'bar') # not affected + A.quux.__name__ = None + self.assertIsNone(A.quux.__name__) + + with self.assertRaisesRegex( + AttributeError, "'property' object has no attribute '__name__'" + ): + property(None, setter).__name__ + + with self.assertRaisesRegex( + AttributeError, "'property' object has no attribute '__name__'" + ): + property(1).__name__ + + class Err: + def __getattr__(self, attr): + raise RuntimeError('fail') + + p = property(Err()) + with self.assertRaisesRegex(RuntimeError, 'fail'): + p.__name__ + + p.__name__ = 'not_fail' + self.assertEqual(p.__name__, 'not_fail') + def test_property_set_name_incorrect_args(self): p = property() @@ -224,6 +295,7 @@ class PropertySubSlots(property): class PropertySubclassTests(unittest.TestCase): + @support.requires_docstrings def test_slots_docstring_copy_exception(self): # A special case error that we preserve despite the GH-98963 behavior # that would otherwise silently ignore this error. diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index 51e3a46d0df1780..3f2bac0155fd9e8 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -1,12 +1,17 @@ -from test.support import verbose, reap_children -from test.support.os_helper import TESTFN, unlink +import sys +import unittest +from test.support import ( + is_apple_mobile, is_emscripten, is_wasi, reap_children, verbose +) from test.support.import_helper import import_module +from test.support.os_helper import TESTFN, unlink -# Skip these tests if termios or fcntl are not available +# Skip these tests if termios is not available import_module('termios') -# fcntl is a proxy for not being one of the wasm32 platforms even though we -# don't use this module... a proper check for what crashes those is needed. -import_module("fcntl") + +# Skip tests on WASM platforms, plus iOS/tvOS/watchOS +if is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest(f"pty tests not required on {sys.platform}") import errno import os @@ -17,7 +22,6 @@ import signal import socket import io # readline -import unittest import warnings TEST_STRING_1 = b"I wish to buy a fish license.\n" diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index c4e6551f6057822..64387296e846213 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -227,7 +227,8 @@ class PyCompileCLITestCase(unittest.TestCase): def setUp(self): self.directory = tempfile.mkdtemp() self.source_path = os.path.join(self.directory, '_test.py') - self.cache_path = importlib.util.cache_from_source(self.source_path) + self.cache_path = importlib.util.cache_from_source(self.source_path, + optimization='' if __debug__ else 1) with open(self.source_path, 'w') as file: file.write('x = 123\n') @@ -250,6 +251,7 @@ def pycompilecmd_failure(self, *args): return script_helper.assert_python_failure('-m', 'py_compile', *args) def test_stdin(self): + self.assertFalse(os.path.exists(self.cache_path)) result = self.pycompilecmd('-', input=self.source_path) self.assertEqual(result.returncode, 0) self.assertEqual(result.stdout, b'') diff --git a/Lib/test/test_pydoc/__init__.py b/Lib/test/test_pydoc/__init__.py new file mode 100644 index 000000000000000..f2a39a3fe29c7f0 --- /dev/null +++ b/Lib/test/test_pydoc/__init__.py @@ -0,0 +1,6 @@ +import os +from test import support + + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/pydoc_mod.py b/Lib/test/test_pydoc/pydoc_mod.py similarity index 100% rename from Lib/test/pydoc_mod.py rename to Lib/test/test_pydoc/pydoc_mod.py diff --git a/Lib/test/pydocfodder.py b/Lib/test/test_pydoc/pydocfodder.py similarity index 72% rename from Lib/test/pydocfodder.py rename to Lib/test/test_pydoc/pydocfodder.py index a3ef22312439549..3cc2d5bd57fe5b3 100644 --- a/Lib/test/pydocfodder.py +++ b/Lib/test/test_pydoc/pydocfodder.py @@ -2,6 +2,12 @@ import types +def global_func(x, y): + """Module global function""" + +def global_func2(x, y): + """Module global function 2""" + class A: "A class." @@ -26,7 +32,7 @@ def A_classmethod(cls, x): "A class method defined in A." A_classmethod = classmethod(A_classmethod) - def A_staticmethod(): + def A_staticmethod(x, y): "A static method defined in A." A_staticmethod = staticmethod(A_staticmethod) @@ -61,6 +67,30 @@ def BD_method(self): def BCD_method(self): "Method defined in B, C and D." + @classmethod + def B_classmethod(cls, x): + "A class method defined in B." + + global_func = global_func # same name + global_func_alias = global_func + global_func2_alias = global_func2 + B_classmethod_alias = B_classmethod + A_classmethod_ref = A.A_classmethod + A_staticmethod = A.A_staticmethod # same name + A_staticmethod_alias = A.A_staticmethod + A_method_ref = A().A_method + A_method_alias = A.A_method + B_method_alias = B_method + count = list.count # same name + list_count = list.count + __repr__ = object.__repr__ # same name + object_repr = object.__repr__ + get = {}.get # same name + dict_get = {}.get + +B.B_classmethod_ref = B.B_classmethod + + class C(A): "A class, derived from A." @@ -136,3 +166,23 @@ def __call__(self, inst): submodule = types.ModuleType(__name__ + '.submodule', """A submodule, which should appear in its parent's summary""") + +global_func_alias = global_func +A_classmethod = A.A_classmethod # same name +A_classmethod2 = A.A_classmethod +A_classmethod3 = B.A_classmethod +A_staticmethod = A.A_staticmethod # same name +A_staticmethod_alias = A.A_staticmethod +A_staticmethod_ref = A().A_staticmethod +A_staticmethod_ref2 = B().A_staticmethod +A_method = A().A_method # same name +A_method2 = A().A_method +A_method3 = B().A_method +B_method = B.B_method # same name +B_method2 = B.B_method +count = list.count # same name +list_count = list.count +__repr__ = object.__repr__ # same name +object_repr = object.__repr__ +get = {}.get # same name +dict_get = {}.get diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py similarity index 84% rename from Lib/test/test_pydoc.py rename to Lib/test/test_pydoc/test_pydoc.py index 99b19d01783a10f..9d40234ed01697f 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -34,7 +34,8 @@ captured_stderr, is_emscripten, is_wasi, requires_docstrings, MISSING_C_DOCSTRINGS) from test.support.os_helper import (TESTFN, rmtree, unlink) -from test import pydoc_mod +from test.test_pydoc import pydoc_mod +from test.test_pydoc import pydocfodder class nonascii: @@ -51,7 +52,7 @@ class nonascii: expected_text_pattern = """ NAME - test.pydoc_mod - This is a test module for test_pydoc + test.test_pydoc.pydoc_mod - This is a test module for test_pydoc %s CLASSES builtins.object @@ -102,7 +103,7 @@ class C(builtins.object) | ---------------------------------------------------------------------- | Class methods defined here: | - | __class_getitem__(item) from builtins.type + | __class_getitem__(item) | | ---------------------------------------------------------------------- | Data descriptors defined here: @@ -124,7 +125,7 @@ class C(builtins.object) DATA __xyz__ = 'X, Y and Z' - c_alias = test.pydoc_mod.C[int] + c_alias = test.test_pydoc.pydoc_mod.C[int] list_alias1 = typing.List[int] list_alias2 = list[int] type_union1 = typing.Union[int, str] @@ -147,7 +148,7 @@ class C(builtins.object) for s in expected_data_docstrings) html2text_of_expected = """ -test.pydoc_mod (version 1.2.3.4) +test.test_pydoc.pydoc_mod (version 1.2.3.4) This is a test module for test_pydoc Modules @@ -166,7 +167,7 @@ class A(builtins.object) Methods defined here: __init__() Wow, I have no function! - + ---------------------------------------------------------------------- Data descriptors defined here: __dict__ dictionary for instance variables @@ -179,6 +180,7 @@ class B(builtins.object) dictionary for instance variables __weakref__ list of weak references to the object + ---------------------------------------------------------------------- Data and other attributes defined here: NO_MEANING = 'eggs' __annotations__ = {'NO_MEANING': <class 'str'>} @@ -191,8 +193,10 @@ class C(builtins.object) is_it_true(self) Return self.get_answer() say_no(self) + ---------------------------------------------------------------------- Class methods defined here: - __class_getitem__(item) from builtins.type + __class_getitem__(item) + ---------------------------------------------------------------------- Data descriptors defined here: __dict__ dictionary for instance variables @@ -209,7 +213,7 @@ class C(builtins.object) Data __xyz__ = 'X, Y and Z' - c_alias = test.pydoc_mod.C[int] + c_alias = test.test_pydoc.pydoc_mod.C[int] list_alias1 = typing.List[int] list_alias2 = list[int] type_union1 = typing.Union[int, str] @@ -330,11 +334,15 @@ def get_pydoc_html(module): loc = "<br><a href=\"" + loc + "\">Module Docs</a>" return output.strip(), loc +def clean_text(doc): + # clean up the extra text formatting that pydoc performs + return re.sub('\b.', '', doc) + def get_pydoc_link(module): "Returns a documentation web link of a module" abspath = os.path.abspath dirname = os.path.dirname - basedir = dirname(dirname(abspath(__file__))) + basedir = dirname(dirname(dirname(abspath(__file__)))) doc = pydoc.TextDoc() loc = doc.getdocloc(module, basedir=basedir) return loc @@ -347,10 +355,7 @@ def get_pydoc_text(module): loc = "\nMODULE DOCS\n " + loc + "\n" output = doc.docmodule(module) - - # clean up the extra text formatting that pydoc performs - patt = re.compile('\b.') - output = patt.sub('', output) + output = clean_text(output) return output.strip(), loc def get_html_title(text): @@ -367,6 +372,7 @@ def html2text(html): Tailored for pydoc tests only. """ html = html.replace("<dd>", "\n") + html = html.replace("<hr>", "-"*70) html = re.sub("<.*?>", "", html) html = pydoc.replace(html, " ", " ", ">", ">", "<", "<") return html @@ -483,7 +489,7 @@ def test_not_here(self): @requires_docstrings def test_not_ascii(self): - result = run_pydoc('test.test_pydoc.nonascii', PYTHONIOENCODING='ascii') + result = run_pydoc('test.test_pydoc.test_pydoc.nonascii', PYTHONIOENCODING='ascii') encoded = nonascii.__doc__.encode('ascii', 'backslashreplace') self.assertIn(encoded, result) @@ -663,9 +669,9 @@ def test_help_output_redirect(self): buf = StringIO() helper = pydoc.Helper(output=buf) unused, doc_loc = get_pydoc_text(pydoc_mod) - module = "test.pydoc_mod" + module = "test.test_pydoc.pydoc_mod" help_header = """ - Help on module test.pydoc_mod in test: + Help on module test.test_pydoc.pydoc_mod in test.test_pydoc: """.lstrip() help_header = textwrap.dedent(help_header) @@ -687,6 +693,30 @@ def test_help_output_redirect(self): finally: pydoc.getpager = getpager_old + def test_lambda_with_return_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"return": int} + with captured_output('stdout') as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a, b, c) -> int", helptext) + + def test_lambda_without_return_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"a": int, "b": int, "c": int} + with captured_output('stdout') as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a: int, b: int, c: int)", helptext) + + def test_lambda_with_return_and_params_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"a": int, "b": int, "c": int, "return": int} + with captured_output('stdout') as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a: int, b: int, c: int) -> int", helptext) + def test_namedtuple_fields(self): Person = namedtuple('Person', ['nickname', 'firstname']) with captured_stdout() as help_io: @@ -798,8 +828,7 @@ def itemconfigure(self, tagOrId, cnf=None, **kw): b_size = A.a_size doc = pydoc.render_doc(B) - # clean up the extra text formatting that pydoc performs - doc = re.sub('\b.', '', doc) + doc = clean_text(doc) self.assertEqual(doc, '''\ Python Library Documentation: class B in module %s @@ -887,8 +916,7 @@ def __init__(self, ... doc = pydoc.render_doc(A) - # clean up the extra text formatting that pydoc performs - doc = re.sub('\b.', '', doc) + doc = clean_text(doc) self.assertEqual(doc, '''Python Library Documentation: class A in module %s class A(builtins.object) @@ -925,8 +953,7 @@ def func( ... doc = pydoc.render_doc(func) - # clean up the extra text formatting that pydoc performs - doc = re.sub('\b.', '', doc) + doc = clean_text(doc) self.assertEqual(doc, '''Python Library Documentation: function func in module %s func( @@ -942,8 +969,7 @@ def function_with_really_long_name_so_annotations_can_be_rather_small( ... doc = pydoc.render_doc(function_with_really_long_name_so_annotations_can_be_rather_small) - # clean up the extra text formatting that pydoc performs - doc = re.sub('\b.', '', doc) + doc = clean_text(doc) self.assertEqual(doc, '''Python Library Documentation: function function_with_really_long_name_so_annotations_can_be_rather_small in module %s function_with_really_long_name_so_annotations_can_be_rather_small( @@ -957,8 +983,7 @@ def function_with_really_long_name_so_annotations_can_be_rather_small( second_very_long_parameter_name: ... doc = pydoc.render_doc(does_not_have_name) - # clean up the extra text formatting that pydoc performs - doc = re.sub('\b.', '', doc) + doc = clean_text(doc) self.assertEqual(doc, '''Python Library Documentation: function <lambda> in module %s <lambda> lambda very_long_parameter_name_that_should_not_fit_into_a_single_line, second_very_long_parameter_name @@ -1137,11 +1162,21 @@ def test_importfile(self): self.assertEqual(loaded_pydoc.__spec__, pydoc.__spec__) +class Rect: + @property + def area(self): + '''Area of the rect''' + return self.w * self.h + + +class Square(Rect): + area = property(lambda self: self.side**2) + + class TestDescriptions(unittest.TestCase): def test_module(self): # Check that pydocfodder module can be described - from test import pydocfodder doc = pydoc.render_doc(pydocfodder) self.assertIn("pydocfodder", doc) @@ -1244,7 +1279,7 @@ def test_unbound_python_method(self): @requires_docstrings def test_unbound_builtin_method(self): self.assertEqual(self._get_summary_line(_pickle.Pickler.dump), - "dump(self, obj, /)") + "dump(self, obj, /) unbound _pickle.Pickler method") # these no longer include "self" def test_bound_python_method(self): @@ -1296,7 +1331,7 @@ def test_module_level_callable_o(self): def test_unbound_builtin_method_noargs(self): self.assertEqual(self._get_summary_line(str.lower), - "lower(self, /)") + "lower(self, /) unbound builtins.str method") def test_bound_builtin_method_noargs(self): self.assertEqual(self._get_summary_line(''.lower), @@ -1304,7 +1339,7 @@ def test_bound_builtin_method_noargs(self): def test_unbound_builtin_method_o(self): self.assertEqual(self._get_summary_line(set.add), - "add(self, object, /)") + "add(self, object, /) unbound builtins.set method") def test_bound_builtin_method_o(self): self.assertEqual(self._get_summary_line(set().add), @@ -1312,7 +1347,7 @@ def test_bound_builtin_method_o(self): def test_unbound_builtin_method_coexist_o(self): self.assertEqual(self._get_summary_line(set.__contains__), - "__contains__(self, object, /)") + "__contains__(self, object, /) unbound builtins.set method") def test_bound_builtin_method_coexist_o(self): self.assertEqual(self._get_summary_line(set().__contains__), @@ -1320,19 +1355,19 @@ def test_bound_builtin_method_coexist_o(self): def test_unbound_builtin_classmethod_noargs(self): self.assertEqual(self._get_summary_line(datetime.datetime.__dict__['utcnow']), - "utcnow(type, /)") + "utcnow(type, /) unbound datetime.datetime method") def test_bound_builtin_classmethod_noargs(self): self.assertEqual(self._get_summary_line(datetime.datetime.utcnow), - "utcnow() method of builtins.type instance") + "utcnow() class method of datetime.datetime") def test_unbound_builtin_classmethod_o(self): self.assertEqual(self._get_summary_line(dict.__dict__['__class_getitem__']), - "__class_getitem__(type, object, /)") + "__class_getitem__(type, object, /) unbound builtins.dict method") def test_bound_builtin_classmethod_o(self): self.assertEqual(self._get_summary_line(dict.__class_getitem__), - "__class_getitem__(object, /) method of builtins.type instance") + "__class_getitem__(object, /) class method of builtins.dict") @support.cpython_only @requires_docstrings @@ -1356,11 +1391,13 @@ def test_builtin_staticmethod_unrepresentable_default(self): @requires_docstrings def test_unbound_builtin_method_unrepresentable_default(self): self.assertEqual(self._get_summary_line(dict.pop), - "pop(self, key, default=<unrepresentable>, /)") + "pop(self, key, default=<unrepresentable>, /) " + "unbound builtins.dict method") import _testcapi cls = _testcapi.DocStringUnrepresentableSignatureTest self.assertEqual(self._get_summary_line(cls.meth), - "meth(self, /, a, b=<x>)") + "meth(self, /, a, b=<x>) unbound " + "_testcapi.DocStringUnrepresentableSignatureTest method") @support.cpython_only @requires_docstrings @@ -1381,7 +1418,8 @@ def test_unbound_builtin_classmethod_unrepresentable_default(self): cls = _testcapi.DocStringUnrepresentableSignatureTest descr = cls.__dict__['classmeth'] self.assertEqual(self._get_summary_line(descr), - "classmeth(type, /, a, b=<x>)") + "classmeth(type, /, a, b=<x>) unbound " + "_testcapi.DocStringUnrepresentableSignatureTest method") @support.cpython_only @requires_docstrings @@ -1389,7 +1427,8 @@ def test_bound_builtin_classmethod_unrepresentable_default(self): import _testcapi cls = _testcapi.DocStringUnrepresentableSignatureTest self.assertEqual(self._get_summary_line(cls.classmeth), - "classmeth(a, b=<x>) method of builtins.type instance") + "classmeth(a, b=<x>) class method of " + "_testcapi.DocStringUnrepresentableSignatureTest") def test_overridden_text_signature(self): class C: @@ -1420,10 +1459,10 @@ def smeth(*args, **kwargs): self.assertEqual(self._get_summary_line(C.meth), "meth" + unbound) self.assertEqual(self._get_summary_line(C().meth), - "meth" + bound + " method of test.test_pydoc.C instance") + "meth" + bound + " method of test.test_pydoc.test_pydoc.C instance") C.cmeth.__func__.__text_signature__ = text_signature self.assertEqual(self._get_summary_line(C.cmeth), - "cmeth" + bound + " method of builtins.type instance") + "cmeth" + bound + " class method of test.test_pydoc.test_pydoc.C") C.smeth.__text_signature__ = text_signature self.assertEqual(self._get_summary_line(C.smeth), "smeth" + unbound) @@ -1460,13 +1499,13 @@ def cm(cls, x): 'cm(...)\n' ' A class method\n') self.assertEqual(self._get_summary_lines(X.cm), """\ -cm(x) method of builtins.type instance +cm(x) class method of test.test_pydoc.test_pydoc.X A class method """) self.assertIn(""" | Class methods defined here: | - | cm(x) from builtins.type + | cm(x) | A class method """, pydoc.plain(pydoc.render_doc(X))) @@ -1522,13 +1561,13 @@ def test_namedtuple_field_descriptor(self): @requires_docstrings def test_property(self): - class Rect: - @property - def area(self): - '''Area of the rect''' - return self.w * self.h - self.assertEqual(self._get_summary_lines(Rect.area), """\ +area + Area of the rect +""") + # inherits the docstring from Rect.area + self.assertEqual(self._get_summary_lines(Square.area), """\ +area Area of the rect """) self.assertIn(""" @@ -1623,6 +1662,140 @@ def a_fn_with_https_link(): ) +class PydocFodderTest(unittest.TestCase): + + def getsection(self, text, beginline, endline): + lines = text.splitlines() + beginindex, endindex = 0, None + if beginline is not None: + beginindex = lines.index(beginline) + if endline is not None: + endindex = lines.index(endline, beginindex) + return lines[beginindex:endindex] + + def test_text_doc_routines_in_class(self, cls=pydocfodder.B): + doc = pydoc.TextDoc() + result = doc.docclass(cls) + result = clean_text(result) + where = 'defined here' if cls is pydocfodder.B else 'inherited from B' + lines = self.getsection(result, f' | Methods {where}:', ' | ' + '-'*70) + self.assertIn(' | A_method_alias = A_method(self)', lines) + self.assertIn(' | B_method_alias = B_method(self)', lines) + self.assertIn(' | A_staticmethod(x, y) from test.test_pydoc.pydocfodder.A', lines) + self.assertIn(' | A_staticmethod_alias = A_staticmethod(x, y)', lines) + self.assertIn(' | global_func(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn(' | global_func_alias = global_func(x, y)', lines) + self.assertIn(' | global_func2_alias = global_func2(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn(' | count(self, value, /) from builtins.list', lines) + self.assertIn(' | list_count = count(self, value, /)', lines) + self.assertIn(' | __repr__(self, /) from builtins.object', lines) + self.assertIn(' | object_repr = __repr__(self, /)', lines) + + lines = self.getsection(result, f' | Static methods {where}:', ' | ' + '-'*70) + self.assertIn(' | A_classmethod_ref = A_classmethod(x) class method of test.test_pydoc.pydocfodder.A', lines) + note = '' if cls is pydocfodder.B else ' class method of test.test_pydoc.pydocfodder.B' + self.assertIn(' | B_classmethod_ref = B_classmethod(x)' + note, lines) + self.assertIn(' | A_method_ref = A_method() method of test.test_pydoc.pydocfodder.A instance', lines) + self.assertIn(' | get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' | dict_get = get(key, default=None, /) method of builtins.dict instance', lines) + + lines = self.getsection(result, f' | Class methods {where}:', ' | ' + '-'*70) + self.assertIn(' | B_classmethod(x)', lines) + self.assertIn(' | B_classmethod_alias = B_classmethod(x)', lines) + + def test_html_doc_routines_in_class(self, cls=pydocfodder.B): + doc = pydoc.HTMLDoc() + result = doc.docclass(cls) + result = html2text(result) + where = 'defined here' if cls is pydocfodder.B else 'inherited from B' + lines = self.getsection(result, f'Methods {where}:', '-'*70) + self.assertIn('A_method_alias = A_method(self)', lines) + self.assertIn('B_method_alias = B_method(self)', lines) + self.assertIn('A_staticmethod(x, y) from test.test_pydoc.pydocfodder.A', lines) + self.assertIn('A_staticmethod_alias = A_staticmethod(x, y)', lines) + self.assertIn('global_func(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn('global_func_alias = global_func(x, y)', lines) + self.assertIn('global_func2_alias = global_func2(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn('count(self, value, /) from builtins.list', lines) + self.assertIn('list_count = count(self, value, /)', lines) + self.assertIn('__repr__(self, /) from builtins.object', lines) + self.assertIn('object_repr = __repr__(self, /)', lines) + + lines = self.getsection(result, f'Static methods {where}:', '-'*70) + self.assertIn('A_classmethod_ref = A_classmethod(x) class method of test.test_pydoc.pydocfodder.A', lines) + note = '' if cls is pydocfodder.B else ' class method of test.test_pydoc.pydocfodder.B' + self.assertIn('B_classmethod_ref = B_classmethod(x)' + note, lines) + self.assertIn('A_method_ref = A_method() method of test.test_pydoc.pydocfodder.A instance', lines) + + lines = self.getsection(result, f'Class methods {where}:', '-'*70) + self.assertIn('B_classmethod(x)', lines) + self.assertIn('B_classmethod_alias = B_classmethod(x)', lines) + + def test_text_doc_inherited_routines_in_class(self): + self.test_text_doc_routines_in_class(pydocfodder.D) + + def test_html_doc_inherited_routines_in_class(self): + self.test_html_doc_routines_in_class(pydocfodder.D) + + def test_text_doc_routines_in_module(self): + doc = pydoc.TextDoc() + result = doc.docmodule(pydocfodder) + result = clean_text(result) + lines = self.getsection(result, 'FUNCTIONS', 'FILE') + # function alias + self.assertIn(' global_func_alias = global_func(x, y)', lines) + self.assertIn(' A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_alias = A_staticmethod(x, y)', lines) + # bound class methods + self.assertIn(' A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod2 = A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod3 = A_classmethod(x) class method of B', lines) + # bound methods + self.assertIn(' A_method() method of A instance', lines) + self.assertIn(' A_method2 = A_method() method of A instance', lines) + self.assertIn(' A_method3 = A_method() method of B instance', lines) + self.assertIn(' A_staticmethod_ref = A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_ref2 = A_staticmethod(y) method of B instance', lines) + self.assertIn(' get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' dict_get = get(key, default=None, /) method of builtins.dict instance', lines) + # unbound methods + self.assertIn(' B_method(self)', lines) + self.assertIn(' B_method2 = B_method(self)', lines) + self.assertIn(' count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(self, /) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(self, /) unbound builtins.object method', lines) + + def test_html_doc_routines_in_module(self): + doc = pydoc.HTMLDoc() + result = doc.docmodule(pydocfodder) + result = html2text(result) + lines = self.getsection(result, ' Functions', None) + # function alias + self.assertIn(' global_func_alias = global_func(x, y)', lines) + self.assertIn(' A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_alias = A_staticmethod(x, y)', lines) + # bound class methods + self.assertIn('A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod2 = A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod3 = A_classmethod(x) class method of B', lines) + # bound methods + self.assertIn(' A_method() method of A instance', lines) + self.assertIn(' A_method2 = A_method() method of A instance', lines) + self.assertIn(' A_method3 = A_method() method of B instance', lines) + self.assertIn(' A_staticmethod_ref = A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_ref2 = A_staticmethod(y) method of B instance', lines) + self.assertIn(' get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' dict_get = get(key, default=None, /) method of builtins.dict instance', lines) + # unbound methods + self.assertIn(' B_method(self)', lines) + self.assertIn(' B_method2 = B_method(self)', lines) + self.assertIn(' count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(self, /) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(self, /) unbound builtins.object method', lines) + + @unittest.skipIf( is_emscripten or is_wasi, "Socket server not available on Emscripten/WASI." diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 33113a72e6b6a9d..92d670ca6f8f5bd 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -2,6 +2,7 @@ # to ensure the Queue locks remain stable. import itertools import random +import sys import threading import time import unittest @@ -241,6 +242,386 @@ def test_shrinking_queue(self): with self.assertRaises(self.queue.Full): q.put_nowait(4) + def test_shutdown_empty(self): + q = self.type2test() + q.shutdown() + with self.assertRaises(self.queue.ShutDown): + q.put("data") + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_nonempty(self): + q = self.type2test() + q.put("data") + q.shutdown() + q.get() + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_immediate(self): + q = self.type2test() + q.put("data") + q.shutdown(immediate=True) + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_allowed_transitions(self): + # allowed transitions would be from alive via shutdown to immediate + q = self.type2test() + self.assertFalse(q.is_shutdown) + + q.shutdown() + self.assertTrue(q.is_shutdown) + + q.shutdown(immediate=True) + self.assertTrue(q.is_shutdown) + + q.shutdown(immediate=False) + + def _shutdown_all_methods_in_one_thread(self, immediate): + q = self.type2test(2) + q.put("L") + q.put_nowait("O") + q.shutdown(immediate) + + with self.assertRaises(self.queue.ShutDown): + q.put("E") + with self.assertRaises(self.queue.ShutDown): + q.put_nowait("W") + if immediate: + with self.assertRaises(self.queue.ShutDown): + q.get() + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() + with self.assertRaises(ValueError): + q.task_done() + q.join() + else: + self.assertIn(q.get(), "LO") + q.task_done() + self.assertIn(q.get(), "LO") + q.task_done() + q.join() + # on shutdown(immediate=False) + # when queue is empty, should raise ShutDown Exception + with self.assertRaises(self.queue.ShutDown): + q.get() # p.get(True) + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() # p.get(False) + with self.assertRaises(self.queue.ShutDown): + q.get(True, 1.0) + + def test_shutdown_all_methods_in_one_thread(self): + return self._shutdown_all_methods_in_one_thread(False) + + def test_shutdown_immediate_all_methods_in_one_thread(self): + return self._shutdown_all_methods_in_one_thread(True) + + def _write_msg_thread(self, q, n, results, delay, + i_when_exec_shutdown, + event_start, event_end): + event_start.wait() + for i in range(1, n+1): + try: + q.put((i, "YDLO")) + results.append(True) + except self.queue.ShutDown: + results.append(False) + # triggers shutdown of queue + if i == i_when_exec_shutdown: + event_end.set() + time.sleep(delay) + # end of all puts + q.join() + + def _read_msg_thread(self, q, nb, results, delay, event_start): + event_start.wait() + block = True + while nb: + time.sleep(delay) + try: + # Get at least one message + q.get(block) + block = False + q.task_done() + results.append(True) + nb -= 1 + except self.queue.ShutDown: + results.append(False) + nb -= 1 + except self.queue.Empty: + pass + q.join() + + def _shutdown_thread(self, q, event_end, immediate): + event_end.wait() + q.shutdown(immediate) + q.join() + + def _join_thread(self, q, delay, event_start): + event_start.wait() + time.sleep(delay) + q.join() + + def _shutdown_all_methods_in_many_threads(self, immediate): + q = self.type2test() + ps = [] + ev_start = threading.Event() + ev_exec_shutdown = threading.Event() + res_puts = [] + res_gets = [] + delay = 1e-4 + read_process = 4 + nb_msgs = read_process * 16 + nb_msgs_r = nb_msgs // read_process + when_exec_shutdown = nb_msgs // 2 + lprocs = ( + (self._write_msg_thread, 1, (q, nb_msgs, res_puts, delay, + when_exec_shutdown, + ev_start, ev_exec_shutdown)), + (self._read_msg_thread, read_process, (q, nb_msgs_r, + res_gets, delay*2, + ev_start)), + (self._join_thread, 2, (q, delay*2, ev_start)), + (self._shutdown_thread, 1, (q, ev_exec_shutdown, immediate)), + ) + # start all threds + for func, n, args in lprocs: + for i in range(n): + ps.append(threading.Thread(target=func, args=args)) + ps[-1].start() + # set event in order to run q.shutdown() + ev_start.set() + + if not immediate: + assert(len(res_gets) == len(res_puts)) + assert(res_gets.count(True) == res_puts.count(True)) + else: + assert(len(res_gets) <= len(res_puts)) + assert(res_gets.count(True) <= res_puts.count(True)) + + for thread in ps[1:]: + thread.join() + + @unittest.skip("test times out (gh-115258)") + def test_shutdown_all_methods_in_many_threads(self): + return self._shutdown_all_methods_in_many_threads(False) + + @unittest.skip("test times out (gh-115258)") + def test_shutdown_immediate_all_methods_in_many_threads(self): + return self._shutdown_all_methods_in_many_threads(True) + + def _get(self, q, go, results, shutdown=False): + go.wait() + try: + msg = q.get() + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _get_shutdown(self, q, go, results): + return self._get(q, go, results, True) + + def _get_task_done(self, q, go, results): + go.wait() + try: + msg = q.get() + q.task_done() + results.append(True) + return msg + except self.queue.ShutDown: + results.append(False) + return False + + def _put(self, q, msg, go, results, shutdown=False): + go.wait() + try: + q.put(msg) + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _put_shutdown(self, q, msg, go, results): + return self._put(q, msg, go, results, True) + + def _join(self, q, results, shutdown=False): + try: + q.join() + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _join_shutdown(self, q, results): + return self._join(q, results, True) + + def _shutdown_get(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + # queue full + + if immediate: + thrds = ( + (self._get_shutdown, (q, go, results)), + (self._get_shutdown, (q, go, results)), + ) + else: + thrds = ( + # on shutdown(immediate=False) + # one of these threads shoud raise Shutdown + (self._get, (q, go, results)), + (self._get, (q, go, results)), + (self._get, (q, go, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + q.shutdown(immediate) + go.set() + for t in threads: + t.join() + if immediate: + self.assertListEqual(results, [True, True]) + else: + self.assertListEqual(sorted(results), [False] + [True]*(len(thrds)-1)) + + def test_shutdown_get(self): + return self._shutdown_get(False) + + def test_shutdown_immediate_get(self): + return self._shutdown_get(True) + + def _shutdown_put(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + # queue fulled + + thrds = ( + (self._put_shutdown, (q, "E", go, results)), + (self._put_shutdown, (q, "W", go, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + q.shutdown() + go.set() + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_put(self): + return self._shutdown_put(False) + + def test_shutdown_immediate_put(self): + return self._shutdown_put(True) + + def _shutdown_join(self, immediate): + q = self.type2test() + results = [] + q.put("Y") + go = threading.Event() + nb = q.qsize() + + thrds = ( + (self._join, (q, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + if not immediate: + res = [] + for i in range(nb): + threads.append(threading.Thread(target=self._get_task_done, args=(q, go, res))) + threads[-1].start() + q.shutdown(immediate) + go.set() + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_immediate_join(self): + return self._shutdown_join(True) + + def test_shutdown_join(self): + return self._shutdown_join(False) + + def _shutdown_put_join(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + nb = q.qsize() + # queue not fulled + + thrds = ( + (self._put_shutdown, (q, "E", go, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + self.assertEqual(q.unfinished_tasks, nb) + for i in range(nb): + t = threading.Thread(target=q.task_done) + t.start() + threads.append(t) + q.shutdown(immediate) + go.set() + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_immediate_put_join(self): + return self._shutdown_put_join(True) + + def test_shutdown_put_join(self): + return self._shutdown_put_join(False) + + def test_shutdown_get_task_done_join(self): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + self.assertEqual(q.unfinished_tasks, q.qsize()) + + thrds = ( + (self._get_task_done, (q, go, results)), + (self._get_task_done, (q, go, results)), + (self._join, (q, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + go.set() + q.shutdown(False) + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + class QueueTest(BaseQueueTestMixin): def setUp(self): diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 993a7d6e264a1f4..b1ac22c28cf7c13 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1,7 +1,7 @@ from test.support import (gc_collect, bigmemtest, _2G, cpython_only, captured_stdout, check_disallow_instantiation, is_emscripten, is_wasi, - warnings_helper, SHORT_TIMEOUT) + warnings_helper, SHORT_TIMEOUT, CPUStopwatch) import locale import re import string @@ -2284,17 +2284,16 @@ def test_bug_40736(self): def test_search_anchor_at_beginning(self): s = 'x'*10**7 - start = time.perf_counter() - for p in r'\Ay', r'^y': - self.assertIsNone(re.search(p, s)) - self.assertEqual(re.split(p, s), [s]) - self.assertEqual(re.findall(p, s), []) - self.assertEqual(list(re.finditer(p, s)), []) - self.assertEqual(re.sub(p, '', s), s) - t = time.perf_counter() - start + with CPUStopwatch() as stopwatch: + for p in r'\Ay', r'^y': + self.assertIsNone(re.search(p, s)) + self.assertEqual(re.split(p, s), [s]) + self.assertEqual(re.findall(p, s), []) + self.assertEqual(list(re.finditer(p, s)), []) + self.assertEqual(re.sub(p, '', s), s) # Without optimization it takes 1 second on my computer. # With optimization -- 0.0003 seconds. - self.assertLess(t, 0.1) + self.assertLess(stopwatch.seconds, 0.1) def test_possessive_quantifiers(self): """Test Possessive Quantifiers diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 89562fa5eac62ca..7e1eaa7d6a515ec 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -399,7 +399,7 @@ def test_unknown_option(self): self.checkError(['--unknown-option'], 'unrecognized arguments: --unknown-option') - def check_ci_mode(self, args, use_resources, rerun=True): + def create_regrtest(self, args): ns = cmdline._parse_args(args) # Check Regrtest attributes which are more reliable than Namespace @@ -411,6 +411,10 @@ def check_ci_mode(self, args, use_resources, rerun=True): regrtest = main.Regrtest(ns) + return regrtest + + def check_ci_mode(self, args, use_resources, rerun=True): + regrtest = self.create_regrtest(args) self.assertEqual(regrtest.num_workers, -1) self.assertEqual(regrtest.want_rerun, rerun) self.assertTrue(regrtest.randomize) @@ -455,6 +459,11 @@ def test_dont_add_python_opts(self): ns = cmdline._parse_args(args) self.assertFalse(ns._add_python_opts) + def test_bisect(self): + args = ['--bisect'] + regrtest = self.create_regrtest(args) + self.assertTrue(regrtest.want_bisect) + @dataclasses.dataclass(slots=True) class Rerun: @@ -1162,8 +1171,8 @@ def check_leak(self, code, what, *, run_workers=False): stderr=subprocess.STDOUT) self.check_executed_tests(output, [test], failed=test, stats=1) - line = 'beginning 6 repetitions\n123456\n......\n' - self.check_line(output, re.escape(line)) + line = r'beginning 6 repetitions. .*\n123:456\n[.0-9X]{3} 111\n' + self.check_line(output, line) line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what) self.assertIn(line2, output) @@ -1192,6 +1201,47 @@ def test_huntrleaks(self): def test_huntrleaks_mp(self): self.check_huntrleaks(run_workers=True) + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') + def test_huntrleaks_bisect(self): + # test --huntrleaks --bisect + code = textwrap.dedent(""" + import unittest + + GLOBAL_LIST = [] + + class RefLeakTest(unittest.TestCase): + def test1(self): + pass + + def test2(self): + pass + + def test3(self): + GLOBAL_LIST.append(object()) + + def test4(self): + pass + """) + + test = self.create_test('huntrleaks', code=code) + + filename = 'reflog.txt' + self.addCleanup(os_helper.unlink, filename) + cmd = ['--huntrleaks', '3:3:', '--bisect', test] + output = self.run_tests(*cmd, + exitcode=EXITCODE_BAD_TEST, + stderr=subprocess.STDOUT) + + self.assertIn(f"Bisect {test}", output) + self.assertIn(f"Bisect {test}: exit code 0", output) + + # test3 is the one which leaks + self.assertIn("Bisection completed in", output) + self.assertIn( + "Tests (1):\n" + f"* {test}.RefLeakTest.test3\n", + output) + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') def test_huntrleaks_fd_leak(self): # test --huntrleaks for file descriptor leak diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 57fe859e366b5b7..9d76764c75be3eb 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -12,7 +12,8 @@ import textwrap import unittest import warnings -from test.support import no_tracing, verbose, requires_subprocess, requires_resource +from test.support import (infinite_recursion, no_tracing, verbose, + requires_subprocess, requires_resource) from test.support.import_helper import forget, make_legacy_pyc, unload from test.support.os_helper import create_empty_file, temp_dir from test.support.script_helper import make_script, make_zip_script @@ -743,7 +744,8 @@ def test_main_recursion_error(self): "runpy.run_path(%r)\n") % dummy_dir script_name = self._make_test_script(script_dir, mod_name, source) zip_name, fname = make_zip_script(script_dir, 'test_zip', script_name) - self.assertRaises(RecursionError, run_path, zip_name) + with infinite_recursion(25): + self.assertRaises(RecursionError, run_path, zip_name) def test_encoding(self): with temp_dir() as script_dir: diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py index 677349c2bfca93d..643775597c56c67 100644 --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -6,8 +6,7 @@ import socket import sys from test import support -from test.support import os_helper -from test.support import socket_helper +from test.support import is_apple, os_helper, socket_helper from time import sleep import unittest import unittest.mock @@ -526,7 +525,7 @@ def test_above_fd_setsize(self): try: fds = s.select() except OSError as e: - if e.errno == errno.EINVAL and sys.platform == 'darwin': + if e.errno == errno.EINVAL and is_apple: # unexplainable errors on macOS don't need to fail the test self.skipTest("Invalid argument error calling poll()") raise diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 8edd75e9907ec02..d96dad4eb9475d2 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -2148,6 +2148,7 @@ def check_chown(path, uid=None, gid=None): check_chown(dirname, uid, gid) +@support.requires_subprocess() class TestWhich(BaseTest, unittest.TestCase): def setUp(self): @@ -3181,6 +3182,7 @@ def test_bad_environ(self): self.assertGreaterEqual(size.lines, 0) @unittest.skipUnless(os.isatty(sys.__stdout__.fileno()), "not on tty") + @support.requires_subprocess() @unittest.skipUnless(hasattr(os, 'get_terminal_size'), 'need os.get_terminal_size()') def test_stty_match(self): diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 637a0ca3b369726..61fb047caf6dab6 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -13,9 +13,10 @@ import time import unittest from test import support -from test.support import os_helper +from test.support import ( + is_apple, is_apple_mobile, os_helper, threading_helper +) from test.support.script_helper import assert_python_ok, spawn_python -from test.support import threading_helper try: import _testcapi except ImportError: @@ -832,7 +833,7 @@ def test_itimer_real(self): self.assertEqual(self.hndl_called, True) # Issue 3864, unknown if this affects earlier versions of freebsd also - @unittest.skipIf(sys.platform in ('netbsd5',), + @unittest.skipIf(sys.platform in ('netbsd5',) or is_apple_mobile, 'itimer not reliable (does not mix well with threading) on some BSDs.') def test_itimer_virtual(self): self.itimer = signal.ITIMER_VIRTUAL @@ -1344,7 +1345,7 @@ def handler(signum, frame): # Python handler self.assertEqual(len(sigs), N, "Some signals were lost") - @unittest.skipIf(sys.platform == "darwin", "crashes due to system bug (FB13453490)") + @unittest.skipIf(is_apple, "crashes due to system bug (FB13453490)") @unittest.skipUnless(hasattr(signal, "SIGUSR1"), "test needs SIGUSR1") @threading_helper.requires_working_threading() diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 231448c75f01db0..b936e9ae91daca0 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1,9 +1,8 @@ import unittest from test import support -from test.support import os_helper -from test.support import socket_helper -from test.support import threading_helper -from test.support import refleak_helper +from test.support import ( + is_apple, os_helper, refleak_helper, socket_helper, threading_helper +) import _thread as thread import array @@ -47,6 +46,7 @@ VSOCKPORT = 1234 AIX = platform.system() == "AIX" +WSL = "microsoft-standard-WSL" in platform.release() try: import _socket @@ -511,6 +511,7 @@ def clientTearDown(self): ThreadableTest.clientTearDown(self) @unittest.skipIf(fcntl is None, "need fcntl") +@unittest.skipIf(WSL, 'VSOCK does not work on Microsoft WSL') @unittest.skipUnless(HAVE_SOCKET_VSOCK, 'VSOCK sockets required for this test.') @unittest.skipUnless(get_cid() != 2, @@ -527,6 +528,7 @@ def setUp(self): self.serv.bind((socket.VMADDR_CID_ANY, VSOCKPORT)) self.serv.listen() self.serverExplicitReady() + self.serv.settimeout(support.LOOPBACK_TIMEOUT) self.conn, self.connaddr = self.serv.accept() self.addCleanup(self.conn.close) @@ -1196,8 +1198,11 @@ def testGetServBy(self): # Find one service that exists, then check all the related interfaces. # I've ordered this by protocols that have both a tcp and udp # protocol, at least for modern Linuxes. - if (sys.platform.startswith(('freebsd', 'netbsd', 'gnukfreebsd')) - or sys.platform in ('linux', 'darwin')): + if ( + sys.platform.startswith(('freebsd', 'netbsd', 'gnukfreebsd')) + or sys.platform == 'linux' + or is_apple + ): # avoid the 'echo' service on this platform, as there is an # assumption breaking non-standard port/protocol entry services = ('daytime', 'qotd', 'domain') @@ -3708,7 +3713,7 @@ def testFDPassCMSG_LEN(self): def _testFDPassCMSG_LEN(self): self.createAndSendFDs(1) - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(AIX, "skipping, see issue #22397") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparate(self): @@ -3719,7 +3724,7 @@ def testFDPassSeparate(self): maxcmsgs=2) @testFDPassSeparate.client_skip - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(AIX, "skipping, see issue #22397") def _testFDPassSeparate(self): fd0, fd1 = self.newFDs(2) @@ -3732,7 +3737,7 @@ def _testFDPassSeparate(self): array.array("i", [fd1]))]), len(MSG)) - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(AIX, "skipping, see issue #22397") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparateMinSpace(self): @@ -3746,7 +3751,7 @@ def testFDPassSeparateMinSpace(self): maxcmsgs=2, ignoreflags=socket.MSG_CTRUNC) @testFDPassSeparateMinSpace.client_skip - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(AIX, "skipping, see issue #22397") def _testFDPassSeparateMinSpace(self): fd0, fd1 = self.newFDs(2) @@ -3770,7 +3775,7 @@ def sendAncillaryIfPossible(self, msg, ancdata): nbytes = self.sendmsgToServer([msg]) self.assertEqual(nbytes, len(msg)) - @unittest.skipIf(sys.platform == "darwin", "see issue #24725") + @unittest.skipIf(is_apple, "skipping, see issue #12958") def testFDPassEmpty(self): # Try to pass an empty FD array. Can receive either no array # or an empty array. diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index f3efe0f52f4fd73..588272448bbfda6 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -31,7 +31,7 @@ from test.support import ( SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess, - is_emscripten, is_wasi + is_apple, is_emscripten, is_wasi ) from test.support import gc_collect from test.support import threading_helper @@ -667,7 +667,7 @@ def test_open_with_path_like_object(self): cx.execute(self._sql) @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(sys.platform == "darwin", "skipped on macOS") + @unittest.skipIf(is_apple, "skipped on Apple platforms") @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_with_undecodable_path(self): @@ -713,7 +713,7 @@ def test_open_uri_readonly(self): cx.execute(self._sql) @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(sys.platform == "darwin", "skipped on macOS") + @unittest.skipIf(is_apple, "skipped on Apple platforms") @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_undecodable_uri(self): diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index 2e1f0b80c10f46e..7261b7f0dc93d0a 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -54,6 +54,76 @@ def test_table_dump(self): [self.assertEqual(expected_sqls[i], actual_sqls[i]) for i in range(len(expected_sqls))] + def test_table_dump_filter(self): + all_table_sqls = [ + """CREATE TABLE "some_table_2" ("id_1" INTEGER);""", + """INSERT INTO "some_table_2" VALUES(3);""", + """INSERT INTO "some_table_2" VALUES(4);""", + """CREATE TABLE "test_table_1" ("id_2" INTEGER);""", + """INSERT INTO "test_table_1" VALUES(1);""", + """INSERT INTO "test_table_1" VALUES(2);""", + ] + all_views_sqls = [ + """CREATE VIEW "view_1" AS SELECT * FROM "some_table_2";""", + """CREATE VIEW "view_2" AS SELECT * FROM "test_table_1";""", + ] + # Create database structure. + for sql in [*all_table_sqls, *all_views_sqls]: + self.cu.execute(sql) + # %_table_% matches all tables. + dump_sqls = list(self.cx.iterdump(filter="%_table_%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_table_sqls, "COMMIT;"], + ) + # view_% matches all views. + dump_sqls = list(self.cx.iterdump(filter="view_%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_views_sqls, "COMMIT;"], + ) + # %_1 matches tables and views with the _1 suffix. + dump_sqls = list(self.cx.iterdump(filter="%_1")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE TABLE "test_table_1" ("id_2" INTEGER);""", + """INSERT INTO "test_table_1" VALUES(1);""", + """INSERT INTO "test_table_1" VALUES(2);""", + """CREATE VIEW "view_1" AS SELECT * FROM "some_table_2";""", + "COMMIT;" + ], + ) + # some_% matches some_table_2. + dump_sqls = list(self.cx.iterdump(filter="some_%")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE TABLE "some_table_2" ("id_1" INTEGER);""", + """INSERT INTO "some_table_2" VALUES(3);""", + """INSERT INTO "some_table_2" VALUES(4);""", + "COMMIT;" + ], + ) + # Only single object. + dump_sqls = list(self.cx.iterdump(filter="view_2")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE VIEW "view_2" AS SELECT * FROM "test_table_1";""", + "COMMIT;" + ], + ) + # % matches all objects. + dump_sqls = list(self.cx.iterdump(filter="%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_table_sqls, *all_views_sqls, "COMMIT;"], + ) + def test_dump_autoincrement(self): expected = [ 'CREATE TABLE "t1" (id integer primary key autoincrement);', diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 3fdfa2960503b8f..1b18230d83577dc 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -2206,14 +2206,15 @@ def _test_get_server_certificate(test, host, port, cert=None): sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) def _test_get_server_certificate_fail(test, host, port): - try: - pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) - except ssl.SSLError as x: - #should fail - if support.verbose: - sys.stdout.write("%s\n" % x) - else: - test.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) + with warnings_helper.check_no_resource_warning(test): + try: + pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + test.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) from test.ssl_servers import make_https_server @@ -3026,6 +3027,16 @@ def test_check_hostname_idn(self): server_hostname="python.example.org") as s: with self.assertRaises(ssl.CertificateError): s.connect((HOST, server.port)) + with ThreadedEchoServer(context=server_context, chatty=True) as server: + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(UnicodeError): + context.wrap_socket(socket.socket(), + server_hostname='.pythontest.net') + with ThreadedEchoServer(context=server_context, chatty=True) as server: + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(UnicodeDecodeError): + context.wrap_socket(socket.socket(), + server_hostname=b'k\xf6nig.idn.pythontest.net') def test_wrong_cert_tls12(self): """Connecting when the server rejects the client's certificate @@ -4983,7 +4994,8 @@ def call_after_accept(conn_to_client): self.assertIsNone(wrap_error.library, msg="attr must exist") finally: # gh-108342: Explicitly break the reference cycle - wrap_error = None + with warnings_helper.check_no_resource_warning(self): + wrap_error = None server = None def test_https_client_non_tls_response_ignored(self): @@ -5032,7 +5044,8 @@ def call_after_accept(conn_to_client): # socket; that fails if the connection is broken. It may seem pointless # to test this. It serves as an illustration of something that we never # want to happen... properly not happening. - with self.assertRaises(OSError): + with warnings_helper.check_no_resource_warning(self), \ + self.assertRaises(OSError): connection.request("HEAD", "/test", headers={"Host": "localhost"}) response = connection.getresponse() diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 4976ac3642bbe46..8bd373976426ef3 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -9,6 +9,13 @@ from _testcapi import get_feature_macros feature_macros = get_feature_macros() + +# Stable ABI is incompatible with Py_TRACE_REFS builds due to PyObject +# layout differences. +# See https://github.com/python/cpython/issues/88299#issuecomment-1113366226 +if feature_macros['Py_TRACE_REFS']: + raise unittest.SkipTest("incompatible with Py_TRACE_REFS.") + ctypes_test = import_module('ctypes') class TestStableABIAvailability(unittest.TestCase): @@ -254,6 +261,7 @@ def test_windows_feature_macros(self): "PyExc_IOError", "PyExc_ImportError", "PyExc_ImportWarning", + "PyExc_IncompleteInputError", "PyExc_IndentationError", "PyExc_IndexError", "PyExc_InterruptedError", @@ -364,6 +372,7 @@ def test_windows_feature_macros(self): "PyList_Append", "PyList_AsTuple", "PyList_GetItem", + "PyList_GetItemRef", "PyList_GetSlice", "PyList_Insert", "PyList_New", @@ -441,7 +450,9 @@ def test_windows_feature_macros(self): "PyModule_AddObjectRef", "PyModule_AddStringConstant", "PyModule_AddType", + "PyModule_Create2", "PyModule_ExecDef", + "PyModule_FromDefAndSpec2", "PyModule_GetDef", "PyModule_GetDict", "PyModule_GetFilename", @@ -911,6 +922,13 @@ def test_windows_feature_macros(self): "_Py_TrueStruct", "_Py_VaBuildValue_SizeT", ) +if feature_macros['HAVE_FORK']: + SYMBOL_NAMES += ( + 'PyOS_AfterFork', + 'PyOS_AfterFork_Child', + 'PyOS_AfterFork_Parent', + 'PyOS_BeforeFork', + ) if feature_macros['MS_WINDOWS']: SYMBOL_NAMES += ( 'PyErr_SetExcFromWindowsErr', @@ -926,17 +944,6 @@ def test_windows_feature_macros(self): 'PyUnicode_DecodeMBCSStateful', 'PyUnicode_EncodeCodePage', ) -if feature_macros['HAVE_FORK']: - SYMBOL_NAMES += ( - 'PyOS_AfterFork', - 'PyOS_AfterFork_Child', - 'PyOS_AfterFork_Parent', - 'PyOS_BeforeFork', - ) -if feature_macros['USE_STACKCHECK']: - SYMBOL_NAMES += ( - 'PyOS_CheckStack', - ) if feature_macros['PY_HAVE_THREAD_NATIVE_ID']: SYMBOL_NAMES += ( 'PyThread_get_thread_native_id', @@ -946,14 +953,23 @@ def test_windows_feature_macros(self): '_Py_NegativeRefcount', '_Py_RefTotal', ) +if feature_macros['Py_TRACE_REFS']: + SYMBOL_NAMES += ( + ) +if feature_macros['USE_STACKCHECK']: + SYMBOL_NAMES += ( + 'PyOS_CheckStack', + ) EXPECTED_FEATURE_MACROS = set(['HAVE_FORK', 'MS_WINDOWS', 'PY_HAVE_THREAD_NATIVE_ID', 'Py_REF_DEBUG', + 'Py_TRACE_REFS', 'USE_STACKCHECK']) WINDOWS_FEATURE_MACROS = {'HAVE_FORK': False, 'MS_WINDOWS': True, 'PY_HAVE_THREAD_NATIVE_ID': True, 'Py_REF_DEBUG': 'maybe', + 'Py_TRACE_REFS': 'maybe', 'USE_STACKCHECK': 'maybe'} diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index d6b6dd6e7417002..49013a4bcd8af6b 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -2,8 +2,7 @@ import os import socket import sys -from test.support import os_helper -from test.support import socket_helper +from test.support import is_apple, os_helper, socket_helper from test.support.import_helper import import_fresh_module from test.support.os_helper import TESTFN @@ -247,7 +246,7 @@ def test_flags_consistent(self): for flag in self.file_flags: if flag.startswith("UF"): self.assertTrue(getattr(self.statmod, flag) & self.statmod.UF_SETTABLE, f"{flag} not in UF_SETTABLE") - elif sys.platform == 'darwin' and self.statmod is c_stat and flag == 'SF_DATALESS': + elif is_apple and self.statmod is c_stat and flag == 'SF_DATALESS': self.assertTrue(self.statmod.SF_DATALESS & self.statmod.SF_SYNTHETIC, "SF_DATALESS not in SF_SYNTHETIC") self.assertFalse(self.statmod.SF_DATALESS & self.statmod.SF_SETTABLE, "SF_DATALESS in SF_SETTABLE") else: diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index bf2c254c9ee7d9a..1cf41638a7f01a3 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2353,6 +2353,66 @@ def test_mixed_int_and_float(self): self.assertAlmostEqual(actual_mean, expected_mean, places=5) +class TestKDE(unittest.TestCase): + + def test_kde(self): + kde = statistics.kde + StatisticsError = statistics.StatisticsError + + kernels = ['normal', 'gauss', 'logistic', 'sigmoid', 'rectangular', + 'uniform', 'triangular', 'parabolic', 'epanechnikov', + 'quartic', 'biweight', 'triweight', 'cosine'] + + sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + + # The approximate integral of a PDF should be close to 1.0 + + def integrate(func, low, high, steps=10_000): + "Numeric approximation of a definite function integral." + dx = (high - low) / steps + midpoints = (low + (i + 1/2) * dx for i in range(steps)) + return sum(map(func, midpoints)) * dx + + for kernel in kernels: + with self.subTest(kernel=kernel): + f_hat = kde(sample, h=1.5, kernel=kernel) + area = integrate(f_hat, -20, 20) + self.assertAlmostEqual(area, 1.0, places=4) + + # Check error cases + + with self.assertRaises(StatisticsError): + kde([], h=1.0) # Empty dataset + with self.assertRaises(TypeError): + kde(['abc', 'def'], 1.5) # Non-numeric data + with self.assertRaises(TypeError): + kde(iter(sample), 1.5) # Data is not a sequence + with self.assertRaises(StatisticsError): + kde(sample, h=0.0) # Zero bandwidth + with self.assertRaises(StatisticsError): + kde(sample, h=0.0) # Negative bandwidth + with self.assertRaises(TypeError): + kde(sample, h='str') # Wrong bandwidth type + with self.assertRaises(StatisticsError): + kde(sample, h=1.0, kernel='bogus') # Invalid kernel + + # Test name and docstring of the generated function + + h = 1.5 + kernel = 'cosine' + f_hat = kde(sample, h, kernel) + self.assertEqual(f_hat.__name__, 'pdf') + self.assertIn(kernel, f_hat.__doc__) + self.assertIn(str(h), f_hat.__doc__) + + # Test closed interval for the support boundaries. + # In particular, 'uniform' should non-zero at the boundaries. + + f_hat = kde([0], 1.0, 'uniform') + self.assertEqual(f_hat(-1.0), 1/2) + self.assertEqual(f_hat(1.0), 1/2) + + class TestQuantiles(unittest.TestCase): def test_specific_cases(self): diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 6c87dfabad9f0f7..38dcabd84d8170a 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -729,7 +729,7 @@ def test_subinterp_intern_dynamically_allocated(self): self.assertIs(t, s) interp = interpreters.create() - interp.exec_sync(textwrap.dedent(f''' + interp.exec(textwrap.dedent(f''' import sys t = sys.intern({s!r}) assert id(t) != {id(s)}, (id(t), {id(s)}) @@ -744,7 +744,7 @@ def test_subinterp_intern_statically_allocated(self): t = sys.intern(s) interp = interpreters.create() - interp.exec_sync(textwrap.dedent(f''' + interp.exec(textwrap.dedent(f''' import sys t = sys.intern({s!r}) assert id(t) == {id(t)}, (id(t), {id(t)}) @@ -1392,6 +1392,7 @@ def setUp(self): self.longdigit = sys.int_info.sizeof_digit import _testinternalcapi self.gc_headsize = _testinternalcapi.SIZEOF_PYGC_HEAD + self.managed_pre_header_size = _testinternalcapi.SIZEOF_MANAGED_PRE_HEADER check_sizeof = test.support.check_sizeof @@ -1427,7 +1428,7 @@ class OverflowSizeof(int): def __sizeof__(self): return int(self) self.assertEqual(sys.getsizeof(OverflowSizeof(sys.maxsize)), - sys.maxsize + self.gc_headsize*2) + sys.maxsize + self.gc_headsize + self.managed_pre_header_size) with self.assertRaises(OverflowError): sys.getsizeof(OverflowSizeof(sys.maxsize + 1)) with self.assertRaises(ValueError): @@ -1650,7 +1651,7 @@ def delx(self): del self.__x # type # static type: PyTypeObject fmt = 'P2nPI13Pl4Pn9Pn12PIPc' - s = vsize('2P' + fmt) + s = vsize(fmt) check(int, s) # class s = vsize(fmt + # PyTypeObject diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index ae6e192a7ab6eff..125f40227118f63 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -7,7 +7,7 @@ import gc from functools import wraps import asyncio -from test.support import import_helper +from test.support import import_helper, requires_subprocess import contextlib import os import tempfile @@ -1810,6 +1810,7 @@ def compare_events(self, line_offset, events, expected_events): def make_tracer(): return Tracer(trace_opcode_events=True) + @requires_subprocess() def test_trace_opcodes_after_settrace(self): """Make sure setting f_trace_opcodes after starting trace works even if it's the first time f_trace_opcodes is being set. GH-103615""" diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index be609a0abd29c87..bb87bf00dc2d1a7 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -154,17 +154,21 @@ def test_posix_venv_scheme(self): 'python%d.%d' % sys.version_info[:2], 'site-packages') - # Resolve the paths in prefix - binpath = os.path.join(sys.prefix, binpath) - incpath = os.path.join(sys.prefix, incpath) - libpath = os.path.join(sys.prefix, libpath) + # Resolve the paths in an imaginary venv/ directory + binpath = os.path.join('venv', binpath) + incpath = os.path.join('venv', incpath) + libpath = os.path.join('venv', libpath) - self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='posix_venv')) - self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='posix_venv')) + # Mimic the venv module, set all bases to the venv directory + bases = ('base', 'platbase', 'installed_base', 'installed_platbase') + vars = {base: 'venv' for base in bases} + + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='posix_venv', vars=vars)) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='posix_venv', vars=vars)) # The include directory on POSIX isn't exactly the same as before, # but it is "within" - sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv') + sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv', vars=vars) self.assertTrue(sysconfig_includedir.startswith(incpath + os.sep)) def test_nt_venv_scheme(self): @@ -174,14 +178,19 @@ def test_nt_venv_scheme(self): incpath = 'Include' libpath = os.path.join('Lib', 'site-packages') - # Resolve the paths in prefix - binpath = os.path.join(sys.prefix, binpath) - incpath = os.path.join(sys.prefix, incpath) - libpath = os.path.join(sys.prefix, libpath) + # Resolve the paths in an imaginary venv\ directory + venv = 'venv' + binpath = os.path.join(venv, binpath) + incpath = os.path.join(venv, incpath) + libpath = os.path.join(venv, libpath) + + # Mimic the venv module, set all bases to the venv directory + bases = ('base', 'platbase', 'installed_base', 'installed_platbase') + vars = {base: 'venv' for base in bases} - self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='nt_venv')) - self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv')) - self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv')) + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='nt_venv', vars=vars)) + self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv', vars=vars)) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv', vars=vars)) def test_venv_scheme(self): if sys.platform == 'win32': diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index da5009126b3815b..a047780fdd7e17c 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -15,6 +15,7 @@ import unittest.mock import tarfile +from test import archiver_tests from test import support from test.support import os_helper from test.support import script_helper @@ -506,14 +507,32 @@ def test_length_zero_header(self): with tarfile.open(support.findfile('recursion.tar', subdir='archivetestdata')): pass - def test_extractfile_name(self): + def test_extractfile_attrs(self): # gh-74468: TarFile.name must name a file, not a parent archive. file = self.tar.getmember('ustar/regtype') with self.tar.extractfile(file) as fobj: self.assertEqual(fobj.name, 'ustar/regtype') + self.assertRaises(AttributeError, fobj.fileno) + self.assertIs(fobj.readable(), True) + self.assertIs(fobj.writable(), False) + if self.is_stream: + self.assertRaises(AttributeError, fobj.seekable) + else: + self.assertIs(fobj.seekable(), True) + self.assertIs(fobj.closed, False) + self.assertIs(fobj.closed, True) + self.assertEqual(fobj.name, 'ustar/regtype') + self.assertRaises(AttributeError, fobj.fileno) + self.assertIs(fobj.readable(), True) + self.assertIs(fobj.writable(), False) + if self.is_stream: + self.assertRaises(AttributeError, fobj.seekable) + else: + self.assertIs(fobj.seekable(), True) class MiscReadTestBase(CommonReadTest): + is_stream = False def requires_name_attribute(self): pass @@ -806,6 +825,7 @@ def requires_name_attribute(self): class StreamReadTest(CommonReadTest, unittest.TestCase): prefix="r|" + is_stream = True def test_read_through(self): # Issue #11224: A poorly designed _FileInFile.read() method @@ -4135,6 +4155,38 @@ def valueerror_filter(tarinfo, path): self.expect_exception(TypeError) # errorlevel is not int +class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase): + testdir = os.path.join(TEMPDIR, "testoverwrite") + + @classmethod + def setUpClass(cls): + p = cls.ar_with_file = os.path.join(TEMPDIR, 'tar-with-file.tar') + cls.addClassCleanup(os_helper.unlink, p) + with tarfile.open(p, 'w') as tar: + t = tarfile.TarInfo('test') + t.size = 10 + tar.addfile(t, io.BytesIO(b'newcontent')) + + p = cls.ar_with_dir = os.path.join(TEMPDIR, 'tar-with-dir.tar') + cls.addClassCleanup(os_helper.unlink, p) + with tarfile.open(p, 'w') as tar: + tar.addfile(tar.gettarinfo(os.curdir, 'test')) + + p = os.path.join(TEMPDIR, 'tar-with-implicit-dir.tar') + cls.ar_with_implicit_dir = p + cls.addClassCleanup(os_helper.unlink, p) + with tarfile.open(p, 'w') as tar: + t = tarfile.TarInfo('test/file') + t.size = 10 + tar.addfile(t, io.BytesIO(b'newcontent')) + + def open(self, path): + return tarfile.open(path, 'r') + + def extractall(self, ar): + ar.extractall(self.testdir, filter='fully_trusted') + + def setUpModule(): os_helper.unlink(TEMPDIR) os.makedirs(TEMPDIR) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 1ab223b81e939e5..3b5c37c948c8c34 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1478,7 +1478,7 @@ def test_threads_join_with_no_main(self): DONE = b'D' interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os import threading import time diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 3b5640abdb6b892..a0aeea515afbd69 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -43,8 +43,8 @@ class _PyTime(enum.IntEnum): ROUND_UP = 3 # _PyTime_t is int64_t -_PyTime_MIN = -2 ** 63 -_PyTime_MAX = 2 ** 63 - 1 +PyTime_MIN = -2 ** 63 +PyTime_MAX = 2 ** 63 - 1 # Rounding modes supported by PyTime ROUNDING_MODES = ( @@ -934,7 +934,7 @@ def test_FromSecondsObject(self): _PyTime_FromSecondsObject(float('nan'), time_rnd) def test_AsSecondsDouble(self): - from _testinternalcapi import _PyTime_AsSecondsDouble + from _testcapi import PyTime_AsSecondsDouble def float_converter(ns): if abs(ns) % SEC_TO_NS == 0: @@ -942,15 +942,10 @@ def float_converter(ns): else: return float(ns) / SEC_TO_NS - self.check_int_rounding(lambda ns, rnd: _PyTime_AsSecondsDouble(ns), + self.check_int_rounding(lambda ns, rnd: PyTime_AsSecondsDouble(ns), float_converter, NS_TO_SEC) - # test nan - for time_rnd, _ in ROUNDING_MODES: - with self.assertRaises(TypeError): - _PyTime_AsSecondsDouble(float('nan')) - def create_decimal_converter(self, denominator): denom = decimal.Decimal(denominator) @@ -1009,7 +1004,7 @@ def test_AsTimeval_clamp(self): tv_sec_max = self.time_t_max tv_sec_min = self.time_t_min - for t in (_PyTime_MIN, _PyTime_MAX): + for t in (PyTime_MIN, PyTime_MAX): ts = _PyTime_AsTimeval_clamp(t, _PyTime.ROUND_CEILING) with decimal.localcontext() as context: context.rounding = decimal.ROUND_CEILING @@ -1028,7 +1023,7 @@ def test_AsTimeval_clamp(self): def test_AsTimespec_clamp(self): from _testinternalcapi import _PyTime_AsTimespec_clamp - for t in (_PyTime_MIN, _PyTime_MAX): + for t in (PyTime_MIN, PyTime_MAX): ts = _PyTime_AsTimespec_clamp(t) tv_sec, tv_nsec = divmod(t, NS_TO_SEC) if self.time_t_max < tv_sec: diff --git a/Lib/test/test_tkinter/support.py b/Lib/test/test_tkinter/support.py index a37705f0ae6febb..ebb9e00ff91bf0e 100644 --- a/Lib/test/test_tkinter/support.py +++ b/Lib/test/test_tkinter/support.py @@ -14,7 +14,7 @@ def setUpClass(cls): # Some window managers can maximize new windows. cls.root.wm_state('normal') try: - cls.root.wm_attributes('-zoomed', False) + cls.root.wm_attributes(zoomed=False) except tkinter.TclError: pass diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index dc8a810235fc9bd..81a20b698a72eb4 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -437,6 +437,61 @@ def test_info_patchlevel(self): self.assertTrue(str(vi).startswith(f'{vi.major}.{vi.minor}')) +class WmTest(AbstractTkTest, unittest.TestCase): + + def test_wm_attribute(self): + w = self.root + attributes = w.wm_attributes(return_python_dict=True) + self.assertIsInstance(attributes, dict) + attributes2 = w.wm_attributes() + self.assertIsInstance(attributes2, tuple) + self.assertEqual(attributes2[::2], + tuple('-' + k for k in attributes)) + self.assertEqual(attributes2[1::2], tuple(attributes.values())) + # silently deprecated + attributes3 = w.wm_attributes(None) + if self.wantobjects: + self.assertEqual(attributes3, attributes2) + else: + self.assertIsInstance(attributes3, str) + + for name in attributes: + self.assertEqual(w.wm_attributes(name), attributes[name]) + # silently deprecated + for name in attributes: + self.assertEqual(w.wm_attributes('-' + name), attributes[name]) + + self.assertIn('alpha', attributes) + self.assertIn('fullscreen', attributes) + self.assertIn('topmost', attributes) + if w._windowingsystem == "win32": + self.assertIn('disabled', attributes) + self.assertIn('toolwindow', attributes) + self.assertIn('transparentcolor', attributes) + if w._windowingsystem == "aqua": + self.assertIn('modified', attributes) + self.assertIn('notify', attributes) + self.assertIn('titlepath', attributes) + self.assertIn('transparent', attributes) + if w._windowingsystem == "x11": + self.assertIn('type', attributes) + self.assertIn('zoomed', attributes) + + w.wm_attributes(alpha=0.5) + self.assertEqual(w.wm_attributes('alpha'), + 0.5 if self.wantobjects else '0.5') + w.wm_attributes(alpha=1.0) + self.assertEqual(w.wm_attributes('alpha'), + 1.0 if self.wantobjects else '1.0') + # silently deprecated + w.wm_attributes('-alpha', 0.5) + self.assertEqual(w.wm_attributes('alpha'), + 0.5 if self.wantobjects else '0.5') + w.wm_attributes(alpha=1.0) + self.assertEqual(w.wm_attributes('alpha'), + 1.0 if self.wantobjects else '1.0') + + class BindTest(AbstractTkTest, unittest.TestCase): def setUp(self): @@ -706,6 +761,101 @@ def test3(e): pass self.assertCommandExist(funcid2) self.assertCommandExist(funcid3) + def _test_tag_bind(self, w): + tag = 'sel' + event = '<Control-Alt-Key-a>' + w.pack() + self.assertRaises(TypeError, w.tag_bind) + tag_bind = w._tag_bind if isinstance(w, tkinter.Text) else w.tag_bind + if isinstance(w, tkinter.Text): + self.assertRaises(TypeError, w.tag_bind, tag) + self.assertRaises(TypeError, w.tag_bind, tag, event) + self.assertEqual(tag_bind(tag), ()) + self.assertEqual(tag_bind(tag, event), '') + def test1(e): pass + def test2(e): pass + + funcid = w.tag_bind(tag, event, test1) + self.assertEqual(tag_bind(tag), (event,)) + script = tag_bind(tag, event) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + + funcid2 = w.tag_bind(tag, event, test2, add=True) + script = tag_bind(tag, event) + self.assertIn(funcid, script) + self.assertIn(funcid2, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + def _test_tag_unbind(self, w): + tag = 'sel' + event = '<Control-Alt-Key-b>' + w.pack() + tag_bind = w._tag_bind if isinstance(w, tkinter.Text) else w.tag_bind + self.assertEqual(tag_bind(tag), ()) + self.assertEqual(tag_bind(tag, event), '') + def test1(e): pass + def test2(e): pass + + funcid = w.tag_bind(tag, event, test1) + funcid2 = w.tag_bind(tag, event, test2, add=True) + + self.assertRaises(TypeError, w.tag_unbind, tag) + w.tag_unbind(tag, event) + self.assertEqual(tag_bind(tag, event), '') + self.assertEqual(tag_bind(tag), ()) + + def _test_tag_bind_rebind(self, w): + tag = 'sel' + event = '<Control-Alt-Key-d>' + w.pack() + tag_bind = w._tag_bind if isinstance(w, tkinter.Text) else w.tag_bind + self.assertEqual(tag_bind(tag), ()) + self.assertEqual(tag_bind(tag, event), '') + def test1(e): pass + def test2(e): pass + def test3(e): pass + + funcid = w.tag_bind(tag, event, test1) + funcid2 = w.tag_bind(tag, event, test2, add=True) + script = tag_bind(tag, event) + self.assertIn(funcid2, script) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + funcid3 = w.tag_bind(tag, event, test3) + script = tag_bind(tag, event) + self.assertNotIn(funcid, script) + self.assertNotIn(funcid2, script) + self.assertIn(funcid3, script) + self.assertCommandExist(funcid3) + + def test_canvas_tag_bind(self): + c = tkinter.Canvas(self.frame) + self._test_tag_bind(c) + + def test_canvas_tag_unbind(self): + c = tkinter.Canvas(self.frame) + self._test_tag_unbind(c) + + def test_canvas_tag_bind_rebind(self): + c = tkinter.Canvas(self.frame) + self._test_tag_bind_rebind(c) + + def test_text_tag_bind(self): + t = tkinter.Text(self.frame) + self._test_tag_bind(t) + + def test_text_tag_unbind(self): + t = tkinter.Text(self.frame) + self._test_tag_unbind(t) + + def test_text_tag_bind_rebind(self): + t = tkinter.Text(self.frame) + self._test_tag_bind_rebind(t) + def test_bindtags(self): f = self.frame self.assertEqual(self.root.bindtags(), ('.', 'Tk', 'all')) diff --git a/Lib/test/test_tkinter/test_text.py b/Lib/test/test_tkinter/test_text.py index f809c4510e3a1f3..b26956930d34028 100644 --- a/Lib/test/test_tkinter/test_text.py +++ b/Lib/test/test_tkinter/test_text.py @@ -52,27 +52,47 @@ def test_count(self): options = ('chars', 'indices', 'lines', 'displaychars', 'displayindices', 'displaylines', 'xpixels', 'ypixels') + self.assertEqual(len(text.count('1.0', 'end', *options, return_ints=True)), 8) self.assertEqual(len(text.count('1.0', 'end', *options)), 8) - self.assertEqual(text.count('1.0', 'end', 'chars', 'lines'), (124, 4)) + self.assertEqual(text.count('1.0', 'end', 'chars', 'lines', return_ints=True), + (124, 4)) self.assertEqual(text.count('1.3', '4.5', 'chars', 'lines'), (92, 3)) + self.assertEqual(text.count('4.5', '1.3', 'chars', 'lines', return_ints=True), + (-92, -3)) self.assertEqual(text.count('4.5', '1.3', 'chars', 'lines'), (-92, -3)) + self.assertEqual(text.count('1.3', '1.3', 'chars', 'lines', return_ints=True), + (0, 0)) self.assertEqual(text.count('1.3', '1.3', 'chars', 'lines'), (0, 0)) - self.assertEqual(text.count('1.0', 'end', 'lines'), 4) - self.assertEqual(text.count('end', '1.0', 'lines'), -4) - self.assertEqual(text.count('1.3', '1.5', 'lines'), 0) - self.assertEqual(text.count('1.3', '1.3', 'lines'), 0) - self.assertEqual(text.count('1.0', 'end'), 124) # 'indices' by default - self.assertEqual(text.count('1.0', 'end', 'indices'), 124) + self.assertEqual(text.count('1.0', 'end', 'lines', return_ints=True), 4) + self.assertEqual(text.count('1.0', 'end', 'lines'), (4,)) + self.assertEqual(text.count('end', '1.0', 'lines', return_ints=True), -4) + self.assertEqual(text.count('end', '1.0', 'lines'), (-4,)) + self.assertEqual(text.count('1.3', '1.5', 'lines', return_ints=True), 0) + self.assertEqual(text.count('1.3', '1.5', 'lines'), None) + self.assertEqual(text.count('1.3', '1.3', 'lines', return_ints=True), 0) + self.assertEqual(text.count('1.3', '1.3', 'lines'), None) + # Count 'indices' by default. + self.assertEqual(text.count('1.0', 'end', return_ints=True), 124) + self.assertEqual(text.count('1.0', 'end'), (124,)) + self.assertEqual(text.count('1.0', 'end', 'indices', return_ints=True), 124) + self.assertEqual(text.count('1.0', 'end', 'indices'), (124,)) self.assertRaises(tkinter.TclError, text.count, '1.0', 'end', 'spam') self.assertRaises(tkinter.TclError, text.count, '1.0', 'end', '-lines') - self.assertIsInstance(text.count('1.3', '1.5', 'ypixels'), int) + self.assertIsInstance(text.count('1.3', '1.5', 'ypixels', return_ints=True), int) + self.assertIsInstance(text.count('1.3', '1.5', 'ypixels'), tuple) + self.assertIsInstance(text.count('1.3', '1.5', 'update', 'ypixels', return_ints=True), int) self.assertIsInstance(text.count('1.3', '1.5', 'update', 'ypixels'), int) - self.assertEqual(text.count('1.3', '1.3', 'update', 'ypixels'), 0) + self.assertEqual(text.count('1.3', '1.3', 'update', 'ypixels', return_ints=True), 0) + self.assertEqual(text.count('1.3', '1.3', 'update', 'ypixels'), None) + self.assertEqual(text.count('1.3', '1.5', 'update', 'indices', return_ints=True), 2) self.assertEqual(text.count('1.3', '1.5', 'update', 'indices'), 2) - self.assertEqual(text.count('1.3', '1.3', 'update', 'indices'), 0) - self.assertEqual(text.count('1.3', '1.5', 'update'), 2) - self.assertEqual(text.count('1.3', '1.3', 'update'), 0) + self.assertEqual(text.count('1.3', '1.3', 'update', 'indices', return_ints=True), 0) + self.assertEqual(text.count('1.3', '1.3', 'update', 'indices'), None) + self.assertEqual(text.count('1.3', '1.5', 'update', return_ints=True), 2) + self.assertEqual(text.count('1.3', '1.5', 'update'), (2,)) + self.assertEqual(text.count('1.3', '1.3', 'update', return_ints=True), 0) + self.assertEqual(text.count('1.3', '1.3', 'update'), None) if __name__ == "__main__": diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 3d7c04172b87a0d..d910889b652f3ea 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1887,6 +1887,43 @@ def test_roundtrip(self): " print('Can not import' # comment2\n)" "else: print('Loaded')\n") + self.check_roundtrip("f'\\N{EXCLAMATION MARK}'") + self.check_roundtrip(r"f'\\N{SNAKE}'") + self.check_roundtrip(r"f'\\N{{SNAKE}}'") + self.check_roundtrip(r"f'\N{SNAKE}'") + self.check_roundtrip(r"f'\\\N{SNAKE}'") + self.check_roundtrip(r"f'\\\\\N{SNAKE}'") + self.check_roundtrip(r"f'\\\\\\\N{SNAKE}'") + + self.check_roundtrip(r"f'\\N{1}'") + self.check_roundtrip(r"f'\\\\N{2}'") + self.check_roundtrip(r"f'\\\\\\N{3}'") + self.check_roundtrip(r"f'\\\\\\\\N{4}'") + + self.check_roundtrip(r"f'\\N{{'") + self.check_roundtrip(r"f'\\\\N{{'") + self.check_roundtrip(r"f'\\\\\\N{{'") + self.check_roundtrip(r"f'\\\\\\\\N{{'") + cases = [ + """ +if 1: + "foo" +"bar" +""", + """ +if 1: + ("foo" + "bar") +""", + """ +if 1: + "foo" + "bar" +""" ] + for case in cases: + self.check_roundtrip(case) + + def test_continuation(self): # Balancing continuation self.check_roundtrip("a = (3,4, \n" @@ -1921,9 +1958,6 @@ def test_random_files(self): tempdir = os.path.dirname(__file__) or os.curdir testfiles = glob.glob(os.path.join(glob.escape(tempdir), "test*.py")) - # TODO: Remove this once we can untokenize PEP 701 syntax - testfiles.remove(os.path.join(tempdir, "test_fstring.py")) - if not support.is_resource_enabled("cpu"): testfiles = random.sample(testfiles, 10) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 372fc48bf81a6a3..dd9b1850adf0860 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -3124,10 +3124,13 @@ def test_smoke_user_exception(self): class MyException(Exception): pass - self.do_test_smoke( - MyException('bad things happened'), - ('test.test_traceback.TestTracebackException.' - 'test_smoke_user_exception.<locals>.MyException')) + if __name__ == '__main__': + expected = ('TestTracebackException.' + 'test_smoke_user_exception.<locals>.MyException') + else: + expected = ('test.test_traceback.TestTracebackException.' + 'test_smoke_user_exception.<locals>.MyException') + self.do_test_smoke(MyException('bad things happened'), expected) def test_from_exception(self): # Check all the parameters are accepted. diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 295df78a17374a3..58572c6f4d31578 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -79,6 +79,19 @@ class C: _clear_type_cache() + def test_per_class_limit(self): + class C: + x = 0 + + type_assign_version(C) + orig_version = type_get_version(C) + for i in range(1001): + C.x = i + type_assign_version(C) + + new_version = type_get_version(C) + self.assertEqual(new_version, 0) + @support.cpython_only class TypeCacheWithSpecializationTests(unittest.TestCase): diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py index 5a911da56f8f8a1..ee8939f62d082cd 100644 --- a/Lib/test/test_type_comments.py +++ b/Lib/test/test_type_comments.py @@ -309,7 +309,7 @@ def test_withstmt(self): self.assertEqual(tree.body[0].type_comment, None) def test_parenthesized_withstmt(self): - for tree in self.parse_all(parenthesized_withstmt, minver=9): + for tree in self.parse_all(parenthesized_withstmt): self.assertEqual(tree.body[0].type_comment, "int") self.assertEqual(tree.body[1].type_comment, "int") tree = self.classic_parse(parenthesized_withstmt) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index b684af4f33ed71d..176623171c98888 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4323,6 +4323,16 @@ class C(B[int]): c.bar = 'abc' self.assertEqual(c.__dict__, {'bar': 'abc'}) + def test_setattr_exceptions(self): + class Immutable[T]: + def __setattr__(self, key, value): + raise RuntimeError("immutable") + + # gh-115165: This used to cause RuntimeError to be raised + # when we tried to set `__orig_class__` on the `Immutable` instance + # returned by the `Immutable[int]()` call + self.assertIsInstance(Immutable[int](), Immutable) + def test_subscripted_generics_as_proxies(self): T = TypeVar('T') class C(Generic[T]): @@ -4920,6 +4930,75 @@ class B(Generic[S]): ... class C(List[int], B): ... self.assertEqual(C.__mro__, (C, list, B, Generic, object)) + def test_multiple_inheritance_non_type_with___mro_entries__(self): + class GoodEntries: + def __mro_entries__(self, bases): + return (object,) + + class A(List[int], GoodEntries()): ... + + self.assertEqual(A.__mro__, (A, list, Generic, object)) + + def test_multiple_inheritance_non_type_without___mro_entries__(self): + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex(TypeError, r"^bases must be types"): + class A(List[int], object()): ... + + def test_multiple_inheritance_non_type_bad___mro_entries__(self): + class BadEntries: + def __mro_entries__(self, bases): + return None + + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex( + TypeError, + r"^__mro_entries__ must return a tuple", + ): + class A(List[int], BadEntries()): ... + + def test_multiple_inheritance___mro_entries___returns_non_type(self): + class BadEntries: + def __mro_entries__(self, bases): + return (object(),) + + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex( + TypeError, + r"^bases must be types", + ): + class A(List[int], BadEntries()): ... + + def test_multiple_inheritance_with_genericalias(self): + class A(typing.Sized, list[int]): ... + + self.assertEqual( + A.__mro__, + (A, collections.abc.Sized, Generic, list, object), + ) + + def test_multiple_inheritance_with_genericalias_2(self): + T = TypeVar("T") + + class BaseSeq(typing.Sequence[T]): ... + class MySeq(List[T], BaseSeq[T]): ... + + self.assertEqual( + MySeq.__mro__, + ( + MySeq, + list, + BaseSeq, + collections.abc.Sequence, + collections.abc.Reversible, + collections.abc.Collection, + collections.abc.Sized, + collections.abc.Iterable, + collections.abc.Container, + Generic, + object, + ), + ) + def test_init_subclass_super_called(self): class FinalException(Exception): pass @@ -6077,8 +6156,6 @@ def test_overload_registry_repeated(self): self.assertEqual(list(get_overloads(impl)), overloads) -# Definitions needed for features introduced in Python 3.6 - from test.typinganndata import ( ann_module, ann_module2, ann_module3, ann_module5, ann_module6, ) @@ -8492,6 +8569,17 @@ def test_instantiate_generic(self): self.assertEqual(MyCount([4, 4, 5]), {4: 2, 5: 1}) self.assertEqual(MyCount[int]([4, 4, 5]), {4: 2, 5: 1}) + def test_instantiate_immutable(self): + class C: + def __setattr__(self, key, value): + raise Exception("should be ignored") + + A = Annotated[C, "a decoration"] + # gh-115165: This used to cause RuntimeError to be raised + # when we tried to set `__orig_class__` on the `C` instance + # returned by the `A()` call + self.assertIsInstance(A(), C) + def test_cannot_instantiate_forward(self): A = Annotated["int", (5, 6)] with self.assertRaises(TypeError): diff --git a/Lib/test/test_unicode_file_functions.py b/Lib/test/test_unicode_file_functions.py index 47619c8807bafe3..25c16e3a0b7e430 100644 --- a/Lib/test/test_unicode_file_functions.py +++ b/Lib/test/test_unicode_file_functions.py @@ -5,7 +5,7 @@ import unittest import warnings from unicodedata import normalize -from test.support import os_helper +from test.support import is_apple, os_helper from test import support @@ -23,13 +23,13 @@ '10_\u1fee\u1ffd', ] -# Mac OS X decomposes Unicode names, using Normal Form D. +# Apple platforms decompose Unicode names, using Normal Form D. # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html # "However, most volume formats do not follow the exact specification for # these normal forms. For example, HFS Plus uses a variant of Normal Form D # in which U+2000 through U+2FFF, U+F900 through U+FAFF, and U+2F800 through # U+2FAFF are not decomposed." -if sys.platform != 'darwin': +if not is_apple: filenames.extend([ # Specific code points: NFC(fn), NFD(fn), NFKC(fn) and NFKD(fn) all different '11_\u0385\u03d3\u03d4', @@ -119,11 +119,11 @@ def test_open(self): os.stat(name) self._apply_failure(os.listdir, name, self._listdir_failure) - # Skip the test on darwin, because darwin does normalize the filename to + # Skip the test on Apple platforms, because they don't normalize the filename to # NFD (a variant of Unicode NFD form). Normalize the filename to NFC, NFKC, # NFKD in Python is useless, because darwin will normalize it later and so # open(), os.stat(), etc. don't raise any exception. - @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') + @unittest.skipIf(is_apple, 'irrelevant test on Apple platforms') @unittest.skipIf( support.is_emscripten or support.is_wasi, "test fails on Emscripten/WASI when host platform is macOS." @@ -142,10 +142,10 @@ def test_normalize(self): self._apply_failure(os.remove, name) self._apply_failure(os.listdir, name) - # Skip the test on darwin, because darwin uses a normalization different + # Skip the test on Apple platforms, because they use a normalization different # than Python NFD normalization: filenames are different even if we use # Python NFD normalization. - @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') + @unittest.skipIf(is_apple, 'irrelevant test on Apple platforms') def test_listdir(self): sf0 = set(self.files) with warnings.catch_warnings(): diff --git a/Lib/test/test_unittest/test_discovery.py b/Lib/test/test_unittest/test_discovery.py index dcb72d73efceabb..004898ed4318348 100644 --- a/Lib/test/test_unittest/test_discovery.py +++ b/Lib/test/test_unittest/test_discovery.py @@ -571,7 +571,7 @@ def _get_module_from_name(name): result = unittest.TestResult() suite.run(result) self.assertEqual(len(result.skipped), 1) - self.assertEqual(result.testsRun, 0) + self.assertEqual(result.testsRun, 1) self.assertEqual(import_calls, ['my_package']) # Check picklability diff --git a/Lib/test/test_unittest/test_skipping.py b/Lib/test/test_unittest/test_skipping.py index 1a6af06d32b4339..f146dcac18ecc09 100644 --- a/Lib/test/test_unittest/test_skipping.py +++ b/Lib/test/test_unittest/test_skipping.py @@ -103,16 +103,16 @@ def test_dont_skip(self): pass result = LoggingResult(events) self.assertIs(suite.run(result), result) self.assertEqual(len(result.skipped), 1) - expected = ['addSkip', 'stopTest', 'startTest', - 'addSuccess', 'stopTest'] + expected = ['startTest', 'addSkip', 'stopTest', + 'startTest', 'addSuccess', 'stopTest'] self.assertEqual(events, expected) - self.assertEqual(result.testsRun, 1) + self.assertEqual(result.testsRun, 2) self.assertEqual(result.skipped, [(test_do_skip, "testing")]) self.assertTrue(result.wasSuccessful()) events = [] result = test_do_skip.run() - self.assertEqual(events, ['startTestRun', 'addSkip', + self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip', 'stopTest', 'stopTestRun']) self.assertEqual(result.skipped, [(test_do_skip, "testing")]) @@ -135,13 +135,13 @@ def test_1(self): test = Foo("test_1") suite = unittest.TestSuite([test]) self.assertIs(suite.run(result), result) - self.assertEqual(events, ['addSkip', 'stopTest']) + self.assertEqual(events, ['startTest', 'addSkip', 'stopTest']) self.assertEqual(result.skipped, [(test, "testing")]) self.assertEqual(record, []) events = [] result = test.run() - self.assertEqual(events, ['startTestRun', 'addSkip', + self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip', 'stopTest', 'stopTestRun']) self.assertEqual(result.skipped, [(test, "testing")]) self.assertEqual(record, []) diff --git a/Lib/test/test_unittest/testmock/testpatch.py b/Lib/test/test_unittest/testmock/testpatch.py index 833d7da1f31a206..d0046d702a53f42 100644 --- a/Lib/test/test_unittest/testmock/testpatch.py +++ b/Lib/test/test_unittest/testmock/testpatch.py @@ -1912,7 +1912,7 @@ def foo(x=0): with patch.object(foo, '__module__', "testpatch2"): self.assertEqual(foo.__module__, "testpatch2") - self.assertEqual(foo.__module__, 'test.test_unittest.testmock.testpatch') + self.assertEqual(foo.__module__, __name__) with patch.object(foo, '__annotations__', dict([('s', 1, )])): self.assertEqual(foo.__annotations__, dict([('s', 1, )])) diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index 6f698a8d8918154..bb15f64c59dbd13 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -370,13 +370,13 @@ def test_slices(self): self.check_ast_roundtrip("a[i:j, k]") def test_invalid_raise(self): - self.check_invalid(ast.Raise(exc=None, cause=ast.Name(id="X"))) + self.check_invalid(ast.Raise(exc=None, cause=ast.Name(id="X", ctx=ast.Load()))) def test_invalid_fstring_value(self): self.check_invalid( ast.JoinedStr( values=[ - ast.Name(id="test"), + ast.Name(id="test", ctx=ast.Load()), ast.Constant(value="test") ] ) @@ -649,6 +649,30 @@ def test_multiquote_joined_string(self): self.check_ast_roundtrip("""f'''""\"''\\'{"\\n\\"'"}''' """) self.check_ast_roundtrip("""f'''""\"''\\'{""\"\\n\\"'''""\" '''\\n'''}''' """) + def test_backslash_in_format_spec(self): + import re + msg = re.escape("invalid escape sequence '\\ '") + with self.assertWarnsRegex(SyntaxWarning, msg): + self.check_ast_roundtrip("""f"{x:\\ }" """) + self.check_ast_roundtrip("""f"{x:\\n}" """) + + self.check_ast_roundtrip("""f"{x:\\\\ }" """) + + with self.assertWarnsRegex(SyntaxWarning, msg): + self.check_ast_roundtrip("""f"{x:\\\\\\ }" """) + self.check_ast_roundtrip("""f"{x:\\\\\\n}" """) + + self.check_ast_roundtrip("""f"{x:\\\\\\\\ }" """) + + def test_quote_in_format_spec(self): + self.check_ast_roundtrip("""f"{x:'}" """) + self.check_ast_roundtrip("""f"{x:\\'}" """) + self.check_ast_roundtrip("""f"{x:\\\\'}" """) + + self.check_ast_roundtrip("""f'\\'{x:"}' """) + self.check_ast_roundtrip("""f'\\'{x:\\"}' """) + self.check_ast_roundtrip("""f'\\'{x:\\\\"}' """) + class ManualASTCreationTestCase(unittest.TestCase): """Test that AST nodes created without a type_params field unparse correctly.""" @@ -694,7 +718,7 @@ def test_function_with_type_params_and_bound(self): body=[ast.Pass()], decorator_list=[], returns=None, - type_params=[ast.TypeVar("T", bound=ast.Name("int"))], + type_params=[ast.TypeVar("T", bound=ast.Name("int", ctx=ast.Load()))], ) ast.fix_missing_locations(node) self.assertEqual(ast.unparse(node), "def f[T: int]():\n pass") diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 99c9e24994732fa..739c15df13de210 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1,6 +1,7 @@ import unittest from test import support from test.support import os_helper +from test.support import requires_subprocess from test.support import warnings_helper from test import test_urllib from unittest import mock @@ -14,10 +15,11 @@ import subprocess import urllib.request -# The proxy bypass method imported below has logic specific to the OSX -# proxy config data structure but is testable on all platforms. +# The proxy bypass method imported below has logic specific to the +# corresponding system but is testable on all platforms. from urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler, HTTPPasswordMgrWithPriorAuth, _parse_proxy, + _proxy_bypass_winreg_override, _proxy_bypass_macosx_sysconf, AbstractDigestAuthHandler) from urllib.parse import urlparse @@ -998,6 +1000,7 @@ def test_http_body_fileobj(self): file_obj.close() + @requires_subprocess() def test_http_body_pipe(self): # A file reading from a pipe. # A pipe cannot be seek'ed. There is no way to determine the @@ -1483,6 +1486,30 @@ def test_proxy_https_proxy_authorization(self): self.assertEqual(req.host, "proxy.example.com:3128") self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") + @unittest.skipUnless(os.name == "nt", "only relevant for Windows") + def test_winreg_proxy_bypass(self): + proxy_override = "www.example.com;*.example.net; 192.168.0.1" + proxy_bypass = _proxy_bypass_winreg_override + for host in ("www.example.com", "www.example.net", "192.168.0.1"): + self.assertTrue(proxy_bypass(host, proxy_override), + "expected bypass of %s to be true" % host) + + for host in ("example.com", "www.example.org", "example.net", + "192.168.0.2"): + self.assertFalse(proxy_bypass(host, proxy_override), + "expected bypass of %s to be False" % host) + + # check intranet address bypass + proxy_override = "example.com; <local>" + self.assertTrue(proxy_bypass("example.com", proxy_override), + "expected bypass of %s to be true" % host) + self.assertFalse(proxy_bypass("example.net", proxy_override), + "expected bypass of %s to be False" % host) + for host in ("test", "localhost"): + self.assertTrue(proxy_bypass(host, proxy_override), + "expect <local> to bypass intranet address '%s'" + % host) + @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") def test_osx_proxy_bypass(self): bypass = { diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 6dda00efd7bbb69..ba31beb81e80b09 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -19,8 +19,8 @@ import tempfile from test.support import (captured_stdout, captured_stderr, skip_if_broken_multiprocessing_synchronize, verbose, - requires_subprocess, is_emscripten, is_wasi, - requires_venv_with_pip, TEST_HOME_DIR, + requires_subprocess, is_apple_mobile, is_emscripten, + is_wasi, requires_venv_with_pip, TEST_HOME_DIR, requires_resource, copy_python_src_ignore) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) import unittest @@ -39,8 +39,10 @@ or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') -if is_emscripten or is_wasi: - raise unittest.SkipTest("venv is not available on Emscripten/WASI.") +# Skip tests on WASM platforms, plus iOS/tvOS/watchOS +if is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest(f"venv tests not required on {sys.platform}") + @requires_subprocess() def check_output(cmd, encoding=None): diff --git a/Lib/test/test_winapi.py b/Lib/test/test_winapi.py new file mode 100644 index 000000000000000..014aeea7239e2b1 --- /dev/null +++ b/Lib/test/test_winapi.py @@ -0,0 +1,94 @@ +# Test the Windows-only _winapi module + +import random +import threading +import time +import unittest +from test.support import import_helper + +_winapi = import_helper.import_module('_winapi', required_on=['win']) + +MAXIMUM_WAIT_OBJECTS = 64 +MAXIMUM_BATCHED_WAIT_OBJECTS = (MAXIMUM_WAIT_OBJECTS - 1) ** 2 + +class WinAPIBatchedWaitForMultipleObjectsTests(unittest.TestCase): + def _events_waitall_test(self, n): + evts = [_winapi.CreateEventW(0, False, False, None) for _ in range(n)] + + with self.assertRaises(TimeoutError): + _winapi.BatchedWaitForMultipleObjects(evts, True, 100) + + # Ensure no errors raised when all are triggered + for e in evts: + _winapi.SetEvent(e) + try: + _winapi.BatchedWaitForMultipleObjects(evts, True, 100) + except TimeoutError: + self.fail("expected wait to complete immediately") + + # Choose 8 events to set, distributed throughout the list, to make sure + # we don't always have them in the first chunk + chosen = [i * (len(evts) // 8) for i in range(8)] + + # Replace events with invalid handles to make sure we fail + for i in chosen: + old_evt = evts[i] + evts[i] = -1 + with self.assertRaises(OSError): + _winapi.BatchedWaitForMultipleObjects(evts, True, 100) + evts[i] = old_evt + + + def _events_waitany_test(self, n): + evts = [_winapi.CreateEventW(0, False, False, None) for _ in range(n)] + + with self.assertRaises(TimeoutError): + _winapi.BatchedWaitForMultipleObjects(evts, False, 100) + + # Choose 8 events to set, distributed throughout the list, to make sure + # we don't always have them in the first chunk + chosen = [i * (len(evts) // 8) for i in range(8)] + + # Trigger one by one. They are auto-reset events, so will only trigger once + for i in chosen: + with self.subTest(f"trigger event {i} of {len(evts)}"): + _winapi.SetEvent(evts[i]) + triggered = _winapi.BatchedWaitForMultipleObjects(evts, False, 10000) + self.assertSetEqual(set(triggered), {i}) + + # Trigger all at once. This may require multiple calls + for i in chosen: + _winapi.SetEvent(evts[i]) + triggered = set() + while len(triggered) < len(chosen): + triggered.update(_winapi.BatchedWaitForMultipleObjects(evts, False, 10000)) + self.assertSetEqual(triggered, set(chosen)) + + # Replace events with invalid handles to make sure we fail + for i in chosen: + with self.subTest(f"corrupt event {i} of {len(evts)}"): + old_evt = evts[i] + evts[i] = -1 + with self.assertRaises(OSError): + _winapi.BatchedWaitForMultipleObjects(evts, False, 100) + evts[i] = old_evt + + + def test_few_events_waitall(self): + self._events_waitall_test(16) + + def test_many_events_waitall(self): + self._events_waitall_test(256) + + def test_max_events_waitall(self): + self._events_waitall_test(MAXIMUM_BATCHED_WAIT_OBJECTS) + + + def test_few_events_waitany(self): + self._events_waitany_test(16) + + def test_many_events_waitany(self): + self._events_waitany_test(256) + + def test_max_events_waitany(self): + self._events_waitany_test(MAXIMUM_BATCHED_WAIT_OBJECTS) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index b9e7937b0bbc00a..c535d631bb646f7 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -13,6 +13,7 @@ import operator import os import pickle +import pyexpat import sys import textwrap import types @@ -120,6 +121,10 @@ </foo> """ +fails_with_expat_2_6_0 = (unittest.expectedFailure + if pyexpat.version_info >= (2, 6, 0) else + lambda test: test) + def checkwarnings(*filters, quiet=False): def decorator(test): def newtest(*args, **kwargs): @@ -536,7 +541,9 @@ def test_iterparse(self): iterparse = ET.iterparse context = iterparse(SIMPLE_XMLFILE) + self.assertIsNone(context.root) action, elem = next(context) + self.assertIsNone(context.root) self.assertEqual((action, elem.tag), ('end', 'element')) self.assertEqual([(action, elem.tag) for action, elem in context], [ ('end', 'element'), @@ -553,6 +560,17 @@ def test_iterparse(self): ('end', '{namespace}root'), ]) + with open(SIMPLE_XMLFILE, 'rb') as source: + context = iterparse(source) + action, elem = next(context) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in context], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(context.root.tag, 'root') + events = () context = iterparse(SIMPLE_XMLFILE, events) self.assertEqual([(action, elem.tag) for action, elem in context], []) @@ -644,12 +662,81 @@ def test_iterparse(self): # Not exhausting the iterator still closes the resource (bpo-43292) with warnings_helper.check_no_resource_warning(self): - it = iterparse(TESTFN) + it = iterparse(SIMPLE_XMLFILE) del it + with warnings_helper.check_no_resource_warning(self): + it = iterparse(SIMPLE_XMLFILE) + it.close() + del it + + with warnings_helper.check_no_resource_warning(self): + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + + with warnings_helper.check_no_resource_warning(self): + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + it.close() + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + with self.assertRaises(FileNotFoundError): iterparse("nonexistent") + def test_iterparse_close(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + it = iterparse(SIMPLE_XMLFILE) + list(it) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + list(it) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + def test_writefile(self): elem = ET.Element("tag") elem.text = "text" @@ -1398,28 +1485,37 @@ def assert_event_tags(self, parser, expected, max_events=None): self.assertEqual([(action, elem.tag) for action, elem in events], expected) - def test_simple_xml(self): - for chunk_size in (None, 1, 5): - with self.subTest(chunk_size=chunk_size): - parser = ET.XMLPullParser() - self.assert_event_tags(parser, []) - self._feed(parser, "<!-- comment -->\n", chunk_size) - self.assert_event_tags(parser, []) - self._feed(parser, - "<root>\n <element key='value'>text</element", - chunk_size) - self.assert_event_tags(parser, []) - self._feed(parser, ">\n", chunk_size) - self.assert_event_tags(parser, [('end', 'element')]) - self._feed(parser, "<element>text</element>tail\n", chunk_size) - self._feed(parser, "<empty-element/>\n", chunk_size) - self.assert_event_tags(parser, [ - ('end', 'element'), - ('end', 'empty-element'), - ]) - self._feed(parser, "</root>\n", chunk_size) - self.assert_event_tags(parser, [('end', 'root')]) - self.assertIsNone(parser.close()) + def test_simple_xml(self, chunk_size=None): + parser = ET.XMLPullParser() + self.assert_event_tags(parser, []) + self._feed(parser, "<!-- comment -->\n", chunk_size) + self.assert_event_tags(parser, []) + self._feed(parser, + "<root>\n <element key='value'>text</element", + chunk_size) + self.assert_event_tags(parser, []) + self._feed(parser, ">\n", chunk_size) + self.assert_event_tags(parser, [('end', 'element')]) + self._feed(parser, "<element>text</element>tail\n", chunk_size) + self._feed(parser, "<empty-element/>\n", chunk_size) + self.assert_event_tags(parser, [ + ('end', 'element'), + ('end', 'empty-element'), + ]) + self._feed(parser, "</root>\n", chunk_size) + self.assert_event_tags(parser, [('end', 'root')]) + self.assertIsNone(parser.close()) + + @fails_with_expat_2_6_0 + def test_simple_xml_chunk_1(self): + self.test_simple_xml(chunk_size=1) + + @fails_with_expat_2_6_0 + def test_simple_xml_chunk_5(self): + self.test_simple_xml(chunk_size=5) + + def test_simple_xml_chunk_22(self): + self.test_simple_xml(chunk_size=22) def test_feed_while_iterating(self): parser = ET.XMLPullParser() @@ -3042,8 +3138,7 @@ def test_basic(self): # With an explicit parser too (issue #9708) sourcefile = serialize(doc, to_string=False) parser = ET.XMLParser(target=ET.TreeBuilder()) - self.assertEqual(next(ET.iterparse(sourcefile, parser=parser))[0], - 'end') + self.assertEqual(next(ET.iterparse(sourcefile, parser=parser))[0], 'end') tree = ET.ElementTree(None) self.assertRaises(AttributeError, tree.iter) diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 9bdb08aeabb7817..7d89f753448dd75 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -4,7 +4,6 @@ import io import itertools import os -import pathlib import posixpath import struct import subprocess @@ -18,13 +17,14 @@ from tempfile import TemporaryFile from random import randint, random, randbytes +from test import archiver_tests from test.support import script_helper from test.support import ( findfile, requires_zlib, requires_bz2, requires_lzma, captured_stdout, captured_stderr, requires_subprocess ) from test.support.os_helper import ( - TESTFN, unlink, rmtree, temp_dir, temp_cwd, fd_count + TESTFN, unlink, rmtree, temp_dir, temp_cwd, fd_count, FakePath ) @@ -159,7 +159,7 @@ def test_open(self): self.zip_open_test(f, self.compression) def test_open_with_pathlike(self): - path = pathlib.Path(TESTFN2) + path = FakePath(TESTFN2) self.zip_open_test(path, self.compression) with zipfile.ZipFile(path, "r", self.compression) as zipfp: self.assertIsInstance(zipfp.filename, str) @@ -446,6 +446,27 @@ def write(self, data): self.assertEqual(zipfp.read('file1'), b'data1') self.assertEqual(zipfp.read('file2'), b'data2') + def test_zipextfile_attrs(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: + zipfp.writestr(fname, "bogus") + + with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: + with zipfp.open(fname) as fid: + self.assertEqual(fid.name, fname) + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertEqual(fid.mode, 'r') + self.assertIs(fid.readable(), True) + self.assertIs(fid.writable(), False) + self.assertIs(fid.seekable(), True) + self.assertIs(fid.closed, False) + self.assertIs(fid.closed, True) + self.assertEqual(fid.name, fname) + self.assertEqual(fid.mode, 'r') + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertRaises(ValueError, fid.readable) + self.assertIs(fid.writable(), False) + self.assertRaises(ValueError, fid.seekable) def tearDown(self): unlink(TESTFN) @@ -577,17 +598,16 @@ def test_write_default_name(self): def test_io_on_closed_zipextfile(self): fname = "somefile.txt" - with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: + with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: zipfp.writestr(fname, "bogus") with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: with zipfp.open(fname) as fid: fid.close() + self.assertIs(fid.closed, True) self.assertRaises(ValueError, fid.read) self.assertRaises(ValueError, fid.seek, 0) self.assertRaises(ValueError, fid.tell) - self.assertRaises(ValueError, fid.readable) - self.assertRaises(ValueError, fid.seekable) def test_write_to_readonly(self): """Check that trying to call write() on a readonly ZipFile object @@ -1284,6 +1304,21 @@ def test_issue44439(self): self.assertEqual(data.write(q), LENGTH) self.assertEqual(zip.getinfo('data').file_size, LENGTH) + def test_zipwritefile_attrs(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: + with zipfp.open(fname, 'w') as fid: + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertIs(fid.readable(), False) + self.assertIs(fid.writable(), True) + self.assertIs(fid.seekable(), False) + self.assertIs(fid.closed, False) + self.assertIs(fid.closed, True) + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertIs(fid.readable(), False) + self.assertIs(fid.writable(), True) + self.assertIs(fid.seekable(), False) + class StoredWriterTests(AbstractWriterTests, unittest.TestCase): compression = zipfile.ZIP_STORED @@ -1486,7 +1521,7 @@ def test_write_pathlike(self): fp.write("print(42)\n") with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: - zipfp.writepy(pathlib.Path(TESTFN2) / "mod1.py") + zipfp.writepy(FakePath(os.path.join(TESTFN2, "mod1.py"))) names = zipfp.namelist() self.assertCompiledIn('mod1.py', names) finally: @@ -1544,7 +1579,7 @@ def test_extract_with_target(self): def test_extract_with_target_pathlike(self): with temp_dir() as extdir: - self._test_extract_with_target(pathlib.Path(extdir)) + self._test_extract_with_target(FakePath(extdir)) def test_extract_all(self): with temp_cwd(): @@ -1579,7 +1614,7 @@ def test_extract_all_with_target(self): def test_extract_all_with_target_pathlike(self): with temp_dir() as extdir: - self._test_extract_all_with_target(pathlib.Path(extdir)) + self._test_extract_all_with_target(FakePath(extdir)) def check_file(self, filename, content): self.assertTrue(os.path.isfile(filename)) @@ -1687,6 +1722,33 @@ def _test_extract_hackers_arcnames(self, hacknames): unlink(TESTFN2) +class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase): + testdir = TESTFN + + @classmethod + def setUpClass(cls): + p = cls.ar_with_file = TESTFN + '-with-file.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.writestr('test', b'newcontent') + + p = cls.ar_with_dir = TESTFN + '-with-dir.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.mkdir('test') + + p = cls.ar_with_implicit_dir = TESTFN + '-with-implicit-dir.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.writestr('test/file', b'newcontent') + + def open(self, path): + return zipfile.ZipFile(path, 'r') + + def extractall(self, ar): + ar.extractall(self.testdir) + + class OtherTests(unittest.TestCase): def test_open_via_zip_info(self): # Create the ZIP archive @@ -1865,7 +1927,7 @@ def test_is_zip_erroneous_file(self): fp.write("this is not a legal zip file\n") self.assertFalse(zipfile.is_zipfile(TESTFN)) # - passing a path-like object - self.assertFalse(zipfile.is_zipfile(pathlib.Path(TESTFN))) + self.assertFalse(zipfile.is_zipfile(FakePath(TESTFN))) # - passing a file object with open(TESTFN, "rb") as fp: self.assertFalse(zipfile.is_zipfile(fp)) @@ -2959,7 +3021,7 @@ def test_create_directory_with_write(self): directory = os.path.join(TESTFN2, "directory2") os.mkdir(directory) - mode = os.stat(directory).st_mode + mode = os.stat(directory).st_mode & 0xFFFF zf.write(directory, arcname="directory2/") zinfo = zf.filelist[1] self.assertEqual(zinfo.filename, "directory2/") @@ -2985,7 +3047,7 @@ def test_from_file(self): self.assertEqual(zi.file_size, os.path.getsize(__file__)) def test_from_file_pathlike(self): - zi = zipfile.ZipInfo.from_file(pathlib.Path(__file__)) + zi = zipfile.ZipInfo.from_file(FakePath(__file__)) self.assertEqual(posixpath.basename(zi.filename), 'test_core.py') self.assertFalse(zi.is_dir()) self.assertEqual(zi.file_size, os.path.getsize(__file__)) diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index 18eab5b33540c9e..8414721555731eb 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -20,7 +20,7 @@ from test.support import MISSING_C_DOCSTRINGS from test.test_zoneinfo import _support as test_support from test.test_zoneinfo._support import OS_ENV_LOCK, TZPATH_TEST_LOCK, ZoneInfoTestBase -from test.support.import_helper import import_module +from test.support.import_helper import import_module, CleanImport lzma = import_module('lzma') py_zoneinfo, c_zoneinfo = test_support.get_modules() @@ -1720,13 +1720,26 @@ def test_env_variable_relative_paths(self): with self.subTest("warning", path_var=path_var): # Note: Per PEP 615 the warning is implementation-defined # behavior, other implementations need not warn. - with self.assertWarns(self.module.InvalidTZPathWarning): + with self.assertWarns(self.module.InvalidTZPathWarning) as w: self.module.reset_tzpath() + self.assertEqual(w.warnings[0].filename, __file__) tzpath = self.module.TZPATH with self.subTest("filtered", path_var=path_var): self.assertSequenceEqual(tzpath, expected_paths) + def test_env_variable_relative_paths_warning_location(self): + path_var = "path/to/somewhere" + + with self.python_tzpath_context(path_var): + with CleanImport("zoneinfo", "zoneinfo._tzpath"): + with self.assertWarns(RuntimeWarning) as w: + import zoneinfo + InvalidTZPathWarning = zoneinfo.InvalidTZPathWarning + self.assertIsInstance(w.warnings[0].message, InvalidTZPathWarning) + # It should represent the current file: + self.assertEqual(w.warnings[0].filename, __file__) + def test_reset_tzpath_kwarg(self): self.module.reset_tzpath(to=[f"{DRIVE}/a/b/c"]) diff --git a/Lib/threading.py b/Lib/threading.py index 00b95f8d92a1f06..b6ff00acadd58fe 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -3,7 +3,6 @@ import os as _os import sys as _sys import _thread -import functools import warnings from time import monotonic as _time @@ -950,7 +949,6 @@ def _after_fork(self, new_ident=None): # This thread is alive. self._ident = new_ident if self._handle is not None: - self._handle.after_fork_alive() assert self._handle.ident == new_ident # bpo-42350: If the fork happens when the thread is already stopped # (ex: after threading._shutdown() has been called), _tstate_lock @@ -966,9 +964,7 @@ def _after_fork(self, new_ident=None): self._is_stopped = True self._tstate_lock = None self._join_lock = None - if self._handle is not None: - self._handle.after_fork_dead() - self._handle = None + self._handle = None def __repr__(self): assert self._initialized, "Thread.__init__() was not called" @@ -1630,8 +1626,7 @@ def _register_atexit(func, *arg, **kwargs): if _SHUTTING_DOWN: raise RuntimeError("can't register atexit after shutdown") - call = functools.partial(func, *arg, **kwargs) - _threading_atexits.append(call) + _threading_atexits.append(lambda: func(*arg, **kwargs)) from _thread import stack_size diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index e0db41dd915ece9..175bfbd7d912d2e 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -1537,16 +1537,19 @@ def unbind(self, sequence, funcid=None): Otherwise destroy the current binding for SEQUENCE, leaving SEQUENCE unbound. """ + self._unbind(('bind', self._w, sequence), funcid) + + def _unbind(self, what, funcid=None): if funcid is None: - self.tk.call('bind', self._w, sequence, '') + self.tk.call(*what, '') else: - lines = self.tk.call('bind', self._w, sequence).split('\n') + lines = self.tk.call(what).split('\n') prefix = f'if {{"[{funcid} ' keep = '\n'.join(line for line in lines if not line.startswith(prefix)) if not keep.strip(): keep = '' - self.tk.call('bind', self._w, sequence, keep) + self.tk.call(*what, keep) self.deletecommand(funcid) def bind_all(self, sequence=None, func=None, add=None): @@ -1558,7 +1561,7 @@ def bind_all(self, sequence=None, func=None, add=None): def unbind_all(self, sequence): """Unbind for all widgets for event SEQUENCE all functions.""" - self.tk.call('bind', 'all' , sequence, '') + self._root()._unbind(('bind', 'all', sequence)) def bind_class(self, className, sequence=None, func=None, add=None): """Bind to widgets with bindtag CLASSNAME at event @@ -1573,7 +1576,7 @@ def bind_class(self, className, sequence=None, func=None, add=None): def unbind_class(self, className, sequence): """Unbind for all widgets with bindtag CLASSNAME for event SEQUENCE all functions.""" - self.tk.call('bind', className , sequence, '') + self._root()._unbind(('bind', className, sequence)) def mainloop(self, n=0): """Call the mainloop of Tk.""" @@ -2105,26 +2108,39 @@ def wm_aspect(self, aspect = wm_aspect - def wm_attributes(self, *args): - """This subcommand returns or sets platform specific attributes - - The first form returns a list of the platform specific flags and - their values. The second form returns the value for the specific - option. The third form sets one or more of the values. The values - are as follows: - - On Windows, -disabled gets or sets whether the window is in a - disabled state. -toolwindow gets or sets the style of the window - to toolwindow (as defined in the MSDN). -topmost gets or sets - whether this is a topmost window (displays above all other - windows). + def wm_attributes(self, *args, return_python_dict=False, **kwargs): + """Return or sets platform specific attributes. - On Macintosh, XXXXX + When called with a single argument return_python_dict=True, + return a dict of the platform specific attributes and their values. + When called without arguments or with a single argument + return_python_dict=False, return a tuple containing intermixed + attribute names with the minus prefix and their values. - On Unix, there are currently no special attribute values. + When called with a single string value, return the value for the + specific option. When called with keyword arguments, set the + corresponding attributes. """ - args = ('wm', 'attributes', self._w) + args - return self.tk.call(args) + if not kwargs: + if not args: + res = self.tk.call('wm', 'attributes', self._w) + if return_python_dict: + return _splitdict(self.tk, res) + else: + return self.tk.splitlist(res) + if len(args) == 1 and args[0] is not None: + option = args[0] + if option[0] == '-': + # TODO: deprecate + option = option[1:] + return self.tk.call('wm', 'attributes', self._w, '-' + option) + # TODO: deprecate + return self.tk.call('wm', 'attributes', self._w, *args) + elif args: + raise TypeError('wm_attribute() options have been specified as ' + 'positional and keyword arguments') + else: + self.tk.call('wm', 'attributes', self._w, *self._options(kwargs)) attributes = wm_attributes @@ -2885,9 +2901,7 @@ def bbox(self, *args): def tag_unbind(self, tagOrId, sequence, funcid=None): """Unbind for all items with TAGORID for event SEQUENCE the function identified with FUNCID.""" - self.tk.call(self._w, 'bind', tagOrId, sequence, '') - if funcid: - self.deletecommand(funcid) + self._unbind((self._w, 'bind', tagOrId, sequence), funcid) def tag_bind(self, tagOrId, sequence=None, func=None, add=None): """Bind to all items with TAGORID at event SEQUENCE a call to function FUNC. @@ -3731,7 +3745,7 @@ def compare(self, index1, op, index2): return self.tk.getboolean(self.tk.call( self._w, 'compare', index1, op, index2)) - def count(self, index1, index2, *options): # new in Tk 8.5 + def count(self, index1, index2, *options, return_ints=False): # new in Tk 8.5 """Counts the number of relevant things between the two indices. If INDEX1 is after INDEX2, the result will be a negative number @@ -3739,19 +3753,26 @@ def count(self, index1, index2, *options): # new in Tk 8.5 The actual items which are counted depends on the options given. The result is a tuple of integers, one for the result of each - counting option given, if more than one option is specified, - otherwise it is an integer. Valid counting options are "chars", - "displaychars", "displayindices", "displaylines", "indices", - "lines", "xpixels" and "ypixels". The default value, if no - option is specified, is "indices". There is an additional possible - option "update", which if given then all subsequent options ensure - that any possible out of date information is recalculated.""" + counting option given, if more than one option is specified or + return_ints is false (default), otherwise it is an integer. + Valid counting options are "chars", "displaychars", + "displayindices", "displaylines", "indices", "lines", "xpixels" + and "ypixels". The default value, if no option is specified, is + "indices". There is an additional possible option "update", + which if given then all subsequent options ensure that any + possible out of date information is recalculated. + """ options = ['-%s' % arg for arg in options] res = self.tk.call(self._w, 'count', *options, index1, index2) if not isinstance(res, int): res = self._getints(res) if len(res) == 1: res, = res + if not return_ints: + if not res: + res = None + elif len(options) <= 1: + res = (res,) return res def debug(self, boolean=None): @@ -3997,9 +4018,7 @@ def tag_add(self, tagName, index1, *args): def tag_unbind(self, tagName, sequence, funcid=None): """Unbind for all characters with TAGNAME for event SEQUENCE the function identified with FUNCID.""" - self.tk.call(self._w, 'tag', 'bind', tagName, sequence, '') - if funcid: - self.deletecommand(funcid) + return self._unbind((self._w, 'tag', 'bind', tagName, sequence), funcid) def tag_bind(self, tagName, sequence, func, add=None): """Bind to all characters with TAGNAME at event SEQUENCE a call to function FUNC. @@ -4010,6 +4029,11 @@ def tag_bind(self, tagName, sequence, func, add=None): return self._bind((self._w, 'tag', 'bind', tagName), sequence, func, add) + def _tag_bind(self, tagName, sequence=None, func=None, add=None): + # For tests only + return self._bind((self._w, 'tag', 'bind', tagName), + sequence, func, add) + def tag_cget(self, tagName, option): """Return the value of OPTION for tag TAGNAME.""" if option[:1] != '-': diff --git a/Lib/tkinter/simpledialog.py b/Lib/tkinter/simpledialog.py index 538bbfc318d7044..0f0dc66460f7987 100644 --- a/Lib/tkinter/simpledialog.py +++ b/Lib/tkinter/simpledialog.py @@ -262,7 +262,7 @@ def _setup_dialog(w): w.tk.call("::tk::unsupported::MacWindowStyle", "style", w, "moveableModal", "") elif w._windowingsystem == "x11": - w.wm_attributes("-type", "dialog") + w.wm_attributes(type="dialog") # -------------------------------------------------------------------- # convenience dialogues diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 12e2532de66d63f..4b39f345426a600 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -171,6 +171,7 @@ def __init__(self): self.tokens = [] self.prev_row = 1 self.prev_col = 0 + self.prev_type = None self.encoding = None def add_whitespace(self, start): @@ -186,6 +187,29 @@ def add_whitespace(self, start): if col_offset: self.tokens.append(" " * col_offset) + def escape_brackets(self, token): + characters = [] + consume_until_next_bracket = False + for character in token: + if character == "}": + if consume_until_next_bracket: + consume_until_next_bracket = False + else: + characters.append(character) + if character == "{": + n_backslashes = sum( + 1 for char in _itertools.takewhile( + "\\".__eq__, + characters[-2::-1] + ) + ) + if n_backslashes % 2 == 0: + characters.append(character) + else: + consume_until_next_bracket = True + characters.append(character) + return "".join(characters) + def untokenize(self, iterable): it = iter(iterable) indents = [] @@ -217,11 +241,13 @@ def untokenize(self, iterable): startline = False elif tok_type == FSTRING_MIDDLE: if '{' in token or '}' in token: + token = self.escape_brackets(token) + last_line = token.splitlines()[-1] end_line, end_col = end - end = (end_line, end_col + token.count('{') + token.count('}')) - token = re.sub('{', '{{', token) - token = re.sub('}', '}}', token) - + extra_chars = last_line.count("{{") + last_line.count("}}") + end = (end_line, end_col + extra_chars) + elif tok_type in (STRING, FSTRING_START) and self.prev_type in (STRING, FSTRING_END): + self.tokens.append(" ") self.add_whitespace(start) self.tokens.append(token) @@ -229,6 +255,7 @@ def untokenize(self, iterable): if tok_type in (NEWLINE, NL): self.prev_row += 1 self.prev_col = 0 + self.prev_type = tok_type return "".join(self.tokens) def compat(self, token, iterable): @@ -236,6 +263,7 @@ def compat(self, token, iterable): toks_append = self.tokens.append startline = token[0] in (NEWLINE, NL) prevstring = False + in_fstring = 0 for tok in _itertools.chain([token], iterable): toknum, tokval = tok[:2] @@ -254,6 +282,10 @@ def compat(self, token, iterable): else: prevstring = False + if toknum == FSTRING_START: + in_fstring += 1 + elif toknum == FSTRING_END: + in_fstring -= 1 if toknum == INDENT: indents.append(tokval) continue @@ -266,11 +298,18 @@ def compat(self, token, iterable): toks_append(indents[-1]) startline = False elif toknum == FSTRING_MIDDLE: - if '{' in tokval or '}' in tokval: - tokval = re.sub('{', '{{', tokval) - tokval = re.sub('}', '}}', tokval) + tokval = self.escape_brackets(tokval) + + # Insert a space between two consecutive brackets if we are in an f-string + if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring: + tokval = ' ' + tokval + + # Insert a space between two consecutive f-strings + if toknum in (STRING, FSTRING_START) and self.prev_type in (STRING, FSTRING_END): + self.tokens.append(" ") toks_append(tokval) + self.prev_type = toknum def untokenize(iterable): diff --git a/Lib/typing.py b/Lib/typing.py index d278b4effc7ebaf..914ddeaf504cd0e 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1127,7 +1127,9 @@ def __call__(self, *args, **kwargs): result = self.__origin__(*args, **kwargs) try: result.__orig_class__ = self - except AttributeError: + # Some objects raise TypeError (or something even more exotic) + # if you try to set attributes on them; we guard against that here + except Exception: pass return result @@ -1135,9 +1137,29 @@ def __mro_entries__(self, bases): res = [] if self.__origin__ not in bases: res.append(self.__origin__) + + # Check if any base that occurs after us in `bases` is either itself a + # subclass of Generic, or something which will add a subclass of Generic + # to `__bases__` via its `__mro_entries__`. If not, add Generic + # ourselves. The goal is to ensure that Generic (or a subclass) will + # appear exactly once in the final bases tuple. If we let it appear + # multiple times, we risk "can't form a consistent MRO" errors. i = bases.index(self) for b in bases[i+1:]: - if isinstance(b, _BaseGenericAlias) or issubclass(b, Generic): + if isinstance(b, _BaseGenericAlias): + break + if not isinstance(b, type): + meth = getattr(b, "__mro_entries__", None) + new_bases = meth(bases) if meth else None + if ( + isinstance(new_bases, tuple) and + any( + isinstance(b2, type) and issubclass(b2, Generic) + for b2 in new_bases + ) + ): + break + elif issubclass(b, Generic): break else: res.append(Generic) diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 811557498bb30ed..001b640dc43ad69 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -606,6 +606,7 @@ def run(self, result=None): else: stopTestRun = None + result.startTest(self) try: testMethod = getattr(self, self._testMethodName) if (getattr(self.__class__, "__unittest_skip__", False) or @@ -616,9 +617,6 @@ def run(self, result=None): _addSkip(result, self, skip_why) return result - # Increase the number of tests only if it hasn't been skipped - result.startTest(self) - expecting_failure = ( getattr(self, "__unittest_expecting_failure__", False) or getattr(testMethod, "__unittest_expecting_failure__", False) diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index 9e56f658027f4de..3ace0a5b7bf2efb 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -97,12 +97,10 @@ def _restoreStdout(self): sys.stdout = self._original_stdout sys.stderr = self._original_stderr - if self._stdout_buffer is not None: - self._stdout_buffer.seek(0) - self._stdout_buffer.truncate() - if self._stderr_buffer is not None: - self._stderr_buffer.seek(0) - self._stderr_buffer.truncate() + self._stdout_buffer.seek(0) + self._stdout_buffer.truncate() + self._stderr_buffer.seek(0) + self._stderr_buffer.truncate() def stopTestRun(self): """Called once after all tests are executed. diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index bca594420f6d9d2..d22af6618d80f16 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -2563,6 +2563,7 @@ def _proxy_bypass_macosx_sysconf(host, proxy_settings): } """ from fnmatch import fnmatch + from ipaddress import AddressValueError, IPv4Address hostonly, port = _splitport(host) @@ -2579,20 +2580,17 @@ def ip2num(ipAddr): return True hostIP = None + try: + hostIP = int(IPv4Address(hostonly)) + except AddressValueError: + pass for value in proxy_settings.get('exceptions', ()): # Items in the list are strings like these: *.local, 169.254/16 if not value: continue m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value) - if m is not None: - if hostIP is None: - try: - hostIP = socket.gethostbyname(hostonly) - hostIP = ip2num(hostIP) - except OSError: - continue - + if m is not None and hostIP is not None: base = ip2num(m.group(1)) mask = m.group(2) if mask is None: @@ -2615,6 +2613,31 @@ def ip2num(ipAddr): return False +# Same as _proxy_bypass_macosx_sysconf, testable on all platforms +def _proxy_bypass_winreg_override(host, override): + """Return True if the host should bypass the proxy server. + + The proxy override list is obtained from the Windows + Internet settings proxy override registry value. + + An example of a proxy override value is: + "www.example.com;*.example.net; 192.168.0.1" + """ + from fnmatch import fnmatch + + host, _ = _splitport(host) + proxy_override = override.split(';') + for test in proxy_override: + test = test.strip() + # "<local>" should bypass the proxy server for all intranet addresses + if test == '<local>': + if '.' not in host: + return True + elif fnmatch(host, test): + return True + return False + + if sys.platform == 'darwin': from _scproxy import _get_proxy_settings, _get_proxies @@ -2713,7 +2736,7 @@ def proxy_bypass_registry(host): import winreg except ImportError: # Std modules, so should be around - but you never know! - return 0 + return False try: internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r'Software\Microsoft\Windows\CurrentVersion\Internet Settings') @@ -2723,40 +2746,10 @@ def proxy_bypass_registry(host): 'ProxyOverride')[0]) # ^^^^ Returned as Unicode but problems if not converted to ASCII except OSError: - return 0 + return False if not proxyEnable or not proxyOverride: - return 0 - # try to make a host list from name and IP address. - rawHost, port = _splitport(host) - host = [rawHost] - try: - addr = socket.gethostbyname(rawHost) - if addr != rawHost: - host.append(addr) - except OSError: - pass - try: - fqdn = socket.getfqdn(rawHost) - if fqdn != rawHost: - host.append(fqdn) - except OSError: - pass - # make a check value list from the registry entry: replace the - # '<local>' string by the localhost entry and the corresponding - # canonical entry. - proxyOverride = proxyOverride.split(';') - # now check if we match one of the registry values. - for test in proxyOverride: - if test == '<local>': - if '.' not in rawHost: - return 1 - test = test.replace(".", r"\.") # mask dots - test = test.replace("*", r".*") # change glob sequence - test = test.replace("?", r".") # change glob char - for val in host: - if re.match(test, val, re.I): - return 1 - return 0 + return False + return _proxy_bypass_winreg_override(host, proxyOverride) def proxy_bypass(host): """Return True, if host should be bypassed. diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat index 2c98122362a060f..dd5ea8eb67b90a7 100644 --- a/Lib/venv/scripts/nt/activate.bat +++ b/Lib/venv/scripts/nt/activate.bat @@ -15,8 +15,8 @@ if not defined PROMPT set PROMPT=$P$G if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT% if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME% -set _OLD_VIRTUAL_PROMPT=%PROMPT% -set PROMPT=(__VENV_PROMPT__) %PROMPT% +set "_OLD_VIRTUAL_PROMPT=%PROMPT%" +set "PROMPT=(__VENV_PROMPT__) %PROMPT%" if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME% set PYTHONHOME= diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index ae6575028be11cb..a37fead41b750e4 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -1248,16 +1248,23 @@ def iterator(source): if close_source: source.close() + gen = iterator(source) class IterParseIterator(collections.abc.Iterator): - __next__ = iterator(source).__next__ + __next__ = gen.__next__ + def close(self): + if close_source: + source.close() + gen.close() def __del__(self): + # TODO: Emit a ResourceWarning if it was not explicitly closed. + # (When the close() method will be supported in all maintained Python versions.) if close_source: source.close() it = IterParseIterator() + it.root = None wr = weakref.ref(it) - del IterParseIterator return it diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 8005b4b34ccf76c..cc08f602fe44e05 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -1802,11 +1802,15 @@ def _extract_member(self, member, targetpath, pwd): # Create all upper directories if necessary. upperdirs = os.path.dirname(targetpath) if upperdirs and not os.path.exists(upperdirs): - os.makedirs(upperdirs) + os.makedirs(upperdirs, exist_ok=True) if member.is_dir(): if not os.path.isdir(targetpath): - os.mkdir(targetpath) + try: + os.mkdir(targetpath) + except FileExistsError: + if not os.path.isdir(targetpath): + raise return targetpath with self.open(member, pwd=pwd) as source, \ diff --git a/Lib/zoneinfo/_tzpath.py b/Lib/zoneinfo/_tzpath.py index 4985dce2dc36d0e..5db17bea045d8c8 100644 --- a/Lib/zoneinfo/_tzpath.py +++ b/Lib/zoneinfo/_tzpath.py @@ -2,7 +2,7 @@ import sysconfig -def reset_tzpath(to=None): +def _reset_tzpath(to=None, stacklevel=4): global TZPATH tzpaths = to @@ -18,17 +18,22 @@ def reset_tzpath(to=None): base_tzpath = tzpaths else: env_var = os.environ.get("PYTHONTZPATH", None) - if env_var is not None: - base_tzpath = _parse_python_tzpath(env_var) - else: - base_tzpath = _parse_python_tzpath( - sysconfig.get_config_var("TZPATH") - ) + if env_var is None: + env_var = sysconfig.get_config_var("TZPATH") + base_tzpath = _parse_python_tzpath(env_var, stacklevel) TZPATH = tuple(base_tzpath) -def _parse_python_tzpath(env_var): +def reset_tzpath(to=None): + """Reset global TZPATH.""" + # We need `_reset_tzpath` helper function because it produces a warning, + # it is used as both a module-level call and a public API. + # This is how we equalize the stacklevel for both calls. + _reset_tzpath(to) + + +def _parse_python_tzpath(env_var, stacklevel): if not env_var: return () @@ -45,6 +50,7 @@ def _parse_python_tzpath(env_var): "Invalid paths specified in PYTHONTZPATH environment variable. " + msg, InvalidTZPathWarning, + stacklevel=stacklevel, ) return new_tzpath @@ -172,4 +178,4 @@ class InvalidTZPathWarning(RuntimeWarning): TZPATH = () -reset_tzpath() +_reset_tzpath(stacklevel=5) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 32de56bcf130863..0af90563cbbb2b7 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.0.11", - url="https://www.openssl.org/source/openssl-3.0.11.tar.gz", - checksum='b3425d3bb4a2218d0697eb41f7fc0cdede016ed19ca49d168b78e8d947887f55', + name="OpenSSL 3.0.13", + url="https://www.openssl.org/source/openssl-3.0.13.tar.gz", + checksum='88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313', buildrecipe=build_universal_openssl, configure=None, install=None, @@ -359,9 +359,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.44.2", - url="https://sqlite.org/2023/sqlite-autoconf-3440200.tar.gz", - checksum="c02f40fd4f809ced95096250adc5764a", + name="SQLite 3.45.1", + url="https://sqlite.org/2024/sqlite-autoconf-3450100.tar.gz", + checksum="cd9c27841b7a5932c9897651e20b86c701dd740556989b01ca596fcfa3d49a0a", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' diff --git a/Makefile.pre.in b/Makefile.pre.in index 37a8b06987c7107..7533a49b4392f04 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -41,7 +41,7 @@ AR= @AR@ READELF= @READELF@ SOABI= @SOABI@ LDVERSION= @LDVERSION@ -LIBPYTHON= @LIBPYTHON@ +MODULE_LDFLAGS=@MODULE_LDFLAGS@ GITVERSION= @GITVERSION@ GITTAG= @GITTAG@ GITBRANCH= @GITBRANCH@ @@ -184,6 +184,8 @@ PYTHONFRAMEWORK= @PYTHONFRAMEWORK@ PYTHONFRAMEWORKDIR= @PYTHONFRAMEWORKDIR@ PYTHONFRAMEWORKPREFIX= @PYTHONFRAMEWORKPREFIX@ PYTHONFRAMEWORKINSTALLDIR= @PYTHONFRAMEWORKINSTALLDIR@ +PYTHONFRAMEWORKINSTALLNAMEPREFIX= @PYTHONFRAMEWORKINSTALLNAMEPREFIX@ +RESSRCDIR= @RESSRCDIR@ # Deployment target selected during configure, to be checked # by distutils. The export statement is needed to ensure that the # deployment target is active during build. @@ -405,6 +407,7 @@ PYTHON_OBJS= \ Python/ast_opt.o \ Python/ast_unparse.o \ Python/bltinmodule.o \ + Python/brc.o \ Python/ceval.o \ Python/codecs.o \ Python/compile.o \ @@ -433,6 +436,7 @@ PYTHON_OBJS= \ Python/initconfig.o \ Python/instrumentation.o \ Python/intrinsics.o \ + Python/jit.o \ Python/legacy_tracing.o \ Python/lock.o \ Python/marshal.o \ @@ -442,6 +446,7 @@ PYTHON_OBJS= \ Python/object_stack.o \ Python/optimizer.o \ Python/optimizer_analysis.o \ + Python/optimizer_symbols.o \ Python/parking_lot.o \ Python/pathconfig.o \ Python/preconfig.o \ @@ -454,6 +459,7 @@ PYTHON_OBJS= \ Python/pystate.o \ Python/pythonrun.o \ Python/pytime.o \ + Python/qsbr.o \ Python/bootstrap_hash.o \ Python/specialize.o \ Python/structmember.o \ @@ -864,7 +870,7 @@ libpython3.so: libpython$(LDVERSION).so $(BLDSHARED) $(NO_AS_NEEDED) -o $@ -Wl,-h$@ $^ libpython$(LDVERSION).dylib: $(LIBRARY_OBJS) - $(CC) -dynamiclib -Wl,-single_module $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(CC) -dynamiclib $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \ libpython$(VERSION).sl: $(LIBRARY_OBJS) @@ -889,14 +895,13 @@ $(BUILDPYTHON)-gdb.py: $(SRC_GDB_HOOKS) # This rule is here for OPENSTEP/Rhapsody/MacOSX. It builds a temporary # minimal framework (not including the Lib directory and such) in the current # directory. -RESSRCDIR=Mac/Resources/framework $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \ $(LIBRARY) \ $(RESSRCDIR)/Info.plist $(INSTALL) -d -m $(DIRMODE) $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION) $(CC) -o $(LDLIBRARY) $(PY_CORE_LDFLAGS) -dynamiclib \ - -all_load $(LIBRARY) -Wl,-single_module \ - -install_name $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK) \ + -all_load $(LIBRARY) \ + -install_name $(DESTDIR)$(PYTHONFRAMEWORKINSTALLNAMEPREFIX)/$(PYTHONFRAMEWORK) \ -compatibility_version $(VERSION) \ -current_version $(VERSION) \ -framework CoreFoundation $(LIBS); @@ -908,6 +913,21 @@ $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \ $(LN) -fsn Versions/Current/$(PYTHONFRAMEWORK) $(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK) $(LN) -fsn Versions/Current/Resources $(PYTHONFRAMEWORKDIR)/Resources +# This rule is for iOS, which requires an annoyingly just slighly different +# format for frameworks to macOS. It *doesn't* use a versioned framework, and +# the Info.plist must be in the root of the framework. +$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK): \ + $(LIBRARY) \ + $(RESSRCDIR)/Info.plist + $(INSTALL) -d -m $(DIRMODE) $(PYTHONFRAMEWORKDIR) + $(CC) -o $(LDLIBRARY) $(PY_CORE_LDFLAGS) -dynamiclib \ + -all_load $(LIBRARY) \ + -install_name $(PYTHONFRAMEWORKINSTALLNAMEPREFIX)/$(PYTHONFRAMEWORK) \ + -compatibility_version $(VERSION) \ + -current_version $(VERSION) \ + -framework CoreFoundation $(LIBS); + $(INSTALL_DATA) $(RESSRCDIR)/Info.plist $(PYTHONFRAMEWORKDIR)/Info.plist + # This rule builds the Cygwin Python DLL and import library if configured # for a shared core library; otherwise, this rule is a noop. $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) @@ -935,6 +955,264 @@ python.html: $(srcdir)/Tools/wasm/python.html python.worker.js python.worker.js: $(srcdir)/Tools/wasm/python.worker.js @cp $(srcdir)/Tools/wasm/python.worker.js $@ +############################################################################ +# Header files + +PYTHON_HEADERS= \ + $(srcdir)/Include/Python.h \ + $(srcdir)/Include/abstract.h \ + $(srcdir)/Include/bltinmodule.h \ + $(srcdir)/Include/boolobject.h \ + $(srcdir)/Include/bytearrayobject.h \ + $(srcdir)/Include/bytesobject.h \ + $(srcdir)/Include/ceval.h \ + $(srcdir)/Include/codecs.h \ + $(srcdir)/Include/compile.h \ + $(srcdir)/Include/complexobject.h \ + $(srcdir)/Include/descrobject.h \ + $(srcdir)/Include/dictobject.h \ + $(srcdir)/Include/dynamic_annotations.h \ + $(srcdir)/Include/enumobject.h \ + $(srcdir)/Include/errcode.h \ + $(srcdir)/Include/exports.h \ + $(srcdir)/Include/fileobject.h \ + $(srcdir)/Include/fileutils.h \ + $(srcdir)/Include/floatobject.h \ + $(srcdir)/Include/frameobject.h \ + $(srcdir)/Include/genericaliasobject.h \ + $(srcdir)/Include/import.h \ + $(srcdir)/Include/interpreteridobject.h \ + $(srcdir)/Include/intrcheck.h \ + $(srcdir)/Include/iterobject.h \ + $(srcdir)/Include/listobject.h \ + $(srcdir)/Include/longobject.h \ + $(srcdir)/Include/marshal.h \ + $(srcdir)/Include/memoryobject.h \ + $(srcdir)/Include/methodobject.h \ + $(srcdir)/Include/modsupport.h \ + $(srcdir)/Include/moduleobject.h \ + $(srcdir)/Include/object.h \ + $(srcdir)/Include/objimpl.h \ + $(srcdir)/Include/opcode.h \ + $(srcdir)/Include/opcode_ids.h \ + $(srcdir)/Include/osdefs.h \ + $(srcdir)/Include/osmodule.h \ + $(srcdir)/Include/patchlevel.h \ + $(srcdir)/Include/pyatomic.h \ + $(srcdir)/Include/pybuffer.h \ + $(srcdir)/Include/pycapsule.h \ + $(srcdir)/Include/pydtrace.h \ + $(srcdir)/Include/pyerrors.h \ + $(srcdir)/Include/pyexpat.h \ + $(srcdir)/Include/pyframe.h \ + $(srcdir)/Include/pyhash.h \ + $(srcdir)/Include/pylifecycle.h \ + $(srcdir)/Include/pymacconfig.h \ + $(srcdir)/Include/pymacro.h \ + $(srcdir)/Include/pymath.h \ + $(srcdir)/Include/pymem.h \ + $(srcdir)/Include/pyport.h \ + $(srcdir)/Include/pystate.h \ + $(srcdir)/Include/pystats.h \ + $(srcdir)/Include/pystrcmp.h \ + $(srcdir)/Include/pystrtod.h \ + $(srcdir)/Include/pythonrun.h \ + $(srcdir)/Include/pythread.h \ + $(srcdir)/Include/pytypedefs.h \ + $(srcdir)/Include/rangeobject.h \ + $(srcdir)/Include/setobject.h \ + $(srcdir)/Include/sliceobject.h \ + $(srcdir)/Include/structmember.h \ + $(srcdir)/Include/structseq.h \ + $(srcdir)/Include/sysmodule.h \ + $(srcdir)/Include/traceback.h \ + $(srcdir)/Include/tupleobject.h \ + $(srcdir)/Include/typeslots.h \ + $(srcdir)/Include/unicodeobject.h \ + $(srcdir)/Include/warnings.h \ + $(srcdir)/Include/weakrefobject.h \ + \ + pyconfig.h \ + $(PARSER_HEADERS) \ + \ + $(srcdir)/Include/cpython/abstract.h \ + $(srcdir)/Include/cpython/bytearrayobject.h \ + $(srcdir)/Include/cpython/bytesobject.h \ + $(srcdir)/Include/cpython/cellobject.h \ + $(srcdir)/Include/cpython/ceval.h \ + $(srcdir)/Include/cpython/classobject.h \ + $(srcdir)/Include/cpython/code.h \ + $(srcdir)/Include/cpython/compile.h \ + $(srcdir)/Include/cpython/complexobject.h \ + $(srcdir)/Include/cpython/context.h \ + $(srcdir)/Include/cpython/descrobject.h \ + $(srcdir)/Include/cpython/dictobject.h \ + $(srcdir)/Include/cpython/fileobject.h \ + $(srcdir)/Include/cpython/fileutils.h \ + $(srcdir)/Include/cpython/floatobject.h \ + $(srcdir)/Include/cpython/frameobject.h \ + $(srcdir)/Include/cpython/funcobject.h \ + $(srcdir)/Include/cpython/genobject.h \ + $(srcdir)/Include/cpython/import.h \ + $(srcdir)/Include/cpython/initconfig.h \ + $(srcdir)/Include/cpython/interpreteridobject.h \ + $(srcdir)/Include/cpython/listobject.h \ + $(srcdir)/Include/cpython/longintrepr.h \ + $(srcdir)/Include/cpython/longobject.h \ + $(srcdir)/Include/cpython/memoryobject.h \ + $(srcdir)/Include/cpython/methodobject.h \ + $(srcdir)/Include/cpython/object.h \ + $(srcdir)/Include/cpython/objimpl.h \ + $(srcdir)/Include/cpython/odictobject.h \ + $(srcdir)/Include/cpython/optimizer.h \ + $(srcdir)/Include/cpython/picklebufobject.h \ + $(srcdir)/Include/cpython/pthread_stubs.h \ + $(srcdir)/Include/cpython/pyatomic.h \ + $(srcdir)/Include/cpython/pyatomic_gcc.h \ + $(srcdir)/Include/cpython/pyatomic_std.h \ + $(srcdir)/Include/cpython/pyctype.h \ + $(srcdir)/Include/cpython/pydebug.h \ + $(srcdir)/Include/cpython/pyerrors.h \ + $(srcdir)/Include/cpython/pyfpe.h \ + $(srcdir)/Include/cpython/pyframe.h \ + $(srcdir)/Include/cpython/pyhash.h \ + $(srcdir)/Include/cpython/pylifecycle.h \ + $(srcdir)/Include/cpython/pymem.h \ + $(srcdir)/Include/cpython/pystate.h \ + $(srcdir)/Include/cpython/pystats.h \ + $(srcdir)/Include/cpython/pythonrun.h \ + $(srcdir)/Include/cpython/pythread.h \ + $(srcdir)/Include/cpython/setobject.h \ + $(srcdir)/Include/cpython/sysmodule.h \ + $(srcdir)/Include/cpython/traceback.h \ + $(srcdir)/Include/cpython/tracemalloc.h \ + $(srcdir)/Include/cpython/tupleobject.h \ + $(srcdir)/Include/cpython/unicodeobject.h \ + $(srcdir)/Include/cpython/warnings.h \ + $(srcdir)/Include/cpython/weakrefobject.h \ + \ + $(MIMALLOC_HEADERS) \ + \ + $(srcdir)/Include/internal/pycore_abstract.h \ + $(srcdir)/Include/internal/pycore_asdl.h \ + $(srcdir)/Include/internal/pycore_ast.h \ + $(srcdir)/Include/internal/pycore_ast_state.h \ + $(srcdir)/Include/internal/pycore_atexit.h \ + $(srcdir)/Include/internal/pycore_bitutils.h \ + $(srcdir)/Include/internal/pycore_blocks_output_buffer.h \ + $(srcdir)/Include/internal/pycore_brc.h \ + $(srcdir)/Include/internal/pycore_bytes_methods.h \ + $(srcdir)/Include/internal/pycore_bytesobject.h \ + $(srcdir)/Include/internal/pycore_call.h \ + $(srcdir)/Include/internal/pycore_capsule.h \ + $(srcdir)/Include/internal/pycore_ceval.h \ + $(srcdir)/Include/internal/pycore_ceval_state.h \ + $(srcdir)/Include/internal/pycore_code.h \ + $(srcdir)/Include/internal/pycore_codecs.h \ + $(srcdir)/Include/internal/pycore_compile.h \ + $(srcdir)/Include/internal/pycore_complexobject.h \ + $(srcdir)/Include/internal/pycore_condvar.h \ + $(srcdir)/Include/internal/pycore_context.h \ + $(srcdir)/Include/internal/pycore_critical_section.h \ + $(srcdir)/Include/internal/pycore_crossinterp.h \ + $(srcdir)/Include/internal/pycore_descrobject.h \ + $(srcdir)/Include/internal/pycore_dict.h \ + $(srcdir)/Include/internal/pycore_dict_state.h \ + $(srcdir)/Include/internal/pycore_dtoa.h \ + $(srcdir)/Include/internal/pycore_exceptions.h \ + $(srcdir)/Include/internal/pycore_faulthandler.h \ + $(srcdir)/Include/internal/pycore_fileutils.h \ + $(srcdir)/Include/internal/pycore_floatobject.h \ + $(srcdir)/Include/internal/pycore_flowgraph.h \ + $(srcdir)/Include/internal/pycore_format.h \ + $(srcdir)/Include/internal/pycore_frame.h \ + $(srcdir)/Include/internal/pycore_freelist.h \ + $(srcdir)/Include/internal/pycore_function.h \ + $(srcdir)/Include/internal/pycore_gc.h \ + $(srcdir)/Include/internal/pycore_genobject.h \ + $(srcdir)/Include/internal/pycore_getopt.h \ + $(srcdir)/Include/internal/pycore_gil.h \ + $(srcdir)/Include/internal/pycore_global_objects.h \ + $(srcdir)/Include/internal/pycore_global_objects_fini_generated.h \ + $(srcdir)/Include/internal/pycore_global_strings.h \ + $(srcdir)/Include/internal/pycore_hamt.h \ + $(srcdir)/Include/internal/pycore_hashtable.h \ + $(srcdir)/Include/internal/pycore_identifier.h \ + $(srcdir)/Include/internal/pycore_import.h \ + $(srcdir)/Include/internal/pycore_importdl.h \ + $(srcdir)/Include/internal/pycore_initconfig.h \ + $(srcdir)/Include/internal/pycore_instruments.h \ + $(srcdir)/Include/internal/pycore_interp.h \ + $(srcdir)/Include/internal/pycore_intrinsics.h \ + $(srcdir)/Include/internal/pycore_jit.h \ + $(srcdir)/Include/internal/pycore_list.h \ + $(srcdir)/Include/internal/pycore_llist.h \ + $(srcdir)/Include/internal/pycore_lock.h \ + $(srcdir)/Include/internal/pycore_long.h \ + $(srcdir)/Include/internal/pycore_memoryobject.h \ + $(srcdir)/Include/internal/pycore_mimalloc.h \ + $(srcdir)/Include/internal/pycore_modsupport.h \ + $(srcdir)/Include/internal/pycore_moduleobject.h \ + $(srcdir)/Include/internal/pycore_namespace.h \ + $(srcdir)/Include/internal/pycore_object.h \ + $(srcdir)/Include/internal/pycore_object_alloc.h \ + $(srcdir)/Include/internal/pycore_object_stack.h \ + $(srcdir)/Include/internal/pycore_object_state.h \ + $(srcdir)/Include/internal/pycore_obmalloc.h \ + $(srcdir)/Include/internal/pycore_obmalloc_init.h \ + $(srcdir)/Include/internal/pycore_opcode_metadata.h \ + $(srcdir)/Include/internal/pycore_opcode_utils.h \ + $(srcdir)/Include/internal/pycore_optimizer.h \ + $(srcdir)/Include/internal/pycore_parking_lot.h \ + $(srcdir)/Include/internal/pycore_parser.h \ + $(srcdir)/Include/internal/pycore_pathconfig.h \ + $(srcdir)/Include/internal/pycore_pyarena.h \ + $(srcdir)/Include/internal/pycore_pyatomic_ft_wrappers.h \ + $(srcdir)/Include/internal/pycore_pybuffer.h \ + $(srcdir)/Include/internal/pycore_pyerrors.h \ + $(srcdir)/Include/internal/pycore_pyhash.h \ + $(srcdir)/Include/internal/pycore_pylifecycle.h \ + $(srcdir)/Include/internal/pycore_pymath.h \ + $(srcdir)/Include/internal/pycore_pymem.h \ + $(srcdir)/Include/internal/pycore_pymem_init.h \ + $(srcdir)/Include/internal/pycore_pystate.h \ + $(srcdir)/Include/internal/pycore_pystats.h \ + $(srcdir)/Include/internal/pycore_pythonrun.h \ + $(srcdir)/Include/internal/pycore_pythread.h \ + $(srcdir)/Include/internal/pycore_qsbr.h \ + $(srcdir)/Include/internal/pycore_range.h \ + $(srcdir)/Include/internal/pycore_runtime.h \ + $(srcdir)/Include/internal/pycore_runtime_init.h \ + $(srcdir)/Include/internal/pycore_runtime_init_generated.h \ + $(srcdir)/Include/internal/pycore_semaphore.h \ + $(srcdir)/Include/internal/pycore_setobject.h \ + $(srcdir)/Include/internal/pycore_signal.h \ + $(srcdir)/Include/internal/pycore_sliceobject.h \ + $(srcdir)/Include/internal/pycore_strhex.h \ + $(srcdir)/Include/internal/pycore_structseq.h \ + $(srcdir)/Include/internal/pycore_symtable.h \ + $(srcdir)/Include/internal/pycore_sysmodule.h \ + $(srcdir)/Include/internal/pycore_time.h \ + $(srcdir)/Include/internal/pycore_token.h \ + $(srcdir)/Include/internal/pycore_traceback.h \ + $(srcdir)/Include/internal/pycore_tracemalloc.h \ + $(srcdir)/Include/internal/pycore_tstate.h \ + $(srcdir)/Include/internal/pycore_tuple.h \ + $(srcdir)/Include/internal/pycore_typeobject.h \ + $(srcdir)/Include/internal/pycore_typevarobject.h \ + $(srcdir)/Include/internal/pycore_ucnhash.h \ + $(srcdir)/Include/internal/pycore_unicodeobject.h \ + $(srcdir)/Include/internal/pycore_unicodeobject_generated.h \ + $(srcdir)/Include/internal/pycore_unionobject.h \ + $(srcdir)/Include/internal/pycore_uop_ids.h \ + $(srcdir)/Include/internal/pycore_uop_metadata.h \ + $(srcdir)/Include/internal/pycore_warnings.h \ + $(srcdir)/Include/internal/pycore_weakref.h \ + $(DTRACE_HEADERS) \ + @PLATFORM_HEADERS@ \ + \ + $(srcdir)/Python/stdlib_module_names.h + ########################################################################## # Build static libmpdec.a LIBMPDEC_CFLAGS=@LIBMPDEC_CFLAGS@ $(PY_STDMODULE_CFLAGS) $(CCSHARED) @@ -1365,10 +1643,10 @@ regen-unicodedata: regen-all: regen-cases regen-typeslots \ regen-token regen-ast regen-keyword regen-sre regen-frozen \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ - regen-test-levenshtein regen-global-objects regen-sbom + regen-test-levenshtein regen-global-objects regen-jit @echo @echo "Note: make regen-stdlib-module-names, make regen-limited-abi, " - @echo "make regen-configure and make regen-unicodedata should be run manually" + @echo "make regen-configure, make regen-sbom, and make regen-unicodedata should be run manually" ############################################################################ # Special rules for object files @@ -1399,7 +1677,7 @@ Modules/getpath.o: $(srcdir)/Modules/getpath.c Python/frozen_modules/getpath.h M Programs/python.o: $(srcdir)/Programs/python.c $(CC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Programs/python.c -Programs/_testembed.o: $(srcdir)/Programs/_testembed.c Programs/test_frozenmain.h +Programs/_testembed.o: $(srcdir)/Programs/_testembed.c Programs/test_frozenmain.h $(PYTHON_HEADERS) $(CC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Programs/_testembed.c Modules/_sre/sre.o: $(srcdir)/Modules/_sre/sre.c $(srcdir)/Modules/_sre/sre.h $(srcdir)/Modules/_sre/sre_constants.h $(srcdir)/Modules/_sre/sre_lib.h @@ -1412,6 +1690,14 @@ Modules/pwdmodule.o: $(srcdir)/Modules/pwdmodule.c $(srcdir)/Modules/posixmodule Modules/signalmodule.o: $(srcdir)/Modules/signalmodule.c $(srcdir)/Modules/posixmodule.h +Modules/_xxsubinterpretersmodule.o: $(srcdir)/Modules/_xxsubinterpretersmodule.c $(srcdir)/Modules/_interpreters_common.h + +Modules/_xxinterpqueuesmodule.o: $(srcdir)/Modules/_xxinterpqueuesmodule.c $(srcdir)/Modules/_interpreters_common.h + +Modules/_xxinterpchannelsmodule.o: $(srcdir)/Modules/_xxinterpchannelsmodule.c $(srcdir)/Modules/_interpreters_common.h + +Python/crossinterp.o: $(srcdir)/Python/crossinterp.c $(srcdir)/Python/crossinterp_data_lookup.h $(srcdir)/Python/crossinterp_exceptions.h + Python/dynload_shlib.o: $(srcdir)/Python/dynload_shlib.c Makefile $(CC) -c $(PY_CORE_CFLAGS) \ -DSOABI='"$(SOABI)"' \ @@ -1604,6 +1890,10 @@ regen-cases: -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier2_generator.py \ -o $(srcdir)/Python/executor_cases.c.h.new $(srcdir)/Python/bytecodes.c + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/optimizer_generator.py \ + -o $(srcdir)/Python/optimizer_cases.c.h.new \ + $(srcdir)/Python/optimizer_bytecodes.c \ + $(srcdir)/Python/bytecodes.c $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_metadata_generator.py \ -o $(srcdir)/Include/internal/pycore_opcode_metadata.h.new $(srcdir)/Python/bytecodes.c $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/uop_metadata_generator.py -o \ @@ -1615,6 +1905,7 @@ regen-cases: $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode_metadata.h $(srcdir)/Include/internal/pycore_opcode_metadata.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_metadata.h $(srcdir)/Include/internal/pycore_uop_metadata.h.new $(UPDATE_FILE) $(srcdir)/Python/executor_cases.c.h $(srcdir)/Python/executor_cases.c.h.new + $(UPDATE_FILE) $(srcdir)/Python/optimizer_cases.c.h $(srcdir)/Python/optimizer_cases.c.h.new $(UPDATE_FILE) $(srcdir)/Lib/_opcode_metadata.py $(srcdir)/Lib/_opcode_metadata.py.new Python/compile.o: $(srcdir)/Include/internal/pycore_opcode_metadata.h @@ -1636,7 +1927,8 @@ Python/optimizer.o: \ Python/optimizer_analysis.o: \ $(srcdir)/Include/internal/pycore_opcode_metadata.h \ - $(srcdir)/Include/internal/pycore_optimizer.h + $(srcdir)/Include/internal/pycore_optimizer.h \ + $(srcdir)/Python/optimizer_cases.c.h Python/frozen.o: $(FROZEN_FILES_OUT) @@ -1668,246 +1960,7 @@ regen-typeslots: $(srcdir)/Objects/typeslots.inc.new $(UPDATE_FILE) $(srcdir)/Objects/typeslots.inc $(srcdir)/Objects/typeslots.inc.new -############################################################################ -# Header files - -PYTHON_HEADERS= \ - $(srcdir)/Include/Python.h \ - $(srcdir)/Include/abstract.h \ - $(srcdir)/Include/bltinmodule.h \ - $(srcdir)/Include/boolobject.h \ - $(srcdir)/Include/bytearrayobject.h \ - $(srcdir)/Include/bytesobject.h \ - $(srcdir)/Include/ceval.h \ - $(srcdir)/Include/codecs.h \ - $(srcdir)/Include/compile.h \ - $(srcdir)/Include/complexobject.h \ - $(srcdir)/Include/descrobject.h \ - $(srcdir)/Include/dictobject.h \ - $(srcdir)/Include/dynamic_annotations.h \ - $(srcdir)/Include/enumobject.h \ - $(srcdir)/Include/errcode.h \ - $(srcdir)/Include/fileobject.h \ - $(srcdir)/Include/fileutils.h \ - $(srcdir)/Include/floatobject.h \ - $(srcdir)/Include/frameobject.h \ - $(srcdir)/Include/import.h \ - $(srcdir)/Include/interpreteridobject.h \ - $(srcdir)/Include/intrcheck.h \ - $(srcdir)/Include/iterobject.h \ - $(srcdir)/Include/listobject.h \ - $(srcdir)/Include/longobject.h \ - $(srcdir)/Include/marshal.h \ - $(srcdir)/Include/memoryobject.h \ - $(srcdir)/Include/methodobject.h \ - $(srcdir)/Include/modsupport.h \ - $(srcdir)/Include/moduleobject.h \ - $(srcdir)/Include/object.h \ - $(srcdir)/Include/objimpl.h \ - $(srcdir)/Include/opcode.h \ - $(srcdir)/Include/opcode_ids.h \ - $(srcdir)/Include/osdefs.h \ - $(srcdir)/Include/osmodule.h \ - $(srcdir)/Include/patchlevel.h \ - $(srcdir)/Include/pybuffer.h \ - $(srcdir)/Include/pycapsule.h \ - $(srcdir)/Include/pydtrace.h \ - $(srcdir)/Include/pyerrors.h \ - $(srcdir)/Include/pyframe.h \ - $(srcdir)/Include/pyhash.h \ - $(srcdir)/Include/pylifecycle.h \ - $(srcdir)/Include/pymacconfig.h \ - $(srcdir)/Include/pymacro.h \ - $(srcdir)/Include/pymath.h \ - $(srcdir)/Include/pymem.h \ - $(srcdir)/Include/pyport.h \ - $(srcdir)/Include/pystate.h \ - $(srcdir)/Include/pystats.h \ - $(srcdir)/Include/pystrcmp.h \ - $(srcdir)/Include/pystrtod.h \ - $(srcdir)/Include/pythonrun.h \ - $(srcdir)/Include/pythread.h \ - $(srcdir)/Include/pytypedefs.h \ - $(srcdir)/Include/rangeobject.h \ - $(srcdir)/Include/setobject.h \ - $(srcdir)/Include/sliceobject.h \ - $(srcdir)/Include/structmember.h \ - $(srcdir)/Include/structseq.h \ - $(srcdir)/Include/sysmodule.h \ - $(srcdir)/Include/traceback.h \ - $(srcdir)/Include/tupleobject.h \ - $(srcdir)/Include/unicodeobject.h \ - $(srcdir)/Include/warnings.h \ - $(srcdir)/Include/weakrefobject.h \ - \ - pyconfig.h \ - $(PARSER_HEADERS) \ - \ - $(srcdir)/Include/cpython/abstract.h \ - $(srcdir)/Include/cpython/bytearrayobject.h \ - $(srcdir)/Include/cpython/bytesobject.h \ - $(srcdir)/Include/cpython/cellobject.h \ - $(srcdir)/Include/cpython/ceval.h \ - $(srcdir)/Include/cpython/classobject.h \ - $(srcdir)/Include/cpython/code.h \ - $(srcdir)/Include/cpython/compile.h \ - $(srcdir)/Include/cpython/complexobject.h \ - $(srcdir)/Include/cpython/context.h \ - $(srcdir)/Include/cpython/descrobject.h \ - $(srcdir)/Include/cpython/dictobject.h \ - $(srcdir)/Include/cpython/fileobject.h \ - $(srcdir)/Include/cpython/fileutils.h \ - $(srcdir)/Include/cpython/floatobject.h \ - $(srcdir)/Include/cpython/frameobject.h \ - $(srcdir)/Include/cpython/funcobject.h \ - $(srcdir)/Include/cpython/genobject.h \ - $(srcdir)/Include/cpython/import.h \ - $(srcdir)/Include/cpython/initconfig.h \ - $(srcdir)/Include/cpython/interpreteridobject.h \ - $(srcdir)/Include/cpython/listobject.h \ - $(srcdir)/Include/cpython/longintrepr.h \ - $(srcdir)/Include/cpython/longobject.h \ - $(srcdir)/Include/cpython/memoryobject.h \ - $(srcdir)/Include/cpython/methodobject.h \ - $(srcdir)/Include/cpython/object.h \ - $(srcdir)/Include/cpython/objimpl.h \ - $(srcdir)/Include/cpython/odictobject.h \ - $(srcdir)/Include/cpython/optimizer.h \ - $(srcdir)/Include/cpython/picklebufobject.h \ - $(srcdir)/Include/cpython/pthread_stubs.h \ - $(srcdir)/Include/cpython/pyatomic.h \ - $(srcdir)/Include/cpython/pyatomic_gcc.h \ - $(srcdir)/Include/cpython/pyatomic_std.h \ - $(srcdir)/Include/cpython/pyctype.h \ - $(srcdir)/Include/cpython/pydebug.h \ - $(srcdir)/Include/cpython/pyerrors.h \ - $(srcdir)/Include/cpython/pyfpe.h \ - $(srcdir)/Include/cpython/pyframe.h \ - $(srcdir)/Include/cpython/pyhash.h \ - $(srcdir)/Include/cpython/pylifecycle.h \ - $(srcdir)/Include/cpython/pymem.h \ - $(srcdir)/Include/cpython/pystate.h \ - $(srcdir)/Include/cpython/pystats.h \ - $(srcdir)/Include/cpython/pythonrun.h \ - $(srcdir)/Include/cpython/pythread.h \ - $(srcdir)/Include/cpython/setobject.h \ - $(srcdir)/Include/cpython/sysmodule.h \ - $(srcdir)/Include/cpython/traceback.h \ - $(srcdir)/Include/cpython/tracemalloc.h \ - $(srcdir)/Include/cpython/tupleobject.h \ - $(srcdir)/Include/cpython/unicodeobject.h \ - $(srcdir)/Include/cpython/warnings.h \ - $(srcdir)/Include/cpython/weakrefobject.h \ - \ - $(MIMALLOC_HEADERS) \ - \ - $(srcdir)/Include/internal/pycore_abstract.h \ - $(srcdir)/Include/internal/pycore_asdl.h \ - $(srcdir)/Include/internal/pycore_ast.h \ - $(srcdir)/Include/internal/pycore_ast_state.h \ - $(srcdir)/Include/internal/pycore_atexit.h \ - $(srcdir)/Include/internal/pycore_bitutils.h \ - $(srcdir)/Include/internal/pycore_bytes_methods.h \ - $(srcdir)/Include/internal/pycore_bytesobject.h \ - $(srcdir)/Include/internal/pycore_call.h \ - $(srcdir)/Include/internal/pycore_capsule.h \ - $(srcdir)/Include/internal/pycore_ceval.h \ - $(srcdir)/Include/internal/pycore_ceval_state.h \ - $(srcdir)/Include/internal/pycore_code.h \ - $(srcdir)/Include/internal/pycore_codecs.h \ - $(srcdir)/Include/internal/pycore_compile.h \ - $(srcdir)/Include/internal/pycore_complexobject.h \ - $(srcdir)/Include/internal/pycore_condvar.h \ - $(srcdir)/Include/internal/pycore_context.h \ - $(srcdir)/Include/internal/pycore_critical_section.h \ - $(srcdir)/Include/internal/pycore_crossinterp.h \ - $(srcdir)/Include/internal/pycore_dict.h \ - $(srcdir)/Include/internal/pycore_dict_state.h \ - $(srcdir)/Include/internal/pycore_descrobject.h \ - $(srcdir)/Include/internal/pycore_dtoa.h \ - $(srcdir)/Include/internal/pycore_exceptions.h \ - $(srcdir)/Include/internal/pycore_faulthandler.h \ - $(srcdir)/Include/internal/pycore_fileutils.h \ - $(srcdir)/Include/internal/pycore_floatobject.h \ - $(srcdir)/Include/internal/pycore_format.h \ - $(srcdir)/Include/internal/pycore_frame.h \ - $(srcdir)/Include/internal/pycore_freelist.h \ - $(srcdir)/Include/internal/pycore_function.h \ - $(srcdir)/Include/internal/pycore_gc.h \ - $(srcdir)/Include/internal/pycore_genobject.h \ - $(srcdir)/Include/internal/pycore_getopt.h \ - $(srcdir)/Include/internal/pycore_gil.h \ - $(srcdir)/Include/internal/pycore_global_objects.h \ - $(srcdir)/Include/internal/pycore_global_objects_fini_generated.h \ - $(srcdir)/Include/internal/pycore_hamt.h \ - $(srcdir)/Include/internal/pycore_hashtable.h \ - $(srcdir)/Include/internal/pycore_identifier.h \ - $(srcdir)/Include/internal/pycore_import.h \ - $(srcdir)/Include/internal/pycore_initconfig.h \ - $(srcdir)/Include/internal/pycore_interp.h \ - $(srcdir)/Include/internal/pycore_intrinsics.h \ - $(srcdir)/Include/internal/pycore_list.h \ - $(srcdir)/Include/internal/pycore_llist.h \ - $(srcdir)/Include/internal/pycore_lock.h \ - $(srcdir)/Include/internal/pycore_long.h \ - $(srcdir)/Include/internal/pycore_modsupport.h \ - $(srcdir)/Include/internal/pycore_moduleobject.h \ - $(srcdir)/Include/internal/pycore_namespace.h \ - $(srcdir)/Include/internal/pycore_object.h \ - $(srcdir)/Include/internal/pycore_object_alloc.h \ - $(srcdir)/Include/internal/pycore_object_stack.h \ - $(srcdir)/Include/internal/pycore_object_state.h \ - $(srcdir)/Include/internal/pycore_obmalloc.h \ - $(srcdir)/Include/internal/pycore_obmalloc_init.h \ - $(srcdir)/Include/internal/pycore_opcode_metadata.h \ - $(srcdir)/Include/internal/pycore_opcode_utils.h \ - $(srcdir)/Include/internal/pycore_optimizer.h \ - $(srcdir)/Include/internal/pycore_parking_lot.h \ - $(srcdir)/Include/internal/pycore_pathconfig.h \ - $(srcdir)/Include/internal/pycore_pyarena.h \ - $(srcdir)/Include/internal/pycore_pybuffer.h \ - $(srcdir)/Include/internal/pycore_pyerrors.h \ - $(srcdir)/Include/internal/pycore_pyhash.h \ - $(srcdir)/Include/internal/pycore_pylifecycle.h \ - $(srcdir)/Include/internal/pycore_pymem.h \ - $(srcdir)/Include/internal/pycore_pymem_init.h \ - $(srcdir)/Include/internal/pycore_pystate.h \ - $(srcdir)/Include/internal/pycore_pystats.h \ - $(srcdir)/Include/internal/pycore_pythonrun.h \ - $(srcdir)/Include/internal/pycore_pythread.h \ - $(srcdir)/Include/internal/pycore_range.h \ - $(srcdir)/Include/internal/pycore_runtime.h \ - $(srcdir)/Include/internal/pycore_runtime_init_generated.h \ - $(srcdir)/Include/internal/pycore_runtime_init.h \ - $(srcdir)/Include/internal/pycore_semaphore.h \ - $(srcdir)/Include/internal/pycore_setobject.h \ - $(srcdir)/Include/internal/pycore_signal.h \ - $(srcdir)/Include/internal/pycore_sliceobject.h \ - $(srcdir)/Include/internal/pycore_strhex.h \ - $(srcdir)/Include/internal/pycore_structseq.h \ - $(srcdir)/Include/internal/pycore_symtable.h \ - $(srcdir)/Include/internal/pycore_sysmodule.h \ - $(srcdir)/Include/internal/pycore_time.h \ - $(srcdir)/Include/internal/pycore_token.h \ - $(srcdir)/Include/internal/pycore_traceback.h \ - $(srcdir)/Include/internal/pycore_tracemalloc.h \ - $(srcdir)/Include/internal/pycore_tstate.h \ - $(srcdir)/Include/internal/pycore_tuple.h \ - $(srcdir)/Include/internal/pycore_typeobject.h \ - $(srcdir)/Include/internal/pycore_typevarobject.h \ - $(srcdir)/Include/internal/pycore_ucnhash.h \ - $(srcdir)/Include/internal/pycore_unionobject.h \ - $(srcdir)/Include/internal/pycore_unicodeobject.h \ - $(srcdir)/Include/internal/pycore_unicodeobject_generated.h \ - $(srcdir)/Include/internal/pycore_uop_metadata.h \ - $(srcdir)/Include/internal/pycore_warnings.h \ - $(srcdir)/Include/internal/pycore_weakref.h \ - $(DTRACE_HEADERS) \ - @PLATFORM_HEADERS@ \ - \ - $(srcdir)/Python/stdlib_module_names.h - -$(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) +$(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) ###################################################################### @@ -1981,7 +2034,7 @@ multissltest: all # which can lead to two parallel `./python setup.py build` processes that # step on each others toes. .PHONY: install -install: @FRAMEWORKINSTALLFIRST@ commoninstall bininstall maninstall @FRAMEWORKINSTALLLAST@ +install: @FRAMEWORKINSTALLFIRST@ @INSTALLTARGETS@ @FRAMEWORKINSTALLLAST@ if test "x$(ENSUREPIP)" != "xno" ; then \ case $(ENSUREPIP) in \ upgrade) ensurepip="--upgrade" ;; \ @@ -2200,13 +2253,13 @@ LIBSUBDIRS= asyncio \ __phello__ TESTSUBDIRS= idlelib/idle_test \ test \ - test/audiodata \ test/archivetestdata \ + test/audiodata \ test/certdata \ test/certdata/capath \ test/cjkencodings \ - test/crashers \ test/configdata \ + test/crashers \ test/data \ test/decimaltestdata \ test/dtracedata \ @@ -2220,8 +2273,10 @@ TESTSUBDIRS= idlelib/idle_test \ test/subprocessdata \ test/support \ test/support/_hypothesis_stubs \ + test/support/interpreters \ test/test_asyncio \ test/test_capi \ + test/test_concurrent_futures \ test/test_cppext \ test/test_ctypes \ test/test_dataclasses \ @@ -2230,7 +2285,6 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_email/data \ test/test_future_stmt \ test/test_gdb \ - test/test_inspect \ test/test_import \ test/test_import/data \ test/test_import/data/circular_imports \ @@ -2283,10 +2337,16 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_importlib/resources/zipdata01 \ test/test_importlib/resources/zipdata02 \ test/test_importlib/source \ + test/test_inspect \ + test/test_interpreters \ test/test_json \ test/test_module \ + test/test_multiprocessing_fork \ + test/test_multiprocessing_forkserver \ + test/test_multiprocessing_spawn \ test/test_pathlib \ test/test_peg_generator \ + test/test_pydoc \ test/test_sqlite3 \ test/test_tkinter \ test/test_tomllib \ @@ -2563,10 +2623,11 @@ frameworkinstall: install # only have to cater for the structural bits of the framework. .PHONY: frameworkinstallframework -frameworkinstallframework: frameworkinstallstructure install frameworkinstallmaclib +frameworkinstallframework: @FRAMEWORKINSTALLFIRST@ install frameworkinstallmaclib -.PHONY: frameworkinstallstructure -frameworkinstallstructure: $(LDLIBRARY) +# macOS uses a versioned frameworks structure that includes a full install +.PHONY: frameworkinstallversionedstructure +frameworkinstallversionedstructure: $(LDLIBRARY) @if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \ echo Not configured with --enable-framework; \ exit 1; \ @@ -2587,6 +2648,27 @@ frameworkinstallstructure: $(LDLIBRARY) $(LN) -fsn Versions/Current/Resources $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Resources $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/$(LDLIBRARY) +# iOS/tvOS/watchOS uses a non-versioned framework with Info.plist in the +# framework root, no .lproj data, and only stub compilation assistance binaries +.PHONY: frameworkinstallunversionedstructure +frameworkinstallunversionedstructure: $(LDLIBRARY) + @if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \ + echo Not configured with --enable-framework; \ + exit 1; \ + else true; \ + fi + if test -d $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include; then \ + echo "Clearing stale header symlink directory"; \ + rm -rf $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include; \ + fi + $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR) + sed 's/%VERSION%/'"`$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import platform; print(platform.python_version())'`"'/g' < $(RESSRCDIR)/Info.plist > $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Info.plist + $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/$(LDLIBRARY) + $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(BINDIR) + for file in $(srcdir)/$(RESSRCDIR)/bin/* ; do \ + $(INSTALL) -m $(EXEMODE) $$file $(DESTDIR)$(BINDIR); \ + done + # This installs Mac/Lib into the framework # Install a number of symlinks to keep software that expects a normal unix # install (which includes python-config) happy. @@ -2627,6 +2709,19 @@ frameworkaltinstallunixtools: frameworkinstallextras: cd Mac && $(MAKE) installextras DESTDIR="$(DESTDIR)" +# On iOS, bin/lib can't live inside the framework; include needs to be called +# "Headers", but *must* be in the framework, and *not* include the `python3.X` +# subdirectory. The install has put these folders in the same folder as +# Python.framework; Move the headers to their final framework-compatible home. +.PHONY: frameworkinstallmobileheaders +frameworkinstallmobileheaders: + if test -d $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers; then \ + echo "Removing old framework headers"; \ + rm -rf $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers; \ + fi + mv "$(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include/python$(VERSION)" "$(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers" + $(LN) -fs "../$(PYTHONFRAMEWORKDIR)/Headers" "$(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include/python$(VERSION)" + # Build the toplevel Makefile Makefile.pre: $(srcdir)/Makefile.pre.in config.status CONFIG_FILES=Makefile.pre CONFIG_HEADERS= ./config.status @@ -2641,6 +2736,23 @@ config.status: $(srcdir)/configure Python/asm_trampoline.o: $(srcdir)/Python/asm_trampoline.S $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< + +JIT_DEPS = \ + $(srcdir)/Tools/jit/*.c \ + $(srcdir)/Tools/jit/*.py \ + $(srcdir)/Python/executor_cases.c.h \ + pyconfig.h + +jit_stencils.h: $(JIT_DEPS) + @REGEN_JIT_COMMAND@ + +Python/jit.o: $(srcdir)/Python/jit.c @JIT_STENCILS_H@ + $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< + +.PHONY: regen-jit +regen-jit: + @REGEN_JIT_COMMAND@ + # Some make's put the object file in the current directory .c.o: $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< @@ -2733,6 +2845,7 @@ clean-retain-profile: pycremoval -rm -f Python/deepfreeze/*.[co] -rm -f Python/frozen_modules/*.h -rm -f Python/frozen_modules/MANIFEST + -rm -f jit_stencils.h -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' -rm -f Include/pydtrace_probes.h -rm -f profile-gen-stamp @@ -2855,8 +2968,11 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h # force rebuild when header file or module build flavor (static/shared) is changed MODULE_DEPS_STATIC=Modules/config.c -MODULE_DEPS_SHARED=$(MODULE_DEPS_STATIC) $(EXPORTSYMS) +MODULE_DEPS_SHARED=@MODULE_DEPS_SHARED@ +MODULE__CURSES_DEPS=$(srcdir)/Include/py_curses.h +MODULE__CURSES_PANEL_DEPS=$(srcdir)/Include/py_curses.h +MODULE__DATETIME_DEPS=$(srcdir)/Include/datetime.h MODULE_CMATH_DEPS=$(srcdir)/Modules/_math.h MODULE_MATH_DEPS=$(srcdir)/Modules/_math.h MODULE_PYEXPAT_DEPS=@LIBEXPAT_INTERNAL@ diff --git a/Misc/ACKS b/Misc/ACKS index 466023f390a421d..f01c7a70a65dc52 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -756,6 +756,7 @@ Raymond Hettinger Lisa Hewus Fresh Kevan Heydon Wouter van Heyst +Derek Higgins Kelsey Hightower Jason Hildebrand Ryan Hileman @@ -1051,6 +1052,7 @@ Mark Lawrence Chris Laws Michael Layzell Michael Lazar +Peter Lazorchak Brian Leair Mathieu Leduc-Hamel Amandine Lee diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index f192bf086ed2598..4739e0fb89a4a8c 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -2722,7 +2722,7 @@ on future on an error - e.g. TimeoutError or KeyboardInterrupt. Fix a :mod:`sqlite3` regression where ``*args`` and ``**kwds`` were incorrectly relayed from :py:func:`~sqlite3.connect` to the :class:`~sqlite3.Connection` factory. The regression was introduced in -3.11a1 with PR 24421 (:gh:`85128`). Patch by Erlend E. Aasland.` +3.11a1 with PR 24421 (:gh:`85128`). Patch by Erlend E. Aasland. .. @@ -2988,7 +2988,7 @@ Kumar Aditya. .. section: Library Fix crash in :class:`struct.Struct` when it was not completely initialized -by initializing it in :meth:`~object.__new__``. Patch by Kumar Aditya. +by initializing it in :meth:`~object.__new__`. Patch by Kumar Aditya. .. diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst index 211513d05d00407..d9804be764c9a90 100644 --- a/Misc/NEWS.d/3.12.0b1.rst +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -563,10 +563,10 @@ Complex function calls are now faster and consume no C stack space. .. nonce: fvgsCl .. section: Core and Builtins -``len()`` for 0-dimensional :class:`memoryview`` objects (such as +``len()`` for 0-dimensional :class:`memoryview` objects (such as ``memoryview(ctypes.c_uint8(42))``) now raises a :exc:`TypeError`. Previously this returned ``1``, which was not consistent with ``mem_0d[0]`` -raising an :exc:`IndexError``. +raising an :exc:`IndexError`. .. @@ -2371,7 +2371,7 @@ Add a new C-API function to eagerly assign a version tag to a PyTypeObject: .. nonce: _paFIF .. section: C API -:c:func:`PyObject_GC_Resize` should calculate preheader size if needed. +:c:macro:`PyObject_GC_Resize` should calculate preheader size if needed. Patch by Donghee Na. .. diff --git a/Misc/NEWS.d/3.13.0a1.rst b/Misc/NEWS.d/3.13.0a1.rst index 102bddcee5c5c2a..16715bee5a8e491 100644 --- a/Misc/NEWS.d/3.13.0a1.rst +++ b/Misc/NEWS.d/3.13.0a1.rst @@ -2276,7 +2276,7 @@ creation. .. nonce: m2H5Bk .. section: Library -Remove unnecessary extra ``__slots__`` in :py:class:`datetime`\'s pure +Remove unnecessary extra ``__slots__`` in :class:`~datetime.datetime`\'s pure python implementation to reduce memory size, as they are defined in the superclass. Patch by James Hilton-Balfe @@ -4380,7 +4380,7 @@ Patch by Victor Stinner. .. nonce: I6MQhb .. section: Library -:pep:`594`: Remove the :mod:`!cgi`` and :mod:`!cgitb` modules, deprecated in +:pep:`594`: Remove the :mod:`!cgi` and :mod:`!cgitb` modules, deprecated in Python 3.11. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.13.0a2.rst b/Misc/NEWS.d/3.13.0a2.rst index d4be4fb8a3d3ab6..e5841e14c02efbf 100644 --- a/Misc/NEWS.d/3.13.0a2.rst +++ b/Misc/NEWS.d/3.13.0a2.rst @@ -565,9 +565,9 @@ part of a :exc:`BaseExceptionGroup`, in addition to the recent support for .. section: Library The :class:`mmap.mmap` class now has an :meth:`~mmap.mmap.seekable` method -that can be used where it requires a file-like object with seekable and the -:meth:`~mmap.mmap.seek` method return the new absolute position. Patch by -Donghee Na. +that can be used when a seekable file-like object is required. +The :meth:`~mmap.mmap.seek` method now returns the new absolute position. +Patch by Donghee Na. .. diff --git a/Misc/NEWS.d/3.13.0a4.rst b/Misc/NEWS.d/3.13.0a4.rst new file mode 100644 index 000000000000000..39af0534cf8fb57 --- /dev/null +++ b/Misc/NEWS.d/3.13.0a4.rst @@ -0,0 +1,1433 @@ +.. date: 2024-02-13-15-14-39 +.. gh-issue: 115399 +.. nonce: xT-scP +.. release date: 2024-02-15 +.. section: Security + +Update bundled libexpat to 2.6.0 + +.. + +.. date: 2024-02-12-00-33-01 +.. gh-issue: 115243 +.. nonce: e1oGX8 +.. section: Security + +Fix possible crashes in :meth:`collections.deque.index` when the deque is +concurrently modified. + +.. + +.. date: 2024-02-14-23-50-55 +.. gh-issue: 112087 +.. nonce: H_4W_v +.. section: Core and Builtins + +For an empty reverse iterator for list will be reduced to :func:`reversed`. +Patch by Donghee Na + +.. + +.. date: 2024-02-12-17-18-26 +.. gh-issue: 114570 +.. nonce: BzwMlJ +.. section: Core and Builtins + +Add :exc:`PythonFinalizationError` exception. This exception derived from +:exc:`RuntimeError` is raised when an operation is blocked during the +:term:`Python finalization <interpreter shutdown>`. Patch by Victor Stinner. + +.. + +.. date: 2024-02-07-18-04-36 +.. gh-issue: 114695 +.. nonce: o9wP5P +.. section: Core and Builtins + +Add :func:`sys._clear_internal_caches`, which clears all internal +performance-related caches (and deprecate the less-general +:func:`sys._clear_type_cache` function). + +.. + +.. date: 2024-02-07-07-50-12 +.. gh-issue: 114828 +.. nonce: nSXwMi +.. section: Core and Builtins + +Fix compilation crashes in uncommon code examples using :func:`super` inside +a comprehension in a class body. + +.. + +.. date: 2024-02-07-00-18-42 +.. gh-issue: 112069 +.. nonce: jRDRR5 +.. section: Core and Builtins + +Adapt :class:`set` and :class:`frozenset` methods to Argument Clinic. + +.. + +.. date: 2024-02-05-12-40-26 +.. gh-issue: 115011 +.. nonce: L1AKF5 +.. section: Core and Builtins + +Setters for members with an unsigned integer type now support the same range +of valid values for objects that has a :meth:`~object.__index__` method as +for :class:`int`. + +.. + +.. date: 2024-02-03-04-07-18 +.. gh-issue: 114887 +.. nonce: uLSFmN +.. section: Core and Builtins + +Changed socket type validation in +:meth:`~asyncio.loop.create_datagram_endpoint` to accept all non-stream +sockets. This fixes a regression in compatibility with raw sockets. + +.. + +.. date: 2024-02-03-01-48-38 +.. gh-issue: 114944 +.. nonce: 4J5ELD +.. section: Core and Builtins + +Fixes a race between ``PyParkingLot_Park`` and ``_PyParkingLot_UnparkAll``. + +.. + +.. date: 2024-02-02-05-27-48 +.. gh-issue: 113462 +.. nonce: VMml8q +.. section: Core and Builtins + +Limit the number of versions that a single class can use. Prevents a few +wayward classes using up all the version numbers. + +.. + +.. date: 2024-02-01-23-43-49 +.. gh-issue: 76763 +.. nonce: o_2J6i +.. section: Core and Builtins + +The :func:`chr` builtin function now always raises :exc:`ValueError` for +values outside the valid range. Previously it raised :exc:`OverflowError` +for very large or small values. + +.. + +.. date: 2024-02-01-18-16-52 +.. gh-issue: 114806 +.. nonce: wrH2J6 +.. section: Core and Builtins + +No longer specialize calls to classes, if those classes have metaclasses. +Fixes bug where the ``__call__`` method of the metaclass was not being +called. + +.. + +.. date: 2024-01-31-09-10-10 +.. gh-issue: 107944 +.. nonce: XWm1B- +.. section: Core and Builtins + +Improve error message for function calls with bad keyword arguments via +getargs + +.. + +.. date: 2024-01-25-18-50-49 +.. gh-issue: 112529 +.. nonce: IbbApA +.. section: Core and Builtins + +The free-threaded build no longer allocates space for the ``PyGC_Head`` +structure in objects that support cyclic garbage collection. A number of +other fields and data structures are used as replacements, including +``ob_gc_bits``, ``ob_tid``, and mimalloc internal data structures. + +.. + +.. date: 2024-01-22-15-10-01 +.. gh-issue: 114456 +.. nonce: fBFEJF +.. section: Core and Builtins + +Lower the recursion limit under a debug build of WASI. + +.. + +.. date: 2024-01-22-09-49-02 +.. gh-issue: 114083 +.. nonce: hf1-ku +.. section: Core and Builtins + +Compiler applies folding of LOAD_CONST with following instruction in a +separate pass before other optimisations. This enables jump threading in +certain circumstances. + +.. + +.. date: 2024-01-21-17-29-32 +.. gh-issue: 114388 +.. nonce: UVGO4K +.. section: Core and Builtins + +Fix a :exc:`RuntimeWarning` emitted when assign an integer-like value that +is not an instance of :class:`int` to an attribute that corresponds to a C +struct member of :ref:`type <PyMemberDef-types>` T_UINT and T_ULONG. Fix a +double :exc:`RuntimeWarning` emitted when assign a negative integer value to +an attribute that corresponds to a C struct member of type T_UINT. + +.. + +.. date: 2024-01-19-13-18-13 +.. gh-issue: 114265 +.. nonce: 7HAi-- +.. section: Core and Builtins + +Compiler propagates line numbers before optimization, leading to more +optimization opportunities and removing the need for the +``guarantee_lineno_for_exits`` hack. + +.. + +.. date: 2024-01-18-20-20-37 +.. gh-issue: 112529 +.. nonce: oVNvDG +.. section: Core and Builtins + +The free-threaded build now has its own thread-safe GC implementation that +uses mimalloc to find GC tracked objects. It is non-generational, unlike the +existing GC implementation. + +.. + +.. date: 2024-01-17-23-39-20 +.. gh-issue: 114050 +.. nonce: Lnv1oq +.. section: Core and Builtins + +Fix segmentation fault caused by an incorrect format string in ``TypeError`` +exception when more than two arguments are passed to ``int``. + +.. + +.. date: 2024-01-17-05-09-32 +.. gh-issue: 112354 +.. nonce: Run9ko +.. section: Core and Builtins + +The ``END_FOR`` instruction now pops only one value. This is to better +support side exits in loops. + +.. + +.. date: 2024-01-17-00-52-57 +.. gh-issue: 113884 +.. nonce: CvEjUE +.. section: Core and Builtins + +Make :class:`queue.SimpleQueue` thread safe when the GIL is disabled. + +.. + +.. date: 2024-01-16-14-41-54 +.. gh-issue: 114058 +.. nonce: Cb2b8h +.. section: Core and Builtins + +Implement the foundations of the Tier 2 redundancy eliminator. + +.. + +.. date: 2024-01-12-16-40-07 +.. gh-issue: 113939 +.. nonce: Yi3L-e +.. section: Core and Builtins + +frame.clear(): Clear frame.f_locals as well, and not only the fast locals. +This is relevant once frame.f_locals was accessed, which would contain also +references to all the locals. + +.. + +.. date: 2024-01-11-22-58-45 +.. gh-issue: 112050 +.. nonce: hDuvDW +.. section: Core and Builtins + +Convert :class:`collections.deque` to use Argument Clinic. + +.. + +.. date: 2024-01-08-21-57-41 +.. gh-issue: 112050 +.. nonce: qwgjx1 +.. section: Core and Builtins + +Make methods on :class:`collections.deque` thread-safe when the GIL is +disabled. + +.. + +.. date: 2023-12-24-03-25-28 +.. gh-issue: 113464 +.. nonce: dvjQmA +.. section: Core and Builtins + +Add an option (``--enable-experimental-jit`` for ``configure``-based builds +or ``--experimental-jit`` for ``PCbuild``-based ones) to build an +*experimental* just-in-time compiler, based on `copy-and-patch +<https://fredrikbk.com/publications/copy-and-patch.pdf>`_ + +.. + +.. date: 2023-12-22-13-21-39 +.. gh-issue: 113055 +.. nonce: 47xBMF +.. section: Core and Builtins + +Make interp->obmalloc a pointer. For interpreters that share state with the +main interpreter, this points to the same static memory structure. For +interpreters with their own obmalloc state, it is heap allocated. Add +free_obmalloc_arenas() which will free the obmalloc arenas and radix tree +structures for interpreters with their own obmalloc state. + +.. + +.. date: 2023-06-06-19-09-00 +.. gh-issue: 55664 +.. nonce: vYYl0V +.. section: Core and Builtins + +Add warning when creating :class:`type` using a namespace dictionary with +non-string keys. Patched by Daniel Urban and Furkan Onder. + +.. + +.. date: 2023-05-16-06-52-34 +.. gh-issue: 104530 +.. nonce: mJnA0W +.. section: Core and Builtins + +Use native Win32 condition variables. + +.. + +.. date: 2024-02-13-18-27-03 +.. gh-issue: 115392 +.. nonce: gle5tp +.. section: Library + +Fix a bug in :mod:`doctest` where incorrect line numbers would be reported +for decorated functions. + +.. + +.. date: 2024-02-11-20-23-36 +.. gh-issue: 114563 +.. nonce: RzxNYT +.. section: Library + +Fix several :func:`format()` bugs when using the C implementation of +:class:`~decimal.Decimal`: * memory leak in some rare cases when using the +``z`` format option (coerce negative 0) * incorrect output when applying the +``z`` format option to type ``F`` (fixed-point with capital ``NAN`` / +``INF``) * incorrect output when applying the ``#`` format option (alternate +form) + +.. + +.. date: 2024-02-10-15-24-20 +.. gh-issue: 102840 +.. nonce: 4mnDq1 +.. section: Library + +Fix confused traceback when floordiv, mod, or divmod operations happens +between instances of :class:`fractions.Fraction` and :class:`complex`. + +.. + +.. date: 2024-02-09-07-20-16 +.. gh-issue: 115165 +.. nonce: yfJLXA +.. section: Library + +Most exceptions are now ignored when attempting to set the +``__orig_class__`` attribute on objects returned when calling :mod:`typing` +generic aliases (including generic aliases created using +:data:`typing.Annotated`). Previously only :exc:`AttributeError` was +ignored. Patch by Dave Shawley. + +.. + +.. date: 2024-02-08-17-04-58 +.. gh-issue: 112903 +.. nonce: SN_vUs +.. section: Library + +Fix "issubclass() arg 1 must be a class" errors in certain cases of multiple +inheritance with generic aliases (regression in early 3.13 alpha releases). + +.. + +.. date: 2024-02-08-14-21-28 +.. gh-issue: 115133 +.. nonce: ycl4ko +.. section: Library + +Fix tests for :class:`~xml.etree.ElementTree.XMLPullParser` with Expat +2.6.0. + +.. + +.. date: 2024-02-08-13-26-14 +.. gh-issue: 115059 +.. nonce: DqP9dr +.. section: Library + +:meth:`io.BufferedRandom.read1` now flushes the underlying write buffer. + +.. + +.. date: 2024-02-07-12-37-52 +.. gh-issue: 79382 +.. nonce: Yz_5WB +.. section: Library + +Trailing ``**`` no longer allows to match files and non-existing paths in +recursive :func:`~glob.glob`. + +.. + +.. date: 2024-02-06-15-16-28 +.. gh-issue: 67837 +.. nonce: _JKa73 +.. section: Library + +Avoid race conditions in the creation of directories during concurrent +extraction in :mod:`tarfile` and :mod:`zipfile`. + +.. + +.. date: 2024-02-06-03-55-46 +.. gh-issue: 115060 +.. nonce: EkWRpP +.. section: Library + +Speed up :meth:`pathlib.Path.glob` by removing redundant regex matching. + +.. + +.. date: 2024-02-05-16-48-06 +.. gh-issue: 97928 +.. nonce: JZCies +.. section: Library + +Partially revert the behavior of :meth:`tkinter.Text.count`. By default it +preserves the behavior of older Python versions, except that setting +``wantobjects`` to 0 no longer has effect. Add a new parameter +*return_ints*: specifying ``return_ints=True`` makes ``Text.count()`` always +returning the single count as an integer instead of a 1-tuple or ``None``. + +.. + +.. date: 2024-02-04-13-17-33 +.. gh-issue: 114628 +.. nonce: WJpqqS +.. section: Library + +When csv.Error is raised when handling TypeError, do not print the TypeError +traceback. + +.. + +.. date: 2024-02-04-02-28-37 +.. gh-issue: 85984 +.. nonce: NHZVTQ +.. section: Library + +Added ``_POSIX_VDISABLE`` from C's ``<unistd.h>`` to :mod:`termios`. + +.. + +.. date: 2024-02-03-17-54-17 +.. gh-issue: 114965 +.. nonce: gHksCK +.. section: Library + +Update bundled pip to 24.0 + +.. + +.. date: 2024-02-03-16-59-25 +.. gh-issue: 114959 +.. nonce: dCfAG2 +.. section: Library + +:mod:`tarfile` no longer ignores errors when trying to extract a directory +on top of a file. + +.. + +.. date: 2024-02-02-15-50-13 +.. gh-issue: 114894 +.. nonce: DF-dSd +.. section: Library + +Add :meth:`array.array.clear`. + +.. + +.. date: 2024-02-01-10-19-11 +.. gh-issue: 114071 +.. nonce: vkm2G_ +.. section: Library + +Support tuple subclasses using auto() for enum member value. + +.. + +.. date: 2024-01-31-20-07-11 +.. gh-issue: 109475 +.. nonce: lmTb9S +.. section: Library + +Fix support of explicit option value "--" in :mod:`argparse` (e.g. +``--option=--``). + +.. + +.. date: 2024-01-30-22-10-50 +.. gh-issue: 49766 +.. nonce: yulJL_ +.. section: Library + +Fix :class:`~datetime.date`-:class:`~datetime.datetime` comparison. Now the +special comparison methods like ``__eq__`` and ``__lt__`` return +:data:`NotImplemented` if one of comparands is :class:`!date` and other is +:class:`!datetime` instead of ignoring the time part and the time zone or +forcefully return "not equal" or raise :exc:`TypeError`. It makes comparison +of :class:`!date` and :class:`!datetime` subclasses more symmetric and +allows to change the default behavior by overriding the special comparison +methods in subclasses. + +.. + +.. date: 2024-01-30-15-34-08 +.. gh-issue: 110190 +.. nonce: Z5PQQX +.. section: Library + +Fix ctypes structs with array on Windows ARM64 platform by setting +``MAX_STRUCT_SIZE`` to 32 in stgdict. Patch by Diego Russo + +.. + +.. date: 2024-01-28-19-40-40 +.. gh-issue: 114678 +.. nonce: kYKcJw +.. section: Library + +Ensure that deprecation warning for 'N' specifier in +:class:`~decimal.Decimal` format is not raised for cases where 'N' appears +in other places in the format specifier. Based on patch by Stefan Krah. + +.. + +.. date: 2024-01-28-18-38-18 +.. gh-issue: 70303 +.. nonce: _Lt_pj +.. section: Library + +Return both files and directories from :meth:`pathlib.Path.glob` if a +pattern ends with "``**``". Previously only directories were returned. + +.. + +.. date: 2024-01-28-00-48-12 +.. gh-issue: 109653 +.. nonce: vF4exe +.. section: Library + +Improve import time of :mod:`importlib.metadata` and :mod:`email.utils`. + +.. + +.. date: 2024-01-27-20-11-24 +.. gh-issue: 113280 +.. nonce: CZPQMf +.. section: Library + +Fix a leak of open socket in rare cases when error occurred in +:class:`ssl.SSLSocket` creation. + +.. + +.. date: 2024-01-26-16-46-21 +.. gh-issue: 77749 +.. nonce: NY_7TS +.. section: Library + +:meth:`email.policy.EmailPolicy.fold` now always encodes non-ASCII +characters in headers if :attr:`~email.policy.EmailPolicy.utf8` is false. + +.. + +.. date: 2024-01-25-19-22-17 +.. gh-issue: 83383 +.. nonce: 3GwO9v +.. section: Library + +Synchronization of the :mod:`dbm.dumb` database is now no-op if there was no +modification since opening or last synchronization. The directory file for a +newly created empty :mod:`dbm.dumb` database is now created immediately +after opening instead of deferring this until synchronizing or closing. + +.. + +.. date: 2024-01-24-20-51-49 +.. gh-issue: 91602 +.. nonce: 8fOH8l +.. section: Library + +Add *filter* keyword-only parameter to :meth:`sqlite3.Connection.iterdump` +for filtering database objects to dump. Patch by Mariusz Felisiak. + +.. + +.. date: 2024-01-24-20-11-46 +.. gh-issue: 112451 +.. nonce: 7YrG4p +.. section: Library + +Prohibit subclassing pure-Python :class:`datetime.timezone`. This is +consistent with C-extension implementation. Patch by Mariusz Felisiak. + +.. + +.. date: 2024-01-24-17-25-18 +.. gh-issue: 69893 +.. nonce: PQq5fR +.. section: Library + +Add the :meth:`!close` method for the iterator returned by +:func:`xml.etree.ElementTree.iterparse`. + +.. + +.. date: 2024-01-23-23-13-47 +.. gh-issue: 109653 +.. nonce: KLBHmT +.. section: Library + +Reduce the import time of :mod:`threading` module by ~50%. Patch by Daniel +Hollas. + +.. + +.. date: 2024-01-23-21-20-40 +.. gh-issue: 114492 +.. nonce: vKxl5o +.. section: Library + +Make the result of :func:`termios.tcgetattr` reproducible on Alpine Linux. +Previously it could leave a random garbage in some fields. + +.. + +.. date: 2024-01-23-14-11-49 +.. gh-issue: 114315 +.. nonce: KeVdzl +.. section: Library + +Make :class:`threading.Lock` a real class, not a factory function. Add +``__new__`` to ``_thread.lock`` type. + +.. + +.. date: 2024-01-23-13-03-22 +.. gh-issue: 100414 +.. nonce: 5kTdU5 +.. section: Library + +Add :mod:`dbm.sqlite3` as a backend to :mod:`dbm`, and make it the new +default :mod:`!dbm` backend. Patch by Raymond Hettinger and Erlend E. +Aasland. + +.. + +.. date: 2024-01-23-11-04-21 +.. gh-issue: 113267 +.. nonce: xe_Pxe +.. section: Library + +Revert changes in :gh:`106584` which made calls of ``TestResult`` methods +``startTest()`` and ``stopTest()`` unbalanced. + +.. + +.. date: 2024-01-22-12-10-34 +.. gh-issue: 75128 +.. nonce: 4FGlRS +.. section: Library + +Ignore an :exc:`OSError` in :meth:`asyncio.BaseEventLoop.create_server` when +IPv6 is available but the interface cannot actually support it. + +.. + +.. date: 2024-01-22-11-43-38 +.. gh-issue: 114423 +.. nonce: 6mMoPH +.. section: Library + +``_DummyThread`` entries in ``threading._active`` are now automatically +removed when the related thread dies. + +.. + +.. date: 2024-01-21-16-32-55 +.. gh-issue: 114257 +.. nonce: bCFld5 +.. section: Library + +Dismiss the :exc:`FileNotFound` error in :func:`ctypes.util.find_library` +and just return ``None`` on Linux. + +.. + +.. date: 2024-01-19-18-41-02 +.. gh-issue: 114321 +.. nonce: yj_Xw3 +.. section: Library + +Expose more platform specific constants in the :mod:`fcntl` module on Linux, +macOS, FreeBSD and NetBSD. + +.. + +.. date: 2024-01-19-15-48-06 +.. gh-issue: 114328 +.. nonce: hixxW3 +.. section: Library + +The :func:`tty.setcbreak` and new :func:`tty.cfmakecbreak` no longer clears +the terminal input ICRLF flag. This fixes a regression introduced in 3.12 +that no longer matched how OSes define cbreak mode in their ``stty(1)`` +manual pages. + +.. + +.. date: 2024-01-19-12-05-22 +.. gh-issue: 114281 +.. nonce: H5JQe4 +.. section: Library + +Remove type hints from ``Lib/asyncio/staggered.py``. The annotations in the +`typeshed <https://github.com/python/typeshed>`__ project should be used +instead. + +.. + +.. date: 2024-01-18-22-29-28 +.. gh-issue: 101438 +.. nonce: 1-uUi_ +.. section: Library + +Avoid reference cycle in ElementTree.iterparse. The iterator returned by +``ElementTree.iterparse`` may hold on to a file descriptor. The reference +cycle prevented prompt clean-up of the file descriptor if the returned +iterator was not exhausted. + +.. + +.. date: 2024-01-18-10-07-52 +.. gh-issue: 114198 +.. nonce: lK4Iif +.. section: Library + +The signature for the ``__replace__`` method on :mod:`dataclasses` now has +the first argument named ``self``, rather than ``obj``. + +.. + +.. date: 2024-01-17-18-53-51 +.. gh-issue: 104522 +.. nonce: 3NyDf4 +.. section: Library + +:exc:`OSError` raised when run a subprocess now only has *filename* +attribute set to *cwd* if the error was caused by a failed attempt to change +the current directory. + +.. + +.. date: 2024-01-16-15-59-06 +.. gh-issue: 114149 +.. nonce: LJ8IPm +.. section: Library + +Enum: correctly handle tuple subclasses in custom ``__new__``. + +.. + +.. date: 2024-01-15-20-21-33 +.. gh-issue: 83648 +.. nonce: HzD_fY +.. section: Library + +Support deprecation of options, positional arguments and subcommands in +:mod:`argparse`. + +.. + +.. date: 2024-01-15-19-54-41 +.. gh-issue: 114087 +.. nonce: Xic5vY +.. section: Library + +Speed up ``dataclasses.asdict`` up to 1.35x. + +.. + +.. date: 2024-01-15-18-42-44 +.. gh-issue: 109534 +.. nonce: wYaLMZ +.. section: Library + +Fix a reference leak in +:class:`asyncio.selector_events.BaseSelectorEventLoop` when SSL handshakes +fail. Patch contributed by Jamie Phan. + +.. + +.. date: 2024-01-12-17-32-36 +.. gh-issue: 79634 +.. nonce: uTSTRI +.. section: Library + +Accept :term:`path-like objects <path-like object>` as patterns in +:meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob`. + +.. + +.. date: 2024-01-12-09-35-07 +.. gh-issue: 112202 +.. nonce: t_0V1m +.. section: Library + +Ensure that a :func:`asyncio.Condition.notify` call does not get lost if the +awakened ``Task`` is simultaneously cancelled or encounters any other error. + +.. + +.. date: 2024-01-11-20-47-49 +.. gh-issue: 113951 +.. nonce: AzlqFK +.. section: Library + +Fix the behavior of ``tag_unbind()`` methods of :class:`tkinter.Text` and +:class:`tkinter.Canvas` classes with three arguments. Previously, +``widget.tag_unbind(tag, sequence, funcid)`` destroyed the current binding +for *sequence*, leaving *sequence* unbound, and deleted the *funcid* +command. Now it removes only *funcid* from the binding for *sequence*, +keeping other commands, and deletes the *funcid* command. It leaves +*sequence* unbound only if *funcid* was the last bound command. + +.. + +.. date: 2024-01-11-15-10-53 +.. gh-issue: 97959 +.. nonce: UOj6d4 +.. section: Library + +Fix rendering class methods, bound methods, method and function aliases in +:mod:`pydoc`. Class methods no longer have "method of builtins.type +instance" note. Corresponding notes are now added for class and unbound +methods. Method and function aliases now have references to the module or +the class where the origin was defined if it differs from the current. Bound +methods are now listed in the static methods section. Methods of builtin +classes are now supported as well as methods of Python classes. + +.. + +.. date: 2024-01-07-21-04-24 +.. gh-issue: 113796 +.. nonce: 6iNsCR +.. section: Library + +Add more validation checks in the :class:`csv.Dialect` constructor. +:exc:`ValueError` is now raised if the same character is used in different +roles. + +.. + +.. date: 2024-01-05-16-27-34 +.. gh-issue: 113732 +.. nonce: fgDRXA +.. section: Library + +Fix support of :data:`~csv.QUOTE_NOTNULL` and :data:`~csv.QUOTE_STRINGS` in +:func:`csv.reader`. + +.. + +.. date: 2024-01-04-20-58-17 +.. gh-issue: 113225 +.. nonce: -nyJM4 +.. section: Library + +Speed up :meth:`pathlib.Path.walk` by using :attr:`os.DirEntry.path` where +possible. + +.. + +.. date: 2023-12-18-20-10-50 +.. gh-issue: 89039 +.. nonce: gqFdtU +.. section: Library + +When replace() method is called on a subclass of datetime, date or time, +properly call derived constructor. Previously, only the base class's +constructor was called. + +Also, make sure to pass non-zero fold values when creating subclasses in +various methods. Previously, fold was silently ignored. + +.. + +.. date: 2023-12-09-23-31-17 +.. gh-issue: 112919 +.. nonce: S5k9QN +.. section: Library + +Speed-up :func:`datetime.datetime.replace`, :func:`datetime.date.replace` +and :func:`datetime.time.replace`. + +.. + +.. date: 2023-11-27-19-54-43 +.. gh-issue: 59013 +.. nonce: chpQ0e +.. section: Library + +Set breakpoint on the first executable line of the function, instead of the +line of function definition when the user do ``break func`` using :mod:`pdb` + +.. + +.. date: 2023-11-24-19-08-50 +.. gh-issue: 112343 +.. nonce: RarGFC +.. section: Library + +Improve handling of pdb convenience variables to avoid replacing string +contents. + +.. + +.. date: 2023-11-18-16-30-21 +.. gh-issue: 112240 +.. nonce: YXS0tj +.. section: Library + +Add option to calendar module CLI to specify the weekday to start each week. +Patch by Steven Ward. + +.. + +.. date: 2023-11-04-22-32-27 +.. gh-issue: 111741 +.. nonce: f1ufr8 +.. section: Library + +Recognise ``image/webp`` as a standard format in the :mod:`mimetypes` +module. + +.. + +.. date: 2023-10-27-19-24-58 +.. gh-issue: 43457 +.. nonce: 84lx9H +.. section: Library + +Fix the :mod:`tkinter` widget method :meth:`!wm_attributes`. It now accepts +the attribute name without the minus prefix to get window attributes and +allows to specify attributes and values to set as keyword arguments. Add new +optional keyword argument *return_python_dict*: calling +``w.wm_attributes(return_python_dict=True)`` returns the attributes as a +dict instead of a tuple. Calling ``w.wm_attributes()`` now returns a tuple +instead of string if *wantobjects* was set to 0. + +.. + +.. date: 2023-10-24-19-19-54 +.. gh-issue: 82626 +.. nonce: _hfLRf +.. section: Library + +Many functions now emit a warning if a boolean value is passed as a file +descriptor argument. + +.. + +.. date: 2023-10-19-02-08-12 +.. gh-issue: 111051 +.. nonce: 8h1Dpk +.. section: Library + +Added check for file modification during debugging with :mod:`pdb` + +.. + +.. date: 2023-10-04-11-09-30 +.. gh-issue: 110345 +.. nonce: fZU1ud +.. section: Library + +Show the Tcl/Tk patchlevel (rather than version) in :meth:`tkinter._test`. + +.. + +.. date: 2023-09-22-22-17-45 +.. gh-issue: 38807 +.. nonce: m9McRN +.. section: Library + +Fix race condition in :mod:`trace`. Instead of checking if a directory +exists and creating it, directly call :func:`os.makedirs` with the kwarg +``exist_ok=True``. + +.. + +.. date: 2023-07-23-12-28-26 +.. gh-issue: 75705 +.. nonce: aB2-Ww +.. section: Library + +Set unixfrom envelope in :class:`mailbox.mbox` and :class:`mailbox.MMDF`. + +.. + +.. date: 2023-06-29-14-26-56 +.. gh-issue: 106233 +.. nonce: Aqw2HI +.. section: Library + +Fix stacklevel in ``InvalidTZPathWarning`` during :mod:`zoneinfo` module +import. + +.. + +.. date: 2023-05-30-18-30-11 +.. gh-issue: 105102 +.. nonce: SnpK04 +.. section: Library + +Allow :class:`ctypes.Union` to be nested in :class:`ctypes.Structure` when +the system endianness is the opposite of the classes. + +.. + +.. date: 2023-05-08-09-30-00 +.. gh-issue: 104282 +.. nonce: h4c6Eb +.. section: Library + +Fix null pointer dereference in :func:`lzma._decode_filter_properties` due +to improper handling of BCJ filters with properties of zero length. Patch by +Radislav Chugunov. + +.. + +.. date: 2023-05-06-04-57-10 +.. gh-issue: 96471 +.. nonce: C9wAU7 +.. section: Library + +Add :py:class:`queue.Queue` termination with +:py:meth:`~queue.Queue.shutdown`. + +.. + +.. date: 2023-04-08-11-41-07 +.. gh-issue: 101599 +.. nonce: PaWNFh +.. section: Library + +Changed argparse flag options formatting to remove redundancy. + +.. + +.. date: 2023-03-15-03-21-18 +.. gh-issue: 85984 +.. nonce: Xaq6ZN +.. section: Library + +Add POSIX pseudo-terminal functions :func:`os.posix_openpt`, +:func:`os.grantpt`, :func:`os.unlockpt`, and :func:`os.ptsname`. + +.. + +.. date: 2023-03-08-00-02-30 +.. gh-issue: 102512 +.. nonce: LiugDr +.. section: Library + +When :func:`os.fork` is called from a foreign thread (aka ``_DummyThread``), +the type of the thread in a child process is changed to ``_MainThread``. +Also changed its name and daemonic status, it can be now joined. + +.. + +.. date: 2022-07-31-01-24-40 +.. gh-issue: 88569 +.. nonce: eU0--b +.. section: Library + +Add :func:`os.path.isreserved`, which identifies reserved pathnames such as +"NUL", "AUX" and "CON". This function is only available on Windows. + +Deprecate :meth:`pathlib.PurePath.is_reserved`. + +.. + +.. bpo: 38364 +.. date: 2019-10-05-22-56-50 +.. nonce: sYTCWF +.. section: Library + +The ``inspect`` functions ``isgeneratorfunction``, ``iscoroutinefunction``, +``isasyncgenfunction`` now support ``functools.partialmethod`` wrapped +functions the same way they support ``functools.partial``. + +.. + +.. date: 2024-02-12-12-26-17 +.. gh-issue: 115233 +.. nonce: aug6r9 +.. section: Documentation + +Fix an example for :class:`~logging.LoggerAdapter` in the Logging Cookbook. + +.. + +.. date: 2024-01-17-11-40-03 +.. gh-issue: 114123 +.. nonce: LuueXf +.. section: Documentation + +Move the :mod:`csv` module docstring to the :mod:`!csv` module instead of +reexporting it from the internal :mod:`!_csv` module, and remove ``__doc__`` +from ``csv.__all__``. + +Move :attr:`!csv.__version__` to the :mod:`!csv` module instead of +reexporting it from the internal :mod:`!_csv` module, and remove +``__version__`` from ``csv.__all__``. + +.. + +.. date: 2024-02-02-13-18-55 +.. gh-issue: 114099 +.. nonce: C_ycWg +.. section: Tests + +Added test exclusions required to run the test suite on iOS. + +.. + +.. date: 2023-06-02-05-04-15 +.. gh-issue: 105089 +.. nonce: KaZFtU +.. section: Tests + +Fix +``test.test_zipfile.test_core.TestWithDirectory.test_create_directory_with_write`` +test in AIX by doing a bitwise AND of 0xFFFF on mode , so that it will be in +sync with ``zinfo.external_attr`` + +.. + +.. date: 2024-02-08-19-36-20 +.. gh-issue: 115167 +.. nonce: LB9nDK +.. section: Build + +Avoid vendoring ``vcruntime140_threads.dll`` when building with Visual +Studio 2022 version 17.8. + +.. + +.. date: 2024-02-08-17-38-56 +.. gh-issue: 113632 +.. nonce: y9KIGb +.. section: Build + +Promote WASI to a tier 2 platform and drop Emscripten from tier 3 in +configure.ac. + +.. + +.. date: 2024-02-07-08-23-48 +.. gh-issue: 114099 +.. nonce: XcEXEZ +.. section: Build + +configure and Makefile were refactored to accomodate framework builds on +Apple platforms other than macOS. + +.. + +.. date: 2024-02-01-20-08-11 +.. gh-issue: 114875 +.. nonce: x_2iZ9 +.. section: Build + +Add :c:func:`!getgrent` as a prerequisite for building the :mod:`grp` +module. + +.. + +.. date: 2024-02-08-21-37-22 +.. gh-issue: 115049 +.. nonce: X1ObpJ +.. section: Windows + +Fixes ``py.exe`` launcher failing when run as users without user profiles. + +.. + +.. date: 2024-02-06-09-05-13 +.. gh-issue: 115009 +.. nonce: ShMjZs +.. section: Windows + +Update Windows installer to use SQLite 3.45.1. + +.. + +.. date: 2024-02-05-16-53-12 +.. gh-issue: 109991 +.. nonce: YqjnDz +.. section: Windows + +Update Windows build to use OpenSSL 3.0.13. + +.. + +.. date: 2024-02-01-14-35-05 +.. gh-issue: 111239 +.. nonce: SO7SUF +.. section: Windows + +Update Windows builds to use zlib v1.3.1. + +.. + +.. date: 2024-01-23-00-05-05 +.. gh-issue: 100107 +.. nonce: lkbP_Q +.. section: Windows + +The ``py.exe`` launcher will no longer attempt to run the Microsoft Store +redirector when launching a script containing a ``/usr/bin/env`` shebang + +.. + +.. date: 2023-12-19-22-32-28 +.. gh-issue: 112984 +.. nonce: F7kFMl +.. section: Windows + +Adds free-threaded binaries to Windows installer as an optional component. + +.. + +.. date: 2023-08-11-18-21-38 +.. gh-issue: 89240 +.. nonce: dtSOLG +.. section: Windows + +Allows :mod:`multiprocessing` to create pools of greater than 62 processes. + +.. + +.. date: 2024-02-06-09-01-10 +.. gh-issue: 115009 +.. nonce: ysau7e +.. section: macOS + +Update macOS installer to use SQLite 3.45.1. + +.. + +.. date: 2024-02-05-18-30-27 +.. gh-issue: 109991 +.. nonce: tun6Yu +.. section: macOS + +Update macOS installer to use OpenSSL 3.0.13. + +.. + +.. date: 2024-01-23-11-35-26 +.. gh-issue: 114490 +.. nonce: FrQOQ0 +.. section: macOS + +Add Mach-O linkage support for :func:`platform.architecture()`. + +.. + +.. date: 2022-11-18-10-05-35 +.. gh-issue: 87804 +.. nonce: rhlDmD +.. section: macOS + +On macOS the result of ``os.statvfs`` and ``os.fstatvfs`` now correctly +report the size of very large disks, in previous versions the reported +number of blocks was wrong for disks with at least 2**32 blocks. + +.. + +.. date: 2024-01-17-23-18-15 +.. gh-issue: 96905 +.. nonce: UYaxoU +.. section: IDLE + +In idlelib code, stop redefining built-ins 'dict' and 'object'. + +.. + +.. date: 2023-04-25-03-01-23 +.. gh-issue: 103820 +.. nonce: LCSpza +.. section: IDLE + +Revise IDLE bindings so that events from mouse button 4/5 on non-X11 +windowing systems (i.e. Win32 and Aqua) are not mistaken for scrolling. + +.. + +.. date: 2024-02-14-15-58-13 +.. gh-issue: 113516 +.. nonce: TyIHWx +.. section: Tools/Demos + +Don't set ``LDSHARED`` when building for WASI. + +.. + +.. date: 2024-02-05-19-00-32 +.. gh-issue: 109991 +.. nonce: yJSEkw +.. section: Tools/Demos + +Update GitHub CI workflows to use OpenSSL 3.0.13 and multissltests to use +1.1.1w, 3.0.13, 3.1.5, and 3.2.1. + +.. + +.. date: 2024-02-05-02-45-51 +.. gh-issue: 115015 +.. nonce: rgtiDB +.. section: Tools/Demos + +Fix a bug in Argument Clinic that generated incorrect code for methods with +no parameters that use the :ref:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS +<METH_METHOD-METH_FASTCALL-METH_KEYWORDS>` calling convention. Only the +positional parameter count was checked; any keyword argument passed would be +silently accepted. + +.. + +.. date: 2024-02-05-17-11-15 +.. gh-issue: 111140 +.. nonce: WMEjid +.. section: C API + +Adds :c:func:`PyLong_AsNativeBytes`, :c:func:`PyLong_FromNativeBytes` and +:c:func:`PyLong_FromUnsignedNativeBytes` functions. + +.. + +.. date: 2024-01-31-15-43-35 +.. gh-issue: 114685 +.. nonce: n7aRmX +.. section: C API + +:c:func:`PyBuffer_FillInfo` now raises a :exc:`SystemError` if called with +:c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE` as flags. These flags should +only be used with the ``PyMemoryView_*`` C API. + +.. + +.. date: 2024-01-29-12-13-24 +.. gh-issue: 114685 +.. nonce: B07RME +.. section: C API + +:c:func:`PyObject_GetBuffer` now raises a :exc:`SystemError` if called with +:c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE` as flags. These flags should +only be used with the ``PyMemoryView_*`` C API. + +.. + +.. date: 2024-01-26-21-54-42 +.. gh-issue: 114626 +.. nonce: SKhbh_ +.. section: C API + +Add ``PyCFunctionFast`` and ``PyCFunctionFastWithKeywords`` typedefs +(identical to the existing ``_PyCFunctionFast`` and +``_PyCFunctionFastWithKeywords`` typedefs, just without a leading ``_`` +prefix). + +.. + +.. date: 2024-01-23-21-45-02 +.. gh-issue: 114329 +.. nonce: YRaBoe +.. section: C API + +Add :c:func:`PyList_GetItemRef`, which is similar to +:c:func:`PyList_GetItem` but returns a :term:`strong reference` instead of a +:term:`borrowed reference`. + +.. + +.. date: 2023-11-16-02-07-48 +.. gh-issue: 110850 +.. nonce: DQGNfF +.. section: C API + +Add PyTime C API: + +* :c:type:`PyTime_t` type. +* :c:var:`PyTime_MIN` and :c:var:`PyTime_MAX` constants. +* :c:func:`PyTime_AsSecondsDouble`, + :c:func:`PyTime_Monotonic`, :c:func:`PyTime_PerfCounter`, and + :c:func:`PyTime_Time` functions. + +Patch by Victor Stinner. + +.. + +.. date: 2023-11-15-13-47-48 +.. gh-issue: 112066 +.. nonce: 22WsqR +.. section: C API + +Add :c:func:`PyDict_SetDefaultRef`: insert a key and value into a dictionary +if the key is not already present. This is similar to +:meth:`dict.setdefault`, but returns an integer value indicating if the key +was already present. It is also similar to :c:func:`PyDict_SetDefault`, but +returns a strong reference instead of a borrowed reference. diff --git a/Misc/NEWS.d/3.5.3.rst b/Misc/NEWS.d/3.5.3.rst index c3fcb67a4563f98..25db389ba5734fd 100644 --- a/Misc/NEWS.d/3.5.3.rst +++ b/Misc/NEWS.d/3.5.3.rst @@ -3,5 +3,6 @@ .. no changes: True .. nonce: zYPqUK .. release date: 2017-01-17 +.. section: Library There were no code changes between 3.5.3rc1 and 3.5.3 final. diff --git a/Misc/NEWS.d/3.6.0.rst b/Misc/NEWS.d/3.6.0.rst index f9805cab28615e7..d5c41f38838d936 100644 --- a/Misc/NEWS.d/3.6.0.rst +++ b/Misc/NEWS.d/3.6.0.rst @@ -3,5 +3,6 @@ .. no changes: True .. nonce: F9ENBV .. release date: 2016-12-23 +.. section: Library No changes since release candidate 2 diff --git a/Misc/NEWS.d/3.6.2.rst b/Misc/NEWS.d/3.6.2.rst index dba43d146df9547..ee50670bd9f4426 100644 --- a/Misc/NEWS.d/3.6.2.rst +++ b/Misc/NEWS.d/3.6.2.rst @@ -3,5 +3,6 @@ .. no changes: True .. nonce: F9ENBV .. release date: 2017-07-17 +.. section: Library No changes since release candidate 2 diff --git a/Misc/NEWS.d/next/Build/2024-02-13-14-52-59.gh-issue-114099.zjXsQr.rst b/Misc/NEWS.d/next/Build/2024-02-13-14-52-59.gh-issue-114099.zjXsQr.rst new file mode 100644 index 000000000000000..e2858bd71d28cb2 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-13-14-52-59.gh-issue-114099.zjXsQr.rst @@ -0,0 +1,2 @@ +Makefile targets were added to support compiling an iOS-compatible framework +build. diff --git a/Misc/NEWS.d/next/Build/2024-02-21-11-58-30.gh-issue-115737.dpNl2T.rst b/Misc/NEWS.d/next/Build/2024-02-21-11-58-30.gh-issue-115737.dpNl2T.rst new file mode 100644 index 000000000000000..112f65258dd84b0 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-21-11-58-30.gh-issue-115737.dpNl2T.rst @@ -0,0 +1,2 @@ +The install name for libPython is now correctly set for non-framework macOS +builds. diff --git a/Misc/NEWS.d/next/Build/2024-02-21-18-22-49.gh-issue-111225.Z8C3av.rst b/Misc/NEWS.d/next/Build/2024-02-21-18-22-49.gh-issue-111225.Z8C3av.rst new file mode 100644 index 000000000000000..8cdeba46ba23136 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-21-18-22-49.gh-issue-111225.Z8C3av.rst @@ -0,0 +1 @@ +Link extension modules against libpython on Android. diff --git a/Misc/NEWS.d/next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst b/Misc/NEWS.d/next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst new file mode 100644 index 000000000000000..5492a804255838c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst @@ -0,0 +1 @@ +Fix building ctypes module with -DWIN32_LEAN_AND_MEAN defined diff --git a/Misc/NEWS.d/next/C API/2024-02-16-15-56-53.gh-issue-114626.ie2esA.rst b/Misc/NEWS.d/next/C API/2024-02-16-15-56-53.gh-issue-114626.ie2esA.rst new file mode 100644 index 000000000000000..763f4cee6d3f0b1 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-02-16-15-56-53.gh-issue-114626.ie2esA.rst @@ -0,0 +1,4 @@ +Add again ``_PyCFunctionFastWithKeywords`` name, removed in Python 3.13 +alpha 4 by mistake. Keep the old private ``_PyCFunctionFastWithKeywords`` +name (Python 3.7) as an alias to the new public name +``PyCFunctionFastWithKeywords`` (Python 3.13a4). Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-04-16-51-56.gh-issue-96497.HTBuIL.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-04-16-51-56.gh-issue-96497.HTBuIL.rst new file mode 100644 index 000000000000000..6881dde2e6cf442 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-09-04-16-51-56.gh-issue-96497.HTBuIL.rst @@ -0,0 +1,2 @@ +Fix incorrect resolution of mangled class variables used in assignment +expressions in comprehensions. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-11-36-50.gh-issue-101860.CKCMbC.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-11-36-50.gh-issue-101860.CKCMbC.rst new file mode 100644 index 000000000000000..5a2743534669737 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-11-36-50.gh-issue-101860.CKCMbC.rst @@ -0,0 +1 @@ +Expose ``__name__`` attribute on property. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst new file mode 100644 index 000000000000000..9338f662374c9ab --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst @@ -0,0 +1,8 @@ +Improve the constructors for :mod:`ast` nodes. Arguments of list types now +default to an empty list if omitted, and optional fields default to ``None``. +AST nodes now have an +``__annotations__`` attribute with the expected types of their attributes. +Passing unrecognized extra arguments to AST nodes is deprecated and will +become an error in Python 3.15. Omitting a required argument to an AST node +is deprecated and will become an error in Python 3.15. Patch by Jelle +Zijlstra. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-16-15-02-47.gh-issue-104090.oMjNa9.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-16-15-02-47.gh-issue-104090.oMjNa9.rst new file mode 100644 index 000000000000000..e581d291d047aee --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-16-15-02-47.gh-issue-104090.oMjNa9.rst @@ -0,0 +1,2 @@ +The multiprocessing resource tracker now exits with non-zero status code if a resource +leak was detected. It still exits with status code 0 otherwise. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-22-13-21-39.gh-issue-113055.47xBMF.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-22-13-21-39.gh-issue-113055.47xBMF.rst deleted file mode 100644 index 90f49272218c960..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-12-22-13-21-39.gh-issue-113055.47xBMF.rst +++ /dev/null @@ -1,5 +0,0 @@ -Make interp->obmalloc a pointer. For interpreters that share state with the -main interpreter, this points to the same static memory structure. For -interpreters with their own obmalloc state, it is heap allocated. Add -free_obmalloc_arenas() which will free the obmalloc arenas and radix tree -structures for interpreters with their own obmalloc state. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-17-00-52-57.gh-issue-113884.CvEjUE.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-17-00-52-57.gh-issue-113884.CvEjUE.rst deleted file mode 100644 index 6a39fd2f60ab813..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-17-00-52-57.gh-issue-113884.CvEjUE.rst +++ /dev/null @@ -1 +0,0 @@ -Make :class:`queue.SimpleQueue` thread safe when the GIL is disabled. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-17-05-09-32.gh-issue-112354.Run9ko.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-17-05-09-32.gh-issue-112354.Run9ko.rst deleted file mode 100644 index ed45ba49c3ad421..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-17-05-09-32.gh-issue-112354.Run9ko.rst +++ /dev/null @@ -1,2 +0,0 @@ -The ``END_FOR`` instruction now pops only one value. This is to better -support side exits in loops. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-17-23-39-20.gh-issue-114050.Lnv1oq.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-17-23-39-20.gh-issue-114050.Lnv1oq.rst deleted file mode 100644 index c35d2508e6bdda6..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-17-23-39-20.gh-issue-114050.Lnv1oq.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix segmentation fault caused by an incorrect format string -in ``TypeError`` exception when more than two arguments are passed to ``int``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-18-20-20-37.gh-issue-112529.oVNvDG.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-18-20-20-37.gh-issue-112529.oVNvDG.rst deleted file mode 100644 index b3aa43801da488a..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-18-20-20-37.gh-issue-112529.oVNvDG.rst +++ /dev/null @@ -1,3 +0,0 @@ -The free-threaded build now has its own thread-safe GC implementation that -uses mimalloc to find GC tracked objects. It is non-generational, unlike the -existing GC implementation. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-19-13-18-13.gh-issue-114265.7HAi--.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-19-13-18-13.gh-issue-114265.7HAi--.rst deleted file mode 100644 index 74affbbd09ffb49..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-19-13-18-13.gh-issue-114265.7HAi--.rst +++ /dev/null @@ -1 +0,0 @@ -Compiler propagates line numbers before optimization, leading to more optimization opportunities and removing the need for the ``guarantee_lineno_for_exits`` hack. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-22-09-49-02.gh-issue-114083.hf1-ku.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-22-09-49-02.gh-issue-114083.hf1-ku.rst deleted file mode 100644 index 79be45e87b90d32..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-22-09-49-02.gh-issue-114083.hf1-ku.rst +++ /dev/null @@ -1 +0,0 @@ -Compiler applies folding of LOAD_CONST with following instruction in a separate pass before other optimisations. This enables jump threading in certain circumstances. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-22-15-10-01.gh-issue-114456.fBFEJF.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-22-15-10-01.gh-issue-114456.fBFEJF.rst deleted file mode 100644 index 2b30ad98fb5c79b..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-22-15-10-01.gh-issue-114456.fBFEJF.rst +++ /dev/null @@ -1 +0,0 @@ -Lower the recursion limit under a debug build of WASI. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-28-02-46-12.gh-issue-112433.FUX-nT.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-28-02-46-12.gh-issue-112433.FUX-nT.rst new file mode 100644 index 000000000000000..fdd11bdf4241b95 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-28-02-46-12.gh-issue-112433.FUX-nT.rst @@ -0,0 +1 @@ +Add ability to force alignment of :mod:`ctypes.Structure` by way of the new ``_align_`` attribute on the class. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-04-07-45-29.gh-issue-107674.q8mCmi.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-04-07-45-29.gh-issue-107674.q8mCmi.rst new file mode 100644 index 000000000000000..f9b96788bfad940 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-04-07-45-29.gh-issue-107674.q8mCmi.rst @@ -0,0 +1 @@ +Improved the performance of :func:`sys.settrace` significantly diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-08-16-01-18.gh-issue-115154.ji96FV.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-08-16-01-18.gh-issue-115154.ji96FV.rst new file mode 100644 index 000000000000000..045596bfcdca431 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-08-16-01-18.gh-issue-115154.ji96FV.rst @@ -0,0 +1,2 @@ +Fix a bug that was causing the :func:`tokenize.untokenize` function to +handle unicode named literals incorrectly. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-09-18-59-22.gh-issue-112175.qglugr.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-09-18-59-22.gh-issue-112175.qglugr.rst new file mode 100644 index 000000000000000..6d919134bf4d9c5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-09-18-59-22.gh-issue-112175.qglugr.rst @@ -0,0 +1 @@ +Every ``PyThreadState`` now has its own ``eval_breaker``, allowing specific threads to be interrupted. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-12-23-29-17.gh-issue-115323.3t6687.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-12-23-29-17.gh-issue-115323.3t6687.rst new file mode 100644 index 000000000000000..171855608fbc6a2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-12-23-29-17.gh-issue-115323.3t6687.rst @@ -0,0 +1,2 @@ +Make error message more meaningful for when :meth:`bytearray.extend` is +called with a :class:`str` object. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-14-23-50-43.gh-issue-115347.VkHvQC.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-14-23-50-43.gh-issue-115347.VkHvQC.rst new file mode 100644 index 000000000000000..16f357a944ed7a5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-14-23-50-43.gh-issue-115347.VkHvQC.rst @@ -0,0 +1,2 @@ +Fix bug where docstring was replaced by a redundant NOP when Python is run +with ``-OO``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-20-12-46-20.gh-issue-115700.KLJ5r4.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-20-12-46-20.gh-issue-115700.KLJ5r4.rst new file mode 100644 index 000000000000000..5b7b8e410b50632 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-20-12-46-20.gh-issue-115700.KLJ5r4.rst @@ -0,0 +1 @@ +The regen-cases build stage now works on Windows. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-20-18-49-02.gh-issue-115733.51Zb85.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-20-18-49-02.gh-issue-115733.51Zb85.rst new file mode 100644 index 000000000000000..5cbb292065b5daa --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-20-18-49-02.gh-issue-115733.51Zb85.rst @@ -0,0 +1 @@ +Fix crash when calling ``next()`` on exhausted list iterators. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-22-11-33-20.gh-issue-115778.jksd1D.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-22-11-33-20.gh-issue-115778.jksd1D.rst new file mode 100644 index 000000000000000..023f6aa3026733e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-22-11-33-20.gh-issue-115778.jksd1D.rst @@ -0,0 +1 @@ +Add ``tierN`` annotation for instruction definition in interpreter DSL. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-22-16-17-53.gh-issue-115823.c1TreJ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-22-16-17-53.gh-issue-115823.c1TreJ.rst new file mode 100644 index 000000000000000..8cda4c9343d4d70 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-22-16-17-53.gh-issue-115823.c1TreJ.rst @@ -0,0 +1,3 @@ +Properly calculate error ranges in the parser when raising +:exc:`SyntaxError` exceptions caused by invalid byte sequences. Patch by +Pablo Galindo diff --git a/Misc/NEWS.d/next/Documentation/2024-01-17-11-40-03.gh-issue-114123.LuueXf.rst b/Misc/NEWS.d/next/Documentation/2024-01-17-11-40-03.gh-issue-114123.LuueXf.rst deleted file mode 100644 index 1d93a4228400770..000000000000000 --- a/Misc/NEWS.d/next/Documentation/2024-01-17-11-40-03.gh-issue-114123.LuueXf.rst +++ /dev/null @@ -1,7 +0,0 @@ -Move the :mod:`csv` module docstring to the :mod:`!csv` module -instead of reexporting it from the internal :mod:`!_csv` module, -and remove ``__doc__`` from ``csv.__all__``. - -Move :attr:`!csv.__version__` to the :mod:`!csv` module -instead of reexporting it from the internal :mod:`!_csv` module, -and remove ``__version__`` from ``csv.__all__``. diff --git a/Misc/NEWS.d/next/Documentation/2024-02-14-20-17-04.gh-issue-115399.fb9a0R.rst b/Misc/NEWS.d/next/Documentation/2024-02-14-20-17-04.gh-issue-115399.fb9a0R.rst new file mode 100644 index 000000000000000..587aea802168bd6 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-02-14-20-17-04.gh-issue-115399.fb9a0R.rst @@ -0,0 +1 @@ +Document CVE-2023-52425 of Expat <2.6.0 under "XML vulnerabilities". diff --git a/Misc/NEWS.d/next/IDLE/2023-12-09-11-04-26.gh-issue-88516.SIIvfs.rst b/Misc/NEWS.d/next/IDLE/2023-12-09-11-04-26.gh-issue-88516.SIIvfs.rst new file mode 100644 index 000000000000000..b6dea5029bf3531 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2023-12-09-11-04-26.gh-issue-88516.SIIvfs.rst @@ -0,0 +1,2 @@ +On macOS show a proxy icon in the title bar of editor windows to match +platform behaviour. diff --git a/Misc/NEWS.d/next/IDLE/2024-01-17-23-18-15.gh-issue-96905.UYaxoU.rst b/Misc/NEWS.d/next/IDLE/2024-01-17-23-18-15.gh-issue-96905.UYaxoU.rst deleted file mode 100644 index fe7dde64c7c7d51..000000000000000 --- a/Misc/NEWS.d/next/IDLE/2024-01-17-23-18-15.gh-issue-96905.UYaxoU.rst +++ /dev/null @@ -1 +0,0 @@ -In idlelib code, stop redefining built-ins 'dict' and 'object'. diff --git a/Misc/NEWS.d/next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst b/Misc/NEWS.d/next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst new file mode 100644 index 000000000000000..2a663ac7940dcb2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst @@ -0,0 +1 @@ +Add 'default' and 'version' help text for localization in argparse. diff --git a/Misc/NEWS.d/next/Library/2020-07-13-23-59-42.bpo-41122.8P_Brh.rst b/Misc/NEWS.d/next/Library/2020-07-13-23-59-42.bpo-41122.8P_Brh.rst new file mode 100644 index 000000000000000..76568d407449f53 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-07-13-23-59-42.bpo-41122.8P_Brh.rst @@ -0,0 +1,3 @@ +Failing to pass arguments properly to :func:`functools.singledispatchmethod` +now throws a TypeError instead of hitting an index out of bounds +internally. diff --git a/Misc/NEWS.d/next/Library/2020-12-15-22-30-49.bpo-42125.UGyseY.rst b/Misc/NEWS.d/next/Library/2020-12-15-22-30-49.bpo-42125.UGyseY.rst new file mode 100644 index 000000000000000..49d4462e2577022 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-12-15-22-30-49.bpo-42125.UGyseY.rst @@ -0,0 +1,2 @@ +linecache: get module name from ``__spec__`` if available. This allows getting +source code for the ``__main__`` module when a custom loader is used. diff --git a/Misc/NEWS.d/next/Library/2021-05-03-11-04-12.bpo-43952.Me7fJe.rst b/Misc/NEWS.d/next/Library/2021-05-03-11-04-12.bpo-43952.Me7fJe.rst new file mode 100644 index 000000000000000..e164619e44a3019 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-05-03-11-04-12.bpo-43952.Me7fJe.rst @@ -0,0 +1,2 @@ +Fix :meth:`multiprocessing.connection.Listener.accept()` to accept empty bytes +as authkey. Not accepting empty bytes as key causes it to hang indefinitely. diff --git a/Misc/NEWS.d/next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst b/Misc/NEWS.d/next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst new file mode 100644 index 000000000000000..ecdb26cdd6edd60 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst @@ -0,0 +1 @@ +Add missing call to localization function in :mod:`argparse`. diff --git a/Misc/NEWS.d/next/Library/2022-01-14-10-50-17.bpo-31116.0bduV9.rst b/Misc/NEWS.d/next/Library/2022-01-14-10-50-17.bpo-31116.0bduV9.rst new file mode 100644 index 000000000000000..d77a96b442bcbb7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-01-14-10-50-17.bpo-31116.0bduV9.rst @@ -0,0 +1 @@ +Add Z85 encoding to ``base64``. diff --git a/Misc/NEWS.d/next/Library/2022-05-25-17-49-04.gh-issue-93205.DjhFVR.rst b/Misc/NEWS.d/next/Library/2022-05-25-17-49-04.gh-issue-93205.DjhFVR.rst new file mode 100644 index 000000000000000..4a280b93d933478 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-05-25-17-49-04.gh-issue-93205.DjhFVR.rst @@ -0,0 +1 @@ +Fixed a bug in :class:`logging.handlers.TimedRotatingFileHandler` where multiple rotating handler instances pointing to files with the same name but different extensions would conflict and not delete the correct files. diff --git a/Misc/NEWS.d/next/Library/2022-07-31-01-24-40.gh-issue-88569.eU0--b.rst b/Misc/NEWS.d/next/Library/2022-07-31-01-24-40.gh-issue-88569.eU0--b.rst deleted file mode 100644 index 31dd985bb5c3b6c..000000000000000 --- a/Misc/NEWS.d/next/Library/2022-07-31-01-24-40.gh-issue-88569.eU0--b.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add :func:`os.path.isreserved`, which identifies reserved pathnames such -as "NUL", "AUX" and "CON". This function is only available on Windows. - -Deprecate :meth:`pathlib.PurePath.is_reserved`. diff --git a/Misc/NEWS.d/next/Library/2022-08-26-15-50-53.gh-issue-96310.0NssDh.rst b/Misc/NEWS.d/next/Library/2022-08-26-15-50-53.gh-issue-96310.0NssDh.rst new file mode 100644 index 000000000000000..f8efb0002e104a4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-08-26-15-50-53.gh-issue-96310.0NssDh.rst @@ -0,0 +1,2 @@ +Fix a traceback in :mod:`argparse` when all options in a mutually exclusive +group are suppressed. diff --git a/Misc/NEWS.d/next/Library/2022-11-22-23-17-43.gh-issue-95782.an_and.rst b/Misc/NEWS.d/next/Library/2022-11-22-23-17-43.gh-issue-95782.an_and.rst new file mode 100644 index 000000000000000..123c3944aa3a3a0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-22-23-17-43.gh-issue-95782.an_and.rst @@ -0,0 +1,4 @@ +Fix :func:`io.BufferedReader.tell`, :func:`io.BufferedReader.seek`, +:func:`_pyio.BufferedReader.tell`, :func:`io.BufferedRandom.tell`, +:func:`io.BufferedRandom.seek` and :func:`_pyio.BufferedRandom.tell` +being able to return negative offsets. diff --git a/Misc/NEWS.d/next/Library/2023-01-09-14-08-02.gh-issue-100884.DcmdLl.rst b/Misc/NEWS.d/next/Library/2023-01-09-14-08-02.gh-issue-100884.DcmdLl.rst new file mode 100644 index 000000000000000..2a3881788108350 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-09-14-08-02.gh-issue-100884.DcmdLl.rst @@ -0,0 +1,2 @@ +email: fix misfolding of comma in address-lists over multiple lines in +combination with unicode encoding. diff --git a/Misc/NEWS.d/next/Library/2023-01-12-14-16-01.gh-issue-100985.GT5Fvd.rst b/Misc/NEWS.d/next/Library/2023-01-12-14-16-01.gh-issue-100985.GT5Fvd.rst new file mode 100644 index 000000000000000..8d8693a5edb3d4f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-12-14-16-01.gh-issue-100985.GT5Fvd.rst @@ -0,0 +1,2 @@ +Update HTTPSConnection to consistently wrap IPv6 Addresses when using a +proxy. diff --git a/Misc/NEWS.d/next/Library/2023-02-14-17-19-59.gh-issue-72249.fv35wU.rst b/Misc/NEWS.d/next/Library/2023-02-14-17-19-59.gh-issue-72249.fv35wU.rst new file mode 100644 index 000000000000000..10cc5a4e7528dd4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-14-17-19-59.gh-issue-72249.fv35wU.rst @@ -0,0 +1,2 @@ +:func:`functools.partial`s of :func:`repr` has been improved to include the +:term:`module` name. Patched by Furkan Onder and Anilyka Barry. diff --git a/Misc/NEWS.d/next/Library/2023-03-03-09-05-42.gh-issue-102389.ucmo0_.rst b/Misc/NEWS.d/next/Library/2023-03-03-09-05-42.gh-issue-102389.ucmo0_.rst new file mode 100644 index 000000000000000..8c11567d79ba7b9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-03-09-05-42.gh-issue-102389.ucmo0_.rst @@ -0,0 +1 @@ +Add ``windows_31j`` to aliases for ``cp932`` codec diff --git a/Misc/NEWS.d/next/Library/2023-03-08-00-02-30.gh-issue-102512.LiugDr.rst b/Misc/NEWS.d/next/Library/2023-03-08-00-02-30.gh-issue-102512.LiugDr.rst deleted file mode 100644 index 659cba73cbf34e9..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-08-00-02-30.gh-issue-102512.LiugDr.rst +++ /dev/null @@ -1,3 +0,0 @@ -When :func:`os.fork` is called from a foreign thread (aka ``_DummyThread``), -the type of the thread in a child process is changed to ``_MainThread``. -Also changed its name and daemonic status, it can be now joined. diff --git a/Misc/NEWS.d/next/Library/2023-04-02-21-20-35.gh-issue-60346.7mjgua.rst b/Misc/NEWS.d/next/Library/2023-04-02-21-20-35.gh-issue-60346.7mjgua.rst new file mode 100644 index 000000000000000..c15bd6ed11d17f9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-02-21-20-35.gh-issue-60346.7mjgua.rst @@ -0,0 +1 @@ +Fix ArgumentParser inconsistent with parse_known_args. diff --git a/Misc/NEWS.d/next/Library/2023-05-01-22-28-57.gh-issue-104061.vxfBXf.rst b/Misc/NEWS.d/next/Library/2023-05-01-22-28-57.gh-issue-104061.vxfBXf.rst new file mode 100644 index 000000000000000..e15a811f9043520 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-01-22-28-57.gh-issue-104061.vxfBXf.rst @@ -0,0 +1 @@ +Add :data:`socket.SO_BINDTOIFINDEX` constant. diff --git a/Misc/NEWS.d/next/Library/2023-05-08-09-30-00.gh-issue-104282.h4c6Eb.rst b/Misc/NEWS.d/next/Library/2023-05-08-09-30-00.gh-issue-104282.h4c6Eb.rst deleted file mode 100644 index 569ce66a5b9d5fd..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-08-09-30-00.gh-issue-104282.h4c6Eb.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix null pointer dereference in :func:`lzma._decode_filter_properties` -due to improper handling of BCJ filters with properties of zero length. -Patch by Radislav Chugunov. diff --git a/Misc/NEWS.d/next/Library/2023-05-17-21-33-21.gh-issue-69990.Blvz9G.rst b/Misc/NEWS.d/next/Library/2023-05-17-21-33-21.gh-issue-69990.Blvz9G.rst new file mode 100644 index 000000000000000..b0cdf44f7b9e391 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-17-21-33-21.gh-issue-69990.Blvz9G.rst @@ -0,0 +1 @@ +:meth:`Profile.print_stats` has been improved to accept multiple sort arguments. Patched by Chiu-Hsiang Hsu and Furkan Onder. diff --git a/Misc/NEWS.d/next/Library/2023-05-30-18-30-11.gh-issue-105102.SnpK04.rst b/Misc/NEWS.d/next/Library/2023-05-30-18-30-11.gh-issue-105102.SnpK04.rst deleted file mode 100644 index 7ca21afefa31325..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-30-18-30-11.gh-issue-105102.SnpK04.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allow :class:`ctypes.Union` to be nested in :class:`ctypes.Structure` when -the system endianness is the opposite of the classes. diff --git a/Misc/NEWS.d/next/Library/2023-08-02-01-17-32.gh-issue-107155.Mj1K9L.rst b/Misc/NEWS.d/next/Library/2023-08-02-01-17-32.gh-issue-107155.Mj1K9L.rst new file mode 100644 index 000000000000000..8362dc0fcfaa748 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-02-01-17-32.gh-issue-107155.Mj1K9L.rst @@ -0,0 +1,3 @@ +Fix incorrect output of ``help(x)`` where ``x`` is a :keyword:`lambda` +function, which has an ``__annotations__`` dictionary attribute with a +``"return"`` key. diff --git a/Misc/NEWS.d/next/Library/2023-09-22-22-17-45.gh-issue-38807.m9McRN.rst b/Misc/NEWS.d/next/Library/2023-09-22-22-17-45.gh-issue-38807.m9McRN.rst deleted file mode 100644 index 4219723d15b9e65..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-09-22-22-17-45.gh-issue-38807.m9McRN.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix race condition in :mod:`trace`. Instead of checking if a directory -exists and creating it, directly call :func:`os.makedirs` with the kwarg -``exist_ok=True``. diff --git a/Misc/NEWS.d/next/Library/2023-10-04-11-09-30.gh-issue-110345.fZU1ud.rst b/Misc/NEWS.d/next/Library/2023-10-04-11-09-30.gh-issue-110345.fZU1ud.rst deleted file mode 100644 index d9ccc0f12de47c2..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-10-04-11-09-30.gh-issue-110345.fZU1ud.rst +++ /dev/null @@ -1 +0,0 @@ -Show the Tcl/Tk patchlevel (rather than version) in :meth:`tkinter._test`. diff --git a/Misc/NEWS.d/next/Library/2023-10-19-02-08-12.gh-issue-111051.8h1Dpk.rst b/Misc/NEWS.d/next/Library/2023-10-19-02-08-12.gh-issue-111051.8h1Dpk.rst deleted file mode 100644 index adb3241b89ae3e6..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-10-19-02-08-12.gh-issue-111051.8h1Dpk.rst +++ /dev/null @@ -1 +0,0 @@ -Added check for file modification during debugging with :mod:`pdb` diff --git a/Misc/NEWS.d/next/Library/2023-11-07-10-22-06.gh-issue-111775.IoVxfX.rst b/Misc/NEWS.d/next/Library/2023-11-07-10-22-06.gh-issue-111775.IoVxfX.rst new file mode 100644 index 000000000000000..2a3bdd640ea67de --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-07-10-22-06.gh-issue-111775.IoVxfX.rst @@ -0,0 +1,2 @@ +Fix :meth:`importlib.resources.simple.ResourceHandle.open` for text mode, +added missed ``stream`` argument. diff --git a/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst b/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst deleted file mode 100644 index aaa50fce3ac962c..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst +++ /dev/null @@ -1 +0,0 @@ -Improve handling of pdb convenience variables to avoid replacing string contents. diff --git a/Misc/NEWS.d/next/Library/2024-01-04-20-58-17.gh-issue-113225.-nyJM4.rst b/Misc/NEWS.d/next/Library/2024-01-04-20-58-17.gh-issue-113225.-nyJM4.rst deleted file mode 100644 index 0c07f42fd065d25..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-04-20-58-17.gh-issue-113225.-nyJM4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up :meth:`pathlib.Path.walk` by using :attr:`os.DirEntry.path` where -possible. diff --git a/Misc/NEWS.d/next/Library/2024-01-07-21-04-24.gh-issue-113796.6iNsCR.rst b/Misc/NEWS.d/next/Library/2024-01-07-21-04-24.gh-issue-113796.6iNsCR.rst deleted file mode 100644 index e9d4aba99066770..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-07-21-04-24.gh-issue-113796.6iNsCR.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add more validation checks in the :class:`csv.Dialect` constructor. -:exc:`ValueError` is now raised if the same character is used in different -roles. diff --git a/Misc/NEWS.d/next/Library/2024-01-12-17-32-36.gh-issue-79634.uTSTRI.rst b/Misc/NEWS.d/next/Library/2024-01-12-17-32-36.gh-issue-79634.uTSTRI.rst deleted file mode 100644 index ba19b5209e648ec..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-12-17-32-36.gh-issue-79634.uTSTRI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Accept :term:`path-like objects <path-like object>` as patterns in -:meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob`. diff --git a/Misc/NEWS.d/next/Library/2024-01-15-19-54-41.gh-issue-114087.Xic5vY.rst b/Misc/NEWS.d/next/Library/2024-01-15-19-54-41.gh-issue-114087.Xic5vY.rst deleted file mode 100644 index 68b27a7b0c9f6c9..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-15-19-54-41.gh-issue-114087.Xic5vY.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up ``dataclasses.asdict`` up to 1.35x. diff --git a/Misc/NEWS.d/next/Library/2024-01-16-15-59-06.gh-issue-114149.LJ8IPm.rst b/Misc/NEWS.d/next/Library/2024-01-16-15-59-06.gh-issue-114149.LJ8IPm.rst deleted file mode 100644 index 1403d78d0d49054..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-16-15-59-06.gh-issue-114149.LJ8IPm.rst +++ /dev/null @@ -1 +0,0 @@ -Enum: correctly handle tuple subclasses in custom ``__new__``. diff --git a/Misc/NEWS.d/next/Library/2024-01-17-18-53-51.gh-issue-104522.3NyDf4.rst b/Misc/NEWS.d/next/Library/2024-01-17-18-53-51.gh-issue-104522.3NyDf4.rst deleted file mode 100644 index ca980945ea12d31..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-17-18-53-51.gh-issue-104522.3NyDf4.rst +++ /dev/null @@ -1,3 +0,0 @@ -:exc:`OSError` raised when run a subprocess now only has *filename* -attribute set to *cwd* if the error was caused by a failed attempt to change -the current directory. diff --git a/Misc/NEWS.d/next/Library/2024-01-18-10-07-52.gh-issue-114198.lK4Iif.rst b/Misc/NEWS.d/next/Library/2024-01-18-10-07-52.gh-issue-114198.lK4Iif.rst deleted file mode 100644 index fa047e288f807ed..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-18-10-07-52.gh-issue-114198.lK4Iif.rst +++ /dev/null @@ -1,2 +0,0 @@ -The signature for the ``__replace__`` method on :mod:`dataclasses` now has -the first argument named ``self``, rather than ``obj``. diff --git a/Misc/NEWS.d/next/Library/2024-01-18-22-29-28.gh-issue-101438.1-uUi_.rst b/Misc/NEWS.d/next/Library/2024-01-18-22-29-28.gh-issue-101438.1-uUi_.rst deleted file mode 100644 index 9b69b5deb1b5a06..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-18-22-29-28.gh-issue-101438.1-uUi_.rst +++ /dev/null @@ -1,4 +0,0 @@ -Avoid reference cycle in ElementTree.iterparse. The iterator returned by -``ElementTree.iterparse`` may hold on to a file descriptor. The reference -cycle prevented prompt clean-up of the file descriptor if the returned -iterator was not exhausted. diff --git a/Misc/NEWS.d/next/Library/2024-01-19-12-05-22.gh-issue-114281.H5JQe4.rst b/Misc/NEWS.d/next/Library/2024-01-19-12-05-22.gh-issue-114281.H5JQe4.rst deleted file mode 100644 index 36c54e8faf214c2..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-19-12-05-22.gh-issue-114281.H5JQe4.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove type hints from ``Lib/asyncio/staggered.py``. -The annotations in the `typeshed <https://github.com/python/typeshed>`__ -project should be used instead. diff --git a/Misc/NEWS.d/next/Library/2024-01-19-15-48-06.gh-issue-114328.hixxW3.rst b/Misc/NEWS.d/next/Library/2024-01-19-15-48-06.gh-issue-114328.hixxW3.rst deleted file mode 100644 index 42262c05fd1fbfb..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-19-15-48-06.gh-issue-114328.hixxW3.rst +++ /dev/null @@ -1,4 +0,0 @@ -The :func:`tty.setcbreak` and new :func:`tty.cfmakecbreak` no longer clears -the terminal input ICRLF flag. This fixes a regression introduced in 3.12 -that no longer matched how OSes define cbreak mode in their ``stty(1)`` -manual pages. diff --git a/Misc/NEWS.d/next/Library/2024-01-19-18-41-02.gh-issue-114321.yj_Xw3.rst b/Misc/NEWS.d/next/Library/2024-01-19-18-41-02.gh-issue-114321.yj_Xw3.rst deleted file mode 100644 index dc2934bd81a42ad..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-19-18-41-02.gh-issue-114321.yj_Xw3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Expose more platform specific constants in the :mod:`fcntl` module on Linux, -macOS, FreeBSD and NetBSD. diff --git a/Misc/NEWS.d/next/Library/2024-01-21-16-32-55.gh-issue-114257.bCFld5.rst b/Misc/NEWS.d/next/Library/2024-01-21-16-32-55.gh-issue-114257.bCFld5.rst deleted file mode 100644 index 6f02ff9e62617d3..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-21-16-32-55.gh-issue-114257.bCFld5.rst +++ /dev/null @@ -1,2 +0,0 @@ -Dismiss the :exc:`FileNotFound` error in :func:`ctypes.util.find_library` and -just return ``None`` on Linux. diff --git a/Misc/NEWS.d/next/Library/2024-01-22-11-43-38.gh-issue-114423.6mMoPH.rst b/Misc/NEWS.d/next/Library/2024-01-22-11-43-38.gh-issue-114423.6mMoPH.rst deleted file mode 100644 index 7b77b73295d9483..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-22-11-43-38.gh-issue-114423.6mMoPH.rst +++ /dev/null @@ -1 +0,0 @@ -``_DummyThread`` entries in ``threading._active`` are now automatically removed when the related thread dies. diff --git a/Misc/NEWS.d/next/Library/2024-01-22-12-10-34.gh-issue-75128.4FGlRS.rst b/Misc/NEWS.d/next/Library/2024-01-22-12-10-34.gh-issue-75128.4FGlRS.rst deleted file mode 100644 index d875148e89b41b0..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-22-12-10-34.gh-issue-75128.4FGlRS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Ignore an :exc:`OSError` in :meth:`asyncio.BaseEventLoop.create_server` when -IPv6 is available but the interface cannot actually support it. diff --git a/Misc/NEWS.d/next/Library/2024-01-23-14-11-49.gh-issue-114315.KeVdzl.rst b/Misc/NEWS.d/next/Library/2024-01-23-14-11-49.gh-issue-114315.KeVdzl.rst deleted file mode 100644 index a8a19fc525d0199..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-23-14-11-49.gh-issue-114315.KeVdzl.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make :class:`threading.Lock` a real class, not a factory function. Add -``__new__`` to ``_thread.lock`` type. diff --git a/Misc/NEWS.d/next/Library/2024-01-23-21-20-40.gh-issue-114492.vKxl5o.rst b/Misc/NEWS.d/next/Library/2024-01-23-21-20-40.gh-issue-114492.vKxl5o.rst deleted file mode 100644 index 8df8299d0dffcd6..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-23-21-20-40.gh-issue-114492.vKxl5o.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make the result of :func:`termios.tcgetattr` reproducible on Alpine Linux. -Previously it could leave a random garbage in some fields. diff --git a/Misc/NEWS.d/next/Library/2024-01-24-20-11-46.gh-issue-112451.7YrG4p.rst b/Misc/NEWS.d/next/Library/2024-01-24-20-11-46.gh-issue-112451.7YrG4p.rst deleted file mode 100644 index 126ca36a3b7cb1a..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-24-20-11-46.gh-issue-112451.7YrG4p.rst +++ /dev/null @@ -1,2 +0,0 @@ -Prohibit subclassing pure-Python :class:`datetime.timezone`. This is consistent -with C-extension implementation. Patch by Mariusz Felisiak. diff --git a/Misc/NEWS.d/next/Library/2024-01-26-16-42-31.gh-issue-114610.S18Vuz.rst b/Misc/NEWS.d/next/Library/2024-01-26-16-42-31.gh-issue-114610.S18Vuz.rst new file mode 100644 index 000000000000000..519aede72aaf29a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-26-16-42-31.gh-issue-114610.S18Vuz.rst @@ -0,0 +1,4 @@ +Fix bug where :meth:`pathlib.PurePath.with_stem` converted a non-empty path +suffix to a stem when given an empty *stem* argument. It now raises +:exc:`ValueError`, just like :meth:`pathlib.PurePath.with_suffix` does when +called on a path with an empty stem, given a non-empty *suffix* argument. diff --git a/Misc/NEWS.d/next/Library/2024-01-26-16-46-21.gh-issue-77749.NY_7TS.rst b/Misc/NEWS.d/next/Library/2024-01-26-16-46-21.gh-issue-77749.NY_7TS.rst deleted file mode 100644 index f1c99c09d2dfe1c..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-26-16-46-21.gh-issue-77749.NY_7TS.rst +++ /dev/null @@ -1,2 +0,0 @@ -:meth:`email.policy.EmailPolicy.fold` now always encodes non-ASCII characters -in headers if :attr:`~email.policy.EmailPolicy.utf8` is false. diff --git a/Misc/NEWS.d/next/Library/2024-01-29-13-46-41.gh-issue-114709.SQ998l.rst b/Misc/NEWS.d/next/Library/2024-01-29-13-46-41.gh-issue-114709.SQ998l.rst new file mode 100644 index 000000000000000..ca0d7902c73d1c0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-29-13-46-41.gh-issue-114709.SQ998l.rst @@ -0,0 +1,5 @@ +:func:`posixpath.commonpath()` now raises a :exc:`ValueError` exception when +passed an empty iterable. Previously, :exc:`IndexError` was raised. + +:func:`posixpath.commonpath()` now raises a :exc:`TypeError` exception when +passed ``None``. Previously, :exc:`ValueError` was raised. diff --git a/Misc/NEWS.d/next/Library/2024-01-30-23-28-29.gh-issue-114763.BRjKkg.rst b/Misc/NEWS.d/next/Library/2024-01-30-23-28-29.gh-issue-114763.BRjKkg.rst new file mode 100644 index 000000000000000..e8bdb83dde61fba --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-30-23-28-29.gh-issue-114763.BRjKkg.rst @@ -0,0 +1,3 @@ +Protect modules loaded with :class:`importlib.util.LazyLoader` from race +conditions when multiple threads try to access attributes before the loading +is complete. diff --git a/Misc/NEWS.d/next/Library/2024-02-09-12-22-47.gh-issue-113812.wOraaG.rst b/Misc/NEWS.d/next/Library/2024-02-09-12-22-47.gh-issue-113812.wOraaG.rst new file mode 100644 index 000000000000000..7ef7bc891cd885d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-09-12-22-47.gh-issue-113812.wOraaG.rst @@ -0,0 +1,3 @@ +:meth:`DatagramTransport.sendto` will now send zero-length datagrams if +called with an empty bytes object. The transport flow control also now +accounts for the datagram header when calculating the buffer size. diff --git a/Misc/NEWS.d/next/Library/2024-02-09-19-41-48.gh-issue-115197.20wkWH.rst b/Misc/NEWS.d/next/Library/2024-02-09-19-41-48.gh-issue-115197.20wkWH.rst new file mode 100644 index 000000000000000..e6ca3cc525d74a1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-09-19-41-48.gh-issue-115197.20wkWH.rst @@ -0,0 +1,2 @@ +``urllib.request`` no longer resolves the hostname before checking it +against the system's proxy bypass list on macOS and Windows. diff --git a/Misc/NEWS.d/next/Library/2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst b/Misc/NEWS.d/next/Library/2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst new file mode 100644 index 000000000000000..2da43a43b787982 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst @@ -0,0 +1,2 @@ +:mod:`pydoc` no longer skips global functions implemented as builtin methods, +such as :class:`~type.MethodDescriptorType` and :class:`~type.WrapperDescriptorType`. diff --git a/Misc/NEWS.d/next/Library/2024-02-15-23-42-54.gh-issue-112006.4wxcK-.rst b/Misc/NEWS.d/next/Library/2024-02-15-23-42-54.gh-issue-112006.4wxcK-.rst new file mode 100644 index 000000000000000..32af2bd24e54f2c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-15-23-42-54.gh-issue-112006.4wxcK-.rst @@ -0,0 +1,3 @@ +Fix :func:`inspect.unwrap` for types with the ``__wrapper__`` data +descriptor. Fix :meth:`inspect.Signature.from_callable` for builtins +:func:`classmethod` and :func:`staticmethod`. diff --git a/Misc/NEWS.d/next/Library/2024-02-16-16-40-10.gh-issue-112720.io6_Ac.rst b/Misc/NEWS.d/next/Library/2024-02-16-16-40-10.gh-issue-112720.io6_Ac.rst new file mode 100644 index 000000000000000..32916ede4dee357 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-16-16-40-10.gh-issue-112720.io6_Ac.rst @@ -0,0 +1,2 @@ +Refactor :class:`dis.ArgResolver` to make it possible to subclass and change +the way jump args are interpreted. diff --git a/Misc/NEWS.d/next/Library/2024-02-17-18-47-12.gh-issue-115618.napiNp.rst b/Misc/NEWS.d/next/Library/2024-02-17-18-47-12.gh-issue-115618.napiNp.rst new file mode 100644 index 000000000000000..cb4b147d5dc6639 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-17-18-47-12.gh-issue-115618.napiNp.rst @@ -0,0 +1,3 @@ +Fix improper decreasing the reference count for ``None`` argument in +:class:`property` methods :meth:`~property.getter`, :meth:`~property.setter` +and :meth:`~property.deleter`. diff --git a/Misc/NEWS.d/next/Library/2024-02-18-12-18-12.gh-issue-111358.9yJUMD.rst b/Misc/NEWS.d/next/Library/2024-02-18-12-18-12.gh-issue-111358.9yJUMD.rst new file mode 100644 index 000000000000000..2e895f8f181ce7e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-18-12-18-12.gh-issue-111358.9yJUMD.rst @@ -0,0 +1,2 @@ +Fix a bug in :meth:`asyncio.BaseEventLoop.shutdown_default_executor` to +ensure the timeout passed to the coroutine behaves as expected. diff --git a/Misc/NEWS.d/next/Library/2024-02-19-15-52-30.gh-issue-114914.M5-1d8.rst b/Misc/NEWS.d/next/Library/2024-02-19-15-52-30.gh-issue-114914.M5-1d8.rst new file mode 100644 index 000000000000000..f7d392c8bceea3c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-19-15-52-30.gh-issue-114914.M5-1d8.rst @@ -0,0 +1,2 @@ +Fix an issue where an abandoned :class:`StreamWriter` would not be garbage +collected. diff --git a/Misc/NEWS.d/next/Library/2024-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst b/Misc/NEWS.d/next/Library/2024-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst new file mode 100644 index 000000000000000..4f97b2d6085ba0a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst @@ -0,0 +1,2 @@ +Stop logging potentially sensitive callback arguments in :mod:`asyncio` +unless debug mode is active. diff --git a/Misc/NEWS.d/next/Library/2024-02-20-07-38-15.gh-issue-112364.EX7uGI.rst b/Misc/NEWS.d/next/Library/2024-02-20-07-38-15.gh-issue-112364.EX7uGI.rst new file mode 100644 index 000000000000000..6af71e60ec2a8ea --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-20-07-38-15.gh-issue-112364.EX7uGI.rst @@ -0,0 +1 @@ +Fixed :func:`ast.unparse` to handle format_spec with ``"``, ``'`` or ``\\``. Patched by Frank Hoffmann. diff --git a/Misc/NEWS.d/next/Library/2024-02-20-16-42-54.gh-issue-115712.EXVMXw.rst b/Misc/NEWS.d/next/Library/2024-02-20-16-42-54.gh-issue-115712.EXVMXw.rst new file mode 100644 index 000000000000000..8b19064dba779d2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-20-16-42-54.gh-issue-115712.EXVMXw.rst @@ -0,0 +1,4 @@ +Restore support of space delimiter with ``skipinitialspace=True`` in +:mod:`csv`. :func:`csv.writer()` now quotes empty fields if delimiter is a +space and skipinitialspace is true and raises exception if quoting is not +possible. diff --git a/Misc/NEWS.d/next/Library/2024-02-20-22-02-34.gh-issue-67044.QF9_Ru.rst b/Misc/NEWS.d/next/Library/2024-02-20-22-02-34.gh-issue-67044.QF9_Ru.rst new file mode 100644 index 000000000000000..095e69b6cadab64 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-20-22-02-34.gh-issue-67044.QF9_Ru.rst @@ -0,0 +1,2 @@ +:func:`csv.writer` now always quotes or escapes ``'\r'`` and ``'\n'``, +regardless of *lineterminator* value. diff --git a/Misc/NEWS.d/next/Library/2024-02-22-12-10-18.gh-issue-115714.P2JsU1.rst b/Misc/NEWS.d/next/Library/2024-02-22-12-10-18.gh-issue-115714.P2JsU1.rst new file mode 100644 index 000000000000000..fb626344c87fdb3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-22-12-10-18.gh-issue-115714.P2JsU1.rst @@ -0,0 +1,4 @@ +On WASI, the :mod:`time` module no longer get process time using ``times()`` +or ``CLOCK_PROCESS_CPUTIME_ID``, system API is that is unreliable and is +likely to be removed from WASI. The affected clock functions fall back to +calling ``clock()``. diff --git a/Misc/NEWS.d/next/Library/2024-02-23-11-08-31.gh-issue-115532.zVd3gK.rst b/Misc/NEWS.d/next/Library/2024-02-23-11-08-31.gh-issue-115532.zVd3gK.rst new file mode 100644 index 000000000000000..fb36c0b2a4fabd2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-23-11-08-31.gh-issue-115532.zVd3gK.rst @@ -0,0 +1 @@ +Add kernel density estimation to the statistics module. diff --git a/Misc/NEWS.d/next/Library/2024-02-24-18-48-14.gh-issue-115886.rgM6AF.rst b/Misc/NEWS.d/next/Library/2024-02-24-18-48-14.gh-issue-115886.rgM6AF.rst new file mode 100644 index 000000000000000..9688f713d5ba7b8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-24-18-48-14.gh-issue-115886.rgM6AF.rst @@ -0,0 +1,2 @@ +Fix silent truncation of the name with an embedded null character in +:class:`multiprocessing.shared_memory.SharedMemory`. diff --git a/Misc/NEWS.d/next/Library/2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst b/Misc/NEWS.d/next/Library/2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst new file mode 100644 index 000000000000000..99bccb265ff80cd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst @@ -0,0 +1,4 @@ +Fix issue where :func:`ast.parse` would incorrectly flag conditional context +managers (such as ``with (x() if y else z()): ...``) as invalid syntax if +``feature_version=(3, 8)`` was passed. This reverts changes to the +grammar made as part of gh-94949. diff --git a/Misc/NEWS.d/next/Library/2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst b/Misc/NEWS.d/next/Library/2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst new file mode 100644 index 000000000000000..7512a09a37cd469 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst @@ -0,0 +1,2 @@ +[Enum] Improve error message when calling super().__new__() in custom +__new__. diff --git a/Misc/NEWS.d/next/Security/2024-01-26-22-14-09.gh-issue-114572.t1QMQD.rst b/Misc/NEWS.d/next/Security/2024-01-26-22-14-09.gh-issue-114572.t1QMQD.rst new file mode 100644 index 000000000000000..b4f9fe64db06157 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-01-26-22-14-09.gh-issue-114572.t1QMQD.rst @@ -0,0 +1,4 @@ +:meth:`ssl.SSLContext.cert_store_stats` and +:meth:`ssl.SSLContext.get_ca_certs` now correctly lock access to the +certificate store, when the :class:`ssl.SSLContext` is shared across +multiple threads. diff --git a/Misc/NEWS.d/next/Tests/2024-02-12-22-35-01.gh-issue-115376.n9vubZ.rst b/Misc/NEWS.d/next/Tests/2024-02-12-22-35-01.gh-issue-115376.n9vubZ.rst new file mode 100644 index 000000000000000..e09d78a9c4b189d --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-12-22-35-01.gh-issue-115376.n9vubZ.rst @@ -0,0 +1 @@ +Fix segfault in ``_testinternalcapi.compiler_codegen`` on bad input. diff --git a/Misc/NEWS.d/next/Tests/2024-02-13-18-24-04.gh-issue-115420.-dlzfI.rst b/Misc/NEWS.d/next/Tests/2024-02-13-18-24-04.gh-issue-115420.-dlzfI.rst new file mode 100644 index 000000000000000..1442ada3490fa09 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-13-18-24-04.gh-issue-115420.-dlzfI.rst @@ -0,0 +1,2 @@ +Fix translation of exception hander targets by +``_testinternalcapi.optimize_cfg``. diff --git a/Misc/NEWS.d/next/Tests/2024-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst b/Misc/NEWS.d/next/Tests/2024-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst new file mode 100644 index 000000000000000..c2811b133d9314f --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst @@ -0,0 +1,2 @@ +On Windows, commas passed in arguments to ``Tools\buildbot\test.bat`` and +``PCbuild\\rt.bat`` are now properly handled. diff --git a/Misc/NEWS.d/next/Tests/2024-02-17-08-25-01.gh-issue-115596.RGPCrR.rst b/Misc/NEWS.d/next/Tests/2024-02-17-08-25-01.gh-issue-115596.RGPCrR.rst new file mode 100644 index 000000000000000..2bcb8b9ac6bcd4d --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-17-08-25-01.gh-issue-115596.RGPCrR.rst @@ -0,0 +1,2 @@ +Fix ``ProgramPriorityTests`` in ``test_os`` permanently changing the process +priority. diff --git a/Misc/NEWS.d/next/Tests/2024-02-18-14-20-52.gh-issue-115122.3rGNo9.rst b/Misc/NEWS.d/next/Tests/2024-02-18-14-20-52.gh-issue-115122.3rGNo9.rst new file mode 100644 index 000000000000000..e187a40a40516b4 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-18-14-20-52.gh-issue-115122.3rGNo9.rst @@ -0,0 +1,2 @@ +Add ``--bisect`` option to regrtest test runner: run failed tests with +``test.bisect_cmd`` to identify failing tests. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2024-02-20-15-47-41.gh-issue-115720.w8i8UG.rst b/Misc/NEWS.d/next/Tests/2024-02-20-15-47-41.gh-issue-115720.w8i8UG.rst new file mode 100644 index 000000000000000..a03ee11d974251c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-20-15-47-41.gh-issue-115720.w8i8UG.rst @@ -0,0 +1,2 @@ +Leak tests (``-R``, ``--huntrleaks``) now show a summary of the number of +leaks found in each iteration. diff --git a/Misc/NEWS.d/next/Tests/2024-02-22-00-17-06.gh-issue-115796.d4hpKy.rst b/Misc/NEWS.d/next/Tests/2024-02-22-00-17-06.gh-issue-115796.d4hpKy.rst new file mode 100644 index 000000000000000..a40be74f73908e5 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-22-00-17-06.gh-issue-115796.d4hpKy.rst @@ -0,0 +1,2 @@ +Make '_testinternalcapi.assemble_code_object' construct the exception table +for the code object. diff --git a/Misc/NEWS.d/next/Tests/2024-02-25-15-58-28.gh-issue-71052.lxBjqY.rst b/Misc/NEWS.d/next/Tests/2024-02-25-15-58-28.gh-issue-71052.lxBjqY.rst new file mode 100644 index 000000000000000..8bac68b5aea78ee --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-25-15-58-28.gh-issue-71052.lxBjqY.rst @@ -0,0 +1,2 @@ +Enable ``test_concurrent_futures`` on platforms that support threading but not +multiprocessing. diff --git a/Misc/NEWS.d/next/Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst b/Misc/NEWS.d/next/Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst new file mode 100644 index 000000000000000..48a09da78229156 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst @@ -0,0 +1 @@ +Add consistency in usage message IO between 2 versions of python-config. diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-02-12-19-28-08.gh-issue-100176.Kzs4Zw.rst b/Misc/NEWS.d/next/Tools-Demos/2023-02-12-19-28-08.gh-issue-100176.Kzs4Zw.rst new file mode 100644 index 000000000000000..1a9fc76d93f2976 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-02-12-19-28-08.gh-issue-100176.Kzs4Zw.rst @@ -0,0 +1 @@ +Remove outdated Tools/{io,cc,string}bench diff --git a/Misc/NEWS.d/next/Windows/2023-12-19-22-32-28.gh-issue-112984.F7kFMl.rst b/Misc/NEWS.d/next/Windows/2023-12-19-22-32-28.gh-issue-112984.F7kFMl.rst deleted file mode 100644 index 429cd5bc923e09a..000000000000000 --- a/Misc/NEWS.d/next/Windows/2023-12-19-22-32-28.gh-issue-112984.F7kFMl.rst +++ /dev/null @@ -1 +0,0 @@ -Adds free-threaded binaries to Windows installer as an optional component. diff --git a/Misc/NEWS.d/next/Windows/2024-01-23-00-05-05.gh-issue-100107.lkbP_Q.rst b/Misc/NEWS.d/next/Windows/2024-01-23-00-05-05.gh-issue-100107.lkbP_Q.rst deleted file mode 100644 index 388d61a2b3bd6db..000000000000000 --- a/Misc/NEWS.d/next/Windows/2024-01-23-00-05-05.gh-issue-100107.lkbP_Q.rst +++ /dev/null @@ -1 +0,0 @@ -The ``py.exe`` launcher will no longer attempt to run the Microsoft Store redirector when launching a script containing a ``/usr/bin/env`` shebang diff --git a/Misc/NEWS.d/next/Windows/2024-02-15-23-16-31.gh-issue-115543.otrWnw.rst b/Misc/NEWS.d/next/Windows/2024-02-15-23-16-31.gh-issue-115543.otrWnw.rst new file mode 100644 index 000000000000000..ebd15c83b83491f --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-02-15-23-16-31.gh-issue-115543.otrWnw.rst @@ -0,0 +1,3 @@ +:ref:`launcher` can now detect Python 3.13 when installed from the Microsoft +Store, and will install Python 3.12 by default when +:envvar:`PYLAUNCHER_ALLOW_INSTALL` is set. diff --git a/Misc/NEWS.d/next/Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst b/Misc/NEWS.d/next/Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst new file mode 100644 index 000000000000000..f2e82bf6a3e0285 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst @@ -0,0 +1,3 @@ +Building extensions intended for free-threaded builds of CPython now require +compiling with ``/DPy_GIL_DISABLED`` manually when using a regular install. This +is expected to change in future releases. diff --git a/Misc/NEWS.d/next/Windows/2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst b/Misc/NEWS.d/next/Windows/2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst new file mode 100644 index 000000000000000..a55e5b1c7b566d3 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst @@ -0,0 +1 @@ +Ensure the value of ``GetLastError()`` is preserved across GIL operations. diff --git a/Misc/NEWS.d/next/macOS/2024-01-23-11-35-26.gh-issue-114490.FrQOQ0.rst b/Misc/NEWS.d/next/macOS/2024-01-23-11-35-26.gh-issue-114490.FrQOQ0.rst deleted file mode 100644 index abd296f86085180..000000000000000 --- a/Misc/NEWS.d/next/macOS/2024-01-23-11-35-26.gh-issue-114490.FrQOQ0.rst +++ /dev/null @@ -1 +0,0 @@ -Add Mach-O linkage support for :func:`platform.architecture()`. diff --git a/Misc/platform_triplet.c b/Misc/platform_triplet.c index 3307260544e8a6d..0b912e332510a6b 100644 --- a/Misc/platform_triplet.c +++ b/Misc/platform_triplet.c @@ -233,7 +233,22 @@ PLATFORM_TRIPLET=i386-gnu # error unknown platform triplet # endif #elif defined(__APPLE__) +# include "TargetConditionals.h" +# if TARGET_OS_IOS +# if TARGET_OS_SIMULATOR +# if __x86_64__ +PLATFORM_TRIPLET=x86_64-iphonesimulator +# else +PLATFORM_TRIPLET=arm64-iphonesimulator +# endif +# else +PLATFORM_TRIPLET=arm64-iphoneos +# endif +# elif TARGET_OS_OSX PLATFORM_TRIPLET=darwin +# else +# error unknown Apple platform +# endif #elif defined(__VXWORKS__) PLATFORM_TRIPLET=vxworks #elif defined(__wasm32__) diff --git a/Misc/python-config.in b/Misc/python-config.in index 81c3316e334a485..dd5d161ab2286f1 100644 --- a/Misc/python-config.in +++ b/Misc/python-config.in @@ -13,7 +13,8 @@ valid_opts = ['prefix', 'exec-prefix', 'includes', 'libs', 'cflags', def exit_with_usage(code=1): print("Usage: {0} [{1}]".format( - sys.argv[0], '|'.join('--'+opt for opt in valid_opts)), file=sys.stderr) + sys.argv[0], '|'.join('--'+opt for opt in valid_opts)), + file=sys.stdout if code == 0 else sys.stderr) sys.exit(code) try: diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index 2602fe24c0402e1..eb02223ddcd2c30 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -4,7 +4,12 @@ exit_with_usage () { - echo "Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" + local USAGE="Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" + if [[ "$1" -eq 0 ]]; then + echo "$USAGE" + else + echo "$USAGE" >&2 + fi exit $1 } diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index 94566772338b108..e28eaea81d6aae1 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -48,29 +48,15 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "ab7bb32514d170592dfb3f76e41bbdc075a4e7e0" + "checksumValue": "90c06411f131e777e2b5c3d22b7ccf50bc46f617" }, { "algorithm": "SHA256", - "checksumValue": "f521acdad222644365b0e81a33bcd6939a98c91b225c47582cc84bd73d96febc" + "checksumValue": "3045f9176950aa13a54e53fa096385670c676c492705d636e977f888e4c72d48" } ], "fileName": "Modules/expat/expat.h" }, - { - "SPDXID": "SPDXRef-FILE-Modules-expat-expat-config.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "73627287302ee3e84347c4fe21f37a9cb828bc3b" - }, - { - "algorithm": "SHA256", - "checksumValue": "f17e59f9d95eeb05694c02508aa284d332616c22cbe2e6a802d8a0710310eaab" - } - ], - "fileName": "Modules/expat/expat_config.h" - }, { "SPDXID": "SPDXRef-FILE-Modules-expat-expat-external.h", "checksums": [ @@ -104,11 +90,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "2790d37e7de2f13dccc4f4fb352cbdf9ed6abaa2" + "checksumValue": "9f6d9211a7b627785d5c48d10cc8eda66255113f" }, { "algorithm": "SHA256", - "checksumValue": "d2efe5a1018449968a689f444cca432e3d5875aba6ad08ee18ca235d64f41bb9" + "checksumValue": "9f0bdd346dd94ac4359c636a4e60bc768f4ae53ce0e836eb05fb9246ee36c7f2" } ], "fileName": "Modules/expat/internal.h" @@ -160,11 +146,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "2b984f806f10fbfbf72d8d1b7ba2992413c15299" + "checksumValue": "4c49b5df2bc702f663ba3b5a52d1940ec363226b" }, { "algorithm": "SHA256", - "checksumValue": "fbce56cd680e690043bbf572188cc2d0a25dbfc0d47ac8cb98eb3de768d4e694" + "checksumValue": "b5ec29f6560acc183f1ee8ab92bb3aea17b87b4c2120cd2e3f78deba7a12491e" } ], "fileName": "Modules/expat/siphash.h" @@ -188,11 +174,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "e774ae6ee9391aa6ffb8f775fb74e48f4b428959" + "checksumValue": "a3a8c44efd55dbf2cfea8fcee009ec63120ec0a3" }, { "algorithm": "SHA256", - "checksumValue": "3c71cea9a6174718542331971a35db317902b2433be9d8dd1cb24239b635c0cc" + "checksumValue": "e70948500d34dfcba4e9f0b305319dfe2a937c7cbfb687905128b56e1a6f8b33" } ], "fileName": "Modules/expat/winconfig.h" @@ -202,11 +188,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "b580e827e16baa6b035586ffcd4d90301e5a353f" + "checksumValue": "3b5de0ed1de33cad85b46230707403247f2851df" }, { "algorithm": "SHA256", - "checksumValue": "483518bbd69338eefc706cd7fc0b6039df2d3e347f64097989059ed6d2385a1e" + "checksumValue": "a03abd531601eef61a87e06113d218ff139b6969e15a3d4668cd85d65fc6f79b" } ], "fileName": "Modules/expat/xmlparse.c" @@ -216,11 +202,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "5ef21312af73deb2428be3fe97a65244608e76de" + "checksumValue": "ef767128d2dda99436712dcf3465dde5dbaab876" }, { "algorithm": "SHA256", - "checksumValue": "6fcf8c72ac0112c1b98bd2039c632a66b4c3dc516ce7c1f981390951121ef3c0" + "checksumValue": "71fb52aa302cf6f56e41943009965804f49ff2210d9bd15b258f70aaf70db772" } ], "fileName": "Modules/expat/xmlrole.c" @@ -230,11 +216,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "c1a4ea6356643d0820edb9c024c20ad2aaf562dc" + "checksumValue": "c961fb1a80f7b0601a63e69fba793fe5f6dff157" }, { "algorithm": "SHA256", - "checksumValue": "2b5d674be6ef20c7e3f69295176d75e68c5616e4dfce0a186fdd5e2ed8315f7a" + "checksumValue": "228470eb9181a9a7575b63137edcb61b817ee4e0923faffdbeba29e07c939713" } ], "fileName": "Modules/expat/xmlrole.h" @@ -244,11 +230,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "e6d66ae9fd61d7950c62c5d87693c30a707e8577" + "checksumValue": "8394790c0199c8f88108542ad78f23095d28a3fe" }, { "algorithm": "SHA256", - "checksumValue": "1110f651bdccfa765ad3d6f3857a35887ab35fc0fe7f3f3488fde2b238b482e3" + "checksumValue": "5b16c671ccc42496374762768e4bf48f614aecfd2025a07925b8d94244aec645" } ], "fileName": "Modules/expat/xmltok.c" @@ -258,11 +244,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "9c2a544875fd08ba9c2397296c97263518a410aa" + "checksumValue": "7d2943a0128094455004b1a98007b98734221bae" }, { "algorithm": "SHA256", - "checksumValue": "4299a03828b98bfe47ec6809f6e279252954a9a911dc7e0f19551bd74e3af971" + "checksumValue": "6b8919dc951606dc6f2b0175f8955a9ced901ce8bd08db47f291b6c04227ae7f" } ], "fileName": "Modules/expat/xmltok.h" @@ -272,11 +258,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "aa96882de8e3d1d3083124b595aa911efe44e5ad" + "checksumValue": "7756f7c0d3625ae7dde6cf7d386685ffacb57c7e" }, { "algorithm": "SHA256", - "checksumValue": "0fbcba7931707c60301305dab78d2298d96447d0a5513926d8b18135228c0818" + "checksumValue": "a3fe18ff32b21fbcb7c190895c68158404e1b9fb449db6431bc08b261dc03938" } ], "fileName": "Modules/expat/xmltok_impl.c" @@ -1568,20 +1554,6 @@ } ], "fileName": "Modules/_decimal/libmpdec/vcdiv64.asm" - }, - { - "SPDXID": "SPDXRef-FILE-Lib-ensurepip-bundled-pip-23.3.2-py3-none-any.whl", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "8e48f55ab2965ee64bd55cc91a8077d184a33e30" - }, - { - "algorithm": "SHA256", - "checksumValue": "5052d7889c1f9d05224cd41741acb7c5d6fa735ab34e339624a614eaaa7e7d76" - } - ], - "fileName": "Lib/ensurepip/_bundled/pip-23.3.2-py3-none-any.whl" } ], "packages": [ @@ -1590,22 +1562,22 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "6b902ab103843592be5e99504f846ec109c1abb692e85347587f237a4ffa1033" + "checksumValue": "a13447b9aa67d7c860783fdf6820f33ebdea996900d6d8bbc50a628f55f099f7" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_5_0/expat-2.5.0.tar.gz", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_6_0/expat-2.6.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.5.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.6.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], - "licenseConcluded": "MIT", + "licenseConcluded": "NOASSERTION", "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.5.0" + "versionInfo": "2.6.0" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", @@ -1623,7 +1595,7 @@ "referenceType": "cpe23Type" } ], - "licenseConcluded": "Apache-2.0", + "licenseConcluded": "NOASSERTION", "name": "hacl-star", "originator": "Organization: HACL* Developers", "primaryPackagePurpose": "SOURCE", @@ -1645,7 +1617,7 @@ "referenceType": "cpe23Type" } ], - "licenseConcluded": "CC0-1.0", + "licenseConcluded": "NOASSERTION", "name": "libb2", "originator": "Organization: BLAKE2 - fast secure hashing", "primaryPackagePurpose": "SOURCE", @@ -1667,7 +1639,7 @@ "referenceType": "purl" } ], - "licenseConcluded": "MIT", + "licenseConcluded": "NOASSERTION", "name": "macholib", "originator": "Person: Ronald Oussoren (ronaldoussoren@mac.com)", "primaryPackagePurpose": "SOURCE", @@ -1689,665 +1661,14 @@ "referenceType": "cpe23Type" } ], - "licenseConcluded": "BSD-2-Clause", + "licenseConcluded": "NOASSERTION", "name": "mpdecimal", "originator": "Organization: bytereef.org", "primaryPackagePurpose": "SOURCE", "versionInfo": "2.5.1" - }, - { - "SPDXID": "SPDXRef-PACKAGE-cachecontrol", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "95dedbec849f46dda3137866dc28b9d133fc9af55f5b805ab1291833e4457aa4" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/1d/e3/a22348e6226dcd585d5a4b5f0175b3a16dabfd3912cbeb02f321d00e56c7/cachecontrol-0.13.1-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/cachecontrol@0.13.1", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "cachecontrol", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "0.13.1" - }, - { - "SPDXID": "SPDXRef-PACKAGE-colorama", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/colorama@0.4.6", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "colorama", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "0.4.6" - }, - { - "SPDXID": "SPDXRef-PACKAGE-distlib", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/76/cb/6bbd2b10170ed991cf64e8c8b85e01f2fb38f95d1bc77617569e0b0b26ac/distlib-0.3.6-py2.py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/distlib@0.3.6", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "distlib", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "0.3.6" - }, - { - "SPDXID": "SPDXRef-PACKAGE-distro", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "99522ca3e365cac527b44bde033f64c6945d90eb9f769703caaec52b09bbd3ff" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/f4/2c/c90a3adaf0ddb70afe193f5ebfb539612af57cffe677c3126be533df3098/distro-1.8.0-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/distro@1.8.0", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "distro", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "1.8.0" - }, - { - "SPDXID": "SPDXRef-PACKAGE-msgpack", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "525228efd79bb831cf6830a732e2e80bc1b05436b086d4264814b4b2955b2fa9" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/9f/4a/36d936e54cf71e23ad276564465f6a54fb129e3d61520b76e13e0bb29167/msgpack-1.0.5-cp310-cp310-macosx_10_9_universal2.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/msgpack@1.0.5", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "msgpack", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "1.0.5" - }, - { - "SPDXID": "SPDXRef-PACKAGE-packaging", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/05/8e/8de486cbd03baba4deef4142bd643a3e7bbe954a784dc1bb17142572d127/packaging-21.3-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/packaging@21.3", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "packaging", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "21.3" - }, - { - "SPDXID": "SPDXRef-PACKAGE-platformdirs", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "cec7b889196b9144d088e4c57d9ceef7374f6c39694ad1577a0aab50d27ea28c" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/9e/d8/563a9fc17153c588c8c2042d2f0f84a89057cdb1c30270f589c88b42d62c/platformdirs-3.8.1-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/platformdirs@3.8.1", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "platformdirs", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.8.1" - }, - { - "SPDXID": "SPDXRef-PACKAGE-pyparsing", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "d554a96d1a7d3ddaf7183104485bc19fd80543ad6ac5bdb6426719d766fb06c1" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/a4/24/6ae4c9c45cf99d96b06b5d99e25526c060303171fb0aea9da2bfd7dbde93/pyparsing-3.1.0-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/pyparsing@3.1.0", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "pyparsing", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.1.0" - }, - { - "SPDXID": "SPDXRef-PACKAGE-pyproject-hooks", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/d5/ea/9ae603de7fbb3df820b23a70f6aff92bf8c7770043254ad8d2dc9d6bcba4/pyproject_hooks-1.0.0-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/pyproject-hooks@1.0.0", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "pyproject-hooks", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "1.0.0" - }, - { - "SPDXID": "SPDXRef-PACKAGE-requests", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/requests@2.31.0", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "requests", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.31.0" - }, - { - "SPDXID": "SPDXRef-PACKAGE-certifi", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/4c/dd/2234eab22353ffc7d94e8d13177aaa050113286e93e7b40eae01fbf7c3d9/certifi-2023.7.22-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/certifi@2023.7.22", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "certifi", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "2023.7.22" - }, - { - "SPDXID": "SPDXRef-PACKAGE-chardet", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "362777fb014af596ad31334fde1e8c327dfdb076e1960d1694662d46a6917ab9" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/74/8f/8fc49109009e8d2169d94d72e6b1f4cd45c13d147ba7d6170fb41f22b08f/chardet-5.1.0-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/chardet@5.1.0", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "chardet", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "5.1.0" - }, - { - "SPDXID": "SPDXRef-PACKAGE-idna", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/idna@3.4", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "idna", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.4" - }, - { - "SPDXID": "SPDXRef-PACKAGE-rich", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "8f87bc7ee54675732fa66a05ebfe489e27264caeeff3728c945d25971b6485ec" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/fc/1e/482e5eec0b89b593e81d78f819a9412849814e22225842b598908e7ac560/rich-13.4.2-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/rich@13.4.2", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "rich", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "13.4.2" - }, - { - "SPDXID": "SPDXRef-PACKAGE-pygments", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/34/a7/37c8d68532ba71549db4212cb036dbd6161b40e463aba336770e80c72f84/Pygments-2.15.1-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/pygments@2.15.1", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "pygments", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.15.1" - }, - { - "SPDXID": "SPDXRef-PACKAGE-typing-extensions", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/ec/6b/63cc3df74987c36fe26157ee12e09e8f9db4de771e0f3404263117e75b95/typing_extensions-4.7.1-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/typing_extensions@4.7.1", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "typing_extensions", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "4.7.1" - }, - { - "SPDXID": "SPDXRef-PACKAGE-resolvelib", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "d2da45d1a8dfee81bdd591647783e340ef3bcb104b54c383f70d422ef5cc7dbf" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/d2/fc/e9ccf0521607bcd244aa0b3fbd574f71b65e9ce6a112c83af988bbbe2e23/resolvelib-1.0.1-py2.py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/resolvelib@1.0.1", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "resolvelib", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "1.0.1" - }, - { - "SPDXID": "SPDXRef-PACKAGE-setuptools", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/c7/42/be1c7bbdd83e1bfb160c94b9cafd8e25efc7400346cf7ccdbdb452c467fa/setuptools-68.0.0-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/setuptools@68.0.0", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "setuptools", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "68.0.0" - }, - { - "SPDXID": "SPDXRef-PACKAGE-six", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/six@1.16.0", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "six", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "1.16.0" - }, - { - "SPDXID": "SPDXRef-PACKAGE-tenacity", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "2f277afb21b851637e8f52e6a613ff08734c347dc19ade928e519d7d2d8569b0" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/e7/b0/c23bd61e1b32c9b96fbca996c87784e196a812da8d621d8d04851f6c8181/tenacity-8.2.2-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/tenacity@8.2.2", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "tenacity", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "8.2.2" - }, - { - "SPDXID": "SPDXRef-PACKAGE-tomli", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/tomli@2.0.1", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "tomli", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.0.1" - }, - { - "SPDXID": "SPDXRef-PACKAGE-truststore", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "e37a5642ae9fc48caa8f120b6283d77225d600d224965a672c9e8ef49ce4bb4c" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/20/56/7811d5439b6a56374f274a8672d8f18b4deadadeb3a9f0c86424b98b6f96/truststore-0.8.0-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/truststore@0.8.0", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "truststore", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "0.8.0" - }, - { - "SPDXID": "SPDXRef-PACKAGE-webencodings", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/webencodings@0.5.1", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "webencodings", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "0.5.1" - }, - { - "SPDXID": "SPDXRef-PACKAGE-urllib3", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/48/fe/a5c6cc46e9fe9171d7ecf0f33ee7aae14642f8d74baa7af4d7840f9358be/urllib3-1.26.17-py2.py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/urllib3@1.26.17", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "urllib3", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "1.26.17" - }, - { - "SPDXID": "SPDXRef-PACKAGE-pip", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "5052d7889c1f9d05224cd41741acb7c5d6fa735ab34e339624a614eaaa7e7d76" - } - ], - "downloadLocation": "https://files.pythonhosted.org/packages/15/aa/3f4c7bcee2057a76562a5b33ecbd199be08cdb4443a02e26bd2c3cf6fc39/pip-23.3.2-py3-none-any.whl", - "externalRefs": [ - { - "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:pypa:pip:23.3.2:*:*:*:*:*:*:*", - "referenceType": "cpe23Type" - }, - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": "pkg:pypi/pip@23.3.2", - "referenceType": "purl" - } - ], - "licenseConcluded": "MIT", - "name": "pip", - "originator": "Organization: Python Packaging Authority", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "23.3.2" } ], "relationships": [ - { - "relatedSpdxElement": "SPDXRef-PACKAGE-cachecontrol", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-certifi", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-chardet", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-colorama", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-distlib", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-distro", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-idna", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-msgpack", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-packaging", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-platformdirs", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-pygments", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-pyparsing", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-pyproject-hooks", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-requests", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-resolvelib", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-rich", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-setuptools", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-six", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-tenacity", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-tomli", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-truststore", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-typing-extensions", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-urllib3", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, - { - "relatedSpdxElement": "SPDXRef-PACKAGE-webencodings", - "relationshipType": "DEPENDS_ON", - "spdxElementId": "SPDXRef-PACKAGE-pip" - }, { "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-COPYING", "relationshipType": "CONTAINS", @@ -2368,11 +1689,6 @@ "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-expat-config.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-expat" - }, { "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-expat-external.h", "relationshipType": "CONTAINS", @@ -2907,11 +2223,6 @@ "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-vcdiv64.asm", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Lib-ensurepip-bundled-pip-23.3.2-py3-none-any.whl", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-pip" } ], "spdxVersion": "SPDX-2.3" diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 22b25dd0ec141fd..ca7cf02961571ee 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -78,6 +78,10 @@ [feature_macro.Py_REF_DEBUG] doc = 'when Python is compiled in debug mode (with Py_REF_DEBUG)' windows = 'maybe' +[feature_macro.Py_TRACE_REFS] + # nb. This mode is not compatible with Stable ABI/Limited API. + doc = 'when Python is compiled with Py_TRACE_REFS' + windows = 'maybe' # Mentioned in PEP 384: @@ -2481,3 +2485,14 @@ [function._Py_SetRefcnt] added = '3.13' abi_only = true +[data.PyExc_IncompleteInputError] + added = '3.13' +[function.PyList_GetItemRef] + added = '3.13' +[typedef.PyCFunctionFast] + added = '3.13' + # "abi-only" since 3.10. (Callback type names aren't used in C code, + # but this function signature was expected with METH_FASTCALL.) +[typedef.PyCFunctionFastWithKeywords] + added = '3.13' + # "abi-only" since 3.10. (Same story as PyCFunctionFast.) diff --git a/Modules/Setup b/Modules/Setup index 8ad9a5aebbfcaad..cd1cf24c25d4064 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -285,6 +285,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_testcapi _testcapimodule.c #_testimportmultiple _testimportmultiple.c #_testmultiphase _testmultiphase.c +#_testexternalinspection _testexternalinspection.c #_testsinglephase _testsinglephase.c # --- diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 8a65a9cffb1b9d8..73b082691a3fd49 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -162,7 +162,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/bytearray.c _testcapi/bytes.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/pyos.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c _testcapi/sys.c _testcapi/hash.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/bytearray.c _testcapi/bytes.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/pyos.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c _testcapi/sys.c _testcapi/hash.c _testcapi/time.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c @@ -171,6 +171,7 @@ @MODULE__TESTIMPORTMULTIPLE_TRUE@_testimportmultiple _testimportmultiple.c @MODULE__TESTMULTIPHASE_TRUE@_testmultiphase _testmultiphase.c @MODULE__TESTMULTIPHASE_TRUE@_testsinglephase _testsinglephase.c +@MODULE__TESTEXTERNALINSPECTION_TRUE@_testexternalinspection _testexternalinspection.c @MODULE__CTYPES_TEST_TRUE@_ctypes_test _ctypes/_ctypes_test.c # Limited API template modules; must be built as shared modules. diff --git a/Modules/_abc.c b/Modules/_abc.c index 9473905243d438d..399ecbbd6a21728 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -8,7 +8,6 @@ #include "pycore_object.h" // _PyType_GetSubclasses() #include "pycore_runtime.h" // _Py_ID() #include "pycore_setobject.h" // _PySet_NextEntry() -#include "pycore_typeobject.h" // _PyType_GetMRO() #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include "clinic/_abc.c.h" @@ -744,18 +743,12 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, Py_DECREF(ok); /* 4. Check if it's a direct subclass. */ - PyObject *mro = _PyType_GetMRO((PyTypeObject *)subclass); - assert(PyTuple_Check(mro)); - for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { - PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); - assert(mro_item != NULL); - if ((PyObject *)self == mro_item) { - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { - goto end; - } - result = Py_True; + if (PyType_IsSubtype((PyTypeObject *)subclass, (PyTypeObject *)self)) { + if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { goto end; } + result = Py_True; + goto end; } /* 5. Check if it's a subclass of a registered class (recursive). */ diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index c8cd53de5e22622..309d63c9bf7cbe7 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -3,6 +3,7 @@ #include "pycore_dict.h" // _PyDict_GetItem_KnownHash() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_typeobject.h" // _PyType_GetModuleState() #include <stddef.h> @@ -44,8 +45,11 @@ find_module_state_by_def(PyTypeObject *type) /*[clinic input] module _collections class _tuplegetter "_tuplegetterobject *" "clinic_state()->tuplegetter_type" +class _collections.deque "dequeobject *" "clinic_state()->deque_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7356042a89862e0e]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a033cc2a8476b3f1]*/ + +typedef struct dequeobject dequeobject; /* We can safely assume type to be the defining class, * since tuplegetter is not a base type */ @@ -53,6 +57,12 @@ class _tuplegetter "_tuplegetterobject *" "clinic_state()->tuplegetter_type" #include "clinic/_collectionsmodule.c.h" #undef clinic_state +/*[python input] +class dequeobject_converter(self_converter): + type = "dequeobject *" +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=b6ae4a3ff852be2f]*/ + /* collections module implementation of a deque() datatype Written and maintained by Raymond D. Hettinger <python@rcn.com> */ @@ -121,7 +131,7 @@ typedef struct BLOCK { struct BLOCK *rightlink; } block; -typedef struct { +struct dequeobject { PyObject_VAR_HEAD block *leftblock; block *rightblock; @@ -132,7 +142,7 @@ typedef struct { Py_ssize_t numfreeblocks; block *freeblocks[MAXFREEBLOCKS]; PyObject *weakreflist; -} dequeobject; +}; /* For debug builds, add error checking to track the endpoints * in the chain of links. The goal is to make sure that link @@ -219,8 +229,18 @@ deque_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *)deque; } +/*[clinic input] +@critical_section +_collections.deque.pop as deque_pop + + deque: dequeobject + +Remove and return the rightmost element. +[clinic start generated code]*/ + static PyObject * -deque_pop(dequeobject *deque, PyObject *unused) +deque_pop_impl(dequeobject *deque) +/*[clinic end generated code: output=2e5f7890c4251f07 input=55c5b6a8ad51d72f]*/ { PyObject *item; block *prevblock; @@ -254,10 +274,18 @@ deque_pop(dequeobject *deque, PyObject *unused) return item; } -PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element."); +/*[clinic input] +@critical_section +_collections.deque.popleft as deque_popleft + + deque: dequeobject + +Remove and return the leftmost element. +[clinic start generated code]*/ static PyObject * -deque_popleft(dequeobject *deque, PyObject *unused) +deque_popleft_impl(dequeobject *deque) +/*[clinic end generated code: output=62b154897097ff68 input=1571ce88fe3053de]*/ { PyObject *item; block *prevblock; @@ -292,8 +320,6 @@ deque_popleft(dequeobject *deque, PyObject *unused) return item; } -PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element."); - /* The deque's size limit is d.maxlen. The limit can be zero or positive. * If there is no limit, then d.maxlen == -1. * @@ -309,7 +335,7 @@ PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element."); #define NEEDS_TRIM(deque, maxlen) ((size_t)(maxlen) < (size_t)(Py_SIZE(deque))) static inline int -deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) +deque_append_lock_held(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) { if (deque->rightindex == BLOCKLEN - 1) { block *b = newblock(deque); @@ -326,7 +352,7 @@ deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; if (NEEDS_TRIM(deque, maxlen)) { - PyObject *olditem = deque_popleft(deque, NULL); + PyObject *olditem = deque_popleft_impl(deque); Py_DECREF(olditem); } else { deque->state++; @@ -334,18 +360,29 @@ deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) return 0; } +/*[clinic input] +@critical_section +_collections.deque.append as deque_append + + deque: dequeobject + item: object + / + +Add an element to the right side of the deque. +[clinic start generated code]*/ + static PyObject * -deque_append(dequeobject *deque, PyObject *item) +deque_append_impl(dequeobject *deque, PyObject *item) +/*[clinic end generated code: output=9c7bcb8b599c6362 input=b0eeeb09b9f5cf18]*/ { - if (deque_append_internal(deque, Py_NewRef(item), deque->maxlen) < 0) + if (deque_append_lock_held(deque, Py_NewRef(item), deque->maxlen) < 0) return NULL; Py_RETURN_NONE; } -PyDoc_STRVAR(append_doc, "Add an element to the right side of the deque."); - static inline int -deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) +deque_appendleft_lock_held(dequeobject *deque, PyObject *item, + Py_ssize_t maxlen) { if (deque->leftindex == 0) { block *b = newblock(deque); @@ -361,8 +398,8 @@ deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) Py_SET_SIZE(deque, Py_SIZE(deque) + 1); deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; - if (NEEDS_TRIM(deque, deque->maxlen)) { - PyObject *olditem = deque_pop(deque, NULL); + if (NEEDS_TRIM(deque, maxlen)) { + PyObject *olditem = deque_pop_impl(deque); Py_DECREF(olditem); } else { deque->state++; @@ -370,16 +407,26 @@ deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) return 0; } +/*[clinic input] +@critical_section +_collections.deque.appendleft as deque_appendleft + + deque: dequeobject + item: object + / + +Add an element to the left side of the deque. +[clinic start generated code]*/ + static PyObject * -deque_appendleft(dequeobject *deque, PyObject *item) +deque_appendleft_impl(dequeobject *deque, PyObject *item) +/*[clinic end generated code: output=9a192edbcd0f20db input=236c2fbceaf08e14]*/ { - if (deque_appendleft_internal(deque, Py_NewRef(item), deque->maxlen) < 0) + if (deque_appendleft_lock_held(deque, Py_NewRef(item), deque->maxlen) < 0) return NULL; Py_RETURN_NONE; } -PyDoc_STRVAR(appendleft_doc, "Add an element to the left side of the deque."); - static PyObject* finalize_iterator(PyObject *it) { @@ -410,8 +457,20 @@ consume_iterator(PyObject *it) return finalize_iterator(it); } +/*[clinic input] +@critical_section +_collections.deque.extend as deque_extend + + deque: dequeobject + iterable: object + / + +Extend the right side of the deque with elements from the iterable. +[clinic start generated code]*/ + static PyObject * -deque_extend(dequeobject *deque, PyObject *iterable) +deque_extend_impl(dequeobject *deque, PyObject *iterable) +/*[clinic end generated code: output=8b5ffa57ce82d980 input=85861954127c81da]*/ { PyObject *it, *item; PyObject *(*iternext)(PyObject *); @@ -445,7 +504,7 @@ deque_extend(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { - if (deque_append_internal(deque, item, maxlen) == -1) { + if (deque_append_lock_held(deque, item, maxlen) == -1) { Py_DECREF(item); Py_DECREF(it); return NULL; @@ -454,11 +513,20 @@ deque_extend(dequeobject *deque, PyObject *iterable) return finalize_iterator(it); } -PyDoc_STRVAR(extend_doc, -"Extend the right side of the deque with elements from the iterable"); +/*[clinic input] +@critical_section +_collections.deque.extendleft as deque_extendleft + + deque: dequeobject + iterable: object + / + +Extend the left side of the deque with elements from the iterable. +[clinic start generated code]*/ static PyObject * -deque_extendleft(dequeobject *deque, PyObject *iterable) +deque_extendleft_impl(dequeobject *deque, PyObject *iterable) +/*[clinic end generated code: output=ba44191aa8e35a26 input=640dabd086115689]*/ { PyObject *it, *item; PyObject *(*iternext)(PyObject *); @@ -470,7 +538,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) PyObject *s = PySequence_List(iterable); if (s == NULL) return NULL; - result = deque_extendleft(deque, s); + result = deque_extendleft_impl(deque, s); Py_DECREF(s); return result; } @@ -492,7 +560,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { - if (deque_appendleft_internal(deque, item, maxlen) == -1) { + if (deque_appendleft_lock_held(deque, item, maxlen) == -1) { Py_DECREF(item); Py_DECREF(it); return NULL; @@ -501,14 +569,12 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) return finalize_iterator(it); } -PyDoc_STRVAR(extendleft_doc, -"Extend the left side of the deque with elements from the iterable"); - static PyObject * deque_inplace_concat(dequeobject *deque, PyObject *other) { PyObject *result; + // deque_extend is thread-safe result = deque_extend(deque, other); if (result == NULL) return result; @@ -517,8 +583,18 @@ deque_inplace_concat(dequeobject *deque, PyObject *other) return (PyObject *)deque; } +/*[clinic input] +@critical_section +_collections.deque.copy as deque_copy + + deque: dequeobject + +Return a shallow copy of a deque. +[clinic start generated code]*/ + static PyObject * -deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored)) +deque_copy_impl(dequeobject *deque) +/*[clinic end generated code: output=6409b3d1ad2898b5 input=51d2ed1a23bab5e2]*/ { PyObject *result; dequeobject *old_deque = (dequeobject *)deque; @@ -532,12 +608,16 @@ deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored)) if (new_deque == NULL) return NULL; new_deque->maxlen = old_deque->maxlen; - /* Fast path for the deque_repeat() common case where len(deque) == 1 */ + /* Fast path for the deque_repeat() common case where len(deque) == 1 + * + * It's safe to not acquire the per-object lock for new_deque; it's + * invisible to other threads. + */ if (Py_SIZE(deque) == 1) { PyObject *item = old_deque->leftblock->data[old_deque->leftindex]; - rv = deque_append(new_deque, item); + rv = deque_append_impl(new_deque, item); } else { - rv = deque_extend(new_deque, deque); + rv = deque_extend_impl(new_deque, (PyObject *)deque); } if (rv != NULL) { Py_DECREF(rv); @@ -547,7 +627,8 @@ deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored)) return NULL; } if (old_deque->maxlen < 0) - result = PyObject_CallOneArg((PyObject *)(Py_TYPE(deque)), deque); + result = PyObject_CallOneArg((PyObject *)(Py_TYPE(deque)), + (PyObject *)deque); else result = PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "Oi", deque, old_deque->maxlen, NULL); @@ -561,10 +642,22 @@ deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored)) return result; } -PyDoc_STRVAR(copy_doc, "Return a shallow copy of a deque."); +/*[clinic input] +@critical_section +_collections.deque.__copy__ as deque___copy__ = _collections.deque.copy + +Return a shallow copy of a deque. +[clinic start generated code]*/ static PyObject * -deque_concat(dequeobject *deque, PyObject *other) +deque___copy___impl(dequeobject *deque) +/*[clinic end generated code: output=7c5821504342bf23 input=f5464036f9686a55]*/ +{ + return deque_copy_impl(deque); +} + +static PyObject * +deque_concat_lock_held(dequeobject *deque, PyObject *other) { PyObject *new_deque, *result; int rv; @@ -580,10 +673,13 @@ deque_concat(dequeobject *deque, PyObject *other) return NULL; } - new_deque = deque_copy((PyObject *)deque, NULL); + new_deque = deque_copy_impl(deque); if (new_deque == NULL) return NULL; - result = deque_extend((dequeobject *)new_deque, other); + + // It's safe to not acquire the per-object lock for new_deque; it's + // invisible to other threads. + result = deque_extend_impl((dequeobject *)new_deque, other); if (result == NULL) { Py_DECREF(new_deque); return NULL; @@ -592,6 +688,16 @@ deque_concat(dequeobject *deque, PyObject *other) return new_deque; } +static PyObject * +deque_concat(dequeobject *deque, PyObject *other) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(deque); + result = deque_concat_lock_held(deque, other); + Py_END_CRITICAL_SECTION(); + return result; +} + static int deque_clear(dequeobject *deque) { @@ -669,24 +775,32 @@ deque_clear(dequeobject *deque) alternate_method: while (Py_SIZE(deque)) { - item = deque_pop(deque, NULL); + item = deque_pop_impl(deque); assert (item != NULL); Py_DECREF(item); } return 0; } +/*[clinic input] +@critical_section +_collections.deque.clear as deque_clearmethod + + deque: dequeobject + +Remove all elements from the deque. +[clinic start generated code]*/ + static PyObject * -deque_clearmethod(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque_clearmethod_impl(dequeobject *deque) +/*[clinic end generated code: output=79b2513e097615c1 input=3a22e9605d20c5e9]*/ { deque_clear(deque); Py_RETURN_NONE; } -PyDoc_STRVAR(clear_doc, "Remove all elements from the deque."); - static PyObject * -deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) +deque_inplace_repeat_lock_held(dequeobject *deque, Py_ssize_t n) { Py_ssize_t i, m, size; PyObject *seq; @@ -750,7 +864,7 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) n = (deque->maxlen + size - 1) / size; for (i = 0 ; i < n-1 ; i++) { - rv = deque_extend(deque, seq); + rv = deque_extend_impl(deque, seq); if (rv == NULL) { Py_DECREF(seq); return NULL; @@ -762,16 +876,30 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) return (PyObject *)deque; } +static PyObject * +deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(deque); + result = deque_inplace_repeat_lock_held(deque, n); + Py_END_CRITICAL_SECTION(); + return result; +} + static PyObject * deque_repeat(dequeobject *deque, Py_ssize_t n) { dequeobject *new_deque; PyObject *rv; - new_deque = (dequeobject *)deque_copy((PyObject *) deque, NULL); + Py_BEGIN_CRITICAL_SECTION(deque); + new_deque = (dequeobject *)deque_copy_impl(deque); + Py_END_CRITICAL_SECTION(); if (new_deque == NULL) return NULL; - rv = deque_inplace_repeat(new_deque, n); + // It's safe to not acquire the per-object lock for new_deque; it's + // invisible to other threads. + rv = deque_inplace_repeat_lock_held(new_deque, n); Py_DECREF(new_deque); return rv; } @@ -925,36 +1053,38 @@ _deque_rotate(dequeobject *deque, Py_ssize_t n) return rv; } -static PyObject * -deque_rotate(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) -{ - Py_ssize_t n=1; +/*[clinic input] +@critical_section +_collections.deque.rotate as deque_rotate - if (!_PyArg_CheckPositional("deque.rotate", nargs, 0, 1)) { - return NULL; - } - if (nargs) { - PyObject *index = _PyNumber_Index(args[0]); - if (index == NULL) { - return NULL; - } - n = PyLong_AsSsize_t(index); - Py_DECREF(index); - if (n == -1 && PyErr_Occurred()) { - return NULL; - } - } + deque: dequeobject + n: Py_ssize_t = 1 + / + +Rotate the deque n steps to the right. If n is negative, rotates left. +[clinic start generated code]*/ +static PyObject * +deque_rotate_impl(dequeobject *deque, Py_ssize_t n) +/*[clinic end generated code: output=96c2402a371eb15d input=5bf834296246e002]*/ +{ if (!_deque_rotate(deque, n)) Py_RETURN_NONE; return NULL; } -PyDoc_STRVAR(rotate_doc, -"Rotate the deque n steps to the right (default n=1). If n is negative, rotates left."); +/*[clinic input] +@critical_section +_collections.deque.reverse as deque_reverse + + deque: dequeobject + +Reverse *IN PLACE*. +[clinic start generated code]*/ static PyObject * -deque_reverse(dequeobject *deque, PyObject *unused) +deque_reverse_impl(dequeobject *deque) +/*[clinic end generated code: output=bdeebc2cf8c1f064 input=26f4167fd623027f]*/ { block *leftblock = deque->leftblock; block *rightblock = deque->rightblock; @@ -991,11 +1121,20 @@ deque_reverse(dequeobject *deque, PyObject *unused) Py_RETURN_NONE; } -PyDoc_STRVAR(reverse_doc, -"D.reverse() -- reverse *IN PLACE*"); +/*[clinic input] +@critical_section +_collections.deque.count as deque_count + + deque: dequeobject + value as v: object + / + +Return number of occurrences of value. +[clinic start generated code]*/ static PyObject * -deque_count(dequeobject *deque, PyObject *v) +deque_count_impl(dequeobject *deque, PyObject *v) +/*[clinic end generated code: output=2ca26c49b6ab0400 input=4ef67ef2b34dc1fc]*/ { block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; @@ -1030,11 +1169,8 @@ deque_count(dequeobject *deque, PyObject *v) return PyLong_FromSsize_t(count); } -PyDoc_STRVAR(count_doc, -"D.count(value) -- return number of occurrences of value"); - static int -deque_contains(dequeobject *deque, PyObject *v) +deque_contains_lock_held(dequeobject *deque, PyObject *v) { block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; @@ -1065,28 +1201,50 @@ deque_contains(dequeobject *deque, PyObject *v) return 0; } +static int +deque_contains(dequeobject *deque, PyObject *v) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(deque); + result = deque_contains_lock_held(deque, v); + Py_END_CRITICAL_SECTION(); + return result; +} + static Py_ssize_t deque_len(dequeobject *deque) { - return Py_SIZE(deque); + return FT_ATOMIC_LOAD_SSIZE(((PyVarObject *)deque)->ob_size); } +/*[clinic input] +@critical_section +@text_signature "($self, value, [start, [stop]])" +_collections.deque.index as deque_index + + deque: dequeobject + value as v: object + start: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='0') = NULL + stop: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='Py_SIZE(deque)') = NULL + / + +Return first index of value. + +Raises ValueError if the value is not present. +[clinic start generated code]*/ + static PyObject * -deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, + Py_ssize_t stop) +/*[clinic end generated code: output=df45132753175ef9 input=90f48833a91e1743]*/ { - Py_ssize_t i, n, start=0, stop=Py_SIZE(deque); - PyObject *v, *item; + Py_ssize_t i, n; + PyObject *item; block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; size_t start_state = deque->state; int cmp; - if (!_PyArg_ParseStack(args, nargs, "O|O&O&:index", &v, - _PyEval_SliceIndexNotNone, &start, - _PyEval_SliceIndexNotNone, &stop)) { - return NULL; - } - if (start < 0) { start += Py_SIZE(deque); if (start < 0) @@ -1117,8 +1275,9 @@ deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) n = stop - i; while (--n >= 0) { CHECK_NOT_END(b); - item = b->data[index]; + item = Py_NewRef(b->data[index]); cmp = PyObject_RichCompareBool(item, v, Py_EQ); + Py_DECREF(item); if (cmp > 0) return PyLong_FromSsize_t(stop - n - 1); if (cmp < 0) @@ -1138,10 +1297,6 @@ deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) return NULL; } -PyDoc_STRVAR(index_doc, -"D.index(value, [start, [stop]]) -- return first index of value.\n" -"Raises ValueError if the value is not present."); - /* insert(), remove(), and delitem() are implemented in terms of rotate() for simplicity and reasonable performance near the end points. If for some reason these methods become popular, it is not @@ -1150,32 +1305,39 @@ PyDoc_STRVAR(index_doc, boost (by moving each pointer only once instead of twice). */ +/*[clinic input] +@critical_section +_collections.deque.insert as deque_insert + + deque: dequeobject + index: Py_ssize_t + value: object + / + +Insert value before index. +[clinic start generated code]*/ + static PyObject * -deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +deque_insert_impl(dequeobject *deque, Py_ssize_t index, PyObject *value) +/*[clinic end generated code: output=ef4d2c15d5532b80 input=dbee706586cc9cde]*/ { - Py_ssize_t index; Py_ssize_t n = Py_SIZE(deque); - PyObject *value; PyObject *rv; - if (!_PyArg_ParseStack(args, nargs, "nO:insert", &index, &value)) { - return NULL; - } - if (deque->maxlen == Py_SIZE(deque)) { PyErr_SetString(PyExc_IndexError, "deque already at its maximum size"); return NULL; } if (index >= n) - return deque_append(deque, value); + return deque_append_impl(deque, value); if (index <= -n || index == 0) - return deque_appendleft(deque, value); + return deque_appendleft_impl(deque, value); if (_deque_rotate(deque, -index)) return NULL; if (index < 0) - rv = deque_append(deque, value); + rv = deque_append_impl(deque, value); else - rv = deque_appendleft(deque, value); + rv = deque_appendleft_impl(deque, value); if (rv == NULL) return NULL; Py_DECREF(rv); @@ -1184,12 +1346,6 @@ deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) Py_RETURN_NONE; } -PyDoc_STRVAR(insert_doc, -"D.insert(index, object) -- insert object before index"); - -PyDoc_STRVAR(remove_doc, -"D.remove(value) -- remove first occurrence of value."); - static int valid_index(Py_ssize_t i, Py_ssize_t limit) { @@ -1199,7 +1355,7 @@ valid_index(Py_ssize_t i, Py_ssize_t limit) } static PyObject * -deque_item(dequeobject *deque, Py_ssize_t i) +deque_item_lock_held(dequeobject *deque, Py_ssize_t i) { block *b; PyObject *item; @@ -1237,6 +1393,16 @@ deque_item(dequeobject *deque, Py_ssize_t i) return Py_NewRef(item); } +static PyObject * +deque_item(dequeobject *deque, Py_ssize_t i) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(deque); + result = deque_item_lock_held(deque, i); + Py_END_CRITICAL_SECTION(); + return result; +} + static int deque_del_item(dequeobject *deque, Py_ssize_t i) { @@ -1246,15 +1412,27 @@ deque_del_item(dequeobject *deque, Py_ssize_t i) assert (i >= 0 && i < Py_SIZE(deque)); if (_deque_rotate(deque, -i)) return -1; - item = deque_popleft(deque, NULL); + item = deque_popleft_impl(deque); rv = _deque_rotate(deque, i); assert (item != NULL); Py_DECREF(item); return rv; } +/*[clinic input] +@critical_section +_collections.deque.remove as deque_remove + + deque: dequeobject + value: object + / + +Remove first occurrence of value. +[clinic start generated code]*/ + static PyObject * -deque_remove(dequeobject *deque, PyObject *value) +deque_remove_impl(dequeobject *deque, PyObject *value) +/*[clinic end generated code: output=54cff28b8ef78c5b input=60eb3f8aa4de532a]*/ { PyObject *item; block *b = deque->leftblock; @@ -1295,7 +1473,7 @@ deque_remove(dequeobject *deque, PyObject *value) } static int -deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) +deque_ass_item_lock_held(dequeobject *deque, Py_ssize_t i, PyObject *v) { block *b; Py_ssize_t n, len=Py_SIZE(deque), halflen=(len+1)>>1, index=i; @@ -1326,6 +1504,16 @@ deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) return 0; } +static int +deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(deque); + result = deque_ass_item_lock_held(deque, i, v); + Py_END_CRITICAL_SECTION(); + return result; +} + static void deque_dealloc(dequeobject *deque) { @@ -1375,8 +1563,17 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg) return 0; } +/*[clinic input] +_collections.deque.__reduce__ as deque___reduce__ + + deque: dequeobject + +Return state information for pickling. +[clinic start generated code]*/ + static PyObject * -deque_reduce(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque___reduce___impl(dequeobject *deque) +/*[clinic end generated code: output=cb85d9e0b7d2c5ad input=991a933a5bc7a526]*/ { PyObject *state, *it; @@ -1391,6 +1588,8 @@ deque_reduce(dequeobject *deque, PyObject *Py_UNUSED(ignored)) return NULL; } + // It's safe to access deque->maxlen here without holding the per object + // lock for deque; deque->maxlen is only assigned during construction. if (deque->maxlen < 0) { return Py_BuildValue("O()NN", Py_TYPE(deque), state, it); } @@ -1510,26 +1709,23 @@ deque_richcompare(PyObject *v, PyObject *w, int op) return NULL; } +/*[clinic input] +@critical_section +@text_signature "([iterable[, maxlen]])" +_collections.deque.__init__ as deque_init + + deque: dequeobject + iterable: object = NULL + maxlen as maxlenobj: object = NULL + +A list-like sequence optimized for data accesses near its endpoints. +[clinic start generated code]*/ + static int -deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs) +deque_init_impl(dequeobject *deque, PyObject *iterable, PyObject *maxlenobj) +/*[clinic end generated code: output=7084a39d71218dcd input=2b9e37af1fd73143]*/ { - PyObject *iterable = NULL; - PyObject *maxlenobj = NULL; Py_ssize_t maxlen = -1; - char *kwlist[] = {"iterable", "maxlen", 0}; - - if (kwdargs == NULL && PyTuple_GET_SIZE(args) <= 2) { - if (PyTuple_GET_SIZE(args) > 0) { - iterable = PyTuple_GET_ITEM(args, 0); - } - if (PyTuple_GET_SIZE(args) > 1) { - maxlenobj = PyTuple_GET_ITEM(args, 1); - } - } else { - if (!PyArg_ParseTupleAndKeywords(args, kwdargs, "|OO:deque", kwlist, - &iterable, &maxlenobj)) - return -1; - } if (maxlenobj != NULL && maxlenobj != Py_None) { maxlen = PyLong_AsSsize_t(maxlenobj); if (maxlen == -1 && PyErr_Occurred()) @@ -1543,7 +1739,7 @@ deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs) if (Py_SIZE(deque) > 0) deque_clear(deque); if (iterable != NULL) { - PyObject *rv = deque_extend(deque, iterable); + PyObject *rv = deque_extend_impl(deque, iterable); if (rv == NULL) return -1; Py_DECREF(rv); @@ -1551,8 +1747,18 @@ deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs) return 0; } +/*[clinic input] +@critical_section +_collections.deque.__sizeof__ as deque___sizeof__ + + deque: dequeobject + +Return the size of the deque in memory, in bytes. +[clinic start generated code]*/ + static PyObject * -deque_sizeof(dequeobject *deque, void *unused) +deque___sizeof___impl(dequeobject *deque) +/*[clinic end generated code: output=4d36e9fb4f30bbaf input=762312f2d4813535]*/ { size_t res = _PyObject_SIZE(Py_TYPE(deque)); size_t blocks; @@ -1563,9 +1769,6 @@ deque_sizeof(dequeobject *deque, void *unused) return PyLong_FromSize_t(res); } -PyDoc_STRVAR(sizeof_doc, -"D.__sizeof__() -- size of D in memory, in bytes"); - static PyObject * deque_get_maxlen(dequeobject *deque, void *Py_UNUSED(ignored)) { @@ -1574,6 +1777,22 @@ deque_get_maxlen(dequeobject *deque, void *Py_UNUSED(ignored)) return PyLong_FromSsize_t(deque->maxlen); } +static PyObject *deque_reviter(dequeobject *deque); + +/*[clinic input] +_collections.deque.__reversed__ as deque___reversed__ + + deque: dequeobject + +Return a reverse iterator over the deque. +[clinic start generated code]*/ + +static PyObject * +deque___reversed___impl(dequeobject *deque) +/*[clinic end generated code: output=3e7e7e715883cf2e input=3d494c25a6fe5c7e]*/ +{ + return deque_reviter(deque); +} /* deque object ********************************************************/ @@ -1584,47 +1803,26 @@ static PyGetSetDef deque_getset[] = { }; static PyObject *deque_iter(dequeobject *deque); -static PyObject *deque_reviter(dequeobject *deque, PyObject *Py_UNUSED(ignored)); -PyDoc_STRVAR(reversed_doc, - "D.__reversed__() -- return a reverse iterator over the deque"); static PyMethodDef deque_methods[] = { - {"append", (PyCFunction)deque_append, - METH_O, append_doc}, - {"appendleft", (PyCFunction)deque_appendleft, - METH_O, appendleft_doc}, - {"clear", (PyCFunction)deque_clearmethod, - METH_NOARGS, clear_doc}, - {"__copy__", deque_copy, - METH_NOARGS, copy_doc}, - {"copy", deque_copy, - METH_NOARGS, copy_doc}, - {"count", (PyCFunction)deque_count, - METH_O, count_doc}, - {"extend", (PyCFunction)deque_extend, - METH_O, extend_doc}, - {"extendleft", (PyCFunction)deque_extendleft, - METH_O, extendleft_doc}, - {"index", _PyCFunction_CAST(deque_index), - METH_FASTCALL, index_doc}, - {"insert", _PyCFunction_CAST(deque_insert), - METH_FASTCALL, insert_doc}, - {"pop", (PyCFunction)deque_pop, - METH_NOARGS, pop_doc}, - {"popleft", (PyCFunction)deque_popleft, - METH_NOARGS, popleft_doc}, - {"__reduce__", (PyCFunction)deque_reduce, - METH_NOARGS, reduce_doc}, - {"remove", (PyCFunction)deque_remove, - METH_O, remove_doc}, - {"__reversed__", (PyCFunction)deque_reviter, - METH_NOARGS, reversed_doc}, - {"reverse", (PyCFunction)deque_reverse, - METH_NOARGS, reverse_doc}, - {"rotate", _PyCFunction_CAST(deque_rotate), - METH_FASTCALL, rotate_doc}, - {"__sizeof__", (PyCFunction)deque_sizeof, - METH_NOARGS, sizeof_doc}, + DEQUE_APPEND_METHODDEF + DEQUE_APPENDLEFT_METHODDEF + DEQUE_CLEARMETHOD_METHODDEF + DEQUE___COPY___METHODDEF + DEQUE_COPY_METHODDEF + DEQUE_COUNT_METHODDEF + DEQUE_EXTEND_METHODDEF + DEQUE_EXTENDLEFT_METHODDEF + DEQUE_INDEX_METHODDEF + DEQUE_INSERT_METHODDEF + DEQUE_POP_METHODDEF + DEQUE_POPLEFT_METHODDEF + DEQUE___REDUCE___METHODDEF + DEQUE_REMOVE_METHODDEF + DEQUE___REVERSED___METHODDEF + DEQUE_REVERSE_METHODDEF + DEQUE_ROTATE_METHODDEF + DEQUE___SIZEOF___METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ @@ -1635,17 +1833,12 @@ static PyMemberDef deque_members[] = { {NULL}, }; -PyDoc_STRVAR(deque_doc, -"deque([iterable[, maxlen]]) --> deque object\n\ -\n\ -A list-like sequence optimized for data accesses near its endpoints."); - static PyType_Slot deque_slots[] = { {Py_tp_dealloc, deque_dealloc}, {Py_tp_repr, deque_repr}, {Py_tp_hash, PyObject_HashNotImplemented}, {Py_tp_getattro, PyObject_GenericGetAttr}, - {Py_tp_doc, (void *)deque_doc}, + {Py_tp_doc, (void *)deque_init__doc__}, {Py_tp_traverse, deque_traverse}, {Py_tp_clear, deque_clear}, {Py_tp_richcompare, deque_richcompare}, @@ -1699,11 +1892,13 @@ deque_iter(dequeobject *deque) it = PyObject_GC_New(dequeiterobject, state->dequeiter_type); if (it == NULL) return NULL; + Py_BEGIN_CRITICAL_SECTION(deque); it->b = deque->leftblock; it->index = deque->leftindex; it->deque = (dequeobject*)Py_NewRef(deque); it->state = deque->state; it->counter = Py_SIZE(deque); + Py_END_CRITICAL_SECTION(); PyObject_GC_Track(it); return (PyObject *)it; } @@ -1735,7 +1930,7 @@ dequeiter_dealloc(dequeiterobject *dio) } static PyObject * -dequeiter_next(dequeiterobject *it) +dequeiter_next_lock_held(dequeiterobject *it, dequeobject *deque) { PyObject *item; @@ -1761,6 +1956,20 @@ dequeiter_next(dequeiterobject *it) return Py_NewRef(item); } +static PyObject * +dequeiter_next(dequeiterobject *it) +{ + PyObject *result; + // It's safe to access it->deque without holding the per-object lock for it + // here; it->deque is only assigned during construction of it. + dequeobject *deque = it->deque; + Py_BEGIN_CRITICAL_SECTION2(it, deque); + result = dequeiter_next_lock_held(it, deque); + Py_END_CRITICAL_SECTION2(); + + return result; +} + static PyObject * dequeiter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -1781,6 +1990,11 @@ dequeiter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (item) { Py_DECREF(item); } else { + /* + * It's safe to read directly from it without acquiring the + * per-object lock; the iterator isn't visible to any other threads + * yet. + */ if (it->counter) { Py_DECREF(it); return NULL; @@ -1794,7 +2008,8 @@ dequeiter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject * dequeiter_len(dequeiterobject *it, PyObject *Py_UNUSED(ignored)) { - return PyLong_FromSsize_t(it->counter); + Py_ssize_t len = FT_ATOMIC_LOAD_SSIZE(it->counter); + return PyLong_FromSsize_t(len); } PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); @@ -1802,7 +2017,16 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list( static PyObject * dequeiter_reduce(dequeiterobject *it, PyObject *Py_UNUSED(ignored)) { - return Py_BuildValue("O(On)", Py_TYPE(it), it->deque, Py_SIZE(it->deque) - it->counter); + PyTypeObject *ty = Py_TYPE(it); + // It's safe to access it->deque without holding the per-object lock for it + // here; it->deque is only assigned during construction of it. + dequeobject *deque = it->deque; + Py_ssize_t size, counter; + Py_BEGIN_CRITICAL_SECTION2(it, deque); + size = Py_SIZE(deque); + counter = it->counter; + Py_END_CRITICAL_SECTION2(); + return Py_BuildValue("O(On)", ty, deque, size - counter); } static PyMethodDef dequeiter_methods[] = { @@ -1834,7 +2058,7 @@ static PyType_Spec dequeiter_spec = { /*********************** Deque Reverse Iterator **************************/ static PyObject * -deque_reviter(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque_reviter(dequeobject *deque) { dequeiterobject *it; collections_state *state = find_module_state_by_def(Py_TYPE(deque)); @@ -1842,17 +2066,19 @@ deque_reviter(dequeobject *deque, PyObject *Py_UNUSED(ignored)) it = PyObject_GC_New(dequeiterobject, state->dequereviter_type); if (it == NULL) return NULL; + Py_BEGIN_CRITICAL_SECTION(deque); it->b = deque->rightblock; it->index = deque->rightindex; it->deque = (dequeobject*)Py_NewRef(deque); it->state = deque->state; it->counter = Py_SIZE(deque); + Py_END_CRITICAL_SECTION(); PyObject_GC_Track(it); return (PyObject *)it; } static PyObject * -dequereviter_next(dequeiterobject *it) +dequereviter_next_lock_held(dequeiterobject *it, dequeobject *deque) { PyObject *item; if (it->counter == 0) @@ -1878,6 +2104,19 @@ dequereviter_next(dequeiterobject *it) return Py_NewRef(item); } +static PyObject * +dequereviter_next(dequeiterobject *it) +{ + PyObject *item; + // It's safe to access it->deque without holding the per-object lock for it + // here; it->deque is only assigned during construction of it. + dequeobject *deque = it->deque; + Py_BEGIN_CRITICAL_SECTION2(it, deque); + item = dequereviter_next_lock_held(it, deque); + Py_END_CRITICAL_SECTION2(); + return item; +} + static PyObject * dequereviter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -1889,7 +2128,7 @@ dequereviter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; assert(type == state->dequereviter_type); - it = (dequeiterobject*)deque_reviter((dequeobject *)deque, NULL); + it = (dequeiterobject *)deque_reviter((dequeobject *)deque); if (!it) return NULL; /* consume items from the queue */ @@ -1898,6 +2137,11 @@ dequereviter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (item) { Py_DECREF(item); } else { + /* + * It's safe to read directly from it without acquiring the + * per-object lock; the iterator isn't visible to any other threads + * yet. + */ if (it->counter) { Py_DECREF(it); return NULL; diff --git a/Modules/_csv.c b/Modules/_csv.c index 929c21584ac2ef0..660c5455af764ef 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -131,7 +131,7 @@ typedef struct { Py_UCS4 *field; /* temporary buffer */ Py_ssize_t field_size; /* size of allocated buffer */ Py_ssize_t field_len; /* length of current field */ - int numeric_field; /* treat field as numeric */ + bool unquoted_field; /* true if no quotes around the current field */ unsigned long line_num; /* Source-file line number */ } ReaderObj; @@ -332,9 +332,9 @@ dialect_check_quoting(int quoting) } static int -dialect_check_char(const char *name, Py_UCS4 c, DialectObj *dialect) +dialect_check_char(const char *name, Py_UCS4 c, DialectObj *dialect, bool allowspace) { - if (c == '\r' || c == '\n' || (dialect->skipinitialspace && c == ' ')) { + if (c == '\r' || c == '\n' || (c == ' ' && !allowspace)) { PyErr_Format(PyExc_ValueError, "bad %s value", name); return -1; } @@ -535,9 +535,11 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_TypeError, "lineterminator must be set"); goto err; } - if (dialect_check_char("delimiter", self->delimiter, self) || - dialect_check_char("escapechar", self->escapechar, self) || - dialect_check_char("quotechar", self->quotechar, self) || + if (dialect_check_char("delimiter", self->delimiter, self, true) || + dialect_check_char("escapechar", self->escapechar, self, + !self->skipinitialspace) || + dialect_check_char("quotechar", self->quotechar, self, + !self->skipinitialspace) || dialect_check_chars("delimiter", "escapechar", self->delimiter, self->escapechar) || dialect_check_chars("delimiter", "quotechar", @@ -644,22 +646,33 @@ _call_dialect(_csvstate *module_state, PyObject *dialect_inst, PyObject *kwargs) static int parse_save_field(ReaderObj *self) { + int quoting = self->dialect->quoting; PyObject *field; - field = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - (void *) self->field, self->field_len); - if (field == NULL) - return -1; - self->field_len = 0; - if (self->numeric_field) { - PyObject *tmp; - - self->numeric_field = 0; - tmp = PyNumber_Float(field); - Py_DECREF(field); - if (tmp == NULL) + if (self->unquoted_field && + self->field_len == 0 && + (quoting == QUOTE_NOTNULL || quoting == QUOTE_STRINGS)) + { + field = Py_NewRef(Py_None); + } + else { + field = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + (void *) self->field, self->field_len); + if (field == NULL) { return -1; - field = tmp; + } + if (self->unquoted_field && + self->field_len != 0 && + (quoting == QUOTE_NONNUMERIC || quoting == QUOTE_STRINGS)) + { + PyObject *tmp = PyNumber_Float(field); + Py_DECREF(field); + if (tmp == NULL) { + return -1; + } + field = tmp; + } + self->field_len = 0; } if (PyList_Append(self->fields, field) < 0) { Py_DECREF(field); @@ -721,6 +734,7 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) /* fallthru */ case START_FIELD: /* expecting field */ + self->unquoted_field = true; if (c == '\n' || c == '\r' || c == EOL) { /* save empty field - return [fields] */ if (parse_save_field(self) < 0) @@ -730,10 +744,12 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) else if (c == dialect->quotechar && dialect->quoting != QUOTE_NONE) { /* start quoted field */ + self->unquoted_field = false; self->state = IN_QUOTED_FIELD; } else if (c == dialect->escapechar) { /* possible escaped character */ + self->unquoted_field = false; self->state = ESCAPED_CHAR; } else if (c == ' ' && dialect->skipinitialspace) @@ -746,8 +762,6 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) } else { /* begin new unquoted field */ - if (dialect->quoting == QUOTE_NONNUMERIC) - self->numeric_field = 1; if (parse_add_char(self, module_state, c) < 0) return -1; self->state = IN_FIELD; @@ -892,7 +906,7 @@ parse_reset(ReaderObj *self) return -1; self->field_len = 0; self->state = START_RECORD; - self->numeric_field = 0; + self->unquoted_field = false; return 0; } @@ -1138,6 +1152,8 @@ join_append_data(WriterObj *self, int field_kind, const void *field_data, if (c == dialect->delimiter || c == dialect->escapechar || c == dialect->quotechar || + c == '\n' || + c == '\r' || PyUnicode_FindChar( dialect->lineterminator, c, 0, PyUnicode_GET_LENGTH(dialect->lineterminator), 1) >= 0) { @@ -1209,6 +1225,7 @@ join_check_rec_size(WriterObj *self, Py_ssize_t rec_len) static int join_append(WriterObj *self, PyObject *field, int quoted) { + DialectObj *dialect = self->dialect; int field_kind = -1; const void *field_data = NULL; Py_ssize_t field_len = 0; @@ -1219,6 +1236,19 @@ join_append(WriterObj *self, PyObject *field, int quoted) field_data = PyUnicode_DATA(field); field_len = PyUnicode_GET_LENGTH(field); } + if (!field_len && dialect->delimiter == ' ' && dialect->skipinitialspace) { + if (dialect->quoting == QUOTE_NONE || + (field == NULL && + (dialect->quoting == QUOTE_STRINGS || + dialect->quoting == QUOTE_NOTNULL))) + { + PyErr_Format(self->error_obj, + "empty field must be quoted if delimiter is a space " + "and skipinitialspace is true"); + return 0; + } + quoted = 1; + } rec_len = join_append_data(self, field_kind, field_data, field_len, "ed, 0); if (rec_len < 0) @@ -1270,6 +1300,7 @@ csv_writerow(WriterObj *self, PyObject *seq) { DialectObj *dialect = self->dialect; PyObject *iter, *field, *line, *result; + bool null_field = false; iter = PyObject_GetIter(seq); if (iter == NULL) { @@ -1306,11 +1337,12 @@ csv_writerow(WriterObj *self, PyObject *seq) break; } + null_field = (field == Py_None); if (PyUnicode_Check(field)) { append_ok = join_append(self, field, quoted); Py_DECREF(field); } - else if (field == Py_None) { + else if (null_field) { append_ok = join_append(self, NULL, quoted); Py_DECREF(field); } @@ -1336,7 +1368,11 @@ csv_writerow(WriterObj *self, PyObject *seq) return NULL; if (self->num_fields > 0 && self->rec_len == 0) { - if (dialect->quoting == QUOTE_NONE) { + if (dialect->quoting == QUOTE_NONE || + (null_field && + (dialect->quoting == QUOTE_STRINGS || + dialect->quoting == QUOTE_NOTNULL))) + { PyErr_Format(self->error_obj, "single empty field record must be quoted"); return NULL; diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 1989723f6f3dbb1..02f48a9ed55843c 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -32,6 +32,10 @@ #endif #endif +#ifdef MS_WIN32 +#include <Unknwn.h> // for IUnknown interface +#endif + typedef struct { PyTypeObject *DictRemover_Type; PyTypeObject *PyCArg_Type; diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 2397015ba658895..32ee414a7a0cddd 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -384,6 +384,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct int bitofs; PyObject *tmp; int pack; + int forced_alignment = 1; Py_ssize_t ffi_ofs; int big_endian; int arrays_seen = 0; @@ -424,6 +425,28 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct pack = 0; } + if (PyObject_GetOptionalAttr(type, &_Py_ID(_align_), &tmp) < 0) { + return -1; + } + if (tmp) { + forced_alignment = PyLong_AsInt(tmp); + Py_DECREF(tmp); + if (forced_alignment < 0) { + if (!PyErr_Occurred() || + PyErr_ExceptionMatches(PyExc_TypeError) || + PyErr_ExceptionMatches(PyExc_OverflowError)) + { + PyErr_SetString(PyExc_ValueError, + "_align_ must be a non-negative integer"); + } + return -1; + } + } + else { + /* Setting `_align_ = 0` amounts to using the default alignment */ + forced_alignment = 1; + } + len = PySequence_Size(fields); if (len == -1) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { @@ -469,6 +492,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct align = basedict->align; union_size = 0; total_align = align ? align : 1; + total_align = max(total_align, forced_alignment); stgdict->ffi_type_pointer.type = FFI_TYPE_STRUCT; stgdict->ffi_type_pointer.elements = PyMem_New(ffi_type *, basedict->length + len + 1); if (stgdict->ffi_type_pointer.elements == NULL) { @@ -488,7 +512,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct size = 0; align = 0; union_size = 0; - total_align = 1; + total_align = forced_alignment; stgdict->ffi_type_pointer.type = FFI_TYPE_STRUCT; stgdict->ffi_type_pointer.elements = PyMem_New(ffi_type *, len + 1); if (stgdict->ffi_type_pointer.elements == NULL) { @@ -707,7 +731,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct /* * The value of MAX_STRUCT_SIZE depends on the platform Python is running on. */ -#if defined(__aarch64__) || defined(__arm__) +#if defined(__aarch64__) || defined(__arm__) || defined(_M_ARM64) # define MAX_STRUCT_SIZE 32 #elif defined(__powerpc64__) # define MAX_STRUCT_SIZE 64 diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index cb5403e8461ff0e..a626bda2ea9be9f 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -14,6 +14,8 @@ #include "Python.h" #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_object.h" // _PyObject_Init() +#include "pycore_time.h" // _PyTime_ObjectToTime_t() + #include "datetime.h" @@ -61,16 +63,6 @@ static datetime_state _datetime_global_state; #define STATIC_STATE() (&_datetime_global_state) -/*[clinic input] -module datetime -class datetime.datetime "PyDateTime_DateTime *" "&PyDateTime_DateTimeType" -class datetime.date "PyDateTime_Date *" "&PyDateTime_DateType" -class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "&PyDateTime_IsoCalendarDateType" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=81bec0fa19837f63]*/ - -#include "clinic/_datetimemodule.c.h" - /* We require that C int be at least 32 bits, and use int virtually * everywhere. In just a few cases we use a temp long, where a Python * API returns a C long. In such cases, we have to ensure that the @@ -161,6 +153,17 @@ static PyTypeObject PyDateTime_TimeZoneType; static int check_tzinfo_subclass(PyObject *p); +/*[clinic input] +module datetime +class datetime.datetime "PyDateTime_DateTime *" "&PyDateTime_DateTimeType" +class datetime.date "PyDateTime_Date *" "&PyDateTime_DateType" +class datetime.time "PyDateTime_Time *" "&PyDateTime_TimeType" +class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "&PyDateTime_IsoCalendarDateType" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6f65a48dd22fa40f]*/ + +#include "clinic/_datetimemodule.c.h" + /* --------------------------------------------------------------------------- * Math utilities. @@ -1044,6 +1047,40 @@ new_datetime_ex(int year, int month, int day, int hour, int minute, new_datetime_ex2(y, m, d, hh, mm, ss, us, tzinfo, fold, \ &PyDateTime_DateTimeType) +static PyObject * +call_subclass_fold(PyObject *cls, int fold, const char *format, ...) +{ + PyObject *kwargs = NULL, *res = NULL; + va_list va; + + va_start(va, format); + PyObject *args = Py_VaBuildValue(format, va); + va_end(va); + if (args == NULL) { + return NULL; + } + if (fold) { + kwargs = PyDict_New(); + if (kwargs == NULL) { + goto Done; + } + PyObject *obj = PyLong_FromLong(fold); + if (obj == NULL) { + goto Done; + } + int err = PyDict_SetItemString(kwargs, "fold", obj); + Py_DECREF(obj); + if (err < 0) { + goto Done; + } + } + res = PyObject_Call(cls, args, kwargs); +Done: + Py_DECREF(args); + Py_XDECREF(kwargs); + return res; +} + static PyObject * new_datetime_subclass_fold_ex(int year, int month, int day, int hour, int minute, int second, int usecond, PyObject *tzinfo, @@ -1053,17 +1090,11 @@ new_datetime_subclass_fold_ex(int year, int month, int day, int hour, int minute // Use the fast path constructor dt = new_datetime(year, month, day, hour, minute, second, usecond, tzinfo, fold); - } else { + } + else { // Subclass - dt = PyObject_CallFunction(cls, "iiiiiiiO", - year, - month, - day, - hour, - minute, - second, - usecond, - tzinfo); + dt = call_subclass_fold(cls, fold, "iiiiiiiO", year, month, day, + hour, minute, second, usecond, tzinfo); } return dt; @@ -1119,6 +1150,24 @@ new_time_ex(int hour, int minute, int second, int usecond, #define new_time(hh, mm, ss, us, tzinfo, fold) \ new_time_ex2(hh, mm, ss, us, tzinfo, fold, &PyDateTime_TimeType) +static PyObject * +new_time_subclass_fold_ex(int hour, int minute, int second, int usecond, + PyObject *tzinfo, int fold, PyObject *cls) +{ + PyObject *t; + if ((PyTypeObject*)cls == &PyDateTime_TimeType) { + // Use the fast path constructor + t = new_time(hour, minute, second, usecond, tzinfo, fold); + } + else { + // Subclass + t = call_subclass_fold(cls, fold, "iiiiO", hour, minute, second, + usecond, tzinfo); + } + + return t; +} + /* Create a timedelta instance. Normalize the members iff normalize is * true. Passing false is a speed optimization, if you know for sure * that seconds and microseconds are already in their proper ranges. In any @@ -1815,16 +1864,6 @@ diff_to_bool(int diff, int op) Py_RETURN_RICHCOMPARE(diff, 0, op); } -/* Raises a "can't compare" TypeError and returns NULL. */ -static PyObject * -cmperror(PyObject *a, PyObject *b) -{ - PyErr_Format(PyExc_TypeError, - "can't compare %s to %s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); - return NULL; -} - /* --------------------------------------------------------------------------- * Class implementations. */ @@ -3447,7 +3486,15 @@ date_isocalendar(PyDateTime_Date *self, PyObject *Py_UNUSED(ignored)) static PyObject * date_richcompare(PyObject *self, PyObject *other, int op) { - if (PyDate_Check(other)) { + /* Since DateTime is a subclass of Date, if the other object is + * a DateTime, it would compute an equality testing or an ordering + * based on the date part alone, and we don't want that. + * So return NotImplemented here in that case. + * If a subclass wants to change this, it's up to the subclass to do so. + * The behavior is the same as if Date and DateTime were independent + * classes. + */ + if (PyDate_Check(other) && !PyDateTime_Check(other)) { int diff = memcmp(((PyDateTime_Date *)self)->data, ((PyDateTime_Date *)other)->data, _PyDateTime_DATE_DATASIZE); @@ -3466,24 +3513,22 @@ date_timetuple(PyDateTime_Date *self, PyObject *Py_UNUSED(ignored)) 0, 0, 0, -1); } +/*[clinic input] +datetime.date.replace + + year: int(c_default="GET_YEAR(self)") = unchanged + month: int(c_default="GET_MONTH(self)") = unchanged + day: int(c_default="GET_DAY(self)") = unchanged + +Return date with new specified fields. +[clinic start generated code]*/ + static PyObject * -date_replace(PyDateTime_Date *self, PyObject *args, PyObject *kw) +datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, + int day) +/*[clinic end generated code: output=2a9430d1e6318aeb input=0d1f02685b3e90f6]*/ { - PyObject *clone; - PyObject *tuple; - int year = GET_YEAR(self); - int month = GET_MONTH(self); - int day = GET_DAY(self); - - if (! PyArg_ParseTupleAndKeywords(args, kw, "|iii:replace", date_kws, - &year, &month, &day)) - return NULL; - tuple = Py_BuildValue("iii", year, month, day); - if (tuple == NULL) - return NULL; - clone = date_new(Py_TYPE(self), tuple, NULL); - Py_DECREF(tuple); - return clone; + return new_date_subclass_ex(year, month, day, (PyObject *)Py_TYPE(self)); } static Py_hash_t @@ -3596,10 +3641,9 @@ static PyMethodDef date_methods[] = { PyDoc_STR("Return the day of the week represented by the date.\n" "Monday == 0 ... Sunday == 6")}, - {"replace", _PyCFunction_CAST(date_replace), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("Return date with new specified fields.")}, + DATETIME_DATE_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(date_replace), METH_VARARGS | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL | METH_KEYWORDS}, {"__reduce__", (PyCFunction)date_reduce, METH_NOARGS, PyDoc_STR("__reduce__() -> (cls, state)")}, @@ -4573,36 +4617,28 @@ time_hash(PyDateTime_Time *self) return self->hashcode; } +/*[clinic input] +datetime.time.replace + + hour: int(c_default="TIME_GET_HOUR(self)") = unchanged + minute: int(c_default="TIME_GET_MINUTE(self)") = unchanged + second: int(c_default="TIME_GET_SECOND(self)") = unchanged + microsecond: int(c_default="TIME_GET_MICROSECOND(self)") = unchanged + tzinfo: object(c_default="HASTZINFO(self) ? self->tzinfo : Py_None") = unchanged + * + fold: int(c_default="TIME_GET_FOLD(self)") = unchanged + +Return time with new specified fields. +[clinic start generated code]*/ + static PyObject * -time_replace(PyDateTime_Time *self, PyObject *args, PyObject *kw) +datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold) +/*[clinic end generated code: output=0b89a44c299e4f80 input=9b6a35b1e704b0ca]*/ { - PyObject *clone; - PyObject *tuple; - int hh = TIME_GET_HOUR(self); - int mm = TIME_GET_MINUTE(self); - int ss = TIME_GET_SECOND(self); - int us = TIME_GET_MICROSECOND(self); - PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; - int fold = TIME_GET_FOLD(self); - - if (! PyArg_ParseTupleAndKeywords(args, kw, "|iiiiO$i:replace", - time_kws, - &hh, &mm, &ss, &us, &tzinfo, &fold)) - return NULL; - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } - tuple = Py_BuildValue("iiiiO", hh, mm, ss, us, tzinfo); - if (tuple == NULL) - return NULL; - clone = time_new(Py_TYPE(self), tuple, NULL); - if (clone != NULL) { - TIME_SET_FOLD(clone, fold); - } - Py_DECREF(tuple); - return clone; + return new_time_subclass_fold_ex(hour, minute, second, microsecond, tzinfo, + fold, (PyObject *)Py_TYPE(self)); } static PyObject * @@ -4732,10 +4768,9 @@ static PyMethodDef time_methods[] = { {"dst", (PyCFunction)time_dst, METH_NOARGS, PyDoc_STR("Return self.tzinfo.dst(self).")}, - {"replace", _PyCFunction_CAST(time_replace), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("Return time with new specified fields.")}, + DATETIME_TIME_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(time_replace), METH_VARARGS | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL | METH_KEYWORDS}, {"fromisoformat", (PyCFunction)time_fromisoformat, METH_O | METH_CLASS, PyDoc_STR("string -> time from a string in ISO 8601 format")}, @@ -5098,7 +5133,11 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, static PyObject * datetime_best_possible(PyObject *cls, TM_FUNC f, PyObject *tzinfo) { - _PyTime_t ts = _PyTime_GetSystemClock(); + PyTime_t ts; + if (PyTime_Time(&ts) < 0) { + return NULL; + } + time_t secs; int us; @@ -5891,21 +5930,7 @@ datetime_richcompare(PyObject *self, PyObject *other, int op) PyObject *offset1, *offset2; int diff; - if (! PyDateTime_Check(other)) { - if (PyDate_Check(other)) { - /* Prevent invocation of date_richcompare. We want to - return NotImplemented here to give the other object - a chance. But since DateTime is a subclass of - Date, if the other object is a Date, it would - compute an ordering based on the date part alone, - and we don't want that. So force unequal or - uncomparable here in that case. */ - if (op == Py_EQ) - Py_RETURN_FALSE; - if (op == Py_NE) - Py_RETURN_TRUE; - return cmperror(self, other); - } + if (!PyDateTime_Check(other)) { Py_RETURN_NOTIMPLEMENTED; } @@ -6042,40 +6067,33 @@ datetime_hash(PyDateTime_DateTime *self) return self->hashcode; } +/*[clinic input] +datetime.datetime.replace + + year: int(c_default="GET_YEAR(self)") = unchanged + month: int(c_default="GET_MONTH(self)") = unchanged + day: int(c_default="GET_DAY(self)") = unchanged + hour: int(c_default="DATE_GET_HOUR(self)") = unchanged + minute: int(c_default="DATE_GET_MINUTE(self)") = unchanged + second: int(c_default="DATE_GET_SECOND(self)") = unchanged + microsecond: int(c_default="DATE_GET_MICROSECOND(self)") = unchanged + tzinfo: object(c_default="HASTZINFO(self) ? self->tzinfo : Py_None") = unchanged + * + fold: int(c_default="DATE_GET_FOLD(self)") = unchanged + +Return datetime with new specified fields. +[clinic start generated code]*/ + static PyObject * -datetime_replace(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) +datetime_datetime_replace_impl(PyDateTime_DateTime *self, int year, + int month, int day, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold) +/*[clinic end generated code: output=00bc96536833fddb input=9b38253d56d9bcad]*/ { - PyObject *clone; - PyObject *tuple; - int y = GET_YEAR(self); - int m = GET_MONTH(self); - int d = GET_DAY(self); - int hh = DATE_GET_HOUR(self); - int mm = DATE_GET_MINUTE(self); - int ss = DATE_GET_SECOND(self); - int us = DATE_GET_MICROSECOND(self); - PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; - int fold = DATE_GET_FOLD(self); - - if (! PyArg_ParseTupleAndKeywords(args, kw, "|iiiiiiiO$i:replace", - datetime_kws, - &y, &m, &d, &hh, &mm, &ss, &us, - &tzinfo, &fold)) - return NULL; - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } - tuple = Py_BuildValue("iiiiiiiO", y, m, d, hh, mm, ss, us, tzinfo); - if (tuple == NULL) - return NULL; - clone = datetime_new(Py_TYPE(self), tuple, NULL); - if (clone != NULL) { - DATE_SET_FOLD(clone, fold); - } - Py_DECREF(tuple); - return clone; + return new_datetime_subclass_fold_ex(year, month, day, hour, minute, + second, microsecond, tzinfo, fold, + (PyObject *)Py_TYPE(self)); } static PyObject * @@ -6597,10 +6615,9 @@ static PyMethodDef datetime_methods[] = { {"dst", (PyCFunction)datetime_dst, METH_NOARGS, PyDoc_STR("Return self.tzinfo.dst(self).")}, - {"replace", _PyCFunction_CAST(datetime_replace), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("Return datetime with new specified fields.")}, + DATETIME_DATETIME_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(datetime_replace), METH_VARARGS | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_datetime_replace), METH_FASTCALL | METH_KEYWORDS}, {"astimezone", _PyCFunction_CAST(datetime_astimezone), METH_VARARGS | METH_KEYWORDS, PyDoc_STR("tz -> convert to local time in new timezone tz\n")}, diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 8b93f8e2cbcf0bb..5b053c73e20bc95 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -82,6 +82,9 @@ typedef struct { /* Convert rationals for comparison */ PyObject *Rational; + /* Invariant: NULL or pointer to _pydecimal.Decimal */ + PyObject *PyDecimal; + PyObject *SignalTuple; struct DecCondMap *signal_map; @@ -3336,56 +3339,6 @@ dotsep_as_utf8(const char *s) return utf8; } -/* copy of libmpdec _mpd_round() */ -static void -_mpd_round(mpd_t *result, const mpd_t *a, mpd_ssize_t prec, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_ssize_t exp = a->exp + a->digits - prec; - - if (prec <= 0) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - if (mpd_isspecial(a) || mpd_iszero(a)) { - mpd_qcopy(result, a, status); - return; - } - - mpd_qrescale_fmt(result, a, exp, ctx, status); - if (result->digits > prec) { - mpd_qrescale_fmt(result, result, exp+1, ctx, status); - } -} - -/* Locate negative zero "z" option within a UTF-8 format spec string. - * Returns pointer to "z", else NULL. - * The portion of the spec we're working with is [[fill]align][sign][z] */ -static const char * -format_spec_z_search(char const *fmt, Py_ssize_t size) { - char const *pos = fmt; - char const *fmt_end = fmt + size; - /* skip over [[fill]align] (fill may be multi-byte character) */ - pos += 1; - while (pos < fmt_end && *pos & 0x80) { - pos += 1; - } - if (pos < fmt_end && strchr("<>=^", *pos) != NULL) { - pos += 1; - } else { - /* fill not present-- skip over [align] */ - pos = fmt; - if (pos < fmt_end && strchr("<>=^", *pos) != NULL) { - pos += 1; - } - } - /* skip over [sign] */ - if (pos < fmt_end && strchr("+- ", *pos) != NULL) { - pos += 1; - } - return pos < fmt_end && *pos == 'z' ? pos : NULL; -} - static int dict_get_item_string(PyObject *dict, const char *key, PyObject **valueobj, const char **valuestr) { @@ -3411,6 +3364,48 @@ dict_get_item_string(PyObject *dict, const char *key, PyObject **valueobj, const return 0; } +/* + * Fallback _pydecimal formatting for new format specifiers that mpdecimal does + * not yet support. As documented, libmpdec follows the PEP-3101 format language: + * https://www.bytereef.org/mpdecimal/doc/libmpdec/assign-convert.html#to-string + */ +static PyObject * +pydec_format(PyObject *dec, PyObject *context, PyObject *fmt, decimal_state *state) +{ + PyObject *result; + PyObject *pydec; + PyObject *u; + + if (state->PyDecimal == NULL) { + state->PyDecimal = _PyImport_GetModuleAttrString("_pydecimal", "Decimal"); + if (state->PyDecimal == NULL) { + return NULL; + } + } + + u = dec_str(dec); + if (u == NULL) { + return NULL; + } + + pydec = PyObject_CallOneArg(state->PyDecimal, u); + Py_DECREF(u); + if (pydec == NULL) { + return NULL; + } + + result = PyObject_CallMethod(pydec, "__format__", "(OO)", fmt, context); + Py_DECREF(pydec); + + if (result == NULL && PyErr_ExceptionMatches(PyExc_ValueError)) { + /* Do not confuse users with the _pydecimal exception */ + PyErr_Clear(); + PyErr_SetString(PyExc_ValueError, "invalid format string"); + } + + return result; +} + /* Formatted representation of a PyDecObject. */ static PyObject * dec_format(PyObject *dec, PyObject *args) @@ -3423,16 +3418,11 @@ dec_format(PyObject *dec, PyObject *args) PyObject *fmtarg; PyObject *context; mpd_spec_t spec; - char const *fmt; - char *fmt_copy = NULL; + char *fmt; char *decstring = NULL; uint32_t status = 0; int replace_fillchar = 0; - int no_neg_0 = 0; Py_ssize_t size; - mpd_t *mpd = MPD(dec); - mpd_uint_t dt[MPD_MINALLOC_MAX]; - mpd_t tmp = {MPD_STATIC|MPD_STATIC_DATA,0,0,0,MPD_MINALLOC_MAX,dt}; decimal_state *state = get_module_state_by_def(Py_TYPE(dec)); @@ -3442,39 +3432,27 @@ dec_format(PyObject *dec, PyObject *args) } if (PyUnicode_Check(fmtarg)) { - fmt = PyUnicode_AsUTF8AndSize(fmtarg, &size); + fmt = (char *)PyUnicode_AsUTF8AndSize(fmtarg, &size); if (fmt == NULL) { return NULL; } - /* NOTE: If https://github.com/python/cpython/pull/29438 lands, the - * format string manipulation below can be eliminated by enhancing - * the forked mpd_parse_fmt_str(). */ + + if (size > 0 && fmt[size-1] == 'N') { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Format specifier 'N' is deprecated", 1) < 0) { + return NULL; + } + } + if (size > 0 && fmt[0] == '\0') { /* NUL fill character: must be replaced with a valid UTF-8 char before calling mpd_parse_fmt_str(). */ replace_fillchar = 1; - fmt = fmt_copy = dec_strdup(fmt, size); - if (fmt_copy == NULL) { + fmt = dec_strdup(fmt, size); + if (fmt == NULL) { return NULL; } - fmt_copy[0] = '_'; - } - /* Strip 'z' option, which isn't understood by mpd_parse_fmt_str(). - * NOTE: fmt is always null terminated by PyUnicode_AsUTF8AndSize() */ - char const *z_position = format_spec_z_search(fmt, size); - if (z_position != NULL) { - no_neg_0 = 1; - size_t z_index = z_position - fmt; - if (fmt_copy == NULL) { - fmt = fmt_copy = dec_strdup(fmt, size); - if (fmt_copy == NULL) { - return NULL; - } - } - /* Shift characters (including null terminator) left, - overwriting the 'z' option. */ - memmove(fmt_copy + z_index, fmt_copy + z_index + 1, size - z_index); - size -= 1; + fmt[0] = '_'; } } else { @@ -3484,10 +3462,13 @@ dec_format(PyObject *dec, PyObject *args) } if (!mpd_parse_fmt_str(&spec, fmt, CtxCaps(context))) { - PyErr_SetString(PyExc_ValueError, - "invalid format string"); - goto finish; + if (replace_fillchar) { + PyMem_Free(fmt); + } + + return pydec_format(dec, context, fmtarg, state); } + if (replace_fillchar) { /* In order to avoid clobbering parts of UTF-8 thousands separators or decimal points when the substitution is reversed later, the actual @@ -3540,45 +3521,8 @@ dec_format(PyObject *dec, PyObject *args) } } - if (no_neg_0 && mpd_isnegative(mpd) && !mpd_isspecial(mpd)) { - /* Round into a temporary (carefully mirroring the rounding - of mpd_qformat_spec()), and check if the result is negative zero. - If so, clear the sign and format the resulting positive zero. */ - mpd_ssize_t prec; - mpd_qcopy(&tmp, mpd, &status); - if (spec.prec >= 0) { - switch (spec.type) { - case 'f': - mpd_qrescale(&tmp, &tmp, -spec.prec, CTX(context), &status); - break; - case '%': - tmp.exp += 2; - mpd_qrescale(&tmp, &tmp, -spec.prec, CTX(context), &status); - break; - case 'g': - prec = (spec.prec == 0) ? 1 : spec.prec; - if (tmp.digits > prec) { - _mpd_round(&tmp, &tmp, prec, CTX(context), &status); - } - break; - case 'e': - if (!mpd_iszero(&tmp)) { - _mpd_round(&tmp, &tmp, spec.prec+1, CTX(context), &status); - } - break; - } - } - if (status & MPD_Errors) { - PyErr_SetString(PyExc_ValueError, "unexpected error when rounding"); - goto finish; - } - if (mpd_iszero(&tmp)) { - mpd_set_positive(&tmp); - mpd = &tmp; - } - } - decstring = mpd_qformat_spec(mpd, &spec, CTX(context), &status); + decstring = mpd_qformat_spec(MPD(dec), &spec, CTX(context), &status); if (decstring == NULL) { if (status & MPD_Malloc_error) { PyErr_NoMemory(); @@ -3593,12 +3537,6 @@ dec_format(PyObject *dec, PyObject *args) if (replace_fillchar) { dec_replace_fillchar(decstring); } - if (strchr(fmt, 'N') != NULL) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Format specifier 'N' is deprecated", 1) < 0) { - goto finish; - } - } result = PyUnicode_DecodeUTF8(decstring, size, NULL); @@ -3607,7 +3545,7 @@ dec_format(PyObject *dec, PyObject *args) Py_XDECREF(grouping); Py_XDECREF(sep); Py_XDECREF(dot); - if (fmt_copy) PyMem_Free(fmt_copy); + if (replace_fillchar) PyMem_Free(fmt); if (decstring) mpd_free(decstring); return result; } @@ -5985,6 +5923,9 @@ _decimal_exec(PyObject *m) Py_CLEAR(collections_abc); Py_CLEAR(MutableMapping); + /* For format specifiers not yet supported by libmpdec */ + state->PyDecimal = NULL; + /* Add types to the module */ CHECK_INT(PyModule_AddType(m, state->PyDec_Type)); CHECK_INT(PyModule_AddType(m, state->PyDecContext_Type)); @@ -6190,6 +6131,7 @@ decimal_clear(PyObject *module) Py_CLEAR(state->extended_context_template); Py_CLEAR(state->Rational); Py_CLEAR(state->SignalTuple); + Py_CLEAR(state->PyDecimal); PyMem_Free(state->signal_map); PyMem_Free(state->cond_map); diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 9ab847165dc0970..d2212d40550a3c7 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -365,6 +365,8 @@ partial_repr(partialobject *pto) { PyObject *result = NULL; PyObject *arglist; + PyObject *mod; + PyObject *name; Py_ssize_t i, n; PyObject *key, *value; int status; @@ -399,13 +401,28 @@ partial_repr(partialobject *pto) if (arglist == NULL) goto done; } - result = PyUnicode_FromFormat("%s(%R%U)", Py_TYPE(pto)->tp_name, - pto->fn, arglist); + + mod = _PyType_GetModuleName(Py_TYPE(pto)); + if (mod == NULL) { + goto error; + } + name = PyType_GetQualName(Py_TYPE(pto)); + if (name == NULL) { + Py_DECREF(mod); + goto error; + } + result = PyUnicode_FromFormat("%S.%S(%R%U)", mod, name, pto->fn, arglist); + Py_DECREF(mod); + Py_DECREF(name); Py_DECREF(arglist); done: Py_ReprLeave((PyObject *)pto); return result; + error: + Py_DECREF(arglist); + Py_ReprLeave((PyObject *)pto); + return NULL; } /* Pickle strategy: diff --git a/Modules/_interpreters_common.h b/Modules/_interpreters_common.h new file mode 100644 index 000000000000000..5661a26d8790d18 --- /dev/null +++ b/Modules/_interpreters_common.h @@ -0,0 +1,13 @@ + +#define _RESOLVE_MODINIT_FUNC_NAME(NAME) \ + PyInit_ ## NAME +#define RESOLVE_MODINIT_FUNC_NAME(NAME) \ + _RESOLVE_MODINIT_FUNC_NAME(NAME) + + +static int +ensure_xid_class(PyTypeObject *cls, crossinterpdatafunc getdata) +{ + //assert(cls->tp_flags & Py_TPFLAGS_HEAPTYPE); + return _PyCrossInterpreterData_RegisterClass(cls, getdata); +} diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index f02207ace9f3d26..b3450eeaf99401f 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1050,6 +1050,16 @@ _io__Buffered_read1_impl(buffered *self, Py_ssize_t n) Py_DECREF(res); return NULL; } + /* Flush the write buffer if necessary */ + if (self->writable) { + PyObject *r = buffered_flush_and_rewind_unlocked(self); + if (r == NULL) { + LEAVE_BUFFERED(self) + Py_DECREF(res); + return NULL; + } + Py_DECREF(r); + } _bufferedreader_reset_buf(self); r = _bufferedreader_raw_read(self, PyBytes_AS_STRING(res), n); LEAVE_BUFFERED(self) @@ -1315,7 +1325,11 @@ _io__Buffered_tell_impl(buffered *self) if (pos == -1) return NULL; pos -= RAW_OFFSET(self); - /* TODO: sanity check (pos >= 0) */ + + // GH-95782 + if (pos < 0) + pos = 0; + return PyLong_FromOff_t(pos); } @@ -1385,6 +1399,11 @@ _io__Buffered_seek_impl(buffered *self, PyObject *targetobj, int whence) offset = target; if (offset >= -self->pos && offset <= avail) { self->pos += offset; + + // GH-95782 + if (current - avail + offset < 0) + return PyLong_FromOff_t(0); + return PyLong_FromOff_t(current - avail + offset); } } diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index d5bec5f71f5be88..64eddcd314a803e 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -96,7 +96,7 @@ _io__BufferedIOBase_detach_impl(PyObject *self, PyTypeObject *cls); static PyObject * _io__BufferedIOBase_detach(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "detach() takes no arguments"); return NULL; } @@ -1245,4 +1245,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=442b05b9a117df6c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4249187a725a3b3e input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index 37023e49087647e..620e9e3b84ea192 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -96,7 +96,7 @@ _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls); static PyObject * _io_BytesIO_getbuffer(bytesio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getbuffer() takes no arguments"); return NULL; } @@ -534,4 +534,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=2be0e05a8871b7e2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ef116925b8b9e535 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index cf3ba28b066cf7f..5b5487d63eba900 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -27,7 +27,7 @@ _io_FileIO_close_impl(fileio *self, PyTypeObject *cls); static PyObject * _io_FileIO_close(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "close() takes no arguments"); return NULL; } @@ -528,4 +528,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=1c0f4a36f76b0c6a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e3d9446b4087020e input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index 6bdfa1444015acc..bae80a265fab075 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -262,7 +262,7 @@ _io__IOBase_fileno_impl(PyObject *self, PyTypeObject *cls); static PyObject * _io__IOBase_fileno(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "fileno() takes no arguments"); return NULL; } @@ -438,4 +438,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=5a22bc5db0ecaacb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e7326fbefc52bfba input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 23b3cc8d71e098c..f04ee729abc9edc 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -27,7 +27,7 @@ _io__TextIOBase_detach_impl(PyObject *self, PyTypeObject *cls); static PyObject * _io__TextIOBase_detach(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "detach() takes no arguments"); return NULL; } @@ -1292,4 +1292,4 @@ _io_TextIOWrapper__CHUNK_SIZE_set(textio *self, PyObject *value, void *Py_UNUSED return return_value; } -/*[clinic end generated code: output=d01aa598647c1385 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=93a5a91a22100a28 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index 6cab295c44611d4..4696ecc5c843e6e 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -29,7 +29,7 @@ _io__WindowsConsoleIO_close_impl(winconsoleio *self, PyTypeObject *cls); static PyObject * _io__WindowsConsoleIO_close(winconsoleio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "close() takes no arguments"); return NULL; } @@ -457,4 +457,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=04108fc26b187386 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2c2bc86713b21dd6 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 9cf268ca0b26c81..6bb156e41fe43cb 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -269,6 +269,13 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, self->fd = -1; } + if (PyBool_Check(nameobj)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return -1; + } + } fd = PyLong_AsInt(nameobj); if (fd < 0) { if (!PyErr_Occurred()) { diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index d794af8de2b8f02..a3239ec0f529609 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2393,7 +2393,7 @@ textiowrapper_parse_cookie(cookie_type *cookie, PyObject *cookieObj) return -1; if (_PyLong_AsByteArray(cookieLong, buffer, sizeof(buffer), - PY_LITTLE_ENDIAN, 0) < 0) { + PY_LITTLE_ENDIAN, 0, 1) < 0) { Py_DECREF(cookieLong); return -1; } diff --git a/Modules/_json.c b/Modules/_json.c index 24b292ce70e5ebd..c55299899e77fe5 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -691,11 +691,10 @@ _parse_object_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ss key = scanstring_unicode(pystr, idx + 1, s->strict, &next_idx); if (key == NULL) goto bail; - memokey = PyDict_SetDefault(memo, key, key); - if (memokey == NULL) { + if (PyDict_SetDefaultRef(memo, key, key, &memokey) < 0) { goto bail; } - Py_SETREF(key, Py_NewRef(memokey)); + Py_SETREF(key, memokey); idx = next_idx; /* skip whitespace between key and : delimiter, read :, skip whitespace */ diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 8f09204097529f4..f1cee7cb6f66bf8 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -6,6 +6,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_SetProfile() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_time.h" // _PyTime_FromLong() #include "rotatingtree.h" @@ -17,8 +18,8 @@ struct _ProfilerEntry; /* represents a function called from another function */ typedef struct _ProfilerSubEntry { rotating_node_t header; - _PyTime_t tt; - _PyTime_t it; + PyTime_t tt; + PyTime_t it; long callcount; long recursivecallcount; long recursionLevel; @@ -28,8 +29,8 @@ typedef struct _ProfilerSubEntry { typedef struct _ProfilerEntry { rotating_node_t header; PyObject *userObj; /* PyCodeObject, or a descriptive str for builtins */ - _PyTime_t tt; /* total time in this entry */ - _PyTime_t it; /* inline time in this entry (not in subcalls) */ + PyTime_t tt; /* total time in this entry */ + PyTime_t it; /* inline time in this entry (not in subcalls) */ long callcount; /* how many times this was called */ long recursivecallcount; /* how many times called recursively */ long recursionLevel; @@ -37,8 +38,8 @@ typedef struct _ProfilerEntry { } ProfilerEntry; typedef struct _ProfilerContext { - _PyTime_t t0; - _PyTime_t subt; + PyTime_t t0; + PyTime_t subt; struct _ProfilerContext *previous; ProfilerEntry *ctxEntry; } ProfilerContext; @@ -84,7 +85,7 @@ _lsprof_get_state(PyObject *module) /*** External Timers ***/ -static _PyTime_t CallExternalTimer(ProfilerObject *pObj) +static PyTime_t CallExternalTimer(ProfilerObject *pObj) { PyObject *o = _PyObject_CallNoArgs(pObj->externalTimer); if (o == NULL) { @@ -92,16 +93,16 @@ static _PyTime_t CallExternalTimer(ProfilerObject *pObj) return 0; } - _PyTime_t result; + PyTime_t result; int err; if (pObj->externalTimerUnit > 0.0) { /* interpret the result as an integer that will be scaled in profiler_getstats() */ - err = _PyTime_FromNanosecondsObject(&result, o); + err = _PyTime_FromLong(&result, o); } else { /* interpret the result as a double measured in seconds. - As the profiler works with _PyTime_t internally + As the profiler works with PyTime_t internally we convert it to a large integer */ err = _PyTime_FromSecondsObject(&result, o, _PyTime_ROUND_FLOOR); } @@ -113,14 +114,14 @@ static _PyTime_t CallExternalTimer(ProfilerObject *pObj) return result; } -static inline _PyTime_t +static inline PyTime_t call_timer(ProfilerObject *pObj) { if (pObj->externalTimer != NULL) { return CallExternalTimer(pObj); } else { - return _PyTime_GetPerfCounter(); + return _PyTime_PerfCounterUnchecked(); } } @@ -311,8 +312,8 @@ initContext(ProfilerObject *pObj, ProfilerContext *self, ProfilerEntry *entry) static void Stop(ProfilerObject *pObj, ProfilerContext *self, ProfilerEntry *entry) { - _PyTime_t tt = call_timer(pObj) - self->t0; - _PyTime_t it = tt - self->subt; + PyTime_t tt = call_timer(pObj) - self->t0; + PyTime_t it = tt - self->subt; if (self->previous) self->previous->subt += tt; pObj->currentProfilerContext = self->previous; @@ -557,7 +558,7 @@ _lsprof_Profiler_getstats_impl(ProfilerObject *self, PyTypeObject *cls) return NULL; } if (!self->externalTimer || self->externalTimerUnit == 0.0) { - _PyTime_t onesec = _PyTime_FromSeconds(1); + PyTime_t onesec = _PyTime_FromSeconds(1); collect.factor = (double)1 / onesec; } else { diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index 425ce10075c1565..4ab15fa65736652 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -11,6 +11,7 @@ posixshmem - A Python extension that provides shm_open() and shm_unlink() #include <Python.h> +#include <string.h> // strlen() #include <errno.h> // EINTR #ifdef HAVE_SYS_MMAN_H # include <sys/mman.h> // shm_open(), shm_unlink() @@ -48,10 +49,15 @@ _posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags, { int fd; int async_err = 0; - const char *name = PyUnicode_AsUTF8AndSize(path, NULL); + Py_ssize_t name_size; + const char *name = PyUnicode_AsUTF8AndSize(path, &name_size); if (name == NULL) { return -1; } + if (strlen(name) != (size_t)name_size) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return -1; + } do { Py_BEGIN_ALLOW_THREADS fd = shm_open(name, flags, mode); @@ -87,10 +93,15 @@ _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path) { int rv; int async_err = 0; - const char *name = PyUnicode_AsUTF8AndSize(path, NULL); + Py_ssize_t name_size; + const char *name = PyUnicode_AsUTF8AndSize(path, &name_size); if (name == NULL) { return NULL; } + if (strlen(name) != (size_t)name_size) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return NULL; + } do { Py_BEGIN_ALLOW_THREADS rv = shm_unlink(name); diff --git a/Modules/_pickle.c b/Modules/_pickle.c index f210c0ca2059911..0d83261168185d9 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -2162,7 +2162,8 @@ save_long(PicklerObject *self, PyObject *obj) pdata = (unsigned char *)PyBytes_AS_STRING(repr); i = _PyLong_AsByteArray((PyLongObject *)obj, pdata, nbytes, - 1 /* little endian */ , 1 /* signed */ ); + 1 /* little endian */ , 1 /* signed */ , + 1 /* with exceptions */); if (i < 0) goto error; /* If the int is negative, this may be a byte more than diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index aa1a300e4378dd9..bcbbe70680b8e76 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -1032,7 +1032,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, PyInterpreterState *interp = _PyInterpreterState_GET(); if ((preexec_fn != Py_None) && interp->finalizing) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_PythonFinalizationError, "preexec_fn not supported at interpreter shutdown"); return NULL; } diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 18b24855c52ad61..5db9b645849fcd8 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -6,7 +6,7 @@ #include "pycore_ceval.h" // Py_MakePendingCalls() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_parking_lot.h" -#include "pycore_time.h" // _PyTime_t +#include "pycore_time.h" // _PyTime_FromSecondsObject() #include <stdbool.h> #include <stddef.h> // offsetof() @@ -372,13 +372,13 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, int block, PyObject *timeout_obj) /*[clinic end generated code: output=5c2cca914cd1e55b input=f7836c65e5839c51]*/ { - _PyTime_t endtime = 0; + PyTime_t endtime = 0; // XXX Use PyThread_ParseTimeoutArg(). if (block != 0 && !Py_IsNone(timeout_obj)) { /* With timeout */ - _PyTime_t timeout; + PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_CEILING) < 0) { return NULL; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 4403e1d132c0570..56b891dfe0f85f8 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -259,13 +259,15 @@ random_seed_urandom(RandomObject *self) return 0; } -static void +static int random_seed_time_pid(RandomObject *self) { - _PyTime_t now; - uint32_t key[5]; + PyTime_t now; + if (PyTime_Time(&now) < 0) { + return -1; + } - now = _PyTime_GetSystemClock(); + uint32_t key[5]; key[0] = (uint32_t)(now & 0xffffffffU); key[1] = (uint32_t)(now >> 32); @@ -277,11 +279,14 @@ random_seed_time_pid(RandomObject *self) key[2] = 0; #endif - now = _PyTime_GetMonotonicClock(); + if (PyTime_Monotonic(&now) < 0) { + return -1; + } key[3] = (uint32_t)(now & 0xffffffffU); key[4] = (uint32_t)(now >> 32); init_by_array(self, key, Py_ARRAY_LENGTH(key)); + return 0; } static int @@ -299,7 +304,9 @@ random_seed(RandomObject *self, PyObject *arg) /* Reading system entropy failed, fall back on the worst entropy: use the current time and process identifier. */ - random_seed_time_pid(self); + if (random_seed_time_pid(self) < 0) { + return -1; + } } return 0; } @@ -342,7 +349,8 @@ random_seed(RandomObject *self, PyObject *arg) res = _PyLong_AsByteArray((PyLongObject *)n, (unsigned char *)key, keyused * 4, PY_LITTLE_ENDIAN, - 0); /* unsigned */ + 0, /* unsigned */ + 1); /* with exceptions */ if (res == -1) { goto Done; } diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index db5eb77891e52e5..811314b5cd8aed8 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -1204,21 +1204,67 @@ pysqlite_connection_interrupt(pysqlite_Connection *self, PyObject *Py_UNUSED(ign } PyDoc_STRVAR(pysqlite_connection_iterdump__doc__, -"iterdump($self, /)\n" +"iterdump($self, /, *, filter=None)\n" "--\n" "\n" -"Returns iterator to the dump of the database in an SQL text format."); +"Returns iterator to the dump of the database in an SQL text format.\n" +"\n" +" filter\n" +" An optional LIKE pattern for database objects to dump"); #define PYSQLITE_CONNECTION_ITERDUMP_METHODDEF \ - {"iterdump", (PyCFunction)pysqlite_connection_iterdump, METH_NOARGS, pysqlite_connection_iterdump__doc__}, + {"iterdump", _PyCFunction_CAST(pysqlite_connection_iterdump), METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_iterdump__doc__}, static PyObject * -pysqlite_connection_iterdump_impl(pysqlite_Connection *self); +pysqlite_connection_iterdump_impl(pysqlite_Connection *self, + PyObject *filter); static PyObject * -pysqlite_connection_iterdump(pysqlite_Connection *self, PyObject *Py_UNUSED(ignored)) +pysqlite_connection_iterdump(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return pysqlite_connection_iterdump_impl(self); + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(filter), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"filter", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "iterdump", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *filter = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + filter = args[0]; +skip_optional_kwonly: + return_value = pysqlite_connection_iterdump_impl(self, filter); + +exit: + return return_value; } PyDoc_STRVAR(pysqlite_connection_backup__doc__, @@ -1551,7 +1597,9 @@ deserialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { @@ -1818,4 +1866,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=90b5b9c14261b8d7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3c6d0b748fac016f input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 0a6633972cc5ef7..f97afcf5fcf16ef 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1979,12 +1979,17 @@ pysqlite_connection_interrupt_impl(pysqlite_Connection *self) /*[clinic input] _sqlite3.Connection.iterdump as pysqlite_connection_iterdump + * + filter: object = None + An optional LIKE pattern for database objects to dump + Returns iterator to the dump of the database in an SQL text format. [clinic start generated code]*/ static PyObject * -pysqlite_connection_iterdump_impl(pysqlite_Connection *self) -/*[clinic end generated code: output=586997aaf9808768 input=1911ca756066da89]*/ +pysqlite_connection_iterdump_impl(pysqlite_Connection *self, + PyObject *filter) +/*[clinic end generated code: output=fd81069c4bdeb6b0 input=4ae6d9a898f108df]*/ { if (!pysqlite_check_connection(self)) { return NULL; @@ -1998,9 +2003,16 @@ pysqlite_connection_iterdump_impl(pysqlite_Connection *self) } return NULL; } - - PyObject *retval = PyObject_CallOneArg(iterdump, (PyObject *)self); + PyObject *args[3] = {NULL, (PyObject *)self, filter}; + PyObject *kwnames = Py_BuildValue("(s)", "filter"); + if (!kwnames) { + Py_DECREF(iterdump); + return NULL; + } + Py_ssize_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; + PyObject *retval = PyObject_Vectorcall(iterdump, args + 1, nargsf, kwnames); Py_DECREF(iterdump); + Py_DECREF(kwnames); return retval; } diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c index 833a666301d8ff9..9e8613ef67916e9 100644 --- a/Modules/_sqlite/util.c +++ b/Modules/_sqlite/util.c @@ -162,7 +162,7 @@ _pysqlite_long_as_int64(PyObject * py_val) sqlite_int64 int64val; if (_PyLong_AsByteArray((PyLongObject *)py_val, (unsigned char *)&int64val, sizeof(int64val), - IS_LITTLE_ENDIAN, 1 /* signed */) >= 0) { + IS_LITTLE_ENDIAN, 1 /* signed */, 0) >= 0) { return int64val; } } diff --git a/Modules/_sre/clinic/sre.c.h b/Modules/_sre/clinic/sre.c.h index cd3fbbc720bdf1e..48336c7a2fca262 100644 --- a/Modules/_sre/clinic/sre.c.h +++ b/Modules/_sre/clinic/sre.c.h @@ -1434,7 +1434,7 @@ _sre_SRE_Scanner_match_impl(ScannerObject *self, PyTypeObject *cls); static PyObject * _sre_SRE_Scanner_match(ScannerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "match() takes no arguments"); return NULL; } @@ -1455,10 +1455,10 @@ _sre_SRE_Scanner_search_impl(ScannerObject *self, PyTypeObject *cls); static PyObject * _sre_SRE_Scanner_search(ScannerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "search() takes no arguments"); return NULL; } return _sre_SRE_Scanner_search_impl(self, cls); } -/*[clinic end generated code: output=ad513f31b99505fa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c3e711f0b2f43d66 input=a9049054013a1b77]*/ diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index d451974b9cf81e6..00fbd9674b8cdda 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -39,13 +39,14 @@ static const char copyright[] = " SRE 2.2.2 Copyright (c) 1997-2002 by Secret Labs AB "; #include "Python.h" -#include "pycore_dict.h" // _PyDict_Next() -#include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION +#include "pycore_dict.h" // _PyDict_Next() +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_moduleobject.h" // _PyModule_GetState() -#include "sre.h" // SRE_CODE +#include "sre.h" // SRE_CODE -#include <ctype.h> // tolower(), toupper(), isalnum() +#include <ctype.h> // tolower(), toupper(), isalnum() #define SRE_CODE_BITS (8 * sizeof(SRE_CODE)) @@ -2349,26 +2350,28 @@ _sre_SRE_Match_groupdict_impl(MatchObject *self, PyObject *default_value) if (!result || !self->pattern->groupindex) return result; + Py_BEGIN_CRITICAL_SECTION(self->pattern->groupindex); while (_PyDict_Next(self->pattern->groupindex, &pos, &key, &value, &hash)) { int status; Py_INCREF(key); value = match_getslice(self, key, default_value); if (!value) { Py_DECREF(key); - goto failed; + Py_CLEAR(result); + goto exit; } status = _PyDict_SetItem_KnownHash(result, key, value, hash); Py_DECREF(value); Py_DECREF(key); - if (status < 0) - goto failed; + if (status < 0) { + Py_CLEAR(result); + goto exit; + } } +exit: + Py_END_CRITICAL_SECTION(); return result; - -failed: - Py_DECREF(result); - return NULL; } /*[clinic input] diff --git a/Modules/_ssl.c b/Modules/_ssl.c index bc3029094242278..d00f407b569fb69 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -28,6 +28,7 @@ #include "Python.h" #include "pycore_fileutils.h" // _PyIsSelectable_fd() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_time.h" // _PyDeadline_Init() #include "pycore_weakref.h" // _PyWeakref_GET_REF() /* Include symbols from _socket module */ @@ -369,7 +370,7 @@ class _ssl.SSLSession "PySSLSession *" "get_state_type(type)->PySSLSession_Type" #include "clinic/_ssl.c.h" -static int PySSL_select(PySocketSockObject *s, int writing, _PyTime_t timeout); +static int PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout); static int PySSL_set_owner(PySSLSocket *, PyObject *, void *); static int PySSL_set_session(PySSLSocket *, PyObject *, void *); @@ -963,7 +964,7 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) _PySSLError err; int sockstate, nonblocking; PySocketSockObject *sock = GET_SOCKET(self); - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; int has_timeout; if (sock) { @@ -2273,12 +2274,12 @@ PySSL_dealloc(PySSLSocket *self) */ static int -PySSL_select(PySocketSockObject *s, int writing, _PyTime_t timeout) +PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout) { int rc; #ifdef HAVE_POLL struct pollfd pollfd; - _PyTime_t ms; + PyTime_t ms; #else int nfds; fd_set fds; @@ -2357,7 +2358,7 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) _PySSLError err; int nonblocking; PySocketSockObject *sock = GET_SOCKET(self); - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; int has_timeout; if (sock != NULL) { @@ -2495,7 +2496,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, _PySSLError err; int nonblocking; PySocketSockObject *sock = GET_SOCKET(self); - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; int has_timeout; if (!group_right_1 && len < 0) { @@ -2627,7 +2628,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) int sockstate, nonblocking, ret; int zeros = 0; PySocketSockObject *sock = GET_SOCKET(self); - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; int has_timeout; if (sock != NULL) { @@ -4553,6 +4554,50 @@ set_sni_callback(PySSLContext *self, PyObject *arg, void *c) return 0; } +#if OPENSSL_VERSION_NUMBER < 0x30300000L +static X509_OBJECT *x509_object_dup(const X509_OBJECT *obj) +{ + int ok; + X509_OBJECT *ret = X509_OBJECT_new(); + if (ret == NULL) { + return NULL; + } + switch (X509_OBJECT_get_type(obj)) { + case X509_LU_X509: + ok = X509_OBJECT_set1_X509(ret, X509_OBJECT_get0_X509(obj)); + break; + case X509_LU_CRL: + /* X509_OBJECT_get0_X509_CRL was not const-correct prior to 3.0.*/ + ok = X509_OBJECT_set1_X509_CRL( + ret, X509_OBJECT_get0_X509_CRL((X509_OBJECT *)obj)); + break; + default: + /* We cannot duplicate unrecognized types in a polyfill, but it is + * safe to leave an empty object. The caller will ignore it. */ + ok = 1; + break; + } + if (!ok) { + X509_OBJECT_free(ret); + return NULL; + } + return ret; +} + +static STACK_OF(X509_OBJECT) * +X509_STORE_get1_objects(X509_STORE *store) +{ + STACK_OF(X509_OBJECT) *ret; + if (!X509_STORE_lock(store)) { + return NULL; + } + ret = sk_X509_OBJECT_deep_copy(X509_STORE_get0_objects(store), + x509_object_dup, X509_OBJECT_free); + X509_STORE_unlock(store); + return ret; +} +#endif + PyDoc_STRVAR(PySSLContext_sni_callback_doc, "Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension.\n\ \n\ @@ -4582,7 +4627,12 @@ _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self) int x509 = 0, crl = 0, ca = 0, i; store = SSL_CTX_get_cert_store(self->ctx); - objs = X509_STORE_get0_objects(store); + objs = X509_STORE_get1_objects(store); + if (objs == NULL) { + PyErr_SetString(PyExc_MemoryError, "failed to query cert store"); + return NULL; + } + for (i = 0; i < sk_X509_OBJECT_num(objs); i++) { obj = sk_X509_OBJECT_value(objs, i); switch (X509_OBJECT_get_type(obj)) { @@ -4596,12 +4646,11 @@ _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self) crl++; break; default: - /* Ignore X509_LU_FAIL, X509_LU_RETRY, X509_LU_PKEY. - * As far as I can tell they are internal states and never - * stored in a cert store */ + /* Ignore unrecognized types. */ break; } } + sk_X509_OBJECT_pop_free(objs, X509_OBJECT_free); return Py_BuildValue("{sisisi}", "x509", x509, "crl", crl, "x509_ca", ca); } @@ -4633,7 +4682,12 @@ _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) } store = SSL_CTX_get_cert_store(self->ctx); - objs = X509_STORE_get0_objects(store); + objs = X509_STORE_get1_objects(store); + if (objs == NULL) { + PyErr_SetString(PyExc_MemoryError, "failed to query cert store"); + goto error; + } + for (i = 0; i < sk_X509_OBJECT_num(objs); i++) { X509_OBJECT *obj; X509 *cert; @@ -4661,9 +4715,11 @@ _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) } Py_CLEAR(ci); } + sk_X509_OBJECT_pop_free(objs, X509_OBJECT_free); return rlist; error: + sk_X509_OBJECT_pop_free(objs, X509_OBJECT_free); Py_XDECREF(ci); Py_XDECREF(rlist); return NULL; diff --git a/Modules/_struct.c b/Modules/_struct.c index bd16fa89f189455..fa2cd37e003e0a2 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1000,9 +1000,10 @@ bp_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) (unsigned char *)p, 8, 0, /* little_endian */ - 1 /* signed */); + 1, /* signed */ + 0 /* !with_exceptions */); Py_DECREF(v); - if (res == -1 && PyErr_Occurred()) { + if (res < 0) { PyErr_Format(state->StructError, "'%c' format requires %lld <= number <= %lld", f->format, @@ -1024,9 +1025,10 @@ bp_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f (unsigned char *)p, 8, 0, /* little_endian */ - 0 /* signed */); + 0, /* signed */ + 0 /* !with_exceptions */); Py_DECREF(v); - if (res == -1 && PyErr_Occurred()) { + if (res < 0) { PyErr_Format(state->StructError, "'%c' format requires 0 <= number <= %llu", f->format, @@ -1260,9 +1262,10 @@ lp_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) (unsigned char *)p, 8, 1, /* little_endian */ - 1 /* signed */); + 1, /* signed */ + 0 /* !with_exceptions */); Py_DECREF(v); - if (res == -1 && PyErr_Occurred()) { + if (res < 0) { PyErr_Format(state->StructError, "'%c' format requires %lld <= number <= %lld", f->format, @@ -1284,9 +1287,10 @@ lp_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f (unsigned char *)p, 8, 1, /* little_endian */ - 0 /* signed */); + 0, /* signed */ + 0 /* !with_exceptions */); Py_DECREF(v); - if (res == -1 && PyErr_Occurred()) { + if (res < 0) { PyErr_Format(state->StructError, "'%c' format requires 0 <= number <= %llu", f->format, diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 5101834cfe13871..5084bcadb10f85a 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -2816,70 +2816,91 @@ static struct PyModuleDef _testbuffermodule = { NULL }; - -PyMODINIT_FUNC -PyInit__testbuffer(void) +static int +_testbuffer_exec(PyObject *mod) { - PyObject *m; - - m = PyModule_Create(&_testbuffermodule); - if (m == NULL) - return NULL; - Py_SET_TYPE(&NDArray_Type, &PyType_Type); - Py_INCREF(&NDArray_Type); - PyModule_AddObject(m, "ndarray", (PyObject *)&NDArray_Type); + if (PyModule_AddType(mod, &NDArray_Type) < 0) { + return -1; + } Py_SET_TYPE(&StaticArray_Type, &PyType_Type); - Py_INCREF(&StaticArray_Type); - PyModule_AddObject(m, "staticarray", (PyObject *)&StaticArray_Type); + if (PyModule_AddType(mod, &StaticArray_Type) < 0) { + return -1; + } structmodule = PyImport_ImportModule("struct"); - if (structmodule == NULL) - return NULL; + if (structmodule == NULL) { + return -1; + } Struct = PyObject_GetAttrString(structmodule, "Struct"); + if (Struct == NULL) { + return -1; + } calcsize = PyObject_GetAttrString(structmodule, "calcsize"); - if (Struct == NULL || calcsize == NULL) - return NULL; + if (calcsize == NULL) { + return -1; + } simple_format = PyUnicode_FromString(simple_fmt); - if (simple_format == NULL) - return NULL; - - PyModule_AddIntMacro(m, ND_MAX_NDIM); - PyModule_AddIntMacro(m, ND_VAREXPORT); - PyModule_AddIntMacro(m, ND_WRITABLE); - PyModule_AddIntMacro(m, ND_FORTRAN); - PyModule_AddIntMacro(m, ND_SCALAR); - PyModule_AddIntMacro(m, ND_PIL); - PyModule_AddIntMacro(m, ND_GETBUF_FAIL); - PyModule_AddIntMacro(m, ND_GETBUF_UNDEFINED); - PyModule_AddIntMacro(m, ND_REDIRECT); - - PyModule_AddIntMacro(m, PyBUF_SIMPLE); - PyModule_AddIntMacro(m, PyBUF_WRITABLE); - PyModule_AddIntMacro(m, PyBUF_FORMAT); - PyModule_AddIntMacro(m, PyBUF_ND); - PyModule_AddIntMacro(m, PyBUF_STRIDES); - PyModule_AddIntMacro(m, PyBUF_INDIRECT); - PyModule_AddIntMacro(m, PyBUF_C_CONTIGUOUS); - PyModule_AddIntMacro(m, PyBUF_F_CONTIGUOUS); - PyModule_AddIntMacro(m, PyBUF_ANY_CONTIGUOUS); - PyModule_AddIntMacro(m, PyBUF_FULL); - PyModule_AddIntMacro(m, PyBUF_FULL_RO); - PyModule_AddIntMacro(m, PyBUF_RECORDS); - PyModule_AddIntMacro(m, PyBUF_RECORDS_RO); - PyModule_AddIntMacro(m, PyBUF_STRIDED); - PyModule_AddIntMacro(m, PyBUF_STRIDED_RO); - PyModule_AddIntMacro(m, PyBUF_CONTIG); - PyModule_AddIntMacro(m, PyBUF_CONTIG_RO); - - PyModule_AddIntMacro(m, PyBUF_READ); - PyModule_AddIntMacro(m, PyBUF_WRITE); - - return m; -} + if (simple_format == NULL) { + return -1; + } +#define ADD_INT_MACRO(mod, macro) \ + do { \ + if (PyModule_AddIntConstant(mod, #macro, macro) < 0) { \ + return -1; \ + } \ + } while (0) + + ADD_INT_MACRO(mod, ND_MAX_NDIM); + ADD_INT_MACRO(mod, ND_VAREXPORT); + ADD_INT_MACRO(mod, ND_WRITABLE); + ADD_INT_MACRO(mod, ND_FORTRAN); + ADD_INT_MACRO(mod, ND_SCALAR); + ADD_INT_MACRO(mod, ND_PIL); + ADD_INT_MACRO(mod, ND_GETBUF_FAIL); + ADD_INT_MACRO(mod, ND_GETBUF_UNDEFINED); + ADD_INT_MACRO(mod, ND_REDIRECT); + + ADD_INT_MACRO(mod, PyBUF_SIMPLE); + ADD_INT_MACRO(mod, PyBUF_WRITABLE); + ADD_INT_MACRO(mod, PyBUF_FORMAT); + ADD_INT_MACRO(mod, PyBUF_ND); + ADD_INT_MACRO(mod, PyBUF_STRIDES); + ADD_INT_MACRO(mod, PyBUF_INDIRECT); + ADD_INT_MACRO(mod, PyBUF_C_CONTIGUOUS); + ADD_INT_MACRO(mod, PyBUF_F_CONTIGUOUS); + ADD_INT_MACRO(mod, PyBUF_ANY_CONTIGUOUS); + ADD_INT_MACRO(mod, PyBUF_FULL); + ADD_INT_MACRO(mod, PyBUF_FULL_RO); + ADD_INT_MACRO(mod, PyBUF_RECORDS); + ADD_INT_MACRO(mod, PyBUF_RECORDS_RO); + ADD_INT_MACRO(mod, PyBUF_STRIDED); + ADD_INT_MACRO(mod, PyBUF_STRIDED_RO); + ADD_INT_MACRO(mod, PyBUF_CONTIG); + ADD_INT_MACRO(mod, PyBUF_CONTIG_RO); + + ADD_INT_MACRO(mod, PyBUF_READ); + ADD_INT_MACRO(mod, PyBUF_WRITE); + +#undef ADD_INT_MACRO + return 0; +} +PyMODINIT_FUNC +PyInit__testbuffer(void) +{ + PyObject *mod = PyModule_Create(&_testbuffermodule); + if (mod == NULL) { + return NULL; + } + if (_testbuffer_exec(mod) < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} diff --git a/Modules/_testcapi/buffer.c b/Modules/_testcapi/buffer.c index 942774156c6c47f..7e2f6e5e29482c2 100644 --- a/Modules/_testcapi/buffer.c +++ b/Modules/_testcapi/buffer.c @@ -54,8 +54,10 @@ static int testbuf_getbuf(testBufObject *self, Py_buffer *view, int flags) { int buf = PyObject_GetBuffer(self->obj, view, flags); - Py_SETREF(view->obj, Py_NewRef(self)); - self->references++; + if (buf == 0) { + Py_SETREF(view->obj, Py_NewRef(self)); + self->references++; + } return buf; } diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c index 42e056b7d07a31f..fe03c24f75e196b 100644 --- a/Modules/_testcapi/dict.c +++ b/Modules/_testcapi/dict.c @@ -225,6 +225,31 @@ dict_setdefault(PyObject *self, PyObject *args) return PyDict_SetDefault(mapping, key, defaultobj); } +static PyObject * +dict_setdefaultref(PyObject *self, PyObject *args) +{ + PyObject *obj, *key, *default_value, *result = UNINITIALIZED_PTR; + if (!PyArg_ParseTuple(args, "OOO", &obj, &key, &default_value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(key); + NULLABLE(default_value); + switch (PyDict_SetDefaultRef(obj, key, default_value, &result)) { + case -1: + assert(result == NULL); + return NULL; + case 0: + assert(result == default_value); + return result; + case 1: + return result; + default: + Py_FatalError("PyDict_SetDefaultRef() returned invalid code"); + Py_UNREACHABLE(); + } +} + static PyObject * dict_delitem(PyObject *self, PyObject *args) { @@ -433,6 +458,7 @@ static PyMethodDef test_methods[] = { {"dict_delitem", dict_delitem, METH_VARARGS}, {"dict_delitemstring", dict_delitemstring, METH_VARARGS}, {"dict_setdefault", dict_setdefault, METH_VARARGS}, + {"dict_setdefaultref", dict_setdefaultref, METH_VARARGS}, {"dict_keys", dict_keys, METH_O}, {"dict_values", dict_values, METH_O}, {"dict_items", dict_items, METH_O}, diff --git a/Modules/_testcapi/list.c b/Modules/_testcapi/list.c index 10e18699f01bc1f..2cb6499e28336df 100644 --- a/Modules/_testcapi/list.c +++ b/Modules/_testcapi/list.c @@ -59,6 +59,18 @@ list_get_item(PyObject *Py_UNUSED(module), PyObject *args) return Py_XNewRef(PyList_GET_ITEM(obj, i)); } +static PyObject * +list_get_item_ref(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return PyList_GetItemRef(obj, i); +} + static PyObject * list_setitem(PyObject *Py_UNUSED(module), PyObject *args) { @@ -191,6 +203,7 @@ static PyMethodDef test_methods[] = { {"list_get_size", list_get_size, METH_O}, {"list_getitem", list_getitem, METH_VARARGS}, {"list_get_item", list_get_item, METH_VARARGS}, + {"list_get_item_ref", list_get_item_ref, METH_VARARGS}, {"list_setitem", list_setitem, METH_VARARGS}, {"list_set_item", list_set_item, METH_VARARGS}, {"list_insert", list_insert, METH_VARARGS}, diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 32ad8d32ab85239..dc21cf9f4752289 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -776,6 +776,51 @@ pylong_asvoidptr(PyObject *module, PyObject *arg) return Py_NewRef((PyObject *)value); } +static PyObject * +pylong_asnativebytes(PyObject *module, PyObject *args) +{ + PyObject *v; + Py_buffer buffer; + Py_ssize_t n, endianness; + if (!PyArg_ParseTuple(args, "Ow*nn", &v, &buffer, &n, &endianness)) { + return NULL; + } + if (buffer.readonly) { + PyErr_SetString(PyExc_TypeError, "buffer must be writable"); + PyBuffer_Release(&buffer); + return NULL; + } + if (buffer.len < n) { + PyErr_SetString(PyExc_ValueError, "buffer must be at least 'n' bytes"); + PyBuffer_Release(&buffer); + return NULL; + } + Py_ssize_t res = PyLong_AsNativeBytes(v, buffer.buf, n, (int)endianness); + PyBuffer_Release(&buffer); + return res >= 0 ? PyLong_FromSsize_t(res) : NULL; +} + +static PyObject * +pylong_fromnativebytes(PyObject *module, PyObject *args) +{ + Py_buffer buffer; + Py_ssize_t n, endianness, signed_; + if (!PyArg_ParseTuple(args, "y*nnn", &buffer, &n, &endianness, &signed_)) { + return NULL; + } + if (buffer.len < n) { + PyErr_SetString(PyExc_ValueError, "buffer must be at least 'n' bytes"); + PyBuffer_Release(&buffer); + return NULL; + } + PyObject *res = signed_ + ? PyLong_FromNativeBytes(buffer.buf, n, (int)endianness) + : PyLong_FromUnsignedNativeBytes(buffer.buf, n, (int)endianness); + PyBuffer_Release(&buffer); + return res; +} + + static PyMethodDef test_methods[] = { _TESTCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF _TESTCAPI_TEST_LONG_API_METHODDEF @@ -804,6 +849,8 @@ static PyMethodDef test_methods[] = { {"pylong_as_size_t", pylong_as_size_t, METH_O}, {"pylong_asdouble", pylong_asdouble, METH_O}, {"pylong_asvoidptr", pylong_asvoidptr, METH_O}, + {"pylong_asnativebytes", pylong_asnativebytes, METH_VARARGS}, + {"pylong_fromnativebytes", pylong_fromnativebytes, METH_VARARGS}, {NULL}, }; @@ -813,6 +860,5 @@ _PyTestCapi_Init_Long(PyObject *mod) if (PyModule_AddFunctions(mod, test_methods) < 0) { return -1; } - return 0; } diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 29817edd69b1348..e8cfb2423500d4b 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -59,6 +59,7 @@ int _PyTestCapi_Init_Immortal(PyObject *module); int _PyTestCapi_Init_GC(PyObject *module); int _PyTestCapi_Init_Sys(PyObject *module); int _PyTestCapi_Init_Hash(PyObject *module); +int _PyTestCapi_Init_Time(PyObject *module); int _PyTestCapi_Init_VectorcallLimited(PyObject *module); int _PyTestCapi_Init_HeaptypeRelative(PyObject *module); diff --git a/Modules/_testcapi/time.c b/Modules/_testcapi/time.c new file mode 100644 index 000000000000000..68f082bf3f3d88a --- /dev/null +++ b/Modules/_testcapi/time.c @@ -0,0 +1,109 @@ +#include "parts.h" + + +static int +pytime_from_nanoseconds(PyTime_t *tp, PyObject *obj) +{ + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expect int, got %s", + Py_TYPE(obj)->tp_name); + return -1; + } + + long long nsec = PyLong_AsLongLong(obj); + if (nsec == -1 && PyErr_Occurred()) { + return -1; + } + + Py_BUILD_ASSERT(sizeof(long long) == sizeof(PyTime_t)); + *tp = (PyTime_t)nsec; + return 0; +} + + +static PyObject * +test_pytime_assecondsdouble(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *obj; + if (!PyArg_ParseTuple(args, "O", &obj)) { + return NULL; + } + PyTime_t ts; + if (pytime_from_nanoseconds(&ts, obj) < 0) { + return NULL; + } + double d = PyTime_AsSecondsDouble(ts); + return PyFloat_FromDouble(d); +} + + +static PyObject* +pytime_as_float(PyTime_t t) +{ + return PyFloat_FromDouble(PyTime_AsSecondsDouble(t)); +} + + + +static PyObject* +test_pytime_monotonic(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res = PyTime_Monotonic(&t); + if (res < 0) { + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_perf_counter(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res = PyTime_PerfCounter(&t); + if (res < 0) { + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_time(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res = PyTime_Time(&t); + if (res < 0) { + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyMethodDef test_methods[] = { + {"PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS}, + {"PyTime_Monotonic", test_pytime_monotonic, METH_NOARGS}, + {"PyTime_PerfCounter", test_pytime_perf_counter, METH_NOARGS}, + {"PyTime_Time", test_pytime_time, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Time(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + Py_BUILD_ASSERT(sizeof(long long) == sizeof(PyTime_t)); + if (PyModule_AddObject(m, "PyTime_MIN", PyLong_FromLongLong(PyTime_MIN)) < 0) { + return 1; + } + if (PyModule_AddObject(m, "PyTime_MAX", PyLong_FromLongLong(PyTime_MAX)) < 0) { + return 1; + } + return 0; +} diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index a763ff46a3c2901..1eb0db2c2e65761 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -15,8 +15,8 @@ module _testcapi /*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ // Test dict watching -static PyObject *g_dict_watch_events; -static int g_dict_watchers_installed; +static PyObject *g_dict_watch_events = NULL; +static int g_dict_watchers_installed = 0; static int dict_watch_callback(PyDict_WatchEvent event, diff --git a/Modules/_testcapi_feature_macros.inc b/Modules/_testcapi_feature_macros.inc index a076e7149800743..f5f3524f2c0177f 100644 --- a/Modules/_testcapi_feature_macros.inc +++ b/Modules/_testcapi_feature_macros.inc @@ -38,6 +38,15 @@ if (res) { Py_DECREF(result); return NULL; } +#ifdef Py_TRACE_REFS + res = PyDict_SetItemString(result, "Py_TRACE_REFS", Py_True); +#else + res = PyDict_SetItemString(result, "Py_TRACE_REFS", Py_False); +#endif +if (res) { + Py_DECREF(result); return NULL; +} + #ifdef USE_STACKCHECK res = PyDict_SetItemString(result, "USE_STACKCHECK", Py_True); #else diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 6def680190b1a63..b03f871b089c8a3 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1261,6 +1261,26 @@ make_memoryview_from_NULL_pointer(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyMemoryView_FromBuffer(&info); } +static PyObject * +buffer_fill_info(PyObject *self, PyObject *args) +{ + Py_buffer info; + const char *data; + Py_ssize_t size; + int readonly; + int flags; + + if (!PyArg_ParseTuple(args, "s#ii:buffer_fill_info", + &data, &size, &readonly, &flags)) { + return NULL; + } + + if (PyBuffer_FillInfo(&info, NULL, (void *)data, size, readonly, flags) < 0) { + return NULL; + } + return PyMemoryView_FromBuffer(&info); +} + static PyObject * test_from_contiguous(PyObject* self, PyObject *Py_UNUSED(ignored)) { @@ -3314,6 +3334,7 @@ static PyMethodDef TestMethods[] = { {"eval_code_ex", eval_eval_code_ex, METH_VARARGS}, {"make_memoryview_from_NULL_pointer", make_memoryview_from_NULL_pointer, METH_NOARGS}, + {"buffer_fill_info", buffer_fill_info, METH_VARARGS}, {"crash_no_current_thread", crash_no_current_thread, METH_NOARGS}, {"test_current_tstate_matches", test_current_tstate_matches, METH_NOARGS}, {"run_in_subinterp", run_in_subinterp, METH_VARARGS}, @@ -4086,6 +4107,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Hash(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Time(m) < 0) { + return NULL; + } PyState_AddModule(m, &_testcapimodule); return m; diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 15e0093f15ba1e6..fb0936bbccd3183 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1213,6 +1213,40 @@ clone_with_conv_f2_impl(PyObject *module, custom_t path) } +/*[clinic input] +class _testclinic.TestClass "PyObject *" "PyObject" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=668a591c65bec947]*/ + +/*[clinic input] +_testclinic.TestClass.meth_method_no_params + cls: defining_class + / +[clinic start generated code]*/ + +static PyObject * +_testclinic_TestClass_meth_method_no_params_impl(PyObject *self, + PyTypeObject *cls) +/*[clinic end generated code: output=c140f100080c2fc8 input=6bd34503d11c63c1]*/ +{ + Py_RETURN_NONE; +} + +static struct PyMethodDef test_class_methods[] = { + _TESTCLINIC_TESTCLASS_METH_METHOD_NO_PARAMS_METHODDEF + {NULL, NULL} +}; + +static PyTypeObject TestClass = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.TestClass", + .tp_basicsize = sizeof(PyObject), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = PyType_GenericNew, + .tp_methods = test_class_methods, +}; + + /*[clinic input] output push destination deprstar new file '{dirname}/clinic/_testclinic_depr.c.h' @@ -1906,6 +1940,9 @@ PyInit__testclinic(void) if (m == NULL) { return NULL; } + if (PyModule_AddType(m, &TestClass) < 0) { + goto error; + } if (PyModule_AddType(m, &DeprStarNew) < 0) { goto error; } diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c new file mode 100644 index 000000000000000..19ee3b8dd1428d6 --- /dev/null +++ b/Modules/_testexternalinspection.c @@ -0,0 +1,629 @@ +#define _GNU_SOURCE + +#ifdef __linux__ +# include <elf.h> +# include <sys/uio.h> +# if INTPTR_MAX == INT64_MAX +# define Elf_Ehdr Elf64_Ehdr +# define Elf_Shdr Elf64_Shdr +# define Elf_Phdr Elf64_Phdr +# else +# define Elf_Ehdr Elf32_Ehdr +# define Elf_Shdr Elf32_Shdr +# define Elf_Phdr Elf32_Phdr +# endif +# include <sys/mman.h> +#endif + +#ifdef __APPLE__ +# include <libproc.h> +# include <mach-o/fat.h> +# include <mach-o/loader.h> +# include <mach-o/nlist.h> +# include <mach/mach.h> +# include <mach/mach_vm.h> +# include <mach/machine.h> +# include <sys/mman.h> +# include <sys/proc.h> +# include <sys/sysctl.h> +#endif + +#include <errno.h> +#include <fcntl.h> +#include <stddef.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/param.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif +#include "Python.h" +#include <internal/pycore_runtime.h> + +#ifndef HAVE_PROCESS_VM_READV +# define HAVE_PROCESS_VM_READV 0 +#endif + +#ifdef __APPLE__ +static void* +analyze_macho64(mach_port_t proc_ref, void* base, void* map) +{ + struct mach_header_64* hdr = (struct mach_header_64*)map; + int ncmds = hdr->ncmds; + + int cmd_cnt = 0; + struct segment_command_64* cmd = map + sizeof(struct mach_header_64); + + mach_vm_size_t size = 0; + mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); + mach_vm_address_t address = (mach_vm_address_t)base; + vm_region_basic_info_data_64_t region_info; + mach_port_t object_name; + + for (int i = 0; cmd_cnt < 2 && i < ncmds; i++) { + if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__DATA") == 0) { + while (cmd->filesize != size) { + address += size; + if (mach_vm_region( + proc_ref, + &address, + &size, + VM_REGION_BASIC_INFO_64, + (vm_region_info_t)®ion_info, // cppcheck-suppress [uninitvar] + &count, + &object_name) + != KERN_SUCCESS) + { + PyErr_SetString(PyExc_RuntimeError, "Cannot get any more VM maps.\n"); + return NULL; + } + } + base = (void*)address - cmd->vmaddr; + + int nsects = cmd->nsects; + struct section_64* sec = + (struct section_64*)((void*)cmd + sizeof(struct segment_command_64)); + for (int j = 0; j < nsects; j++) { + if (strcmp(sec[j].sectname, "PyRuntime") == 0) { + return base + sec[j].addr; + } + } + cmd_cnt++; + } + + cmd = (struct segment_command_64*)((void*)cmd + cmd->cmdsize); + } + return NULL; +} + +static void* +analyze_macho(char* path, void* base, mach_vm_size_t size, mach_port_t proc_ref) +{ + int fd = open(path, O_RDONLY); + if (fd == -1) { + PyErr_Format(PyExc_RuntimeError, "Cannot open binary %s\n", path); + return NULL; + } + + struct stat fs; + if (fstat(fd, &fs) == -1) { + PyErr_Format(PyExc_RuntimeError, "Cannot get size of binary %s\n", path); + close(fd); + return NULL; + } + + void* map = mmap(0, fs.st_size, PROT_READ, MAP_SHARED, fd, 0); + if (map == MAP_FAILED) { + PyErr_Format(PyExc_RuntimeError, "Cannot map binary %s\n", path); + close(fd); + return NULL; + } + + void* result = NULL; + + struct mach_header_64* hdr = (struct mach_header_64*)map; + switch (hdr->magic) { + case MH_MAGIC: + case MH_CIGAM: + case FAT_MAGIC: + case FAT_CIGAM: + PyErr_SetString(PyExc_RuntimeError, "32-bit Mach-O binaries are not supported"); + break; + case MH_MAGIC_64: + case MH_CIGAM_64: + result = analyze_macho64(proc_ref, base, map); + break; + default: + PyErr_SetString(PyExc_RuntimeError, "Unknown Mach-O magic"); + break; + } + + munmap(map, fs.st_size); + if (close(fd) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + } + return result; +} + +static mach_port_t +pid_to_task(pid_t pid) +{ + mach_port_t task; + kern_return_t result; + + result = task_for_pid(mach_task_self(), pid, &task); + if (result != KERN_SUCCESS) { + PyErr_Format(PyExc_PermissionError, "Cannot get task for PID %d", pid); + return 0; + } + return task; +} + +static void* +get_py_runtime_macos(pid_t pid) +{ + mach_vm_address_t address = 0; + mach_vm_size_t size = 0; + mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); + vm_region_basic_info_data_64_t region_info; + mach_port_t object_name; + + mach_port_t proc_ref = pid_to_task(pid); + if (proc_ref == 0) { + PyErr_SetString(PyExc_PermissionError, "Cannot get task for PID"); + return NULL; + } + + int match_found = 0; + char map_filename[MAXPATHLEN + 1]; + void* result_address = NULL; + while (mach_vm_region( + proc_ref, + &address, + &size, + VM_REGION_BASIC_INFO_64, + (vm_region_info_t)®ion_info, + &count, + &object_name) + == KERN_SUCCESS) + { + int path_len = proc_regionfilename(pid, address, map_filename, MAXPATHLEN); + if (path_len == 0) { + address += size; + continue; + } + + char* filename = strrchr(map_filename, '/'); + if (filename != NULL) { + filename++; // Move past the '/' + } else { + filename = map_filename; // No path, use the whole string + } + + // Check if the filename starts with "python" or "libpython" + if (!match_found && strncmp(filename, "python", 6) == 0) { + match_found = 1; + result_address = analyze_macho(map_filename, (void*)address, size, proc_ref); + } + if (strncmp(filename, "libpython", 9) == 0) { + match_found = 1; + result_address = analyze_macho(map_filename, (void*)address, size, proc_ref); + break; + } + + address += size; + } + return result_address; +} +#endif + +#ifdef __linux__ +void* +find_python_map_start_address(pid_t pid, char* result_filename) +{ + char maps_file_path[64]; + sprintf(maps_file_path, "/proc/%d/maps", pid); + + FILE* maps_file = fopen(maps_file_path, "r"); + if (maps_file == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + int match_found = 0; + + char line[256]; + char map_filename[PATH_MAX]; + void* result_address = 0; + while (fgets(line, sizeof(line), maps_file) != NULL) { + unsigned long start_address = 0; + sscanf(line, "%lx-%*x %*s %*s %*s %*s %s", &start_address, map_filename); + char* filename = strrchr(map_filename, '/'); + if (filename != NULL) { + filename++; // Move past the '/' + } else { + filename = map_filename; // No path, use the whole string + } + + // Check if the filename starts with "python" or "libpython" + if (!match_found && strncmp(filename, "python", 6) == 0) { + match_found = 1; + result_address = (void*)start_address; + strcpy(result_filename, map_filename); + } + if (strncmp(filename, "libpython", 9) == 0) { + match_found = 1; + result_address = (void*)start_address; + strcpy(result_filename, map_filename); + break; + } + } + + fclose(maps_file); + + if (!match_found) { + map_filename[0] = '\0'; + } + + return result_address; +} + +void* +get_py_runtime_linux(pid_t pid) +{ + char elf_file[256]; + void* start_address = (void*)find_python_map_start_address(pid, elf_file); + + if (start_address == 0) { + PyErr_SetString(PyExc_RuntimeError, "No memory map associated with python or libpython found"); + return NULL; + } + + void* result = NULL; + void* file_memory = NULL; + + int fd = open(elf_file, O_RDONLY); + if (fd < 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto exit; + } + + struct stat file_stats; + if (fstat(fd, &file_stats) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto exit; + } + + file_memory = mmap(NULL, file_stats.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (file_memory == MAP_FAILED) { + PyErr_SetFromErrno(PyExc_OSError); + goto exit; + } + + Elf_Ehdr* elf_header = (Elf_Ehdr*)file_memory; + + Elf_Shdr* section_header_table = (Elf_Shdr*)(file_memory + elf_header->e_shoff); + + Elf_Shdr* shstrtab_section = §ion_header_table[elf_header->e_shstrndx]; + char* shstrtab = (char*)(file_memory + shstrtab_section->sh_offset); + + Elf_Shdr* py_runtime_section = NULL; + for (int i = 0; i < elf_header->e_shnum; i++) { + if (strcmp(".PyRuntime", shstrtab + section_header_table[i].sh_name) == 0) { + py_runtime_section = §ion_header_table[i]; + break; + } + } + + Elf_Phdr* program_header_table = (Elf_Phdr*)(file_memory + elf_header->e_phoff); + // Find the first PT_LOAD segment + Elf_Phdr* first_load_segment = NULL; + for (int i = 0; i < elf_header->e_phnum; i++) { + if (program_header_table[i].p_type == PT_LOAD) { + first_load_segment = &program_header_table[i]; + break; + } + } + + if (py_runtime_section != NULL && first_load_segment != NULL) { + uintptr_t elf_load_addr = first_load_segment->p_vaddr + - (first_load_segment->p_vaddr % first_load_segment->p_align); + result = start_address + py_runtime_section->sh_addr - elf_load_addr; + } + +exit: + if (close(fd) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + } + if (file_memory != NULL) { + munmap(file_memory, file_stats.st_size); + } + return result; +} +#endif + +ssize_t +read_memory(pid_t pid, void* remote_address, size_t len, void* dst) +{ + ssize_t total_bytes_read = 0; +#ifdef __linux__ + struct iovec local[1]; + struct iovec remote[1]; + ssize_t result = 0; + ssize_t read = 0; + + do { + local[0].iov_base = dst + result; + local[0].iov_len = len - result; + remote[0].iov_base = (void*)(remote_address + result); + remote[0].iov_len = len - result; + + read = process_vm_readv(pid, local, 1, remote, 1, 0); + if (read < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + result += read; + } while ((size_t)read != local[0].iov_len); + total_bytes_read = result; +#elif defined(__APPLE__) + ssize_t result = -1; + kern_return_t kr = mach_vm_read_overwrite( + pid_to_task(pid), + (mach_vm_address_t)remote_address, + len, + (mach_vm_address_t)dst, + (mach_vm_size_t*)&result); + + if (kr != KERN_SUCCESS) { + switch (kr) { + case KERN_PROTECTION_FAILURE: + PyErr_SetString(PyExc_PermissionError, "Not enough permissions to read memory"); + break; + case KERN_INVALID_ARGUMENT: + PyErr_SetString(PyExc_PermissionError, "Invalid argument to mach_vm_read_overwrite"); + break; + default: + PyErr_SetString(PyExc_RuntimeError, "Unknown error reading memory"); + } + return -1; + } + total_bytes_read = len; +#else + return -1; +#endif + return total_bytes_read; +} + +int +read_string(pid_t pid, _Py_DebugOffsets* debug_offsets, void* address, char* buffer, Py_ssize_t size) +{ + Py_ssize_t len; + ssize_t bytes_read = + read_memory(pid, address + debug_offsets->unicode_object.length, sizeof(Py_ssize_t), &len); + if (bytes_read == -1) { + return -1; + } + if (len >= size) { + PyErr_SetString(PyExc_RuntimeError, "Buffer too small"); + return -1; + } + size_t offset = debug_offsets->unicode_object.asciiobject_size; + bytes_read = read_memory(pid, address + offset, len, buffer); + if (bytes_read == -1) { + return -1; + } + buffer[len] = '\0'; + return 0; +} + +void* +get_py_runtime(pid_t pid) +{ +#if defined(__linux__) + return get_py_runtime_linux(pid); +#elif defined(__APPLE__) + return get_py_runtime_macos(pid); +#else + return NULL; +#endif +} + +static int +parse_code_object( + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + void* address, + void** previous_frame) +{ + void* address_of_function_name; + read_memory( + pid, + (void*)(address + offsets->code_object.name), + sizeof(void*), + &address_of_function_name); + + if (address_of_function_name == NULL) { + PyErr_SetString(PyExc_RuntimeError, "No function name found"); + return -1; + } + + char function_name[256]; + if (read_string(pid, offsets, address_of_function_name, function_name, sizeof(function_name)) != 0) { + return -1; + } + + PyObject* py_function_name = PyUnicode_FromString(function_name); + if (py_function_name == NULL) { + return -1; + } + + if (PyList_Append(result, py_function_name) == -1) { + Py_DECREF(py_function_name); + return -1; + } + Py_DECREF(py_function_name); + + return 0; +} + +static int +parse_frame_object( + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + void* address, + void** previous_frame) +{ + ssize_t bytes_read = read_memory( + pid, + (void*)(address + offsets->interpreter_frame.previous), + sizeof(void*), + previous_frame); + if (bytes_read == -1) { + return -1; + } + + char owner; + bytes_read = + read_memory(pid, (void*)(address + offsets->interpreter_frame.owner), sizeof(char), &owner); + if (bytes_read < 0) { + return -1; + } + + if (owner == FRAME_OWNED_BY_CSTACK) { + return 0; + } + + void* address_of_code_object; + bytes_read = read_memory( + pid, + (void*)(address + offsets->interpreter_frame.executable), + sizeof(void*), + &address_of_code_object); + if (bytes_read == -1) { + return -1; + } + + if (address_of_code_object == NULL) { + return 0; + } + return parse_code_object(pid, result, offsets, address_of_code_object, previous_frame); +} + +static PyObject* +get_stack_trace(PyObject* self, PyObject* args) +{ +#if (!defined(__linux__) && !defined(__APPLE__)) || (defined(__linux__) && !HAVE_PROCESS_VM_READV) + PyErr_SetString(PyExc_RuntimeError, "get_stack_trace is not supported on this platform"); + return NULL; +#endif + int pid; + + if (!PyArg_ParseTuple(args, "i", &pid)) { + return NULL; + } + + void* runtime_start_address = get_py_runtime(pid); + if (runtime_start_address == NULL) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, "Failed to get .PyRuntime address"); + } + return NULL; + } + size_t size = sizeof(struct _Py_DebugOffsets); + struct _Py_DebugOffsets local_debug_offsets; + + ssize_t bytes_read = read_memory(pid, runtime_start_address, size, &local_debug_offsets); + if (bytes_read == -1) { + return NULL; + } + off_t thread_state_list_head = local_debug_offsets.runtime_state.interpreters_head; + + void* address_of_interpreter_state; + bytes_read = read_memory( + pid, + (void*)(runtime_start_address + thread_state_list_head), + sizeof(void*), + &address_of_interpreter_state); + if (bytes_read == -1) { + return NULL; + } + + if (address_of_interpreter_state == NULL) { + PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); + return NULL; + } + + void* address_of_thread; + bytes_read = read_memory( + pid, + (void*)(address_of_interpreter_state + local_debug_offsets.interpreter_state.threads_head), + sizeof(void*), + &address_of_thread); + if (bytes_read == -1) { + return NULL; + } + + PyObject* result = PyList_New(0); + if (result == NULL) { + return NULL; + } + + // No Python frames are available for us (can happen at tear-down). + if (address_of_thread != NULL) { + void* address_of_current_frame; + (void)read_memory( + pid, + (void*)(address_of_thread + local_debug_offsets.thread_state.current_frame), + sizeof(void*), + &address_of_current_frame); + while (address_of_current_frame != NULL) { + if (parse_frame_object( + pid, + result, + &local_debug_offsets, + address_of_current_frame, + &address_of_current_frame) + < 0) + { + Py_DECREF(result); + return NULL; + } + } + } + + return result; +} + +static PyMethodDef methods[] = { + {"get_stack_trace", get_stack_trace, METH_VARARGS, "Get the Python stack from a given PID"}, + {NULL, NULL, 0, NULL}, +}; + +static struct PyModuleDef module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_testexternalinspection", + .m_size = -1, + .m_methods = methods, +}; + +PyMODINIT_FUNC +PyInit__testexternalinspection(void) +{ + PyObject* mod = PyModule_Create(&module); + int rc = PyModule_AddIntConstant(mod, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); + if (rc < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index c4a648a1816392f..db8817418950b92 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -24,6 +24,7 @@ #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() #include "pycore_object.h" // _PyObject_IsFreed() +#include "pycore_optimizer.h" // _Py_UopsSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -960,13 +961,13 @@ iframe_getlasti(PyObject *self, PyObject *frame) } static PyObject * -get_counter_optimizer(PyObject *self, PyObject *arg) +new_counter_optimizer(PyObject *self, PyObject *arg) { return PyUnstable_Optimizer_NewCounter(); } static PyObject * -get_uop_optimizer(PyObject *self, PyObject *arg) +new_uop_optimizer(PyObject *self, PyObject *arg) { return PyUnstable_Optimizer_NewUOpOptimizer(); } @@ -977,7 +978,9 @@ set_optimizer(PyObject *self, PyObject *opt) if (opt == Py_None) { opt = NULL; } - PyUnstable_SetOptimizer((_PyOptimizerObject*)opt); + if (PyUnstable_SetOptimizer((_PyOptimizerObject*)opt) < 0) { + return NULL; + } Py_RETURN_NONE; } @@ -1033,7 +1036,7 @@ static PyObject * invalidate_executors(PyObject *self, PyObject *obj) { PyInterpreterState *interp = PyInterpreterState_Get(); - _Py_Executors_InvalidateDependency(interp, obj); + _Py_Executors_InvalidateDependency(interp, obj, 1); Py_RETURN_NONE; } @@ -1650,6 +1653,20 @@ get_rare_event_counters(PyObject *self, PyObject *type) ); } +static PyObject * +reset_rare_event_counters(PyObject *self, PyObject *Py_UNUSED(type)) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + + interp->rare_events.set_class = 0; + interp->rare_events.set_bases = 0; + interp->rare_events.set_eval_frame_func = 0; + interp->rare_events.builtin_dict = 0; + interp->rare_events.func_modification = 0; + + return Py_None; +} + #ifdef Py_GIL_DISABLED static PyObject * @@ -1661,7 +1678,6 @@ get_py_thread_id(PyObject *self, PyObject *Py_UNUSED(ignored)) } #endif - static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1695,8 +1711,8 @@ static PyMethodDef module_functions[] = { {"get_optimizer", get_optimizer, METH_NOARGS, NULL}, {"set_optimizer", set_optimizer, METH_O, NULL}, {"get_executor", _PyCFunction_CAST(get_executor), METH_FASTCALL, NULL}, - {"get_counter_optimizer", get_counter_optimizer, METH_NOARGS, NULL}, - {"get_uop_optimizer", get_uop_optimizer, METH_NOARGS, NULL}, + {"new_counter_optimizer", new_counter_optimizer, METH_NOARGS, NULL}, + {"new_uop_optimizer", new_uop_optimizer, METH_NOARGS, NULL}, {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, {"invalidate_executors", invalidate_executors, METH_O, NULL}, {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc), @@ -1727,9 +1743,11 @@ static PyMethodDef module_functions[] = { _TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF {"get_type_module_name", get_type_module_name, METH_O}, {"get_rare_event_counters", get_rare_event_counters, METH_NOARGS}, + {"reset_rare_event_counters", reset_rare_event_counters, METH_NOARGS}, #ifdef Py_GIL_DISABLED {"py_thread_id", get_py_thread_id, METH_NOARGS}, #endif + {"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; @@ -1752,8 +1770,18 @@ module_exec(PyObject *module) return 1; } + Py_ssize_t sizeof_gc_head = 0; +#ifndef Py_GIL_DISABLED + sizeof_gc_head = sizeof(PyGC_Head); +#endif + if (PyModule_Add(module, "SIZEOF_PYGC_HEAD", - PyLong_FromSsize_t(sizeof(PyGC_Head))) < 0) { + PyLong_FromSsize_t(sizeof_gc_head)) < 0) { + return 1; + } + + if (PyModule_Add(module, "SIZEOF_MANAGED_PRE_HEADER", + PyLong_FromSsize_t(2 * sizeof(PyObject*))) < 0) { return 1; } diff --git a/Modules/_testinternalcapi/pytime.c b/Modules/_testinternalcapi/pytime.c index 2b5f9eb0ef28511..2b0a205d158a963 100644 --- a/Modules/_testinternalcapi/pytime.c +++ b/Modules/_testinternalcapi/pytime.c @@ -2,10 +2,10 @@ #include "parts.h" -#include "pycore_time.h" +#include "pycore_time.h" // _PyTime_FromSeconds() #ifdef MS_WINDOWS -# include <winsock2.h> // struct timeval +# include <winsock2.h> // struct timeval #endif @@ -16,8 +16,8 @@ test_pytime_fromseconds(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "i", &seconds)) { return NULL; } - _PyTime_t ts = _PyTime_FromSeconds(seconds); - return _PyTime_AsNanosecondsObject(ts); + PyTime_t ts = _PyTime_FromSeconds(seconds); + return _PyTime_AsLong(ts); } static int @@ -45,26 +45,11 @@ test_pytime_fromsecondsobject(PyObject *self, PyObject *args) if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t ts; + PyTime_t ts; if (_PyTime_FromSecondsObject(&ts, obj, round) == -1) { return NULL; } - return _PyTime_AsNanosecondsObject(ts); -} - -static PyObject * -test_pytime_assecondsdouble(PyObject *self, PyObject *args) -{ - PyObject *obj; - if (!PyArg_ParseTuple(args, "O", &obj)) { - return NULL; - } - _PyTime_t ts; - if (_PyTime_FromNanosecondsObject(&ts, obj) < 0) { - return NULL; - } - double d = _PyTime_AsSecondsDouble(ts); - return PyFloat_FromDouble(d); + return _PyTime_AsLong(ts); } static PyObject * @@ -78,8 +63,8 @@ test_PyTime_AsTimeval(PyObject *self, PyObject *args) if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } struct timeval tv; @@ -105,8 +90,8 @@ test_PyTime_AsTimeval_clamp(PyObject *self, PyObject *args) if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } struct timeval tv; @@ -127,8 +112,8 @@ test_PyTime_AsTimespec(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O", &obj)) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } struct timespec ts; @@ -145,8 +130,8 @@ test_PyTime_AsTimespec_clamp(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O", &obj)) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } struct timespec ts; @@ -163,16 +148,15 @@ test_PyTime_AsMilliseconds(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t ms = _PyTime_AsMilliseconds(t, round); - _PyTime_t ns = _PyTime_FromNanoseconds(ms); - return _PyTime_AsNanosecondsObject(ns); + PyTime_t ms = _PyTime_AsMilliseconds(t, round); + return _PyTime_AsLong(ms); } static PyObject * @@ -183,16 +167,15 @@ test_PyTime_AsMicroseconds(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t us = _PyTime_AsMicroseconds(t, round); - _PyTime_t ns = _PyTime_FromNanoseconds(us); - return _PyTime_AsNanosecondsObject(ns); + PyTime_t us = _PyTime_AsMicroseconds(t, round); + return _PyTime_AsLong(us); } static PyObject * @@ -254,7 +237,6 @@ test_pytime_object_to_timespec(PyObject *self, PyObject *args) static PyMethodDef TestMethods[] = { {"_PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS}, {"_PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS}, - {"_PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS}, #ifdef HAVE_CLOCK_GETTIME {"_PyTime_AsTimespec", test_PyTime_AsTimespec, METH_VARARGS}, {"_PyTime_AsTimespec_clamp", test_PyTime_AsTimespec_clamp, METH_VARARGS}, diff --git a/Modules/_testinternalcapi/test_critical_sections.c b/Modules/_testinternalcapi/test_critical_sections.c index 1f7e311558b27cb..94da0468fcf149a 100644 --- a/Modules/_testinternalcapi/test_critical_sections.c +++ b/Modules/_testinternalcapi/test_critical_sections.c @@ -49,6 +49,15 @@ test_critical_sections(PyObject *self, PyObject *Py_UNUSED(args)) Py_END_CRITICAL_SECTION2(); assert_nogil(!PyMutex_IsLocked(&d2->ob_mutex)); + // Optional variant behaves the same if the object is non-NULL + Py_XBEGIN_CRITICAL_SECTION(d1); + assert_nogil(PyMutex_IsLocked(&d1->ob_mutex)); + Py_XEND_CRITICAL_SECTION(); + + // No-op + Py_XBEGIN_CRITICAL_SECTION(NULL); + Py_XEND_CRITICAL_SECTION(); + Py_DECREF(d2); Py_DECREF(d1); Py_RETURN_NONE; diff --git a/Modules/_testinternalcapi/test_lock.c b/Modules/_testinternalcapi/test_lock.c index 83081f73a72f648..1c5048170e9f2ea 100644 --- a/Modules/_testinternalcapi/test_lock.c +++ b/Modules/_testinternalcapi/test_lock.c @@ -1,8 +1,9 @@ // C Extension module to test pycore_lock.h API #include "parts.h" - #include "pycore_lock.h" +#include "pycore_time.h" // _PyTime_MonotonicUnchecked() + #include "clinic/test_lock.c.h" #ifdef MS_WINDOWS @@ -289,7 +290,7 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, goto exit; } - _PyTime_t start = _PyTime_GetMonotonicClock(); + PyTime_t start = _PyTime_MonotonicUnchecked(); for (Py_ssize_t i = 0; i < num_threads; i++) { thread_data[i].bench_data = &bench_data; @@ -306,7 +307,7 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, } Py_ssize_t total_iters = bench_data.total_iters; - _PyTime_t end = _PyTime_GetMonotonicClock(); + PyTime_t end = _PyTime_MonotonicUnchecked(); // Return the total number of acquisitions and the number of acquisitions // for each thread. diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index c42a15a0eff4944..092673a9ea43e11 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -8,11 +8,10 @@ //#include <time.h> #include "Python.h" #include "pycore_namespace.h" // _PyNamespace_New() -#include "pycore_time.h" // _PyTime_t typedef struct { - _PyTime_t initialized; + PyTime_t initialized; PyObject *error; PyObject *int_const; PyObject *str_const; @@ -67,17 +66,17 @@ clear_state(module_state *state) } static int -_set_initialized(_PyTime_t *initialized) +_set_initialized(PyTime_t *initialized) { /* We go strictly monotonic to ensure each time is unique. */ - _PyTime_t prev; - if (_PyTime_GetMonotonicClockWithInfo(&prev, NULL) != 0) { + PyTime_t prev; + if (PyTime_Monotonic(&prev) != 0) { return -1; } /* We do a busy sleep since the interval should be super short. */ - _PyTime_t t; + PyTime_t t; do { - if (_PyTime_GetMonotonicClockWithInfo(&t, NULL) != 0) { + if (PyTime_Monotonic(&t) != 0) { return -1; } } while (t == prev); @@ -136,7 +135,7 @@ init_module(PyObject *module, module_state *state) return -1; } - double d = _PyTime_AsSecondsDouble(state->initialized); + double d = PyTime_AsSecondsDouble(state->initialized); if (PyModule_Add(module, "_module_initialized", PyFloat_FromDouble(d)) < 0) { return -1; } @@ -157,7 +156,7 @@ common_state_initialized(PyObject *self, PyObject *Py_UNUSED(ignored)) if (state == NULL) { Py_RETURN_NONE; } - double d = _PyTime_AsSecondsDouble(state->initialized); + double d = PyTime_AsSecondsDouble(state->initialized); return PyFloat_FromDouble(d); } diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 5cceb84658deb78..4c2185cc7ea1fd3 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -9,8 +9,10 @@ #include "pycore_pylifecycle.h" #include "pycore_pystate.h" // _PyThreadState_SetCurrent() #include "pycore_sysmodule.h" // _PySys_GetAttr() +#include "pycore_time.h" // _PyTime_FromSeconds() #include "pycore_weakref.h" // _PyWeakref_GET_REF() +#include <stdbool.h> #include <stddef.h> // offsetof() #ifdef HAVE_SIGNAL_H # include <signal.h> // SIGINT @@ -44,6 +46,7 @@ get_thread_state(PyObject *module) typedef struct { PyObject_HEAD + struct llist_node node; // linked list node (see _pythread_runtime_state) PyThread_ident_t ident; PyThread_handle_t handle; char joinable; @@ -59,6 +62,11 @@ new_thread_handle(thread_module_state* state) self->ident = 0; self->handle = 0; self->joinable = 0; + + HEAD_LOCK(&_PyRuntime); + llist_insert_tail(&_PyRuntime.threads.handles, &self->node); + HEAD_UNLOCK(&_PyRuntime); + return self; } @@ -66,6 +74,14 @@ static void ThreadHandle_dealloc(ThreadHandleObject *self) { PyObject *tp = (PyObject *) Py_TYPE(self); + + // Remove ourself from the global list of handles + HEAD_LOCK(&_PyRuntime); + if (self->node.next != NULL) { + llist_remove(&self->node); + } + HEAD_UNLOCK(&_PyRuntime); + if (self->joinable) { int ret = PyThread_detach_thread(self->handle); if (ret) { @@ -77,6 +93,28 @@ ThreadHandle_dealloc(ThreadHandleObject *self) Py_DECREF(tp); } +void +_PyThread_AfterFork(struct _pythread_runtime_state *state) +{ + // gh-115035: We mark ThreadHandles as not joinable early in the child's + // after-fork handler. We do this before calling any Python code to ensure + // that it happens before any ThreadHandles are deallocated, such as by a + // GC cycle. + PyThread_ident_t current = PyThread_get_thread_ident_ex(); + + struct llist_node *node; + llist_for_each_safe(node, &state->handles) { + ThreadHandleObject *hobj = llist_data(node, ThreadHandleObject, node); + if (hobj->ident == current) { + continue; + } + + // Disallow calls to detach() and join() as they could crash. + hobj->joinable = 0; + llist_remove(node); + } +} + static PyObject * ThreadHandle_repr(ThreadHandleObject *self) { @@ -91,21 +129,6 @@ ThreadHandle_get_ident(ThreadHandleObject *self, void *ignored) } -static PyObject * -ThreadHandle_after_fork_alive(ThreadHandleObject *self, void* ignored) -{ - PyThread_update_thread_after_fork(&self->ident, &self->handle); - Py_RETURN_NONE; -} - -static PyObject * -ThreadHandle_after_fork_dead(ThreadHandleObject *self, void* ignored) -{ - // Disallow calls to detach() and join() as they could crash. - self->joinable = 0; - Py_RETURN_NONE; -} - static PyObject * ThreadHandle_detach(ThreadHandleObject *self, void* ignored) { @@ -157,8 +180,6 @@ static PyGetSetDef ThreadHandle_getsetlist[] = { static PyMethodDef ThreadHandle_methods[] = { - {"after_fork_alive", (PyCFunction)ThreadHandle_after_fork_alive, METH_NOARGS}, - {"after_fork_dead", (PyCFunction)ThreadHandle_after_fork_dead, METH_NOARGS}, {"detach", (PyCFunction)ThreadHandle_detach, METH_NOARGS}, {"join", (PyCFunction)ThreadHandle_join, METH_NOARGS}, {0, 0} @@ -215,14 +236,14 @@ lock_dealloc(lockobject *self) } static inline PyLockStatus -acquire_timed(PyThread_type_lock lock, _PyTime_t timeout) +acquire_timed(PyThread_type_lock lock, PyTime_t timeout) { return PyThread_acquire_lock_timed_with_retries(lock, timeout); } static int lock_acquire_parse_args(PyObject *args, PyObject *kwds, - _PyTime_t *timeout) + PyTime_t *timeout) { char *kwlist[] = {"blocking", "timeout", NULL}; int blocking = 1; @@ -233,7 +254,7 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, // XXX Use PyThread_ParseTimeoutArg(). - const _PyTime_t unset_timeout = _PyTime_FromSeconds(-1); + const PyTime_t unset_timeout = _PyTime_FromSeconds(-1); *timeout = unset_timeout; if (timeout_obj @@ -254,7 +275,7 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, if (!blocking) *timeout = 0; else if (*timeout != unset_timeout) { - _PyTime_t microseconds; + PyTime_t microseconds; microseconds = _PyTime_AsMicroseconds(*timeout, _PyTime_ROUND_TIMEOUT); if (microseconds > PY_TIMEOUT_MAX) { @@ -269,7 +290,7 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, static PyObject * lock_PyThread_acquire_lock(lockobject *self, PyObject *args, PyObject *kwds) { - _PyTime_t timeout; + PyTime_t timeout; if (lock_acquire_parse_args(args, kwds, &timeout) < 0) return NULL; @@ -470,10 +491,18 @@ rlock_dealloc(rlockobject *self) Py_DECREF(tp); } +static bool +rlock_is_owned_by(rlockobject *self, PyThread_ident_t tid) +{ + PyThread_ident_t owner_tid = + _Py_atomic_load_ullong_relaxed(&self->rlock_owner); + return owner_tid == tid && self->rlock_count > 0; +} + static PyObject * rlock_acquire(rlockobject *self, PyObject *args, PyObject *kwds) { - _PyTime_t timeout; + PyTime_t timeout; PyThread_ident_t tid; PyLockStatus r = PY_LOCK_ACQUIRED; @@ -481,7 +510,7 @@ rlock_acquire(rlockobject *self, PyObject *args, PyObject *kwds) return NULL; tid = PyThread_get_thread_ident_ex(); - if (self->rlock_count > 0 && tid == self->rlock_owner) { + if (rlock_is_owned_by(self, tid)) { unsigned long count = self->rlock_count + 1; if (count <= self->rlock_count) { PyErr_SetString(PyExc_OverflowError, @@ -494,7 +523,7 @@ rlock_acquire(rlockobject *self, PyObject *args, PyObject *kwds) r = acquire_timed(self->rlock_lock, timeout); if (r == PY_LOCK_ACQUIRED) { assert(self->rlock_count == 0); - self->rlock_owner = tid; + _Py_atomic_store_ullong_relaxed(&self->rlock_owner, tid); self->rlock_count = 1; } else if (r == PY_LOCK_INTR) { @@ -525,13 +554,13 @@ rlock_release(rlockobject *self, PyObject *Py_UNUSED(ignored)) { PyThread_ident_t tid = PyThread_get_thread_ident_ex(); - if (self->rlock_count == 0 || self->rlock_owner != tid) { + if (!rlock_is_owned_by(self, tid)) { PyErr_SetString(PyExc_RuntimeError, "cannot release un-acquired lock"); return NULL; } if (--self->rlock_count == 0) { - self->rlock_owner = 0; + _Py_atomic_store_ullong_relaxed(&self->rlock_owner, 0); PyThread_release_lock(self->rlock_lock); } Py_RETURN_NONE; @@ -570,7 +599,7 @@ rlock_acquire_restore(rlockobject *self, PyObject *args) return NULL; } assert(self->rlock_count == 0); - self->rlock_owner = owner; + _Py_atomic_store_ullong_relaxed(&self->rlock_owner, owner); self->rlock_count = count; Py_RETURN_NONE; } @@ -595,7 +624,7 @@ rlock_release_save(rlockobject *self, PyObject *Py_UNUSED(ignored)) owner = self->rlock_owner; count = self->rlock_count; self->rlock_count = 0; - self->rlock_owner = 0; + _Py_atomic_store_ullong_relaxed(&self->rlock_owner, 0); PyThread_release_lock(self->rlock_lock); return Py_BuildValue("k" Py_PARSE_THREAD_IDENT_T, count, owner); } @@ -609,8 +638,9 @@ static PyObject * rlock_recursion_count(rlockobject *self, PyObject *Py_UNUSED(ignored)) { PyThread_ident_t tid = PyThread_get_thread_ident_ex(); - return PyLong_FromUnsignedLong( - self->rlock_owner == tid ? self->rlock_count : 0UL); + PyThread_ident_t owner = + _Py_atomic_load_ullong_relaxed(&self->rlock_owner); + return PyLong_FromUnsignedLong(owner == tid ? self->rlock_count : 0UL); } PyDoc_STRVAR(rlock_recursion_count_doc, @@ -623,7 +653,7 @@ rlock_is_owned(rlockobject *self, PyObject *Py_UNUSED(ignored)) { PyThread_ident_t tid = PyThread_get_thread_ident_ex(); - if (self->rlock_count > 0 && self->rlock_owner == tid) { + if (rlock_is_owned_by(self, tid)) { Py_RETURN_TRUE; } Py_RETURN_FALSE; @@ -657,10 +687,12 @@ rlock_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject * rlock_repr(rlockobject *self) { + PyThread_ident_t owner = + _Py_atomic_load_ullong_relaxed(&self->rlock_owner); return PyUnicode_FromFormat( "<%s %s object owner=%" PY_FORMAT_THREAD_IDENT_T " count=%lu at %p>", self->rlock_count ? "locked" : "unlocked", - Py_TYPE(self)->tp_name, self->rlock_owner, + Py_TYPE(self)->tp_name, owner, self->rlock_count, self); } @@ -1225,7 +1257,7 @@ thread_run(void *boot_raw) _PyThreadState_Bind(tstate); PyEval_AcquireThread(tstate); - tstate->interp->threads.count++; + _Py_atomic_add_ssize(&tstate->interp->threads.count, 1); PyObject *res = PyObject_Call(boot->func, boot->args, boot->kwargs); if (res == NULL) { @@ -1243,7 +1275,7 @@ thread_run(void *boot_raw) thread_bootstate_free(boot, 1); - tstate->interp->threads.count--; + _Py_atomic_add_ssize(&tstate->interp->threads.count, -1); PyThreadState_Clear(tstate); _PyThreadState_DeleteCurrent(tstate); @@ -1285,7 +1317,7 @@ do_start_new_thread(thread_module_state* state, return -1; } if (interp->finalizing) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_PythonFinalizationError, "can't create new thread at interpreter shutdown"); return -1; } @@ -1520,7 +1552,7 @@ static PyObject * thread__count(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyInterpreterState *interp = _PyInterpreterState_GET(); - return PyLong_FromLong(interp->threads.count); + return PyLong_FromSsize_t(_Py_atomic_load_ssize(&interp->threads.count)); } PyDoc_STRVAR(_count_doc, @@ -1916,7 +1948,7 @@ thread_module_exec(PyObject *module) // TIMEOUT_MAX double timeout_max = (double)PY_TIMEOUT_MAX * 1e-6; - double time_max = _PyTime_AsSecondsDouble(_PyTime_MAX); + double time_max = PyTime_AsSecondsDouble(PyTime_MAX); timeout_max = Py_MIN(timeout_max, time_max); // Round towards minus infinity timeout_max = floor(timeout_max); diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index f6181168a85ae15..e3789867dc085fd 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -926,7 +926,8 @@ AsObj(PyObject *value) (unsigned char *)(void *)&wideValue, sizeof(wideValue), PY_LITTLE_ENDIAN, - /* signed */ 1) == 0) { + /* signed */ 1, + /* with_exceptions */ 1) == 0) { return Tcl_NewWideIntObj(wideValue); } PyErr_Clear(); diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 5e5eb123c4ccfff..8f9b8520bb3f349 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -139,7 +139,7 @@ overlapped_dealloc(OverlappedObject *self) { /* The operation is still pending -- give a warning. This will probably only happen on Windows XP. */ - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_PythonFinalizationError, "I/O operations still in flight while destroying " "Overlapped object, the process may crash"); PyErr_WriteUnraisable(NULL); @@ -438,6 +438,39 @@ _winapi_ConnectNamedPipe_impl(PyObject *module, HANDLE handle, Py_RETURN_NONE; } +/*[clinic input] +_winapi.CreateEventW -> HANDLE + + security_attributes: LPSECURITY_ATTRIBUTES + manual_reset: BOOL + initial_state: BOOL + name: LPCWSTR(accept={str, NoneType}) +[clinic start generated code]*/ + +static HANDLE +_winapi_CreateEventW_impl(PyObject *module, + LPSECURITY_ATTRIBUTES security_attributes, + BOOL manual_reset, BOOL initial_state, + LPCWSTR name) +/*[clinic end generated code: output=2d4c7d5852ecb298 input=4187cee28ac763f8]*/ +{ + HANDLE handle; + + if (PySys_Audit("_winapi.CreateEventW", "bbu", manual_reset, initial_state, name) < 0) { + return INVALID_HANDLE_VALUE; + } + + Py_BEGIN_ALLOW_THREADS + handle = CreateEventW(security_attributes, manual_reset, initial_state, name); + Py_END_ALLOW_THREADS + + if (handle == INVALID_HANDLE_VALUE) { + PyErr_SetFromWindowsErr(0); + } + + return handle; +} + /*[clinic input] _winapi.CreateFile -> HANDLE @@ -674,6 +707,37 @@ _winapi_CreateJunction_impl(PyObject *module, LPCWSTR src_path, Py_RETURN_NONE; } +/*[clinic input] +_winapi.CreateMutexW -> HANDLE + + security_attributes: LPSECURITY_ATTRIBUTES + initial_owner: BOOL + name: LPCWSTR(accept={str, NoneType}) +[clinic start generated code]*/ + +static HANDLE +_winapi_CreateMutexW_impl(PyObject *module, + LPSECURITY_ATTRIBUTES security_attributes, + BOOL initial_owner, LPCWSTR name) +/*[clinic end generated code: output=31b9ee8fc37e49a5 input=7d54b921e723254a]*/ +{ + HANDLE handle; + + if (PySys_Audit("_winapi.CreateMutexW", "bu", initial_owner, name) < 0) { + return INVALID_HANDLE_VALUE; + } + + Py_BEGIN_ALLOW_THREADS + handle = CreateMutexW(security_attributes, initial_owner, name); + Py_END_ALLOW_THREADS + + if (handle == INVALID_HANDLE_VALUE) { + PyErr_SetFromWindowsErr(0); + } + + return handle; +} + /*[clinic input] _winapi.CreateNamedPipe -> HANDLE @@ -1590,6 +1654,67 @@ _winapi_UnmapViewOfFile_impl(PyObject *module, LPCVOID address) Py_RETURN_NONE; } +/*[clinic input] +_winapi.OpenEventW -> HANDLE + + desired_access: DWORD + inherit_handle: BOOL + name: LPCWSTR +[clinic start generated code]*/ + +static HANDLE +_winapi_OpenEventW_impl(PyObject *module, DWORD desired_access, + BOOL inherit_handle, LPCWSTR name) +/*[clinic end generated code: output=c4a45e95545a4bd2 input=dec26598748d35aa]*/ +{ + HANDLE handle; + + if (PySys_Audit("_winapi.OpenEventW", "Iu", desired_access, name) < 0) { + return INVALID_HANDLE_VALUE; + } + + Py_BEGIN_ALLOW_THREADS + handle = OpenEventW(desired_access, inherit_handle, name); + Py_END_ALLOW_THREADS + + if (handle == INVALID_HANDLE_VALUE) { + PyErr_SetFromWindowsErr(0); + } + + return handle; +} + + +/*[clinic input] +_winapi.OpenMutexW -> HANDLE + + desired_access: DWORD + inherit_handle: BOOL + name: LPCWSTR +[clinic start generated code]*/ + +static HANDLE +_winapi_OpenMutexW_impl(PyObject *module, DWORD desired_access, + BOOL inherit_handle, LPCWSTR name) +/*[clinic end generated code: output=dda39d7844397bf0 input=f3a7b466c5307712]*/ +{ + HANDLE handle; + + if (PySys_Audit("_winapi.OpenMutexW", "Iu", desired_access, name) < 0) { + return INVALID_HANDLE_VALUE; + } + + Py_BEGIN_ALLOW_THREADS + handle = OpenMutexW(desired_access, inherit_handle, name); + Py_END_ALLOW_THREADS + + if (handle == INVALID_HANDLE_VALUE) { + PyErr_SetFromWindowsErr(0); + } + + return handle; +} + /*[clinic input] _winapi.OpenFileMapping -> HANDLE @@ -1820,6 +1945,75 @@ _winapi_ReadFile_impl(PyObject *module, HANDLE handle, DWORD size, return Py_BuildValue("NI", buf, err); } +/*[clinic input] +_winapi.ReleaseMutex + + mutex: HANDLE +[clinic start generated code]*/ + +static PyObject * +_winapi_ReleaseMutex_impl(PyObject *module, HANDLE mutex) +/*[clinic end generated code: output=5b9001a72dd8af37 input=49e9d20de3559d84]*/ +{ + int err = 0; + + Py_BEGIN_ALLOW_THREADS + if (!ReleaseMutex(mutex)) { + err = GetLastError(); + } + Py_END_ALLOW_THREADS + if (err) { + return PyErr_SetFromWindowsErr(err); + } + Py_RETURN_NONE; +} + +/*[clinic input] +_winapi.ResetEvent + + event: HANDLE +[clinic start generated code]*/ + +static PyObject * +_winapi_ResetEvent_impl(PyObject *module, HANDLE event) +/*[clinic end generated code: output=81c8501d57c0530d input=e2d42d990322e87a]*/ +{ + int err = 0; + + Py_BEGIN_ALLOW_THREADS + if (!ResetEvent(event)) { + err = GetLastError(); + } + Py_END_ALLOW_THREADS + if (err) { + return PyErr_SetFromWindowsErr(err); + } + Py_RETURN_NONE; +} + +/*[clinic input] +_winapi.SetEvent + + event: HANDLE +[clinic start generated code]*/ + +static PyObject * +_winapi_SetEvent_impl(PyObject *module, HANDLE event) +/*[clinic end generated code: output=c18ba09eb9aa774d input=e660e830a37c09f8]*/ +{ + int err = 0; + + Py_BEGIN_ALLOW_THREADS + if (!SetEvent(event)) { + err = GetLastError(); + } + Py_END_ALLOW_THREADS + if (err) { + return PyErr_SetFromWindowsErr(err); + } + Py_RETURN_NONE; +} + /*[clinic input] _winapi.SetNamedPipeHandleState @@ -1942,6 +2136,310 @@ _winapi_WaitNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD timeout) Py_RETURN_NONE; } + +typedef struct { + HANDLE handles[MAXIMUM_WAIT_OBJECTS]; + HANDLE cancel_event; + DWORD handle_base; + DWORD handle_count; + HANDLE thread; + volatile DWORD result; +} BatchedWaitData; + +static DWORD WINAPI +_batched_WaitForMultipleObjects_thread(LPVOID param) +{ + BatchedWaitData *data = (BatchedWaitData *)param; + data->result = WaitForMultipleObjects( + data->handle_count, + data->handles, + FALSE, + INFINITE + ); + if (data->result == WAIT_FAILED) { + DWORD err = GetLastError(); + SetEvent(data->cancel_event); + return err; + } else if (data->result >= WAIT_ABANDONED_0 && data->result < WAIT_ABANDONED_0 + MAXIMUM_WAIT_OBJECTS) { + data->result = WAIT_FAILED; + SetEvent(data->cancel_event); + return ERROR_ABANDONED_WAIT_0; + } + return 0; +} + +/*[clinic input] +_winapi.BatchedWaitForMultipleObjects + + handle_seq: object + wait_all: BOOL + milliseconds: DWORD(c_default='INFINITE') = _winapi.INFINITE + +Supports a larger number of handles than WaitForMultipleObjects + +Note that the handles may be waited on other threads, which could cause +issues for objects like mutexes that become associated with the thread +that was waiting for them. Objects may also be left signalled, even if +the wait fails. + +It is recommended to use WaitForMultipleObjects whenever possible, and +only switch to BatchedWaitForMultipleObjects for scenarios where you +control all the handles involved, such as your own thread pool or +files, and all wait objects are left unmodified by a wait (for example, +manual reset events, threads, and files/pipes). + +Overlapped handles returned from this module use manual reset events. +[clinic start generated code]*/ + +static PyObject * +_winapi_BatchedWaitForMultipleObjects_impl(PyObject *module, + PyObject *handle_seq, + BOOL wait_all, DWORD milliseconds) +/*[clinic end generated code: output=d21c1a4ad0a252fd input=7e196f29005dc77b]*/ +{ + Py_ssize_t thread_count = 0, handle_count = 0, i, j; + Py_ssize_t nhandles; + BatchedWaitData *thread_data[MAXIMUM_WAIT_OBJECTS]; + HANDLE handles[MAXIMUM_WAIT_OBJECTS]; + HANDLE sigint_event = NULL; + HANDLE cancel_event = NULL; + DWORD result; + + const Py_ssize_t _MAXIMUM_TOTAL_OBJECTS = (MAXIMUM_WAIT_OBJECTS - 1) * (MAXIMUM_WAIT_OBJECTS - 1); + + if (!PySequence_Check(handle_seq)) { + PyErr_Format(PyExc_TypeError, + "sequence type expected, got '%s'", + Py_TYPE(handle_seq)->tp_name); + return NULL; + } + nhandles = PySequence_Length(handle_seq); + if (nhandles == -1) { + return NULL; + } + if (nhandles == 0) { + return wait_all ? Py_NewRef(Py_None) : PyList_New(0); + } + + /* If this is the main thread then make the wait interruptible + by Ctrl-C. When waiting for *all* handles, it is only checked + in between batches. */ + if (_PyOS_IsMainThread()) { + sigint_event = _PyOS_SigintEvent(); + assert(sigint_event != NULL); + } + + if (nhandles < 0 || nhandles > _MAXIMUM_TOTAL_OBJECTS) { + PyErr_Format(PyExc_ValueError, + "need at most %zd handles, got a sequence of length %zd", + _MAXIMUM_TOTAL_OBJECTS, nhandles); + return NULL; + } + + if (!wait_all) { + cancel_event = CreateEventW(NULL, TRUE, FALSE, NULL); + if (!cancel_event) { + PyErr_SetExcFromWindowsErr(PyExc_OSError, 0); + return NULL; + } + } + + i = 0; + while (i < nhandles) { + BatchedWaitData *data = (BatchedWaitData*)PyMem_Malloc(sizeof(BatchedWaitData)); + if (!data) { + goto error; + } + thread_data[thread_count++] = data; + data->thread = NULL; + data->cancel_event = cancel_event; + data->handle_base = Py_SAFE_DOWNCAST(i, Py_ssize_t, DWORD); + data->handle_count = Py_SAFE_DOWNCAST(nhandles - i, Py_ssize_t, DWORD); + if (data->handle_count > MAXIMUM_WAIT_OBJECTS - 1) { + data->handle_count = MAXIMUM_WAIT_OBJECTS - 1; + } + for (j = 0; j < data->handle_count; ++i, ++j) { + PyObject *v = PySequence_GetItem(handle_seq, i); + if (!v || !PyArg_Parse(v, F_HANDLE, &data->handles[j])) { + Py_XDECREF(v); + goto error; + } + Py_DECREF(v); + } + if (!wait_all) { + data->handles[data->handle_count++] = cancel_event; + } + } + + DWORD err = 0; + + /* We need to use different strategies when waiting for ALL handles + as opposed to ANY handle. This is because there is no way to + (safely) interrupt a thread that is waiting for all handles in a + group. So for ALL handles, we loop over each set and wait. For + ANY handle, we use threads and wait on them. */ + if (wait_all) { + Py_BEGIN_ALLOW_THREADS + long long deadline = 0; + if (milliseconds != INFINITE) { + deadline = (long long)GetTickCount64() + milliseconds; + } + + for (i = 0; !err && i < thread_count; ++i) { + DWORD timeout = milliseconds; + if (deadline) { + long long time_to_deadline = deadline - GetTickCount64(); + if (time_to_deadline <= 0) { + err = WAIT_TIMEOUT; + break; + } else if (time_to_deadline < UINT_MAX) { + timeout = (DWORD)time_to_deadline; + } + } + result = WaitForMultipleObjects(thread_data[i]->handle_count, + thread_data[i]->handles, TRUE, timeout); + // ABANDONED is not possible here because we own all the handles + if (result == WAIT_FAILED) { + err = GetLastError(); + } else if (result == WAIT_TIMEOUT) { + err = WAIT_TIMEOUT; + } + + if (!err && sigint_event) { + result = WaitForSingleObject(sigint_event, 0); + if (result == WAIT_OBJECT_0) { + err = ERROR_CONTROL_C_EXIT; + } else if (result == WAIT_FAILED) { + err = GetLastError(); + } + } + } + + CloseHandle(cancel_event); + + Py_END_ALLOW_THREADS + } else { + Py_BEGIN_ALLOW_THREADS + + for (i = 0; i < thread_count; ++i) { + BatchedWaitData *data = thread_data[i]; + data->thread = CreateThread( + NULL, + 1, // smallest possible initial stack + _batched_WaitForMultipleObjects_thread, + (LPVOID)data, + CREATE_SUSPENDED, + NULL + ); + if (!data->thread) { + err = GetLastError(); + break; + } + handles[handle_count++] = data->thread; + } + Py_END_ALLOW_THREADS + + if (err) { + PyErr_SetExcFromWindowsErr(PyExc_OSError, err); + goto error; + } + if (handle_count > MAXIMUM_WAIT_OBJECTS - 1) { + // basically an assert, but stronger + PyErr_SetString(PyExc_SystemError, "allocated too many wait objects"); + goto error; + } + + Py_BEGIN_ALLOW_THREADS + + // Once we start resuming threads, can no longer "goto error" + for (i = 0; i < thread_count; ++i) { + ResumeThread(thread_data[i]->thread); + } + if (sigint_event) { + handles[handle_count++] = sigint_event; + } + result = WaitForMultipleObjects((DWORD)handle_count, handles, wait_all, milliseconds); + // ABANDONED is not possible here because we own all the handles + if (result == WAIT_FAILED) { + err = GetLastError(); + } else if (result == WAIT_TIMEOUT) { + err = WAIT_TIMEOUT; + } else if (sigint_event && result == WAIT_OBJECT_0 + handle_count) { + err = ERROR_CONTROL_C_EXIT; + } + + SetEvent(cancel_event); + + // Wait for all threads to finish before we start freeing their memory + if (sigint_event) { + handle_count -= 1; + } + WaitForMultipleObjects((DWORD)handle_count, handles, TRUE, INFINITE); + + for (i = 0; i < thread_count; ++i) { + if (!err && thread_data[i]->result == WAIT_FAILED) { + if (!GetExitCodeThread(thread_data[i]->thread, &err)) { + err = GetLastError(); + } + } + CloseHandle(thread_data[i]->thread); + } + + CloseHandle(cancel_event); + + Py_END_ALLOW_THREADS + + } + + PyObject *triggered_indices; + if (sigint_event != NULL && err == ERROR_CONTROL_C_EXIT) { + errno = EINTR; + PyErr_SetFromErrno(PyExc_OSError); + triggered_indices = NULL; + } else if (err) { + PyErr_SetExcFromWindowsErr(PyExc_OSError, err); + triggered_indices = NULL; + } else if (wait_all) { + triggered_indices = Py_NewRef(Py_None); + } else { + triggered_indices = PyList_New(0); + if (triggered_indices) { + for (i = 0; i < thread_count; ++i) { + Py_ssize_t triggered = (Py_ssize_t)thread_data[i]->result - WAIT_OBJECT_0; + if (triggered >= 0 && triggered < thread_data[i]->handle_count - 1) { + PyObject *v = PyLong_FromSsize_t(thread_data[i]->handle_base + triggered); + if (!v || PyList_Append(triggered_indices, v) < 0) { + Py_XDECREF(v); + Py_CLEAR(triggered_indices); + break; + } + Py_DECREF(v); + } + } + } + } + + for (i = 0; i < thread_count; ++i) { + PyMem_Free((void *)thread_data[i]); + } + + return triggered_indices; + +error: + // We should only enter here before any threads start running. + // Once we start resuming threads, different cleanup is required + CloseHandle(cancel_event); + while (--thread_count >= 0) { + HANDLE t = thread_data[thread_count]->thread; + if (t) { + TerminateThread(t, WAIT_ABANDONED_0); + CloseHandle(t); + } + PyMem_Free((void *)thread_data[thread_count]); + } + return NULL; +} + /*[clinic input] _winapi.WaitForMultipleObjects @@ -2335,8 +2833,10 @@ _winapi_CopyFile2_impl(PyObject *module, LPCWSTR existing_file_name, static PyMethodDef winapi_functions[] = { _WINAPI_CLOSEHANDLE_METHODDEF _WINAPI_CONNECTNAMEDPIPE_METHODDEF + _WINAPI_CREATEEVENTW_METHODDEF _WINAPI_CREATEFILE_METHODDEF _WINAPI_CREATEFILEMAPPING_METHODDEF + _WINAPI_CREATEMUTEXW_METHODDEF _WINAPI_CREATENAMEDPIPE_METHODDEF _WINAPI_CREATEPIPE_METHODDEF _WINAPI_CREATEPROCESS_METHODDEF @@ -2350,17 +2850,23 @@ static PyMethodDef winapi_functions[] = { _WINAPI_GETSTDHANDLE_METHODDEF _WINAPI_GETVERSION_METHODDEF _WINAPI_MAPVIEWOFFILE_METHODDEF + _WINAPI_OPENEVENTW_METHODDEF _WINAPI_OPENFILEMAPPING_METHODDEF + _WINAPI_OPENMUTEXW_METHODDEF _WINAPI_OPENPROCESS_METHODDEF _WINAPI_PEEKNAMEDPIPE_METHODDEF _WINAPI_LCMAPSTRINGEX_METHODDEF _WINAPI_READFILE_METHODDEF + _WINAPI_RELEASEMUTEX_METHODDEF + _WINAPI_RESETEVENT_METHODDEF + _WINAPI_SETEVENT_METHODDEF _WINAPI_SETNAMEDPIPEHANDLESTATE_METHODDEF _WINAPI_TERMINATEPROCESS_METHODDEF _WINAPI_UNMAPVIEWOFFILE_METHODDEF _WINAPI_VIRTUALQUERYSIZE_METHODDEF _WINAPI_WAITNAMEDPIPE_METHODDEF _WINAPI_WAITFORMULTIPLEOBJECTS_METHODDEF + _WINAPI_BATCHEDWAITFORMULTIPLEOBJECTS_METHODDEF _WINAPI_WAITFORSINGLEOBJECT_METHODDEF _WINAPI_WRITEFILE_METHODDEF _WINAPI_GETACP_METHODDEF diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 4e9b8a82a3f630b..82d2ae7fc4c9634 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -17,6 +17,8 @@ #include <sched.h> // sched_yield() #endif +#include "_interpreters_common.h" + /* This module has the following process-global state: @@ -80,7 +82,9 @@ channel's queue, which are safely managed via the _PyCrossInterpreterData_*() API.. The module does not create any objects that are shared globally. */ -#define MODULE_NAME "_xxinterpchannels" +#define MODULE_NAME _xxinterpchannels +#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME) +#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME) #define GLOBAL_MALLOC(TYPE) \ @@ -101,7 +105,7 @@ static int register_xid_class(PyTypeObject *cls, crossinterpdatafunc shared, struct xid_class_registry *classes) { - int res = _PyCrossInterpreterData_RegisterClass(cls, shared); + int res = ensure_xid_class(cls, shared); if (res == 0) { assert(classes->count < MAX_XID_CLASSES); // The class has refs elsewhere, so we need to incref here. @@ -167,7 +171,7 @@ _get_current_interp(void) static PyObject * _get_current_module(void) { - PyObject *name = PyUnicode_FromString(MODULE_NAME); + PyObject *name = PyUnicode_FromString(MODULE_NAME_STR); if (name == NULL) { return NULL; } @@ -217,7 +221,7 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base) } #define ADD_NEW_EXCEPTION(MOD, NAME, BASE) \ - add_new_exception(MOD, MODULE_NAME "." Py_STRINGIFY(NAME), BASE) + add_new_exception(MOD, MODULE_NAME_STR "." Py_STRINGIFY(NAME), BASE) static PyTypeObject * add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared, @@ -299,7 +303,7 @@ _get_current_module_state(void) if (mod == NULL) { // XXX import it? PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME " module not imported yet"); + MODULE_NAME_STR " module not imported yet"); return NULL; } module_state *state = get_module_state(mod); @@ -784,7 +788,7 @@ _channelqueue_clear_interpreter(_channelqueue *queue, int64_t interpid) while (next != NULL) { _channelitem *item = next; next = item->next; - if (item->data->interpid == interpid) { + if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) { if (prev == NULL) { queue->first = item->next; } @@ -2126,7 +2130,7 @@ static PyStructSequence_Field channel_info_fields[] = { }; static PyStructSequence_Desc channel_info_desc = { - .name = MODULE_NAME ".ChannelInfo", + .name = MODULE_NAME_STR ".ChannelInfo", .doc = channel_info_doc, .fields = channel_info_fields, .n_in_sequence = 8, @@ -2154,7 +2158,7 @@ new_channel_info(PyObject *mod, struct channel_info *info) do { \ PyObject *obj = PyLong_FromLongLong(val); \ if (obj == NULL) { \ - Py_CLEAR(info); \ + Py_CLEAR(self); \ return NULL; \ } \ PyStructSequence_SET_ITEM(self, pos++, obj); \ @@ -2474,10 +2478,11 @@ struct _channelid_xid { static PyObject * _channelid_from_xid(_PyCrossInterpreterData *data) { - struct _channelid_xid *xid = (struct _channelid_xid *)data->data; + struct _channelid_xid *xid = \ + (struct _channelid_xid *)_PyCrossInterpreterData_DATA(data); // It might not be imported yet, so we can't use _get_current_module(). - PyObject *mod = PyImport_ImportModule(MODULE_NAME); + PyObject *mod = PyImport_ImportModule(MODULE_NAME_STR); if (mod == NULL) { return NULL; } @@ -2530,7 +2535,8 @@ _channelid_shared(PyThreadState *tstate, PyObject *obj, { return -1; } - struct _channelid_xid *xid = (struct _channelid_xid *)data->data; + struct _channelid_xid *xid = \ + (struct _channelid_xid *)_PyCrossInterpreterData_DATA(data); xid->cid = ((channelid *)obj)->cid; xid->end = ((channelid *)obj)->end; xid->resolve = ((channelid *)obj)->resolve; @@ -2601,7 +2607,7 @@ static PyType_Slot channelid_typeslots[] = { }; static PyType_Spec channelid_typespec = { - .name = MODULE_NAME ".ChannelID", + .name = MODULE_NAME_STR ".ChannelID", .basicsize = sizeof(channelid), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), @@ -2680,7 +2686,7 @@ _channelend_shared(PyThreadState *tstate, PyObject *obj, if (res < 0) { return -1; } - data->new_object = _channelend_from_xid; + _PyCrossInterpreterData_SET_NEW_OBJECT(data, _channelend_from_xid); return 0; } @@ -3379,7 +3385,7 @@ module_free(void *mod) static struct PyModuleDef moduledef = { .m_base = PyModuleDef_HEAD_INIT, - .m_name = MODULE_NAME, + .m_name = MODULE_NAME_STR, .m_doc = module_doc, .m_size = sizeof(module_state), .m_methods = module_functions, @@ -3390,7 +3396,7 @@ static struct PyModuleDef moduledef = { }; PyMODINIT_FUNC -PyInit__xxinterpchannels(void) +MODINIT_FUNC_NAME(void) { return PyModuleDef_Init(&moduledef); } diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index 537ba9188055dd1..715bb766cac624e 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -8,8 +8,12 @@ #include "Python.h" #include "pycore_crossinterp.h" // struct _xid +#include "_interpreters_common.h" -#define MODULE_NAME "_xxinterpqueues" + +#define MODULE_NAME _xxinterpqueues +#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME) +#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME) #define GLOBAL_MALLOC(TYPE) \ @@ -64,7 +68,7 @@ _get_current_interp(void) static PyObject * _get_current_module(void) { - PyObject *name = PyUnicode_FromString(MODULE_NAME); + PyObject *name = PyUnicode_FromString(MODULE_NAME_STR); if (name == NULL) { return NULL; } @@ -290,6 +294,8 @@ handle_queue_error(int err, PyObject *mod, int64_t qid) case ERR_QUEUES_ALLOC: PyErr_NoMemory(); break; + case -1: + return -1; default: state = get_module_state(mod); assert(state->QueueError != NULL); @@ -316,14 +322,17 @@ struct _queueitem; typedef struct _queueitem { _PyCrossInterpreterData *data; + int fmt; struct _queueitem *next; } _queueitem; static void -_queueitem_init(_queueitem *item, _PyCrossInterpreterData *data) +_queueitem_init(_queueitem *item, + _PyCrossInterpreterData *data, int fmt) { *item = (_queueitem){ .data = data, + .fmt = fmt, }; } @@ -340,14 +349,14 @@ _queueitem_clear(_queueitem *item) } static _queueitem * -_queueitem_new(_PyCrossInterpreterData *data) +_queueitem_new(_PyCrossInterpreterData *data, int fmt) { _queueitem *item = GLOBAL_MALLOC(_queueitem); if (item == NULL) { PyErr_NoMemory(); return NULL; } - _queueitem_init(item, data); + _queueitem_init(item, data, fmt); return item; } @@ -369,9 +378,11 @@ _queueitem_free_all(_queueitem *item) } static void -_queueitem_popped(_queueitem *item, _PyCrossInterpreterData **p_data) +_queueitem_popped(_queueitem *item, + _PyCrossInterpreterData **p_data, int *p_fmt) { *p_data = item->data; + *p_fmt = item->fmt; // We clear them here, so they won't be released in _queueitem_clear(). item->data = NULL; _queueitem_free(item); @@ -389,10 +400,11 @@ typedef struct _queue { _queueitem *first; _queueitem *last; } items; + int fmt; } _queue; static int -_queue_init(_queue *queue, Py_ssize_t maxsize) +_queue_init(_queue *queue, Py_ssize_t maxsize, int fmt) { PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { @@ -404,6 +416,7 @@ _queue_init(_queue *queue, Py_ssize_t maxsize) .items = { .maxsize = maxsize, }, + .fmt = fmt, }; return 0; } @@ -482,7 +495,7 @@ _queue_unlock(_queue *queue) } static int -_queue_add(_queue *queue, _PyCrossInterpreterData *data) +_queue_add(_queue *queue, _PyCrossInterpreterData *data, int fmt) { int err = _queue_lock(queue); if (err < 0) { @@ -498,7 +511,7 @@ _queue_add(_queue *queue, _PyCrossInterpreterData *data) return ERR_QUEUE_FULL; } - _queueitem *item = _queueitem_new(data); + _queueitem *item = _queueitem_new(data, fmt); if (item == NULL) { _queue_unlock(queue); return -1; @@ -518,7 +531,8 @@ _queue_add(_queue *queue, _PyCrossInterpreterData *data) } static int -_queue_next(_queue *queue, _PyCrossInterpreterData **p_data) +_queue_next(_queue *queue, + _PyCrossInterpreterData **p_data, int *p_fmt) { int err = _queue_lock(queue); if (err < 0) { @@ -537,7 +551,7 @@ _queue_next(_queue *queue, _PyCrossInterpreterData **p_data) } queue->items.count -= 1; - _queueitem_popped(item, p_data); + _queueitem_popped(item, p_data, p_fmt); _queue_unlock(queue); return 0; @@ -602,7 +616,7 @@ _queue_clear_interpreter(_queue *queue, int64_t interpid) while (next != NULL) { _queueitem *item = next; next = item->next; - if (item->data->interpid == interpid) { + if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) { if (prev == NULL) { queue->items.first = item->next; } @@ -839,18 +853,26 @@ _queues_decref(_queues *queues, int64_t qid) PyThread_release_lock(queues->mutex); } -static int64_t * +struct queue_id_and_fmt { + int64_t id; + int fmt; +}; + +static struct queue_id_and_fmt * _queues_list_all(_queues *queues, int64_t *count) { - int64_t *qids = NULL; + struct queue_id_and_fmt *qids = NULL; PyThread_acquire_lock(queues->mutex, WAIT_LOCK); - int64_t *ids = PyMem_NEW(int64_t, (Py_ssize_t)(queues->count)); + struct queue_id_and_fmt *ids = PyMem_NEW(struct queue_id_and_fmt, + (Py_ssize_t)(queues->count)); if (ids == NULL) { goto done; } _queueref *ref = queues->head; for (int64_t i=0; ref != NULL; ref = ref->next, i++) { - ids[i] = ref->qid; + ids[i].id = ref->qid; + assert(ref->queue != NULL); + ids[i].fmt = ref->queue->fmt; } *count = queues->count; @@ -886,13 +908,13 @@ _queue_free(_queue *queue) // Create a new queue. static int64_t -queue_create(_queues *queues, Py_ssize_t maxsize) +queue_create(_queues *queues, Py_ssize_t maxsize, int fmt) { _queue *queue = GLOBAL_MALLOC(_queue); if (queue == NULL) { return ERR_QUEUE_ALLOC; } - int err = _queue_init(queue, maxsize); + int err = _queue_init(queue, maxsize, fmt); if (err < 0) { GLOBAL_FREE(queue); return (int64_t)err; @@ -921,7 +943,7 @@ queue_destroy(_queues *queues, int64_t qid) // Push an object onto the queue. static int -queue_put(_queues *queues, int64_t qid, PyObject *obj) +queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt) { // Look up the queue. _queue *queue = NULL; @@ -944,7 +966,7 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj) } // Add the data to the queue. - int res = _queue_add(queue, data); + int res = _queue_add(queue, data, fmt); _queue_unmark_waiter(queue, queues->mutex); if (res != 0) { // We may chain an exception here: @@ -959,7 +981,7 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj) // Pop the next object off the queue. Fail if empty. // XXX Support a "wait" mutex? static int -queue_get(_queues *queues, int64_t qid, PyObject **res) +queue_get(_queues *queues, int64_t qid, PyObject **res, int *p_fmt) { int err; *res = NULL; @@ -975,7 +997,7 @@ queue_get(_queues *queues, int64_t qid, PyObject **res) // Pop off the next item from the queue. _PyCrossInterpreterData *data = NULL; - err = _queue_next(queue, &data); + err = _queue_next(queue, &data, p_fmt); _queue_unmark_waiter(queue, queues->mutex); if (err != 0) { return err; @@ -1062,7 +1084,7 @@ set_external_queue_type(PyObject *module, PyTypeObject *queue_type) } state->queue_type = (PyTypeObject *)Py_NewRef(queue_type); - if (_PyCrossInterpreterData_RegisterClass(queue_type, _queueobj_shared) < 0) { + if (ensure_xid_class(queue_type, _queueobj_shared) < 0) { return -1; } @@ -1130,7 +1152,7 @@ _queueid_xid_free(void *data) static PyObject * _queueobj_from_xid(_PyCrossInterpreterData *data) { - int64_t qid = *(int64_t *)data->data; + int64_t qid = *(int64_t *)_PyCrossInterpreterData_DATA(data); PyObject *qidobj = PyLong_FromLongLong(qid); if (qidobj == NULL) { return NULL; @@ -1140,7 +1162,7 @@ _queueobj_from_xid(_PyCrossInterpreterData *data) if (mod == NULL) { // XXX import it? PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME " module not imported yet"); + MODULE_NAME_STR " module not imported yet"); return NULL; } @@ -1181,7 +1203,7 @@ _queueobj_shared(PyThreadState *tstate, PyObject *queueobj, _PyCrossInterpreterData_Init(data, tstate->interp, raw, NULL, _queueobj_from_xid); Py_DECREF(qidobj); - data->free = _queueid_xid_free; + _PyCrossInterpreterData_SET_FREE(data, _queueid_xid_free); return 0; } @@ -1263,14 +1285,15 @@ qidarg_converter(PyObject *arg, void *ptr) static PyObject * queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"maxsize", NULL}; - Py_ssize_t maxsize = -1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|n:create", kwlist, - &maxsize)) { + static char *kwlist[] = {"maxsize", "fmt", NULL}; + Py_ssize_t maxsize; + int fmt; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "ni:create", kwlist, + &maxsize, &fmt)) { return NULL; } - int64_t qid = queue_create(&_globals.queues, maxsize); + int64_t qid = queue_create(&_globals.queues, maxsize, fmt); if (qid < 0) { (void)handle_queue_error((int)qid, self, qid); return NULL; @@ -1325,7 +1348,7 @@ static PyObject * queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) { int64_t count = 0; - int64_t *qids = _queues_list_all(&_globals.queues, &count); + struct queue_id_and_fmt *qids = _queues_list_all(&_globals.queues, &count); if (qids == NULL) { if (count == 0) { return PyList_New(0); @@ -1336,14 +1359,14 @@ queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) if (ids == NULL) { goto finally; } - int64_t *cur = qids; + struct queue_id_and_fmt *cur = qids; for (int64_t i=0; i < count; cur++, i++) { - PyObject *qidobj = PyLong_FromLongLong(*cur); - if (qidobj == NULL) { + PyObject *item = Py_BuildValue("Li", cur->id, cur->fmt); + if (item == NULL) { Py_SETREF(ids, NULL); break; } - PyList_SET_ITEM(ids, (Py_ssize_t)i, qidobj); + PyList_SET_ITEM(ids, (Py_ssize_t)i, item); } finally: @@ -1359,17 +1382,18 @@ Return the list of IDs for all queues."); static PyObject * queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"qid", "obj", NULL}; + static char *kwlist[] = {"qid", "obj", "fmt", NULL}; qidarg_converter_data qidarg; PyObject *obj; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O:put", kwlist, - qidarg_converter, &qidarg, &obj)) { + int fmt; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&Oi:put", kwlist, + qidarg_converter, &qidarg, &obj, &fmt)) { return NULL; } int64_t qid = qidarg.id; /* Queue up the object. */ - int err = queue_put(&_globals.queues, qid, obj); + int err = queue_put(&_globals.queues, qid, obj, fmt); if (handle_queue_error(err, self, qid)) { return NULL; } @@ -1378,7 +1402,7 @@ queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(queuesmod_put_doc, -"put(qid, obj)\n\ +"put(qid, obj, sharedonly=False)\n\ \n\ Add the object's data to the queue."); @@ -1395,7 +1419,8 @@ queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) int64_t qid = qidarg.id; PyObject *obj = NULL; - int err = queue_get(&_globals.queues, qid, &obj); + int fmt; + int err = queue_get(&_globals.queues, qid, &obj, &fmt); if (err == ERR_QUEUE_EMPTY && dflt != NULL) { assert(obj == NULL); obj = Py_NewRef(dflt); @@ -1403,7 +1428,10 @@ queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) else if (handle_queue_error(err, self, qid)) { return NULL; } - return obj; + + PyObject *res = Py_BuildValue("Oi", obj, fmt); + Py_DECREF(obj); + return res; } PyDoc_STRVAR(queuesmod_get_doc, @@ -1495,6 +1523,33 @@ PyDoc_STRVAR(queuesmod_get_maxsize_doc, \n\ Return the maximum number of items in the queue."); +static PyObject * +queuesmod_get_default_fmt(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:get_default_fmt", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + _queue *queue = NULL; + int err = _queues_lookup(&_globals.queues, qid, &queue); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + int fmt = queue->fmt; + _queue_unmark_waiter(queue, _globals.queues.mutex); + return PyLong_FromLong(fmt); +} + +PyDoc_STRVAR(queuesmod_get_default_fmt_doc, +"get_default_fmt(qid)\n\ +\n\ +Return the default format to use for the queue."); + static PyObject * queuesmod_is_full(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1589,6 +1644,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, queuesmod_release_doc}, {"get_maxsize", _PyCFunction_CAST(queuesmod_get_maxsize), METH_VARARGS | METH_KEYWORDS, queuesmod_get_maxsize_doc}, + {"get_default_fmt", _PyCFunction_CAST(queuesmod_get_default_fmt), + METH_VARARGS | METH_KEYWORDS, queuesmod_get_default_fmt_doc}, {"is_full", _PyCFunction_CAST(queuesmod_is_full), METH_VARARGS | METH_KEYWORDS, queuesmod_is_full_doc}, {"get_count", _PyCFunction_CAST(queuesmod_get_count), @@ -1670,7 +1727,7 @@ module_free(void *mod) static struct PyModuleDef moduledef = { .m_base = PyModuleDef_HEAD_INIT, - .m_name = MODULE_NAME, + .m_name = MODULE_NAME_STR, .m_doc = module_doc, .m_size = sizeof(module_state), .m_methods = module_functions, @@ -1681,7 +1738,7 @@ static struct PyModuleDef moduledef = { }; PyMODINIT_FUNC -PyInit__xxinterpqueues(void) +MODINIT_FUNC_NAME(void) { return PyModuleDef_Init(&moduledef); } diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 4e9e13457a9eb3f..28c2f9c08bc0daf 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -19,8 +19,12 @@ #include "interpreteridobject.h" #include "marshal.h" // PyMarshal_ReadObjectFromString() +#include "_interpreters_common.h" -#define MODULE_NAME "_xxsubinterpreters" + +#define MODULE_NAME _xxsubinterpreters +#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME) +#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME) static PyInterpreterState * @@ -125,7 +129,7 @@ get_interpid_obj(PyInterpreterState *interp) static PyObject * _get_current_module(void) { - PyObject *name = PyUnicode_FromString(MODULE_NAME); + PyObject *name = PyUnicode_FromString(MODULE_NAME_STR); if (name == NULL) { return NULL; } @@ -152,16 +156,16 @@ typedef struct { static PyObject * xibufferview_from_xid(PyTypeObject *cls, _PyCrossInterpreterData *data) { - assert(data->data != NULL); - assert(data->obj == NULL); - assert(data->interpid >= 0); + assert(_PyCrossInterpreterData_DATA(data) != NULL); + assert(_PyCrossInterpreterData_OBJ(data) == NULL); + assert(_PyCrossInterpreterData_INTERPID(data) >= 0); XIBufferViewObject *self = PyObject_Malloc(sizeof(XIBufferViewObject)); if (self == NULL) { return NULL; } PyObject_Init((PyObject *)self, cls); - self->view = (Py_buffer *)data->data; - self->interpid = data->interpid; + self->view = (Py_buffer *)_PyCrossInterpreterData_DATA(data); + self->interpid = _PyCrossInterpreterData_INTERPID(data); return (PyObject *)self; } @@ -209,7 +213,7 @@ static PyType_Slot XIBufferViewType_slots[] = { }; static PyType_Spec XIBufferViewType_spec = { - .name = MODULE_NAME ".CrossInterpreterBufferView", + .name = MODULE_NAME_STR ".CrossInterpreterBufferView", .basicsize = sizeof(XIBufferViewObject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), @@ -267,8 +271,7 @@ register_memoryview_xid(PyObject *mod, PyTypeObject **p_state) *p_state = cls; // Register XID for the builtin memoryview type. - if (_PyCrossInterpreterData_RegisterClass( - &PyMemoryView_Type, _memoryview_shared) < 0) { + if (ensure_xid_class(&PyMemoryView_Type, _memoryview_shared) < 0) { return -1; } // We don't ever bother un-registering memoryview. @@ -303,7 +306,7 @@ _get_current_module_state(void) if (mod == NULL) { // XXX import it? PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME " module not imported yet"); + MODULE_NAME_STR " module not imported yet"); return NULL; } module_state *state = get_module_state(mod); @@ -372,9 +375,7 @@ check_code_object(PyCodeObject *code) } // We trust that no code objects under co_consts have unbound cell vars. - if (code->co_executors != NULL - || code->_co_instrumentation_version > 0) - { + if (_PyCode_HAS_EXECUTORS(code) || _PyCode_HAS_INSTRUMENTATION(code)) { return "only basic functions are supported"; } if (code->_co_monitoring != NULL) { @@ -602,7 +603,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) // Destroy the interpreter. PyThreadState *tstate = PyThreadState_New(interp); - tstate->_whence = _PyThreadState_WHENCE_INTERP; + _PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_INTERP); // XXX Possible GILState issues? PyThreadState *save_tstate = PyThreadState_Swap(tstate); Py_EndInterpreter(tstate); @@ -691,7 +692,7 @@ static PyObject * interp_set___main___attrs(PyObject *self, PyObject *args) { PyObject *id, *updates; - if (!PyArg_ParseTuple(args, "OO:" MODULE_NAME ".set___main___attrs", + if (!PyArg_ParseTuple(args, "OO:" MODULE_NAME_STR ".set___main___attrs", &id, &updates)) { return NULL; @@ -856,18 +857,18 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) PyObject *id, *code; PyObject *shared = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O:" MODULE_NAME ".exec", kwlist, + "OO|O:" MODULE_NAME_STR ".exec", kwlist, &id, &code, &shared)) { return NULL; } const char *expected = "a string, a function, or a code object"; if (PyUnicode_Check(code)) { - code = (PyObject *)convert_script_arg(code, MODULE_NAME ".exec", + code = (PyObject *)convert_script_arg(code, MODULE_NAME_STR ".exec", "argument 2", expected); } else { - code = (PyObject *)convert_code_arg(code, MODULE_NAME ".exec", + code = (PyObject *)convert_code_arg(code, MODULE_NAME_STR ".exec", "argument 2", expected); } if (code == NULL) { @@ -901,6 +902,56 @@ The code/function must not take any arguments or be a closure\n\ If a function is provided, its code object is used and all its state\n\ is ignored, including its __globals__ dict."); +static PyObject * +interp_call(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "callable", "args", "kwargs", NULL}; + PyObject *id, *callable; + PyObject *args_obj = NULL; + PyObject *kwargs_obj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OO|OO:" MODULE_NAME_STR ".call", kwlist, + &id, &callable, &args_obj, &kwargs_obj)) { + return NULL; + } + + if (args_obj != NULL) { + PyErr_SetString(PyExc_ValueError, "got unexpected args"); + return NULL; + } + if (kwargs_obj != NULL) { + PyErr_SetString(PyExc_ValueError, "got unexpected kwargs"); + return NULL; + } + + PyObject *code = (PyObject *)convert_code_arg(callable, MODULE_NAME_STR ".call", + "argument 2", "a function"); + if (code == NULL) { + return NULL; + } + + PyObject *excinfo = NULL; + int res = _interp_exec(self, id, code, NULL, &excinfo); + Py_DECREF(code); + if (res < 0) { + assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); + return excinfo; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(call_doc, +"call(id, callable, args=None, kwargs=None)\n\ +\n\ +Call the provided object in the identified interpreter.\n\ +Pass the given args and kwargs, if possible.\n\ +\n\ +\"callable\" may be a plain function with no free vars that takes\n\ +no arguments.\n\ +\n\ +The function's code object is used and all its state\n\ +is ignored, including its __globals__ dict."); + static PyObject * interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) { @@ -908,12 +959,12 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) PyObject *id, *script; PyObject *shared = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OU|O:" MODULE_NAME ".run_string", kwlist, + "OU|O:" MODULE_NAME_STR ".run_string", kwlist, &id, &script, &shared)) { return NULL; } - script = (PyObject *)convert_script_arg(script, MODULE_NAME ".exec", + script = (PyObject *)convert_script_arg(script, MODULE_NAME_STR ".exec", "argument 2", "a string"); if (script == NULL) { return NULL; @@ -934,7 +985,7 @@ PyDoc_STRVAR(run_string_doc, \n\ Execute the provided string in the identified interpreter.\n\ \n\ -(See " MODULE_NAME ".exec()."); +(See " MODULE_NAME_STR ".exec()."); static PyObject * interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) @@ -943,12 +994,12 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) PyObject *id, *func; PyObject *shared = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O:" MODULE_NAME ".run_func", kwlist, + "OO|O:" MODULE_NAME_STR ".run_func", kwlist, &id, &func, &shared)) { return NULL; } - PyCodeObject *code = convert_code_arg(func, MODULE_NAME ".exec", + PyCodeObject *code = convert_code_arg(func, MODULE_NAME_STR ".exec", "argument 2", "a function or a code object"); if (code == NULL) { @@ -972,7 +1023,7 @@ Execute the body of the provided function in the identified interpreter.\n\ Code objects are also supported. In both cases, closures and args\n\ are not supported. Methods and other callables are not supported either.\n\ \n\ -(See " MODULE_NAME ".exec()."); +(See " MODULE_NAME_STR ".exec()."); static PyObject * @@ -1084,6 +1135,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, is_running_doc}, {"exec", _PyCFunction_CAST(interp_exec), METH_VARARGS | METH_KEYWORDS, exec_doc}, + {"call", _PyCFunction_CAST(interp_call), + METH_VARARGS | METH_KEYWORDS, call_doc}, {"run_string", _PyCFunction_CAST(interp_run_string), METH_VARARGS | METH_KEYWORDS, run_string_doc}, {"run_func", _PyCFunction_CAST(interp_run_func), @@ -1112,6 +1165,7 @@ The 'interpreters' module provides a more convenient interface."); static int module_exec(PyObject *mod) { + PyInterpreterState *interp = PyInterpreterState_Get(); module_state *state = get_module_state(mod); // exceptions @@ -1121,6 +1175,11 @@ module_exec(PyObject *mod) if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterNotFoundError) < 0) { goto error; } + PyObject *PyExc_NotShareableError = \ + _PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError; + if (PyModule_AddType(mod, (PyTypeObject *)PyExc_NotShareableError) < 0) { + goto error; + } if (register_memoryview_xid(mod, &state->XIBufferViewType) < 0) { goto error; @@ -1166,7 +1225,7 @@ module_free(void *mod) static struct PyModuleDef moduledef = { .m_base = PyModuleDef_HEAD_INIT, - .m_name = MODULE_NAME, + .m_name = MODULE_NAME_STR, .m_doc = module_doc, .m_size = sizeof(module_state), .m_methods = module_functions, @@ -1177,7 +1236,7 @@ static struct PyModuleDef moduledef = { }; PyMODINIT_FUNC -PyInit__xxsubinterpreters(void) +MODINIT_FUNC_NAME(void) { return PyModuleDef_Init(&moduledef); } diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index e133b4d3c444809..6ea9f64d6285304 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -502,7 +502,6 @@ static int fuzz_elementtree_parsewhole(const char* data, size_t size) { } #define MAX_PYCOMPILE_TEST_SIZE 16384 -static char pycompile_scratch[MAX_PYCOMPILE_TEST_SIZE]; static const int start_vals[] = {Py_eval_input, Py_single_input, Py_file_input}; const size_t NUM_START_VALS = sizeof(start_vals) / sizeof(start_vals[0]); @@ -531,6 +530,8 @@ static int fuzz_pycompile(const char* data, size_t size) { unsigned char optimize_idx = (unsigned char) data[1]; int optimize = optimize_vals[optimize_idx % NUM_OPTIMIZE_VALS]; + char pycompile_scratch[MAX_PYCOMPILE_TEST_SIZE]; + // Create a NUL-terminated C string from the remaining input memcpy(pycompile_scratch, data + 2, size - 2); // Put a NUL terminator just after the copied data. (Space was reserved already.) @@ -549,7 +550,13 @@ static int fuzz_pycompile(const char* data, size_t size) { PyObject *result = Py_CompileStringExFlags(pycompile_scratch, "<fuzz input>", start, flags, optimize); if (result == NULL) { - /* compilation failed, most likely from a syntax error */ + /* Compilation failed, most likely from a syntax error. If it was a + SystemError we abort. There's no non-bug reason to raise a + SystemError. */ + if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_SystemError)) { + PyErr_Print(); + abort(); + } PyErr_Clear(); } else { Py_DECREF(result); diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index b97ade6126fa08f..df09d9d84789f73 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -868,6 +868,21 @@ array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) return (PyObject *)np; } +/*[clinic input] +array.array.clear + +Remove all items from the array. +[clinic start generated code]*/ + +static PyObject * +array_array_clear_impl(arrayobject *self) +/*[clinic end generated code: output=5efe0417062210a9 input=5dffa30e94e717a4]*/ +{ + if (array_resize(self, 0) == -1) { + return NULL; + } + Py_RETURN_NONE; +} /*[clinic input] array.array.__copy__ @@ -2342,6 +2357,7 @@ static PyMethodDef array_methods[] = { ARRAY_ARRAY_APPEND_METHODDEF ARRAY_ARRAY_BUFFER_INFO_METHODDEF ARRAY_ARRAY_BYTESWAP_METHODDEF + ARRAY_ARRAY_CLEAR_METHODDEF ARRAY_ARRAY___COPY___METHODDEF ARRAY_ARRAY_COUNT_METHODDEF ARRAY_ARRAY___DEEPCOPY___METHODDEF diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index 305ade17b1f1aa8..b5639d5cf10a22a 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -668,7 +668,7 @@ _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *se static PyObject * _multibytecodec_MultibyteStreamWriter_reset(MultibyteStreamWriterObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "reset() takes no arguments"); return NULL; } @@ -682,4 +682,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=219a363662d2fbff input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ee767a6d93c7108a input=a9049054013a1b77]*/ diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 5d3c16a98423baa..2125da437963d2d 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -973,7 +973,8 @@ _multibytecodec_MultibyteIncrementalEncoder_setstate_impl(MultibyteIncrementalEn if (_PyLong_AsByteArray(statelong, statebytes, sizeof(statebytes), 1 /* little-endian */ , - 0 /* unsigned */ ) < 0) { + 0 /* unsigned */ , + 1 /* with_exceptions */) < 0) { goto errorexit; } @@ -1255,7 +1256,8 @@ _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDe if (_PyLong_AsByteArray(statelong, statebytes, sizeof(statebytes), 1 /* little-endian */ , - 0 /* unsigned */ ) < 0) { + 0 /* unsigned */ , + 1 /* with_exceptions */) < 0) { return NULL; } diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index d941c280a4300ba..6a9c8ff6d8fdd9e 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -120,7 +120,7 @@ _asyncio_Future_exception_impl(FutureObj *self, PyTypeObject *cls); static PyObject * _asyncio_Future_exception(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "exception() takes no arguments"); return NULL; } @@ -453,7 +453,7 @@ _asyncio_Future_get_loop_impl(FutureObj *self, PyTypeObject *cls); static PyObject * _asyncio_Future_get_loop(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_loop() takes no arguments"); return NULL; } @@ -1487,4 +1487,4 @@ _asyncio_current_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=f3864d8e2af7635f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b26155080c82c472 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 12fea806ab52093..1c0f37442ab3509 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -297,7 +297,9 @@ _codecs_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { @@ -1099,7 +1101,9 @@ _codecs_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_ if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { @@ -1175,7 +1179,9 @@ _codecs_raw_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ss if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { @@ -1644,7 +1650,9 @@ _codecs_readbuffer_encode(PyObject *module, PyObject *const *args, Py_ssize_t na if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { @@ -2738,4 +2746,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=d8d9e372f7ccba35 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e50d5fdf65bd45fa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_collectionsmodule.c.h b/Modules/clinic/_collectionsmodule.c.h index 591ab50c76a8e8f..fa0ff2133a05d2b 100644 --- a/Modules/clinic/_collectionsmodule.c.h +++ b/Modules/clinic/_collectionsmodule.c.h @@ -2,9 +2,566 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +PyDoc_STRVAR(deque_pop__doc__, +"pop($self, /)\n" +"--\n" +"\n" +"Remove and return the rightmost element."); + +#define DEQUE_POP_METHODDEF \ + {"pop", (PyCFunction)deque_pop, METH_NOARGS, deque_pop__doc__}, + +static PyObject * +deque_pop_impl(dequeobject *deque); + +static PyObject * +deque_pop(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_pop_impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_popleft__doc__, +"popleft($self, /)\n" +"--\n" +"\n" +"Remove and return the leftmost element."); + +#define DEQUE_POPLEFT_METHODDEF \ + {"popleft", (PyCFunction)deque_popleft, METH_NOARGS, deque_popleft__doc__}, + +static PyObject * +deque_popleft_impl(dequeobject *deque); + +static PyObject * +deque_popleft(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_popleft_impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_append__doc__, +"append($self, item, /)\n" +"--\n" +"\n" +"Add an element to the right side of the deque."); + +#define DEQUE_APPEND_METHODDEF \ + {"append", (PyCFunction)deque_append, METH_O, deque_append__doc__}, + +static PyObject * +deque_append_impl(dequeobject *deque, PyObject *item); + +static PyObject * +deque_append(dequeobject *deque, PyObject *item) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_append_impl(deque, item); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_appendleft__doc__, +"appendleft($self, item, /)\n" +"--\n" +"\n" +"Add an element to the left side of the deque."); + +#define DEQUE_APPENDLEFT_METHODDEF \ + {"appendleft", (PyCFunction)deque_appendleft, METH_O, deque_appendleft__doc__}, + +static PyObject * +deque_appendleft_impl(dequeobject *deque, PyObject *item); + +static PyObject * +deque_appendleft(dequeobject *deque, PyObject *item) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_appendleft_impl(deque, item); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_extend__doc__, +"extend($self, iterable, /)\n" +"--\n" +"\n" +"Extend the right side of the deque with elements from the iterable."); + +#define DEQUE_EXTEND_METHODDEF \ + {"extend", (PyCFunction)deque_extend, METH_O, deque_extend__doc__}, + +static PyObject * +deque_extend_impl(dequeobject *deque, PyObject *iterable); + +static PyObject * +deque_extend(dequeobject *deque, PyObject *iterable) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_extend_impl(deque, iterable); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_extendleft__doc__, +"extendleft($self, iterable, /)\n" +"--\n" +"\n" +"Extend the left side of the deque with elements from the iterable."); + +#define DEQUE_EXTENDLEFT_METHODDEF \ + {"extendleft", (PyCFunction)deque_extendleft, METH_O, deque_extendleft__doc__}, + +static PyObject * +deque_extendleft_impl(dequeobject *deque, PyObject *iterable); + +static PyObject * +deque_extendleft(dequeobject *deque, PyObject *iterable) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_extendleft_impl(deque, iterable); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of a deque."); + +#define DEQUE_COPY_METHODDEF \ + {"copy", (PyCFunction)deque_copy, METH_NOARGS, deque_copy__doc__}, + +static PyObject * +deque_copy_impl(dequeobject *deque); + +static PyObject * +deque_copy(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_copy_impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque___copy____doc__, +"__copy__($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of a deque."); + +#define DEQUE___COPY___METHODDEF \ + {"__copy__", (PyCFunction)deque___copy__, METH_NOARGS, deque___copy____doc__}, + +static PyObject * +deque___copy___impl(dequeobject *deque); + +static PyObject * +deque___copy__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque___copy___impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_clearmethod__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all elements from the deque."); + +#define DEQUE_CLEARMETHOD_METHODDEF \ + {"clear", (PyCFunction)deque_clearmethod, METH_NOARGS, deque_clearmethod__doc__}, + +static PyObject * +deque_clearmethod_impl(dequeobject *deque); + +static PyObject * +deque_clearmethod(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_clearmethod_impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_rotate__doc__, +"rotate($self, n=1, /)\n" +"--\n" +"\n" +"Rotate the deque n steps to the right. If n is negative, rotates left."); + +#define DEQUE_ROTATE_METHODDEF \ + {"rotate", _PyCFunction_CAST(deque_rotate), METH_FASTCALL, deque_rotate__doc__}, + +static PyObject * +deque_rotate_impl(dequeobject *deque, Py_ssize_t n); + +static PyObject * +deque_rotate(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t n = 1; + + if (!_PyArg_CheckPositional("rotate", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + n = ival; + } +skip_optional: + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_rotate_impl(deque, n); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(deque_reverse__doc__, +"reverse($self, /)\n" +"--\n" +"\n" +"Reverse *IN PLACE*."); + +#define DEQUE_REVERSE_METHODDEF \ + {"reverse", (PyCFunction)deque_reverse, METH_NOARGS, deque_reverse__doc__}, + +static PyObject * +deque_reverse_impl(dequeobject *deque); + +static PyObject * +deque_reverse(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_reverse_impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_count__doc__, +"count($self, value, /)\n" +"--\n" +"\n" +"Return number of occurrences of value."); + +#define DEQUE_COUNT_METHODDEF \ + {"count", (PyCFunction)deque_count, METH_O, deque_count__doc__}, + +static PyObject * +deque_count_impl(dequeobject *deque, PyObject *v); + +static PyObject * +deque_count(dequeobject *deque, PyObject *v) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_count_impl(deque, v); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_index__doc__, +"index($self, value, [start, [stop]])\n" +"--\n" +"\n" +"Return first index of value.\n" +"\n" +"Raises ValueError if the value is not present."); + +#define DEQUE_INDEX_METHODDEF \ + {"index", _PyCFunction_CAST(deque_index), METH_FASTCALL, deque_index__doc__}, + +static PyObject * +deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, + Py_ssize_t stop); + +static PyObject * +deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *v; + Py_ssize_t start = 0; + Py_ssize_t stop = Py_SIZE(deque); + + if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { + goto exit; + } + v = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[2], &stop)) { + goto exit; + } +skip_optional: + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_index_impl(deque, v, start, stop); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(deque_insert__doc__, +"insert($self, index, value, /)\n" +"--\n" +"\n" +"Insert value before index."); + +#define DEQUE_INSERT_METHODDEF \ + {"insert", _PyCFunction_CAST(deque_insert), METH_FASTCALL, deque_insert__doc__}, + +static PyObject * +deque_insert_impl(dequeobject *deque, Py_ssize_t index, PyObject *value); + +static PyObject * +deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t index; + PyObject *value; + + if (!_PyArg_CheckPositional("insert", nargs, 2, 2)) { + goto exit; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + index = ival; + } + value = args[1]; + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_insert_impl(deque, index, value); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(deque_remove__doc__, +"remove($self, value, /)\n" +"--\n" +"\n" +"Remove first occurrence of value."); + +#define DEQUE_REMOVE_METHODDEF \ + {"remove", (PyCFunction)deque_remove, METH_O, deque_remove__doc__}, + +static PyObject * +deque_remove_impl(dequeobject *deque, PyObject *value); + +static PyObject * +deque_remove(dequeobject *deque, PyObject *value) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_remove_impl(deque, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n" +"Return state information for pickling."); + +#define DEQUE___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)deque___reduce__, METH_NOARGS, deque___reduce____doc__}, + +static PyObject * +deque___reduce___impl(dequeobject *deque); + +static PyObject * +deque___reduce__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque___reduce___impl(deque); +} + +PyDoc_STRVAR(deque_init__doc__, +"deque([iterable[, maxlen]])\n" +"--\n" +"\n" +"A list-like sequence optimized for data accesses near its endpoints."); + +static int +deque_init_impl(dequeobject *deque, PyObject *iterable, PyObject *maxlenobj); + +static int +deque_init(PyObject *deque, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(iterable), &_Py_ID(maxlen), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"iterable", "maxlen", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "deque", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *iterable = NULL; + PyObject *maxlenobj = NULL; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 2, 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[0]) { + iterable = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + maxlenobj = fastargs[1]; +skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_init_impl((dequeobject *)deque, iterable, maxlenobj); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(deque___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"Return the size of the deque in memory, in bytes."); + +#define DEQUE___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)deque___sizeof__, METH_NOARGS, deque___sizeof____doc__}, + +static PyObject * +deque___sizeof___impl(dequeobject *deque); + +static PyObject * +deque___sizeof__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque___sizeof___impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque___reversed____doc__, +"__reversed__($self, /)\n" +"--\n" +"\n" +"Return a reverse iterator over the deque."); + +#define DEQUE___REVERSED___METHODDEF \ + {"__reversed__", (PyCFunction)deque___reversed__, METH_NOARGS, deque___reversed____doc__}, + +static PyObject * +deque___reversed___impl(dequeobject *deque); + +static PyObject * +deque___reversed__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque___reversed___impl(deque); +} + PyDoc_STRVAR(_collections__count_elements__doc__, "_count_elements($module, mapping, iterable, /)\n" "--\n" @@ -72,4 +629,4 @@ tuplegetter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=c896a72f8c45930d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=64c32b16df7be07a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_curses_panel.c.h b/Modules/clinic/_curses_panel.c.h index 7945d93b5433f7d..457f71370afda9f 100644 --- a/Modules/clinic/_curses_panel.c.h +++ b/Modules/clinic/_curses_panel.c.h @@ -19,7 +19,7 @@ _curses_panel_panel_bottom_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * _curses_panel_panel_bottom(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "bottom() takes no arguments"); return NULL; } @@ -43,7 +43,7 @@ _curses_panel_panel_hide_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * _curses_panel_panel_hide(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "hide() takes no arguments"); return NULL; } @@ -65,7 +65,7 @@ _curses_panel_panel_show_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * _curses_panel_panel_show(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "show() takes no arguments"); return NULL; } @@ -87,7 +87,7 @@ _curses_panel_panel_top_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * _curses_panel_panel_top(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "top() takes no arguments"); return NULL; } @@ -327,7 +327,7 @@ _curses_panel_panel_userptr_impl(PyCursesPanelObject *self, static PyObject * _curses_panel_panel_userptr(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "userptr() takes no arguments"); return NULL; } @@ -418,4 +418,4 @@ _curses_panel_update_panels(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _curses_panel_update_panels_impl(module); } -/*[clinic end generated code: output=636beecf71d96ff1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7bac14e9a1194c87 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_datetimemodule.c.h b/Modules/clinic/_datetimemodule.c.h index 1ee50fc2a137621..48499e0aaf7783d 100644 --- a/Modules/clinic/_datetimemodule.c.h +++ b/Modules/clinic/_datetimemodule.c.h @@ -82,6 +82,207 @@ iso_calendar_date_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(datetime_date_replace__doc__, +"replace($self, /, year=unchanged, month=unchanged, day=unchanged)\n" +"--\n" +"\n" +"Return date with new specified fields."); + +#define DATETIME_DATE_REPLACE_METHODDEF \ + {"replace", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL|METH_KEYWORDS, datetime_date_replace__doc__}, + +static PyObject * +datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, + int day); + +static PyObject * +datetime_date_replace(PyDateTime_Date *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "month", "day", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int year = GET_YEAR(self); + int month = GET_MONTH(self); + int day = GET_DAY(self); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + year = PyLong_AsInt(args[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + month = PyLong_AsInt(args[1]); + if (month == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + day = PyLong_AsInt(args[2]); + if (day == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_pos: + return_value = datetime_date_replace_impl(self, year, month, day); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time_replace__doc__, +"replace($self, /, hour=unchanged, minute=unchanged, second=unchanged,\n" +" microsecond=unchanged, tzinfo=unchanged, *, fold=unchanged)\n" +"--\n" +"\n" +"Return time with new specified fields."); + +#define DATETIME_TIME_REPLACE_METHODDEF \ + {"replace", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL|METH_KEYWORDS, datetime_time_replace__doc__}, + +static PyObject * +datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold); + +static PyObject * +datetime_time_replace(PyDateTime_Time *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[6]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int hour = TIME_GET_HOUR(self); + int minute = TIME_GET_MINUTE(self); + int second = TIME_GET_SECOND(self); + int microsecond = TIME_GET_MICROSECOND(self); + PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; + int fold = TIME_GET_FOLD(self); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 5, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + hour = PyLong_AsInt(args[0]); + if (hour == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + minute = PyLong_AsInt(args[1]); + if (minute == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[2]) { + second = PyLong_AsInt(args[2]); + if (second == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[3]) { + microsecond = PyLong_AsInt(args[3]); + if (microsecond == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[4]) { + tzinfo = args[4]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + fold = PyLong_AsInt(args[5]); + if (fold == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = datetime_time_replace_impl(self, hour, minute, second, microsecond, tzinfo, fold); + +exit: + return return_value; +} + PyDoc_STRVAR(datetime_datetime_now__doc__, "now($type, /, tz=None)\n" "--\n" @@ -146,4 +347,153 @@ datetime_datetime_now(PyTypeObject *type, PyObject *const *args, Py_ssize_t narg exit: return return_value; } -/*[clinic end generated code: output=562813dd3e164794 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(datetime_datetime_replace__doc__, +"replace($self, /, year=unchanged, month=unchanged, day=unchanged,\n" +" hour=unchanged, minute=unchanged, second=unchanged,\n" +" microsecond=unchanged, tzinfo=unchanged, *, fold=unchanged)\n" +"--\n" +"\n" +"Return datetime with new specified fields."); + +#define DATETIME_DATETIME_REPLACE_METHODDEF \ + {"replace", _PyCFunction_CAST(datetime_datetime_replace), METH_FASTCALL|METH_KEYWORDS, datetime_datetime_replace__doc__}, + +static PyObject * +datetime_datetime_replace_impl(PyDateTime_DateTime *self, int year, + int month, int day, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold); + +static PyObject * +datetime_datetime_replace(PyDateTime_DateTime *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 9 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "month", "day", "hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[9]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int year = GET_YEAR(self); + int month = GET_MONTH(self); + int day = GET_DAY(self); + int hour = DATE_GET_HOUR(self); + int minute = DATE_GET_MINUTE(self); + int second = DATE_GET_SECOND(self); + int microsecond = DATE_GET_MICROSECOND(self); + PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; + int fold = DATE_GET_FOLD(self); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 8, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + year = PyLong_AsInt(args[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + month = PyLong_AsInt(args[1]); + if (month == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[2]) { + day = PyLong_AsInt(args[2]); + if (day == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[3]) { + hour = PyLong_AsInt(args[3]); + if (hour == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[4]) { + minute = PyLong_AsInt(args[4]); + if (minute == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[5]) { + second = PyLong_AsInt(args[5]); + if (second == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[6]) { + microsecond = PyLong_AsInt(args[6]); + if (microsecond == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[7]) { + tzinfo = args[7]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + fold = PyLong_AsInt(args[8]); + if (fold == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = datetime_datetime_replace_impl(self, year, month, day, hour, minute, second, microsecond, tzinfo, fold); + +exit: + return return_value; +} +/*[clinic end generated code: output=c7a04b865b1e0890 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_dbmmodule.c.h b/Modules/clinic/_dbmmodule.c.h index 5a4aba2825e03a6..d06271e18a49b25 100644 --- a/Modules/clinic/_dbmmodule.c.h +++ b/Modules/clinic/_dbmmodule.c.h @@ -37,7 +37,7 @@ _dbm_dbm_keys_impl(dbmobject *self, PyTypeObject *cls); static PyObject * _dbm_dbm_keys(dbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "keys() takes no arguments"); return NULL; } @@ -149,7 +149,7 @@ _dbm_dbm_clear_impl(dbmobject *self, PyTypeObject *cls); static PyObject * _dbm_dbm_clear(dbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "clear() takes no arguments"); return NULL; } @@ -218,4 +218,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=96fdd4bd7bd256c5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=743ce0cea116747e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index 02375c8a61e73e4..9622591a1aa8552 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -87,7 +87,7 @@ _elementtree_Element___copy___impl(ElementObject *self, PyTypeObject *cls); static PyObject * _elementtree_Element___copy__(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__copy__() takes no arguments"); return NULL; } @@ -644,7 +644,7 @@ _elementtree_Element_itertext_impl(ElementObject *self, PyTypeObject *cls); static PyObject * _elementtree_Element_itertext(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "itertext() takes no arguments"); return NULL; } @@ -1219,4 +1219,4 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=8fdaa17d3262800a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=218ec9e6a889f796 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index c7164e519d0e7de..626e4678809d4f5 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -106,7 +106,7 @@ _gdbm_gdbm_keys_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_keys(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "keys() takes no arguments"); return NULL; } @@ -132,7 +132,7 @@ _gdbm_gdbm_firstkey_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_firstkey(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "firstkey() takes no arguments"); return NULL; } @@ -211,7 +211,7 @@ _gdbm_gdbm_reorganize_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_reorganize(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "reorganize() takes no arguments"); return NULL; } @@ -236,7 +236,7 @@ _gdbm_gdbm_sync_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_sync(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "sync() takes no arguments"); return NULL; } @@ -258,7 +258,7 @@ _gdbm_gdbm_clear_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_clear(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "clear() takes no arguments"); return NULL; } @@ -340,4 +340,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=c5ee922363d5a81f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6b4c19905ac9967d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lsprof.c.h b/Modules/clinic/_lsprof.c.h index dfc003eb54774ca..b3b7fda5660bfd5 100644 --- a/Modules/clinic/_lsprof.c.h +++ b/Modules/clinic/_lsprof.c.h @@ -39,10 +39,10 @@ _lsprof_Profiler_getstats_impl(ProfilerObject *self, PyTypeObject *cls); static PyObject * _lsprof_Profiler_getstats(ProfilerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getstats() takes no arguments"); return NULL; } return _lsprof_Profiler_getstats_impl(self, cls); } -/*[clinic end generated code: output=0615a53cce828f06 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5c9d87d89863dc83 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_pickle.c.h b/Modules/clinic/_pickle.c.h index fb086925e3941d6..5a6ae7be6b6ea77 100644 --- a/Modules/clinic/_pickle.c.h +++ b/Modules/clinic/_pickle.c.h @@ -328,7 +328,7 @@ _pickle_Unpickler_load_impl(UnpicklerObject *self, PyTypeObject *cls); static PyObject * _pickle_Unpickler_load(UnpicklerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "load() takes no arguments"); return NULL; } @@ -1077,4 +1077,4 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=ebe78653233827a6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bd63c85a8737b0aa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index b3b6b8e96c135e4..6f4c715c7229651 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -278,7 +278,7 @@ _queue_SimpleQueue_get_nowait(simplequeueobject *self, PyTypeObject *cls, PyObje { PyObject *return_value = NULL; - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_nowait() takes no arguments"); goto exit; } @@ -349,4 +349,4 @@ _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=242950edc8f7dfd7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=44a718f40072018a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 19c0f619b92f45c..2940f16a2cb7f6a 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -1297,7 +1297,9 @@ _ssl_RAND_add(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&view, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&view, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &view, PyBUF_SIMPLE) != 0) { @@ -1662,4 +1664,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=6342ea0062ab16c7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fd1c3378fbba5240 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index fea30e778381dea..bb516be37ec3f09 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -3141,4 +3141,26 @@ clone_with_conv_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py exit: return return_value; } -/*[clinic end generated code: output=90743ac900d60f9f input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_testclinic_TestClass_meth_method_no_params__doc__, +"meth_method_no_params($self, /)\n" +"--\n" +"\n"); + +#define _TESTCLINIC_TESTCLASS_METH_METHOD_NO_PARAMS_METHODDEF \ + {"meth_method_no_params", _PyCFunction_CAST(_testclinic_TestClass_meth_method_no_params), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testclinic_TestClass_meth_method_no_params__doc__}, + +static PyObject * +_testclinic_TestClass_meth_method_no_params_impl(PyObject *self, + PyTypeObject *cls); + +static PyObject * +_testclinic_TestClass_meth_method_no_params(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "meth_method_no_params() takes no arguments"); + return NULL; + } + return _testclinic_TestClass_meth_method_no_params_impl(self, cls); +} +/*[clinic end generated code: output=6520c1ca5392a3f0 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testmultiphase.c.h b/Modules/clinic/_testmultiphase.c.h index c0a00954c16cbe2..7ac361ece1acc3e 100644 --- a/Modules/clinic/_testmultiphase.c.h +++ b/Modules/clinic/_testmultiphase.c.h @@ -27,7 +27,7 @@ _testmultiphase_StateAccessType_get_defining_module_impl(StateAccessTypeObject * static PyObject * _testmultiphase_StateAccessType_get_defining_module(StateAccessTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_defining_module() takes no arguments"); return NULL; } @@ -50,7 +50,7 @@ _testmultiphase_StateAccessType_getmodulebydef_bad_def_impl(StateAccessTypeObjec static PyObject * _testmultiphase_StateAccessType_getmodulebydef_bad_def(StateAccessTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getmodulebydef_bad_def() takes no arguments"); return NULL; } @@ -156,10 +156,10 @@ _testmultiphase_StateAccessType_get_count_impl(StateAccessTypeObject *self, static PyObject * _testmultiphase_StateAccessType_get_count(StateAccessTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_count() takes no arguments"); return NULL; } return _testmultiphase_StateAccessType_get_count_impl(self, cls); } -/*[clinic end generated code: output=d8c262af27b3b98d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2c199bad52e9cda7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index d1052f38919ddef..468457e624c6916 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -151,6 +151,76 @@ _winapi_ConnectNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t nar return return_value; } +PyDoc_STRVAR(_winapi_CreateEventW__doc__, +"CreateEventW($module, /, security_attributes, manual_reset,\n" +" initial_state, name)\n" +"--\n" +"\n"); + +#define _WINAPI_CREATEEVENTW_METHODDEF \ + {"CreateEventW", _PyCFunction_CAST(_winapi_CreateEventW), METH_FASTCALL|METH_KEYWORDS, _winapi_CreateEventW__doc__}, + +static HANDLE +_winapi_CreateEventW_impl(PyObject *module, + LPSECURITY_ATTRIBUTES security_attributes, + BOOL manual_reset, BOOL initial_state, + LPCWSTR name); + +static PyObject * +_winapi_CreateEventW(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(security_attributes), &_Py_ID(manual_reset), &_Py_ID(initial_state), &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"security_attributes", "manual_reset", "initial_state", "name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "" F_POINTER "iiO&:CreateEventW", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + LPSECURITY_ATTRIBUTES security_attributes; + BOOL manual_reset; + BOOL initial_state; + LPCWSTR name = NULL; + HANDLE _return_value; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &security_attributes, &manual_reset, &initial_state, _PyUnicode_WideCharString_Opt_Converter, &name)) { + goto exit; + } + _return_value = _winapi_CreateEventW_impl(module, security_attributes, manual_reset, initial_state, name); + if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) { + goto exit; + } + if (_return_value == NULL) { + Py_RETURN_NONE; + } + return_value = HANDLE_TO_PYNUM(_return_value); + +exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + + return return_value; +} + PyDoc_STRVAR(_winapi_CreateFile__doc__, "CreateFile($module, file_name, desired_access, share_mode,\n" " security_attributes, creation_disposition,\n" @@ -297,6 +367,73 @@ _winapi_CreateJunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } +PyDoc_STRVAR(_winapi_CreateMutexW__doc__, +"CreateMutexW($module, /, security_attributes, initial_owner, name)\n" +"--\n" +"\n"); + +#define _WINAPI_CREATEMUTEXW_METHODDEF \ + {"CreateMutexW", _PyCFunction_CAST(_winapi_CreateMutexW), METH_FASTCALL|METH_KEYWORDS, _winapi_CreateMutexW__doc__}, + +static HANDLE +_winapi_CreateMutexW_impl(PyObject *module, + LPSECURITY_ATTRIBUTES security_attributes, + BOOL initial_owner, LPCWSTR name); + +static PyObject * +_winapi_CreateMutexW(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(security_attributes), &_Py_ID(initial_owner), &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"security_attributes", "initial_owner", "name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "" F_POINTER "iO&:CreateMutexW", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + LPSECURITY_ATTRIBUTES security_attributes; + BOOL initial_owner; + LPCWSTR name = NULL; + HANDLE _return_value; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &security_attributes, &initial_owner, _PyUnicode_WideCharString_Opt_Converter, &name)) { + goto exit; + } + _return_value = _winapi_CreateMutexW_impl(module, security_attributes, initial_owner, name); + if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) { + goto exit; + } + if (_return_value == NULL) { + Py_RETURN_NONE; + } + return_value = HANDLE_TO_PYNUM(_return_value); + +exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + + return return_value; +} + PyDoc_STRVAR(_winapi_CreateNamedPipe__doc__, "CreateNamedPipe($module, name, open_mode, pipe_mode, max_instances,\n" " out_buffer_size, in_buffer_size, default_timeout,\n" @@ -771,6 +908,138 @@ _winapi_UnmapViewOfFile(PyObject *module, PyObject *arg) return return_value; } +PyDoc_STRVAR(_winapi_OpenEventW__doc__, +"OpenEventW($module, /, desired_access, inherit_handle, name)\n" +"--\n" +"\n"); + +#define _WINAPI_OPENEVENTW_METHODDEF \ + {"OpenEventW", _PyCFunction_CAST(_winapi_OpenEventW), METH_FASTCALL|METH_KEYWORDS, _winapi_OpenEventW__doc__}, + +static HANDLE +_winapi_OpenEventW_impl(PyObject *module, DWORD desired_access, + BOOL inherit_handle, LPCWSTR name); + +static PyObject * +_winapi_OpenEventW(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(desired_access), &_Py_ID(inherit_handle), &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"desired_access", "inherit_handle", "name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "kiO&:OpenEventW", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + DWORD desired_access; + BOOL inherit_handle; + LPCWSTR name = NULL; + HANDLE _return_value; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &desired_access, &inherit_handle, _PyUnicode_WideCharString_Converter, &name)) { + goto exit; + } + _return_value = _winapi_OpenEventW_impl(module, desired_access, inherit_handle, name); + if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) { + goto exit; + } + if (_return_value == NULL) { + Py_RETURN_NONE; + } + return_value = HANDLE_TO_PYNUM(_return_value); + +exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + + return return_value; +} + +PyDoc_STRVAR(_winapi_OpenMutexW__doc__, +"OpenMutexW($module, /, desired_access, inherit_handle, name)\n" +"--\n" +"\n"); + +#define _WINAPI_OPENMUTEXW_METHODDEF \ + {"OpenMutexW", _PyCFunction_CAST(_winapi_OpenMutexW), METH_FASTCALL|METH_KEYWORDS, _winapi_OpenMutexW__doc__}, + +static HANDLE +_winapi_OpenMutexW_impl(PyObject *module, DWORD desired_access, + BOOL inherit_handle, LPCWSTR name); + +static PyObject * +_winapi_OpenMutexW(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(desired_access), &_Py_ID(inherit_handle), &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"desired_access", "inherit_handle", "name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "kiO&:OpenMutexW", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + DWORD desired_access; + BOOL inherit_handle; + LPCWSTR name = NULL; + HANDLE _return_value; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &desired_access, &inherit_handle, _PyUnicode_WideCharString_Converter, &name)) { + goto exit; + } + _return_value = _winapi_OpenMutexW_impl(module, desired_access, inherit_handle, name); + if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) { + goto exit; + } + if (_return_value == NULL) { + Py_RETURN_NONE; + } + return_value = HANDLE_TO_PYNUM(_return_value); + +exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + + return return_value; +} + PyDoc_STRVAR(_winapi_OpenFileMapping__doc__, "OpenFileMapping($module, desired_access, inherit_handle, name, /)\n" "--\n" @@ -991,6 +1260,162 @@ _winapi_ReadFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb return return_value; } +PyDoc_STRVAR(_winapi_ReleaseMutex__doc__, +"ReleaseMutex($module, /, mutex)\n" +"--\n" +"\n"); + +#define _WINAPI_RELEASEMUTEX_METHODDEF \ + {"ReleaseMutex", _PyCFunction_CAST(_winapi_ReleaseMutex), METH_FASTCALL|METH_KEYWORDS, _winapi_ReleaseMutex__doc__}, + +static PyObject * +_winapi_ReleaseMutex_impl(PyObject *module, HANDLE mutex); + +static PyObject * +_winapi_ReleaseMutex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(mutex), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"mutex", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "" F_HANDLE ":ReleaseMutex", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + HANDLE mutex; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &mutex)) { + goto exit; + } + return_value = _winapi_ReleaseMutex_impl(module, mutex); + +exit: + return return_value; +} + +PyDoc_STRVAR(_winapi_ResetEvent__doc__, +"ResetEvent($module, /, event)\n" +"--\n" +"\n"); + +#define _WINAPI_RESETEVENT_METHODDEF \ + {"ResetEvent", _PyCFunction_CAST(_winapi_ResetEvent), METH_FASTCALL|METH_KEYWORDS, _winapi_ResetEvent__doc__}, + +static PyObject * +_winapi_ResetEvent_impl(PyObject *module, HANDLE event); + +static PyObject * +_winapi_ResetEvent(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(event), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"event", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "" F_HANDLE ":ResetEvent", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + HANDLE event; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &event)) { + goto exit; + } + return_value = _winapi_ResetEvent_impl(module, event); + +exit: + return return_value; +} + +PyDoc_STRVAR(_winapi_SetEvent__doc__, +"SetEvent($module, /, event)\n" +"--\n" +"\n"); + +#define _WINAPI_SETEVENT_METHODDEF \ + {"SetEvent", _PyCFunction_CAST(_winapi_SetEvent), METH_FASTCALL|METH_KEYWORDS, _winapi_SetEvent__doc__}, + +static PyObject * +_winapi_SetEvent_impl(PyObject *module, HANDLE event); + +static PyObject * +_winapi_SetEvent(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(event), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"event", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "" F_HANDLE ":SetEvent", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + HANDLE event; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &event)) { + goto exit; + } + return_value = _winapi_SetEvent_impl(module, event); + +exit: + return return_value; +} + PyDoc_STRVAR(_winapi_SetNamedPipeHandleState__doc__, "SetNamedPipeHandleState($module, named_pipe, mode,\n" " max_collection_count, collect_data_timeout, /)\n" @@ -1114,6 +1539,77 @@ _winapi_WaitNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(_winapi_BatchedWaitForMultipleObjects__doc__, +"BatchedWaitForMultipleObjects($module, /, handle_seq, wait_all,\n" +" milliseconds=_winapi.INFINITE)\n" +"--\n" +"\n" +"Supports a larger number of handles than WaitForMultipleObjects\n" +"\n" +"Note that the handles may be waited on other threads, which could cause\n" +"issues for objects like mutexes that become associated with the thread\n" +"that was waiting for them. Objects may also be left signalled, even if\n" +"the wait fails.\n" +"\n" +"It is recommended to use WaitForMultipleObjects whenever possible, and\n" +"only switch to BatchedWaitForMultipleObjects for scenarios where you\n" +"control all the handles involved, such as your own thread pool or\n" +"files, and all wait objects are left unmodified by a wait (for example,\n" +"manual reset events, threads, and files/pipes).\n" +"\n" +"Overlapped handles returned from this module use manual reset events."); + +#define _WINAPI_BATCHEDWAITFORMULTIPLEOBJECTS_METHODDEF \ + {"BatchedWaitForMultipleObjects", _PyCFunction_CAST(_winapi_BatchedWaitForMultipleObjects), METH_FASTCALL|METH_KEYWORDS, _winapi_BatchedWaitForMultipleObjects__doc__}, + +static PyObject * +_winapi_BatchedWaitForMultipleObjects_impl(PyObject *module, + PyObject *handle_seq, + BOOL wait_all, DWORD milliseconds); + +static PyObject * +_winapi_BatchedWaitForMultipleObjects(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(handle_seq), &_Py_ID(wait_all), &_Py_ID(milliseconds), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"handle_seq", "wait_all", "milliseconds", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "Oi|k:BatchedWaitForMultipleObjects", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *handle_seq; + BOOL wait_all; + DWORD milliseconds = INFINITE; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &handle_seq, &wait_all, &milliseconds)) { + goto exit; + } + return_value = _winapi_BatchedWaitForMultipleObjects_impl(module, handle_seq, wait_all, milliseconds); + +exit: + return return_value; +} + PyDoc_STRVAR(_winapi_WaitForMultipleObjects__doc__, "WaitForMultipleObjects($module, handle_seq, wait_flag,\n" " milliseconds=_winapi.INFINITE, /)\n" @@ -1482,4 +1978,4 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -/*[clinic end generated code: output=2350d4f2275d3a6f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1f5bbcfa8d1847c5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index dbce03135416499..60a03fe012550ee 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -5,6 +5,24 @@ preserve #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +PyDoc_STRVAR(array_array_clear__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all items from the array."); + +#define ARRAY_ARRAY_CLEAR_METHODDEF \ + {"clear", (PyCFunction)array_array_clear, METH_NOARGS, array_array_clear__doc__}, + +static PyObject * +array_array_clear_impl(arrayobject *self); + +static PyObject * +array_array_clear(arrayobject *self, PyObject *Py_UNUSED(ignored)) +{ + return array_array_clear_impl(self); +} + PyDoc_STRVAR(array_array___copy____doc__, "__copy__($self, /)\n" "--\n" @@ -652,7 +670,7 @@ array_arrayiterator___reduce___impl(arrayiterobject *self, PyTypeObject *cls); static PyObject * array_arrayiterator___reduce__(arrayiterobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__reduce__() takes no arguments"); return NULL; } @@ -667,4 +685,4 @@ PyDoc_STRVAR(array_arrayiterator___setstate____doc__, #define ARRAY_ARRAYITERATOR___SETSTATE___METHODDEF \ {"__setstate__", (PyCFunction)array_arrayiterator___setstate__, METH_O, array_arrayiterator___setstate____doc__}, -/*[clinic end generated code: output=bf086c01e7e482bf input=a9049054013a1b77]*/ +/*[clinic end generated code: output=52c55d9b1d026c1c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index d50d170589a2cd5..9fff4da616ba003 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -469,6 +469,25 @@ PyDoc_STRVAR(gc_is_tracked__doc__, #define GC_IS_TRACKED_METHODDEF \ {"is_tracked", (PyCFunction)gc_is_tracked, METH_O, gc_is_tracked__doc__}, +static int +gc_is_tracked_impl(PyObject *module, PyObject *obj); + +static PyObject * +gc_is_tracked(PyObject *module, PyObject *obj) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = gc_is_tracked_impl(module, obj); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(gc_is_finalized__doc__, "is_finalized($module, obj, /)\n" "--\n" @@ -478,6 +497,25 @@ PyDoc_STRVAR(gc_is_finalized__doc__, #define GC_IS_FINALIZED_METHODDEF \ {"is_finalized", (PyCFunction)gc_is_finalized, METH_O, gc_is_finalized__doc__}, +static int +gc_is_finalized_impl(PyObject *module, PyObject *obj); + +static PyObject * +gc_is_finalized(PyObject *module, PyObject *obj) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = gc_is_finalized_impl(module, obj); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(gc_freeze__doc__, "freeze($module, /)\n" "--\n" @@ -547,4 +585,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=258f92524c1141fc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0a7e91917adcb937 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/md5module.c.h b/Modules/clinic/md5module.c.h index 7d4d3108dab9b6c..ee7fb3d7c613f22 100644 --- a/Modules/clinic/md5module.c.h +++ b/Modules/clinic/md5module.c.h @@ -23,7 +23,7 @@ MD5Type_copy_impl(MD5object *self, PyTypeObject *cls); static PyObject * MD5Type_copy(MD5object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -148,4 +148,4 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw exit: return return_value; } -/*[clinic end generated code: output=bfadda44914804a8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4dbca39332d3f52f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index ba3e1cfa8dbc21b..b49d64d4281889e 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -4465,6 +4465,156 @@ os_sched_getaffinity(PyObject *module, PyObject *arg) #endif /* defined(HAVE_SCHED_H) && defined(HAVE_SCHED_SETAFFINITY) */ +#if defined(HAVE_POSIX_OPENPT) + +PyDoc_STRVAR(os_posix_openpt__doc__, +"posix_openpt($module, oflag, /)\n" +"--\n" +"\n" +"Open and return a file descriptor for a master pseudo-terminal device.\n" +"\n" +"Performs a posix_openpt() C function call. The oflag argument is used to\n" +"set file status flags and file access modes as specified in the manual page\n" +"of posix_openpt() of your system."); + +#define OS_POSIX_OPENPT_METHODDEF \ + {"posix_openpt", (PyCFunction)os_posix_openpt, METH_O, os_posix_openpt__doc__}, + +static int +os_posix_openpt_impl(PyObject *module, int oflag); + +static PyObject * +os_posix_openpt(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int oflag; + int _return_value; + + oflag = PyLong_AsInt(arg); + if (oflag == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = os_posix_openpt_impl(module, oflag); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} + +#endif /* defined(HAVE_POSIX_OPENPT) */ + +#if defined(HAVE_GRANTPT) + +PyDoc_STRVAR(os_grantpt__doc__, +"grantpt($module, fd, /)\n" +"--\n" +"\n" +"Grant access to the slave pseudo-terminal device.\n" +"\n" +" fd\n" +" File descriptor of a master pseudo-terminal device.\n" +"\n" +"Performs a grantpt() C function call."); + +#define OS_GRANTPT_METHODDEF \ + {"grantpt", (PyCFunction)os_grantpt, METH_O, os_grantpt__doc__}, + +static PyObject * +os_grantpt_impl(PyObject *module, int fd); + +static PyObject * +os_grantpt(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + + if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + goto exit; + } + return_value = os_grantpt_impl(module, fd); + +exit: + return return_value; +} + +#endif /* defined(HAVE_GRANTPT) */ + +#if defined(HAVE_UNLOCKPT) + +PyDoc_STRVAR(os_unlockpt__doc__, +"unlockpt($module, fd, /)\n" +"--\n" +"\n" +"Unlock a pseudo-terminal master/slave pair.\n" +"\n" +" fd\n" +" File descriptor of a master pseudo-terminal device.\n" +"\n" +"Performs an unlockpt() C function call."); + +#define OS_UNLOCKPT_METHODDEF \ + {"unlockpt", (PyCFunction)os_unlockpt, METH_O, os_unlockpt__doc__}, + +static PyObject * +os_unlockpt_impl(PyObject *module, int fd); + +static PyObject * +os_unlockpt(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + + if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + goto exit; + } + return_value = os_unlockpt_impl(module, fd); + +exit: + return return_value; +} + +#endif /* defined(HAVE_UNLOCKPT) */ + +#if (defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R)) + +PyDoc_STRVAR(os_ptsname__doc__, +"ptsname($module, fd, /)\n" +"--\n" +"\n" +"Return the name of the slave pseudo-terminal device.\n" +"\n" +" fd\n" +" File descriptor of a master pseudo-terminal device.\n" +"\n" +"If the ptsname_r() C function is available, it is called;\n" +"otherwise, performs a ptsname() C function call."); + +#define OS_PTSNAME_METHODDEF \ + {"ptsname", (PyCFunction)os_ptsname, METH_O, os_ptsname__doc__}, + +static PyObject * +os_ptsname_impl(PyObject *module, int fd); + +static PyObject * +os_ptsname(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + + if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + goto exit; + } + return_value = os_ptsname_impl(module, fd); + +exit: + return return_value; +} + +#endif /* (defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R)) */ + #if (defined(HAVE_OPENPTY) || defined(HAVE__GETPTY) || defined(HAVE_DEV_PTMX)) PyDoc_STRVAR(os_openpty__doc__, @@ -11062,7 +11212,7 @@ os_DirEntry_is_symlink(DirEntry *self, PyTypeObject *defining_class, PyObject *c PyObject *return_value = NULL; int _return_value; - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "is_symlink() takes no arguments"); goto exit; } @@ -11991,6 +12141,22 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #define OS_SCHED_GETAFFINITY_METHODDEF #endif /* !defined(OS_SCHED_GETAFFINITY_METHODDEF) */ +#ifndef OS_POSIX_OPENPT_METHODDEF + #define OS_POSIX_OPENPT_METHODDEF +#endif /* !defined(OS_POSIX_OPENPT_METHODDEF) */ + +#ifndef OS_GRANTPT_METHODDEF + #define OS_GRANTPT_METHODDEF +#endif /* !defined(OS_GRANTPT_METHODDEF) */ + +#ifndef OS_UNLOCKPT_METHODDEF + #define OS_UNLOCKPT_METHODDEF +#endif /* !defined(OS_UNLOCKPT_METHODDEF) */ + +#ifndef OS_PTSNAME_METHODDEF + #define OS_PTSNAME_METHODDEF +#endif /* !defined(OS_PTSNAME_METHODDEF) */ + #ifndef OS_OPENPTY_METHODDEF #define OS_OPENPTY_METHODDEF #endif /* !defined(OS_OPENPTY_METHODDEF) */ @@ -12422,4 +12588,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=18c128534c355d84 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=268af5cbc8baa9d4 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha1module.c.h b/Modules/clinic/sha1module.c.h index ee391656fb67c31..b89c7e505c788e9 100644 --- a/Modules/clinic/sha1module.c.h +++ b/Modules/clinic/sha1module.c.h @@ -23,7 +23,7 @@ SHA1Type_copy_impl(SHA1object *self, PyTypeObject *cls); static PyObject * SHA1Type_copy(SHA1object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -148,4 +148,4 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=41fc7579213b57b4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=af5a640df662066f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha2module.c.h b/Modules/clinic/sha2module.c.h index ec31d5545be4c15..cf4b88d52856b86 100644 --- a/Modules/clinic/sha2module.c.h +++ b/Modules/clinic/sha2module.c.h @@ -23,7 +23,7 @@ SHA256Type_copy_impl(SHA256object *self, PyTypeObject *cls); static PyObject * SHA256Type_copy(SHA256object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -45,7 +45,7 @@ SHA512Type_copy_impl(SHA512object *self, PyTypeObject *cls); static PyObject * SHA512Type_copy(SHA512object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -437,4 +437,4 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject exit: return return_value; } -/*[clinic end generated code: output=1482d9de086e45c4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b46da764024b1764 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index 6b09abe309bf486..7ff3edf5a557f80 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -637,7 +637,7 @@ zlib_Compress_copy_impl(compobject *self, PyTypeObject *cls); static PyObject * zlib_Compress_copy(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -662,7 +662,7 @@ zlib_Compress___copy___impl(compobject *self, PyTypeObject *cls); static PyObject * zlib_Compress___copy__(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__copy__() takes no arguments"); return NULL; } @@ -735,7 +735,7 @@ zlib_Decompress_copy_impl(compobject *self, PyTypeObject *cls); static PyObject * zlib_Decompress_copy(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -760,7 +760,7 @@ zlib_Decompress___copy___impl(compobject *self, PyTypeObject *cls); static PyObject * zlib_Decompress___copy__(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__copy__() takes no arguments"); return NULL; } @@ -1098,4 +1098,4 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=6dd97dc851c39031 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8bb840fb6af43dd4 input=a9049054013a1b77]*/ diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 1c83563cbf68e74..95464b0dd177351 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -11,11 +11,13 @@ Copyright (c) 2000-2005 Fred L. Drake, Jr. <fdrake@users.sourceforge.net> Copyright (c) 2001-2002 Greg Stein <gstein@users.sourceforge.net> Copyright (c) 2002-2016 Karl Waclawek <karl@waclawek.net> - Copyright (c) 2016-2022 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2016-2024 Sebastian Pipping <sebastian@pipping.org> Copyright (c) 2016 Cristian Rodríguez <crrodriguez@opensuse.org> Copyright (c) 2016 Thomas Beutlich <tc@tbeu.de> Copyright (c) 2017 Rhodri James <rhodri@wildebeest.org.uk> Copyright (c) 2022 Thijs Schreijer <thijs@thijsschreijer.nl> + Copyright (c) 2023 Hanno Böck <hanno@gentoo.org> + Copyright (c) 2023 Sony Corporation / Snild Dolkow <snild@sony.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -269,7 +271,7 @@ XML_ParserCreate_MM(const XML_Char *encoding, const XML_Memory_Handling_Suite *memsuite, const XML_Char *namespaceSeparator); -/* Prepare a parser object to be re-used. This is particularly +/* Prepare a parser object to be reused. This is particularly valuable when memory allocation overhead is disproportionately high, such as when a large number of small documnents need to be parsed. All handlers are cleared from the parser, except for the @@ -951,7 +953,7 @@ XMLPARSEAPI(XML_Index) XML_GetCurrentByteIndex(XML_Parser parser); XMLPARSEAPI(int) XML_GetCurrentByteCount(XML_Parser parser); -/* If XML_CONTEXT_BYTES is defined, returns the input buffer, sets +/* If XML_CONTEXT_BYTES is >=1, returns the input buffer, sets the integer pointed to by offset to the offset within this buffer of the current parse position, and sets the integer pointed to by size to the size of this buffer (the number of input bytes). Otherwise @@ -1025,7 +1027,9 @@ enum XML_FeatureEnum { XML_FEATURE_ATTR_INFO, /* Added in Expat 2.4.0. */ XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, - XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT + XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT, + /* Added in Expat 2.6.0. */ + XML_FEATURE_GE /* Additional features must be added to the end of this enum. */ }; @@ -1038,23 +1042,29 @@ typedef struct { XMLPARSEAPI(const XML_Feature *) XML_GetFeatureList(void); -#ifdef XML_DTD -/* Added in Expat 2.4.0. */ +#if XML_GE == 1 +/* Added in Expat 2.4.0 for XML_DTD defined and + * added in Expat 2.6.0 for XML_GE == 1. */ XMLPARSEAPI(XML_Bool) XML_SetBillionLaughsAttackProtectionMaximumAmplification( XML_Parser parser, float maximumAmplificationFactor); -/* Added in Expat 2.4.0. */ +/* Added in Expat 2.4.0 for XML_DTD defined and + * added in Expat 2.6.0 for XML_GE == 1. */ XMLPARSEAPI(XML_Bool) XML_SetBillionLaughsAttackProtectionActivationThreshold( XML_Parser parser, unsigned long long activationThresholdBytes); #endif +/* Added in Expat 2.6.0. */ +XMLPARSEAPI(XML_Bool) +XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); + /* Expat follows the semantic versioning convention. - See http://semver.org. + See https://semver.org */ #define XML_MAJOR_VERSION 2 -#define XML_MINOR_VERSION 5 +#define XML_MINOR_VERSION 6 #define XML_MICRO_VERSION 0 #ifdef __cplusplus diff --git a/Modules/expat/expat_config.h b/Modules/expat/expat_config.h index 6671f7b689ba973..e7d9499d9078d95 100644 --- a/Modules/expat/expat_config.h +++ b/Modules/expat/expat_config.h @@ -16,6 +16,7 @@ #define XML_NS 1 #define XML_DTD 1 +#define XML_GE 1 #define XML_CONTEXT_BYTES 1024 // bpo-30947: Python uses best available entropy sources to diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index e09f533b23c9df0..cce71e4c5164b51 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -28,9 +28,10 @@ Copyright (c) 2002-2003 Fred L. Drake, Jr. <fdrake@users.sourceforge.net> Copyright (c) 2002-2006 Karl Waclawek <karl@waclawek.net> Copyright (c) 2003 Greg Stein <gstein@users.sourceforge.net> - Copyright (c) 2016-2022 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2016-2023 Sebastian Pipping <sebastian@pipping.org> Copyright (c) 2018 Yury Gribov <tetra2005@gmail.com> Copyright (c) 2019 David Loffredo <loffredo@steptools.com> + Copyright (c) 2023 Sony Corporation / Snild Dolkow <snild@sony.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -154,12 +155,15 @@ extern "C" { void _INTERNAL_trim_to_complete_utf8_characters(const char *from, const char **fromLimRef); -#if defined(XML_DTD) +#if XML_GE == 1 unsigned long long testingAccountingGetCountBytesDirect(XML_Parser parser); unsigned long long testingAccountingGetCountBytesIndirect(XML_Parser parser); const char *unsignedCharToPrintable(unsigned char c); #endif +extern XML_Bool g_reparseDeferralEnabledDefault; // written ONLY in runtests.c +extern unsigned int g_parseAttempts; // used for testing only + #ifdef __cplusplus } #endif diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h index 303283ad2de98d0..a1ed99e687bd6ee 100644 --- a/Modules/expat/siphash.h +++ b/Modules/expat/siphash.h @@ -106,7 +106,7 @@ * if this code is included and compiled as C++; related GCC warning is: * warning: use of C++11 long long integer constant [-Wlong-long] */ -#define _SIP_ULL(high, low) ((((uint64_t)high) << 32) | (low)) +#define SIP_ULL(high, low) ((((uint64_t)high) << 32) | (low)) #define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) @@ -190,10 +190,10 @@ sip_round(struct siphash *H, const int rounds) { static struct siphash * sip24_init(struct siphash *H, const struct sipkey *key) { - H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0]; - H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1]; - H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0]; - H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1]; + H->v0 = SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0]; + H->v1 = SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1]; + H->v2 = SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0]; + H->v3 = SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1]; H->p = H->buf; H->c = 0; diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h index 2ecd61b5b94820c..05805514ec7fa21 100644 --- a/Modules/expat/winconfig.h +++ b/Modules/expat/winconfig.h @@ -9,7 +9,8 @@ Copyright (c) 2000 Clark Cooper <coopercc@users.sourceforge.net> Copyright (c) 2002 Greg Stein <gstein@users.sourceforge.net> Copyright (c) 2005 Karl Waclawek <karl@waclawek.net> - Copyright (c) 2017-2021 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2017-2023 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2023 Orgad Shaneh <orgad.shaneh@audiocodes.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -35,7 +36,9 @@ #ifndef WINCONFIG_H #define WINCONFIG_H -#define WIN32_LEAN_AND_MEAN +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif #include <windows.h> #undef WIN32_LEAN_AND_MEAN diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index b6c2eca97567baa..aaf0fa9c8f96d1e 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* 5ab094ffadd6edfc94c3eee53af44a86951f9f1f0933ada3114bbce2bfb02c99 (2.5.0+) +/* 628e24d4966bedbd4800f6ed128d06d29703765b4bce12d3b7f099f90f842fc9 (2.6.0+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -13,7 +13,7 @@ Copyright (c) 2002-2016 Karl Waclawek <karl@waclawek.net> Copyright (c) 2005-2009 Steven Solie <steven@solie.ca> Copyright (c) 2016 Eric Rahm <erahm@mozilla.com> - Copyright (c) 2016-2022 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2016-2024 Sebastian Pipping <sebastian@pipping.org> Copyright (c) 2016 Gaurav <g.gupta@samsung.com> Copyright (c) 2016 Thomas Beutlich <tc@tbeu.de> Copyright (c) 2016 Gustavo Grieco <gustavo.grieco@imag.fr> @@ -32,10 +32,13 @@ Copyright (c) 2019 David Loffredo <loffredo@steptools.com> Copyright (c) 2019-2020 Ben Wagner <bungeman@chromium.org> Copyright (c) 2019 Vadim Zeitlin <vadim@zeitlins.org> - Copyright (c) 2021 Dong-hee Na <donghee.na@python.org> + Copyright (c) 2021 Donghee Na <donghee.na@python.org> Copyright (c) 2022 Samanta Navarro <ferivoz@riseup.net> Copyright (c) 2022 Jeffrey Walton <noloader@gmail.com> Copyright (c) 2022 Jann Horn <jannh@google.com> + Copyright (c) 2022 Sean McBride <sean@rogue-research.com> + Copyright (c) 2023 Owain Davies <owaind@bath.edu> + Copyright (c) 2023 Sony Corporation / Snild Dolkow <snild@sony.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -60,10 +63,25 @@ #define XML_BUILDING_EXPAT 1 -#include <expat_config.h> +#include "expat_config.h" -#if ! defined(_GNU_SOURCE) -# define _GNU_SOURCE 1 /* syscall prototype */ +#if ! defined(XML_GE) || (1 - XML_GE - 1 == 2) || (XML_GE < 0) || (XML_GE > 1) +# error XML_GE (for general entities) must be defined, non-empty, either 1 or 0 (0 to disable, 1 to enable; 1 is a common default) +#endif + +#if defined(XML_DTD) && XML_GE == 0 +# error Either undefine XML_DTD or define XML_GE to 1. +#endif + +#if ! defined(XML_CONTEXT_BYTES) || (1 - XML_CONTEXT_BYTES - 1 == 2) \ + || (XML_CONTEXT_BYTES + 0 < 0) +# error XML_CONTEXT_BYTES must be defined, non-empty and >=0 (0 to disable, >=1 to enable; 1024 is a common default) +#endif + +#if defined(HAVE_SYSCALL_GETRANDOM) +# if ! defined(_GNU_SOURCE) +# define _GNU_SOURCE 1 /* syscall prototype */ +# endif #endif #ifdef _WIN32 @@ -73,6 +91,7 @@ # endif #endif +#include <stdbool.h> #include <stddef.h> #include <string.h> /* memset(), memcpy() */ #include <assert.h> @@ -131,8 +150,8 @@ Your options include: \ * Linux >=3.17 + glibc >=2.25 (getrandom): HAVE_GETRANDOM, \ * Linux >=3.17 + glibc (including <2.25) (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ - * BSD / macOS >=10.7 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ - * BSD / macOS (including <10.7) (arc4random): HAVE_ARC4RANDOM, \ + * BSD / macOS >=10.7 / glibc >=2.36 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ + * BSD / macOS (including <10.7) / glibc >=2.36 (arc4random): HAVE_ARC4RANDOM, \ * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \ * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \ * Linux (including <3.17) / BSD / macOS (including <10.7) / Solaris >=8 (/dev/urandom): XML_DEV_URANDOM, \ @@ -196,6 +215,8 @@ typedef char ICHAR; /* Do safe (NULL-aware) pointer arithmetic */ #define EXPAT_SAFE_PTR_DIFF(p, q) (((p) && (q)) ? ((p) - (q)) : 0) +#define EXPAT_MIN(a, b) (((a) < (b)) ? (a) : (b)) + #include "internal.h" #include "xmltok.h" #include "xmlrole.h" @@ -279,7 +300,7 @@ typedef struct { XML_Parse()/XML_ParseBuffer(), the buffer is re-allocated to contain the 'raw' name as well. - A parser re-uses these structures, maintaining a list of allocated + A parser reuses these structures, maintaining a list of allocated TAG objects in a free list. */ typedef struct tag { @@ -408,12 +429,12 @@ enum XML_Account { XML_ACCOUNT_NONE /* i.e. do not account, was accounted already */ }; -#ifdef XML_DTD +#if XML_GE == 1 typedef unsigned long long XmlBigCount; typedef struct accounting { XmlBigCount countBytesDirect; XmlBigCount countBytesIndirect; - int debugLevel; + unsigned long debugLevel; float maximumAmplificationFactor; // >=1.0 unsigned long long activationThresholdBytes; } ACCOUNTING; @@ -422,9 +443,9 @@ typedef struct entity_stats { unsigned int countEverOpened; unsigned int currentDepth; unsigned int maximumDepthSeen; - int debugLevel; + unsigned long debugLevel; } ENTITY_STATS; -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ typedef enum XML_Error PTRCALL Processor(XML_Parser parser, const char *start, const char *end, const char **endPtr); @@ -464,41 +485,47 @@ static enum XML_Error doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const char *start, const char *end, const char **endPtr, XML_Bool haveMore, enum XML_Account account); -static enum XML_Error doCdataSection(XML_Parser parser, const ENCODING *, +static enum XML_Error doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, const char *end, const char **nextPtr, XML_Bool haveMore, enum XML_Account account); #ifdef XML_DTD -static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *, +static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, const char *end, const char **nextPtr, XML_Bool haveMore); #endif /* XML_DTD */ static void freeBindings(XML_Parser parser, BINDING *bindings); -static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *, - const char *s, TAG_NAME *tagNamePtr, +static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *enc, + const char *attStr, TAG_NAME *tagNamePtr, BINDING **bindingsPtr, enum XML_Account account); static enum XML_Error addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, const XML_Char *uri, BINDING **bindingsPtr); -static int defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *, XML_Bool isCdata, - XML_Bool isId, const XML_Char *dfltValue, - XML_Parser parser); -static enum XML_Error storeAttributeValue(XML_Parser parser, const ENCODING *, - XML_Bool isCdata, const char *, - const char *, STRING_POOL *, +static int defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, + XML_Bool isCdata, XML_Bool isId, + const XML_Char *value, XML_Parser parser); +static enum XML_Error storeAttributeValue(XML_Parser parser, + const ENCODING *enc, XML_Bool isCdata, + const char *ptr, const char *end, + STRING_POOL *pool, enum XML_Account account); -static enum XML_Error appendAttributeValue(XML_Parser parser, const ENCODING *, - XML_Bool isCdata, const char *, - const char *, STRING_POOL *, +static enum XML_Error appendAttributeValue(XML_Parser parser, + const ENCODING *enc, + XML_Bool isCdata, const char *ptr, + const char *end, STRING_POOL *pool, enum XML_Account account); static ATTRIBUTE_ID *getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, const char *end); -static int setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *); +static int setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType); +#if XML_GE == 1 static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc, const char *start, const char *end, enum XML_Account account); +#else +static enum XML_Error storeSelfEntityValue(XML_Parser parser, ENTITY *entity); +#endif static int reportProcessingInstruction(XML_Parser parser, const ENCODING *enc, const char *start, const char *end); static int reportComment(XML_Parser parser, const ENCODING *enc, @@ -518,21 +545,22 @@ static void dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms); static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_Handling_Suite *ms); -static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *, STRING_POOL *, - const HASH_TABLE *); +static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, + STRING_POOL *newPool, const HASH_TABLE *oldTable); static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize); -static void FASTCALL hashTableInit(HASH_TABLE *, +static void FASTCALL hashTableInit(HASH_TABLE *table, const XML_Memory_Handling_Suite *ms); -static void FASTCALL hashTableClear(HASH_TABLE *); -static void FASTCALL hashTableDestroy(HASH_TABLE *); -static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *, const HASH_TABLE *); -static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *); +static void FASTCALL hashTableClear(HASH_TABLE *table); +static void FASTCALL hashTableDestroy(HASH_TABLE *table); +static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *iter, + const HASH_TABLE *table); +static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *iter); -static void FASTCALL poolInit(STRING_POOL *, +static void FASTCALL poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms); -static void FASTCALL poolClear(STRING_POOL *); -static void FASTCALL poolDestroy(STRING_POOL *); +static void FASTCALL poolClear(STRING_POOL *pool); +static void FASTCALL poolDestroy(STRING_POOL *pool); static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end); static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc, @@ -562,7 +590,7 @@ static XML_Parser parserCreate(const XML_Char *encodingName, static void parserInit(XML_Parser parser, const XML_Char *encodingName); -#ifdef XML_DTD +#if XML_GE == 1 static float accountingGetCurrentAmplification(XML_Parser rootParser); static void accountingReportStats(XML_Parser originParser, const char *epilog); static void accountingOnAbort(XML_Parser originParser); @@ -585,13 +613,12 @@ static void entityTrackingOnClose(XML_Parser parser, ENTITY *entity, static XML_Parser getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff); -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ static unsigned long getDebugLevel(const char *variableName, unsigned long defaultDebugLevel); #define poolStart(pool) ((pool)->start) -#define poolEnd(pool) ((pool)->ptr) #define poolLength(pool) ((pool)->ptr - (pool)->start) #define poolChop(pool) ((void)--(pool->ptr)) #define poolLastChar(pool) (((pool)->ptr)[-1]) @@ -602,21 +629,35 @@ static unsigned long getDebugLevel(const char *variableName, ? 0 \ : ((*((pool)->ptr)++ = c), 1)) +XML_Bool g_reparseDeferralEnabledDefault = XML_TRUE; // write ONLY in runtests.c +unsigned int g_parseAttempts = 0; // used for testing only + struct XML_ParserStruct { /* The first member must be m_userData so that the XML_GetUserData macro works. */ void *m_userData; void *m_handlerArg; - char *m_buffer; + + // How the four parse buffer pointers below relate in time and space: + // + // m_buffer <= m_bufferPtr <= m_bufferEnd <= m_bufferLim + // | | | | + // <--parsed-->| | | + // <---parsing--->| | + // <--unoccupied-->| + // <---------total-malloced/realloced-------->| + + char *m_buffer; // malloc/realloc base pointer of parse buffer const XML_Memory_Handling_Suite m_mem; - /* first character to be parsed */ - const char *m_bufferPtr; - /* past last character to be parsed */ - char *m_bufferEnd; - /* allocated end of m_buffer */ - const char *m_bufferLim; + const char *m_bufferPtr; // first character to be parsed + char *m_bufferEnd; // past last character to be parsed + const char *m_bufferLim; // allocated end of m_buffer + XML_Index m_parseEndByteIndex; const char *m_parseEndPtr; + size_t m_partialTokenBytesBefore; /* used in heuristic to avoid O(n^2) */ + XML_Bool m_reparseDeferralEnabled; + int m_lastBufferRequestSize; XML_Char *m_dataBuf; XML_Char *m_dataBufEnd; XML_StartElementHandler m_startElementHandler; @@ -703,7 +744,7 @@ struct XML_ParserStruct { enum XML_ParamEntityParsing m_paramEntityParsing; #endif unsigned long m_hash_secret_salt; -#ifdef XML_DTD +#if XML_GE == 1 ACCOUNTING m_accounting; ENTITY_STATS m_entity_stats; #endif @@ -948,6 +989,47 @@ get_hash_secret_salt(XML_Parser parser) { return parser->m_hash_secret_salt; } +static enum XML_Error +callProcessor(XML_Parser parser, const char *start, const char *end, + const char **endPtr) { + const size_t have_now = EXPAT_SAFE_PTR_DIFF(end, start); + + if (parser->m_reparseDeferralEnabled + && ! parser->m_parsingStatus.finalBuffer) { + // Heuristic: don't try to parse a partial token again until the amount of + // available data has increased significantly. + const size_t had_before = parser->m_partialTokenBytesBefore; + // ...but *do* try anyway if we're close to causing a reallocation. + size_t available_buffer + = EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); +#if XML_CONTEXT_BYTES > 0 + available_buffer -= EXPAT_MIN(available_buffer, XML_CONTEXT_BYTES); +#endif + available_buffer + += EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd); + // m_lastBufferRequestSize is never assigned a value < 0, so the cast is ok + const bool enough + = (have_now >= 2 * had_before) + || ((size_t)parser->m_lastBufferRequestSize > available_buffer); + + if (! enough) { + *endPtr = start; // callers may expect this to be set + return XML_ERROR_NONE; + } + } + g_parseAttempts += 1; + const enum XML_Error ret = parser->m_processor(parser, start, end, endPtr); + if (ret == XML_ERROR_NONE) { + // if we consumed nothing, remember what we had on this parse attempt. + if (*endPtr == start) { + parser->m_partialTokenBytesBefore = have_now; + } else { + parser->m_partialTokenBytesBefore = 0; + } + } + return ret; +} + static XML_Bool /* only valid for root parser */ startParsing(XML_Parser parser) { /* hash functions must be initialized before setContext() is called */ @@ -1129,6 +1211,9 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_bufferEnd = parser->m_buffer; parser->m_parseEndByteIndex = 0; parser->m_parseEndPtr = NULL; + parser->m_partialTokenBytesBefore = 0; + parser->m_reparseDeferralEnabled = g_reparseDeferralEnabledDefault; + parser->m_lastBufferRequestSize = 0; parser->m_declElementType = NULL; parser->m_declAttributeId = NULL; parser->m_declEntity = NULL; @@ -1163,7 +1248,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { #endif parser->m_hash_secret_salt = 0; -#ifdef XML_DTD +#if XML_GE == 1 memset(&parser->m_accounting, 0, sizeof(ACCOUNTING)); parser->m_accounting.debugLevel = getDebugLevel("EXPAT_ACCOUNTING_DEBUG", 0u); parser->m_accounting.maximumAmplificationFactor @@ -1298,6 +1383,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, to worry which hash secrets each table has. */ unsigned long oldhash_secret_salt; + XML_Bool oldReparseDeferralEnabled; /* Validate the oldParser parameter before we pull everything out of it */ if (oldParser == NULL) @@ -1342,6 +1428,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, to worry which hash secrets each table has. */ oldhash_secret_salt = parser->m_hash_secret_salt; + oldReparseDeferralEnabled = parser->m_reparseDeferralEnabled; #ifdef XML_DTD if (! context) @@ -1394,6 +1481,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, parser->m_defaultExpandInternalEntities = oldDefaultExpandInternalEntities; parser->m_ns_triplets = oldns_triplets; parser->m_hash_secret_salt = oldhash_secret_salt; + parser->m_reparseDeferralEnabled = oldReparseDeferralEnabled; parser->m_parentParser = oldParser; #ifdef XML_DTD parser->m_paramEntityParsing = oldParamEntityParsing; @@ -1848,55 +1936,8 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_PARSING; } - if (len == 0) { - parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal; - if (! isFinal) - return XML_STATUS_OK; - parser->m_positionPtr = parser->m_bufferPtr; - parser->m_parseEndPtr = parser->m_bufferEnd; - - /* If data are left over from last buffer, and we now know that these - data are the final chunk of input, then we have to check them again - to detect errors based on that fact. - */ - parser->m_errorCode - = parser->m_processor(parser, parser->m_bufferPtr, - parser->m_parseEndPtr, &parser->m_bufferPtr); - - if (parser->m_errorCode == XML_ERROR_NONE) { - switch (parser->m_parsingStatus.parsing) { - case XML_SUSPENDED: - /* It is hard to be certain, but it seems that this case - * cannot occur. This code is cleaning up a previous parse - * with no new data (since len == 0). Changing the parsing - * state requires getting to execute a handler function, and - * there doesn't seem to be an opportunity for that while in - * this circumstance. - * - * Given the uncertainty, we retain the code but exclude it - * from coverage tests. - * - * LCOV_EXCL_START - */ - XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, - parser->m_bufferPtr, &parser->m_position); - parser->m_positionPtr = parser->m_bufferPtr; - return XML_STATUS_SUSPENDED; - /* LCOV_EXCL_STOP */ - case XML_INITIALIZED: - case XML_PARSING: - parser->m_parsingStatus.parsing = XML_FINISHED; - /* fall through */ - default: - return XML_STATUS_OK; - } - } - parser->m_eventEndPtr = parser->m_eventPtr; - parser->m_processor = errorProcessor; - return XML_STATUS_ERROR; - } -#ifndef XML_CONTEXT_BYTES - else if (parser->m_bufferPtr == parser->m_bufferEnd) { +#if XML_CONTEXT_BYTES == 0 + if (parser->m_bufferPtr == parser->m_bufferEnd) { const char *end; int nLeftOver; enum XML_Status result; @@ -1907,12 +1948,15 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_processor = errorProcessor; return XML_STATUS_ERROR; } + // though this isn't a buffer request, we assume that `len` is the app's + // preferred buffer fill size, and therefore save it here. + parser->m_lastBufferRequestSize = len; parser->m_parseEndByteIndex += len; parser->m_positionPtr = s; parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal; parser->m_errorCode - = parser->m_processor(parser, s, parser->m_parseEndPtr = s + len, &end); + = callProcessor(parser, s, parser->m_parseEndPtr = s + len, &end); if (parser->m_errorCode != XML_ERROR_NONE) { parser->m_eventEndPtr = parser->m_eventPtr; @@ -1939,23 +1983,25 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { &parser->m_position); nLeftOver = s + len - end; if (nLeftOver) { - if (parser->m_buffer == NULL - || nLeftOver > parser->m_bufferLim - parser->m_buffer) { - /* avoid _signed_ integer overflow */ - char *temp = NULL; - const int bytesToAllocate = (int)((unsigned)len * 2U); - if (bytesToAllocate > 0) { - temp = (char *)REALLOC(parser, parser->m_buffer, bytesToAllocate); - } - if (temp == NULL) { - parser->m_errorCode = XML_ERROR_NO_MEMORY; - parser->m_eventPtr = parser->m_eventEndPtr = NULL; - parser->m_processor = errorProcessor; - return XML_STATUS_ERROR; - } - parser->m_buffer = temp; - parser->m_bufferLim = parser->m_buffer + bytesToAllocate; + // Back up and restore the parsing status to avoid XML_ERROR_SUSPENDED + // (and XML_ERROR_FINISHED) from XML_GetBuffer. + const enum XML_Parsing originalStatus = parser->m_parsingStatus.parsing; + parser->m_parsingStatus.parsing = XML_PARSING; + void *const temp = XML_GetBuffer(parser, nLeftOver); + parser->m_parsingStatus.parsing = originalStatus; + // GetBuffer may have overwritten this, but we want to remember what the + // app requested, not how many bytes were left over after parsing. + parser->m_lastBufferRequestSize = len; + if (temp == NULL) { + // NOTE: parser->m_errorCode has already been set by XML_GetBuffer(). + parser->m_eventPtr = parser->m_eventEndPtr = NULL; + parser->m_processor = errorProcessor; + return XML_STATUS_ERROR; } + // Since we know that the buffer was empty and XML_CONTEXT_BYTES is 0, we + // don't have any data to preserve, and can copy straight into the start + // of the buffer rather than the GetBuffer return pointer (which may be + // pointing further into the allocated buffer). memcpy(parser->m_buffer, end, nLeftOver); } parser->m_bufferPtr = parser->m_buffer; @@ -1966,16 +2012,15 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_eventEndPtr = parser->m_bufferPtr; return result; } -#endif /* not defined XML_CONTEXT_BYTES */ - else { - void *buff = XML_GetBuffer(parser, len); - if (buff == NULL) - return XML_STATUS_ERROR; - else { - memcpy(buff, s, len); - return XML_ParseBuffer(parser, len, isFinal); - } +#endif /* XML_CONTEXT_BYTES == 0 */ + void *buff = XML_GetBuffer(parser, len); + if (buff == NULL) + return XML_STATUS_ERROR; + if (len > 0) { + assert(s != NULL); // make sure s==NULL && len!=0 was rejected above + memcpy(buff, s, len); } + return XML_ParseBuffer(parser, len, isFinal); } enum XML_Status XMLCALL @@ -2015,8 +2060,8 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_parseEndByteIndex += len; parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal; - parser->m_errorCode = parser->m_processor( - parser, start, parser->m_parseEndPtr, &parser->m_bufferPtr); + parser->m_errorCode = callProcessor(parser, start, parser->m_parseEndPtr, + &parser->m_bufferPtr); if (parser->m_errorCode != XML_ERROR_NONE) { parser->m_eventEndPtr = parser->m_eventPtr; @@ -2061,10 +2106,14 @@ XML_GetBuffer(XML_Parser parser, int len) { default:; } - if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd)) { -#ifdef XML_CONTEXT_BYTES + // whether or not the request succeeds, `len` seems to be the app's preferred + // buffer fill size; remember it. + parser->m_lastBufferRequestSize = len; + if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd) + || parser->m_buffer == NULL) { +#if XML_CONTEXT_BYTES > 0 int keep; -#endif /* defined XML_CONTEXT_BYTES */ +#endif /* XML_CONTEXT_BYTES > 0 */ /* Do not invoke signed arithmetic overflow: */ int neededSize = (int)((unsigned)len + (unsigned)EXPAT_SAFE_PTR_DIFF( @@ -2073,7 +2122,7 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } -#ifdef XML_CONTEXT_BYTES +#if XML_CONTEXT_BYTES > 0 keep = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); if (keep > XML_CONTEXT_BYTES) keep = XML_CONTEXT_BYTES; @@ -2083,10 +2132,11 @@ XML_GetBuffer(XML_Parser parser, int len) { return NULL; } neededSize += keep; -#endif /* defined XML_CONTEXT_BYTES */ - if (neededSize - <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) { -#ifdef XML_CONTEXT_BYTES +#endif /* XML_CONTEXT_BYTES > 0 */ + if (parser->m_buffer && parser->m_bufferPtr + && neededSize + <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) { +#if XML_CONTEXT_BYTES > 0 if (keep < EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)) { int offset = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer) @@ -2099,19 +2149,17 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_bufferPtr -= offset; } #else - if (parser->m_buffer && parser->m_bufferPtr) { - memmove(parser->m_buffer, parser->m_bufferPtr, - EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)); - parser->m_bufferEnd - = parser->m_buffer - + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); - parser->m_bufferPtr = parser->m_buffer; - } -#endif /* not defined XML_CONTEXT_BYTES */ + memmove(parser->m_buffer, parser->m_bufferPtr, + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)); + parser->m_bufferEnd + = parser->m_buffer + + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); + parser->m_bufferPtr = parser->m_buffer; +#endif /* XML_CONTEXT_BYTES > 0 */ } else { char *newBuf; int bufferSize - = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferPtr); + = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer); if (bufferSize == 0) bufferSize = INIT_BUFFER_SIZE; do { @@ -2128,7 +2176,7 @@ XML_GetBuffer(XML_Parser parser, int len) { return NULL; } parser->m_bufferLim = newBuf + bufferSize; -#ifdef XML_CONTEXT_BYTES +#if XML_CONTEXT_BYTES > 0 if (parser->m_bufferPtr) { memcpy(newBuf, &parser->m_bufferPtr[-keep], EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) @@ -2158,7 +2206,7 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_bufferEnd = newBuf; } parser->m_bufferPtr = parser->m_buffer = newBuf; -#endif /* not defined XML_CONTEXT_BYTES */ +#endif /* XML_CONTEXT_BYTES > 0 */ } parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_positionPtr = NULL; @@ -2208,7 +2256,7 @@ XML_ResumeParser(XML_Parser parser) { } parser->m_parsingStatus.parsing = XML_PARSING; - parser->m_errorCode = parser->m_processor( + parser->m_errorCode = callProcessor( parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr); if (parser->m_errorCode != XML_ERROR_NONE) { @@ -2272,7 +2320,7 @@ XML_GetCurrentByteCount(XML_Parser parser) { const char *XMLCALL XML_GetInputContext(XML_Parser parser, int *offset, int *size) { -#ifdef XML_CONTEXT_BYTES +#if XML_CONTEXT_BYTES > 0 if (parser == NULL) return NULL; if (parser->m_eventPtr && parser->m_buffer) { @@ -2286,7 +2334,7 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) { (void)parser; (void)offset; (void)size; -#endif /* defined XML_CONTEXT_BYTES */ +#endif /* XML_CONTEXT_BYTES > 0 */ return (const char *)0; } @@ -2506,7 +2554,7 @@ XML_GetFeatureList(void) { #ifdef XML_DTD {XML_FEATURE_DTD, XML_L("XML_DTD"), 0}, #endif -#ifdef XML_CONTEXT_BYTES +#if XML_CONTEXT_BYTES > 0 {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"), XML_CONTEXT_BYTES}, #endif @@ -2522,8 +2570,9 @@ XML_GetFeatureList(void) { #ifdef XML_ATTR_INFO {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0}, #endif -#ifdef XML_DTD - /* Added in Expat 2.4.0. */ +#if XML_GE == 1 + /* Added in Expat 2.4.0 for XML_DTD defined and + * added in Expat 2.6.0 for XML_GE == 1. */ {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, XML_L("XML_BLAP_MAX_AMP"), (long int) @@ -2531,13 +2580,15 @@ XML_GetFeatureList(void) { {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT, XML_L("XML_BLAP_ACT_THRES"), EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT}, + /* Added in Expat 2.6.0. */ + {XML_FEATURE_GE, XML_L("XML_GE"), 0}, #endif {XML_FEATURE_END, NULL, 0}}; return features; } -#ifdef XML_DTD +#if XML_GE == 1 XML_Bool XMLCALL XML_SetBillionLaughsAttackProtectionMaximumAmplification( XML_Parser parser, float maximumAmplificationFactor) { @@ -2559,7 +2610,16 @@ XML_SetBillionLaughsAttackProtectionActivationThreshold( parser->m_accounting.activationThresholdBytes = activationThresholdBytes; return XML_TRUE; } -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ + +XML_Bool XMLCALL +XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) { + if (parser != NULL && (enabled == XML_TRUE || enabled == XML_FALSE)) { + parser->m_reparseDeferralEnabled = enabled; + return XML_TRUE; + } + return XML_FALSE; +} /* Initially tag->rawName always points into the parse buffer; for those TAG instances opened while the current parse buffer was @@ -2581,7 +2641,7 @@ storeRawNames(XML_Parser parser) { */ if (tag->rawName == rawNameBuf) break; - /* For re-use purposes we need to ensure that the + /* For reuse purposes we need to ensure that the size of tag->buf is a multiple of sizeof(XML_Char). */ rawNameLen = ROUND_UP(tag->rawNameLength, sizeof(XML_Char)); @@ -2645,13 +2705,13 @@ externalEntityInitProcessor2(XML_Parser parser, const char *start, int tok = XmlContentTok(parser->m_encoding, start, end, &next); switch (tok) { case XML_TOK_BOM: -#ifdef XML_DTD +#if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, start, next, __LINE__, XML_ACCOUNT_DIRECT)) { accountingOnAbort(parser); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ /* If we are at the end of the buffer, this would cause the next stage, i.e. externalEntityInitProcessor3, to pass control directly to @@ -2765,7 +2825,7 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, for (;;) { const char *next = s; /* XmlContentTok doesn't always set the last arg */ int tok = XmlContentTok(enc, s, end, &next); -#ifdef XML_DTD +#if XML_GE == 1 const char *accountAfter = ((tok == XML_TOK_TRAILING_RSQB) || (tok == XML_TOK_TRAILING_CR)) ? (haveMore ? s /* i.e. 0 bytes */ : end) @@ -2831,14 +2891,14 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, XML_Char ch = (XML_Char)XmlPredefinedEntityName( enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar); if (ch) { -#ifdef XML_DTD +#if XML_GE == 1 /* NOTE: We are replacing 4-6 characters original input for 1 character * so there is no amplification and hence recording without * protection. */ accountingDiffTolerated(parser, tok, (char *)&ch, ((char *)&ch) + sizeof(XML_Char), __LINE__, XML_ACCOUNT_ENTITY_EXPANSION); -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ if (parser->m_characterDataHandler) parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1); else if (parser->m_defaultHandler) @@ -3039,13 +3099,13 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (parser->m_ns && localPart) { /* localPart and prefix may have been overwritten in tag->name.str, since this points to the binding->uri - buffer which gets re-used; so we have to add them again + buffer which gets reused; so we have to add them again */ uri = (XML_Char *)tag->name.str + tag->name.uriLen; /* don't need to check for space - already done in storeAtts() */ while (*localPart) *uri++ = *localPart++; - prefix = (XML_Char *)tag->name.prefix; + prefix = tag->name.prefix; if (parser->m_ns_triplets && prefix) { *uri++ = parser->m_namespaceSeparator; while (*prefix) @@ -3112,7 +3172,7 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, However, now we have a start/endCdataSectionHandler, so it seems easier to let the user deal with this. */ - else if (0 && parser->m_characterDataHandler) + else if ((0) && parser->m_characterDataHandler) parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); /* END disabled code */ @@ -3141,8 +3201,8 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); } else parser->m_characterDataHandler( - parser->m_handlerArg, (XML_Char *)s, - (int)((XML_Char *)end - (XML_Char *)s)); + parser->m_handlerArg, (const XML_Char *)s, + (int)((const XML_Char *)end - (const XML_Char *)s)); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, end); /* We are at the end of the final buffer, should we check for @@ -3175,8 +3235,8 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, *eventPP = s; } } else - charDataHandler(parser->m_handlerArg, (XML_Char *)s, - (int)((XML_Char *)next - (XML_Char *)s)); + charDataHandler(parser->m_handlerArg, (const XML_Char *)s, + (int)((const XML_Char *)next - (const XML_Char *)s)); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -4040,7 +4100,7 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, for (;;) { const char *next = s; /* in case of XML_TOK_NONE or XML_TOK_PARTIAL */ int tok = XmlCdataSectionTok(enc, s, end, &next); -#ifdef XML_DTD +#if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, account)) { accountingOnAbort(parser); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; @@ -4055,7 +4115,7 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, parser->m_endCdataSectionHandler(parser->m_handlerArg); /* BEGIN disabled code */ /* see comment under XML_TOK_CDATA_SECT_OPEN */ - else if (0 && parser->m_characterDataHandler) + else if ((0) && parser->m_characterDataHandler) parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); /* END disabled code */ @@ -4091,8 +4151,8 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *eventPP = s; } } else - charDataHandler(parser->m_handlerArg, (XML_Char *)s, - (int)((XML_Char *)next - (XML_Char *)s)); + charDataHandler(parser->m_handlerArg, (const XML_Char *)s, + (int)((const XML_Char *)next - (const XML_Char *)s)); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -4192,7 +4252,7 @@ doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *eventPP = s; *startPtr = NULL; tok = XmlIgnoreSectionTok(enc, s, end, &next); -# ifdef XML_DTD +# if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { accountingOnAbort(parser); @@ -4284,7 +4344,7 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s, const XML_Char *storedversion = NULL; int standalone = -1; -#ifdef XML_DTD +#if XML_GE == 1 if (! accountingDiffTolerated(parser, XML_TOK_XML_DECL, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { accountingOnAbort(parser); @@ -4482,16 +4542,16 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, parser->m_processor = entityValueProcessor; return entityValueProcessor(parser, next, end, nextPtr); } - /* If we are at the end of the buffer, this would cause XmlPrologTok to - return XML_TOK_NONE on the next call, which would then cause the - function to exit with *nextPtr set to s - that is what we want for other - tokens, but not for the BOM - we would rather like to skip it; - then, when this routine is entered the next time, XmlPrologTok will - return XML_TOK_INVALID, since the BOM is still in the buffer + /* XmlPrologTok has now set the encoding based on the BOM it found, and we + must move s and nextPtr forward to consume the BOM. + + If we didn't, and got XML_TOK_NONE from the next XmlPrologTok call, we + would leave the BOM in the buffer and return. On the next call to this + function, our XmlPrologTok call would return XML_TOK_INVALID, since it + is not valid to have multiple BOMs. */ - else if (tok == XML_TOK_BOM && next == end - && ! parser->m_parsingStatus.finalBuffer) { -# ifdef XML_DTD + else if (tok == XML_TOK_BOM) { +# if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { accountingOnAbort(parser); @@ -4500,7 +4560,7 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, # endif *nextPtr = next; - return XML_ERROR_NONE; + s = next; } /* If we get this token, we have the start of what might be a normal tag, but not a declaration (i.e. it doesn't begin with @@ -4707,11 +4767,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } } role = XmlTokenRole(&parser->m_prologState, tok, s, next, enc); -#ifdef XML_DTD +#if XML_GE == 1 switch (role) { case XML_ROLE_INSTANCE_START: // bytes accounted in contentProcessor case XML_ROLE_XML_DECL: // bytes accounted in processXmlDecl - case XML_ROLE_TEXT_DECL: // bytes accounted in processXmlDecl +# ifdef XML_DTD + case XML_ROLE_TEXT_DECL: // bytes accounted in processXmlDecl +# endif break; default: if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, account)) { @@ -5029,6 +5091,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, break; case XML_ROLE_ENTITY_VALUE: if (dtd->keepProcessing) { +#if XML_GE == 1 + // This will store the given replacement text in + // parser->m_declEntity->textPtr. enum XML_Error result = storeEntityValue(parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar, XML_ACCOUNT_NONE); @@ -5049,6 +5114,25 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolDiscard(&dtd->entityValuePool); if (result != XML_ERROR_NONE) return result; +#else + // This will store "&entity123;" in parser->m_declEntity->textPtr + // to end up as "&entity123;" in the handler. + if (parser->m_declEntity != NULL) { + const enum XML_Error result + = storeSelfEntityValue(parser, parser->m_declEntity); + if (result != XML_ERROR_NONE) + return result; + + if (parser->m_entityDeclHandler) { + *eventEndPP = s; + parser->m_entityDeclHandler( + parser->m_handlerArg, parser->m_declEntity->name, + parser->m_declEntity->is_param, parser->m_declEntity->textPtr, + parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + handleDefault = XML_FALSE; + } + } +#endif } break; case XML_ROLE_DOCTYPE_SYSTEM_ID: @@ -5107,6 +5191,16 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } break; case XML_ROLE_ENTITY_COMPLETE: +#if XML_GE == 0 + // This will store "&entity123;" in entity->textPtr + // to end up as "&entity123;" in the handler. + if (parser->m_declEntity != NULL) { + const enum XML_Error result + = storeSelfEntityValue(parser, parser->m_declEntity); + if (result != XML_ERROR_NONE) + return result; + } +#endif if (dtd->keepProcessing && parser->m_declEntity && parser->m_entityDeclHandler) { *eventEndPP = s; @@ -5648,7 +5742,7 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, for (;;) { const char *next = NULL; int tok = XmlPrologTok(parser->m_encoding, s, end, &next); -#ifdef XML_DTD +#if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { accountingOnAbort(parser); @@ -5728,7 +5822,7 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { return XML_ERROR_NO_MEMORY; } entity->open = XML_TRUE; -#ifdef XML_DTD +#if XML_GE == 1 entityTrackingOnOpen(parser, entity, __LINE__); #endif entity->processed = 0; @@ -5761,10 +5855,10 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) { entity->processed = (int)(next - textStart); parser->m_processor = internalEntityProcessor; - } else { -#ifdef XML_DTD + } else if (parser->m_openInternalEntities->entity == entity) { +#if XML_GE == 1 entityTrackingOnClose(parser, entity, __LINE__); -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ entity->open = XML_FALSE; parser->m_openInternalEntities = openEntity->next; /* put openEntity back in list of free instances */ @@ -5813,7 +5907,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, return result; } -#ifdef XML_DTD +#if XML_GE == 1 entityTrackingOnClose(parser, entity, __LINE__); #endif entity->open = XML_FALSE; @@ -5892,7 +5986,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, const char *next = ptr; /* XmlAttributeValueTok doesn't always set the last arg */ int tok = XmlAttributeValueTok(enc, ptr, end, &next); -#ifdef XML_DTD +#if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, ptr, next, __LINE__, account)) { accountingOnAbort(parser); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; @@ -5957,14 +6051,14 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, XML_Char ch = (XML_Char)XmlPredefinedEntityName( enc, ptr + enc->minBytesPerChar, next - enc->minBytesPerChar); if (ch) { -#ifdef XML_DTD +#if XML_GE == 1 /* NOTE: We are replacing 4-6 characters original input for 1 character * so there is no amplification and hence recording without * protection. */ accountingDiffTolerated(parser, tok, (char *)&ch, ((char *)&ch) + sizeof(XML_Char), __LINE__, XML_ACCOUNT_ENTITY_EXPANSION); -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ if (! poolAppendChar(pool, ch)) return XML_ERROR_NO_MEMORY; break; @@ -6042,14 +6136,14 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, enum XML_Error result; const XML_Char *textEnd = entity->textPtr + entity->textLen; entity->open = XML_TRUE; -#ifdef XML_DTD +#if XML_GE == 1 entityTrackingOnOpen(parser, entity, __LINE__); #endif result = appendAttributeValue(parser, parser->m_internalEncoding, isCdata, (const char *)entity->textPtr, (const char *)textEnd, pool, XML_ACCOUNT_ENTITY_EXPANSION); -#ifdef XML_DTD +#if XML_GE == 1 entityTrackingOnClose(parser, entity, __LINE__); #endif entity->open = XML_FALSE; @@ -6079,6 +6173,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, /* not reached */ } +#if XML_GE == 1 static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc, const char *entityTextPtr, const char *entityTextEnd, @@ -6086,12 +6181,12 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, DTD *const dtd = parser->m_dtd; /* save one level of indirection */ STRING_POOL *pool = &(dtd->entityValuePool); enum XML_Error result = XML_ERROR_NONE; -#ifdef XML_DTD +# ifdef XML_DTD int oldInEntityValue = parser->m_prologState.inEntityValue; parser->m_prologState.inEntityValue = 1; -#else +# else UNUSED_P(account); -#endif /* XML_DTD */ +# endif /* XML_DTD */ /* never return Null for the value argument in EntityDeclHandler, since this would indicate an external entity; therefore we have to make sure that entityValuePool.start is not null */ @@ -6105,18 +6200,16 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, = entityTextPtr; /* XmlEntityValueTok doesn't always set the last arg */ int tok = XmlEntityValueTok(enc, entityTextPtr, entityTextEnd, &next); -#ifdef XML_DTD if (! accountingDiffTolerated(parser, tok, entityTextPtr, next, __LINE__, account)) { accountingOnAbort(parser); result = XML_ERROR_AMPLIFICATION_LIMIT_BREACH; goto endEntityValue; } -#endif switch (tok) { case XML_TOK_PARAM_ENTITY_REF: -#ifdef XML_DTD +# ifdef XML_DTD if (parser->m_isParamEntity || enc != parser->m_encoding) { const XML_Char *name; ENTITY *entity; @@ -6178,7 +6271,7 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, } break; } -#endif /* XML_DTD */ +# endif /* XML_DTD */ /* In the internal subset, PE references are not legal within markup declarations, e.g entity values in this case. */ parser->m_eventPtr = entityTextPtr; @@ -6259,12 +6352,38 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, entityTextPtr = next; } endEntityValue: -#ifdef XML_DTD +# ifdef XML_DTD parser->m_prologState.inEntityValue = oldInEntityValue; -#endif /* XML_DTD */ +# endif /* XML_DTD */ return result; } +#else /* XML_GE == 0 */ + +static enum XML_Error +storeSelfEntityValue(XML_Parser parser, ENTITY *entity) { + // This will store "&entity123;" in entity->textPtr + // to end up as "&entity123;" in the handler. + const char *const entity_start = "&"; + const char *const entity_end = ";"; + + STRING_POOL *const pool = &(parser->m_dtd->entityValuePool); + if (! poolAppendString(pool, entity_start) + || ! poolAppendString(pool, entity->name) + || ! poolAppendString(pool, entity_end)) { + poolDiscard(pool); + return XML_ERROR_NO_MEMORY; + } + + entity->textPtr = poolStart(pool); + entity->textLen = (int)(poolLength(pool)); + poolFinish(pool); + + return XML_ERROR_NONE; +} + +#endif /* XML_GE == 0 */ + static void FASTCALL normalizeLines(XML_Char *s) { XML_Char *p; @@ -6375,8 +6494,9 @@ reportDefault(XML_Parser parser, const ENCODING *enc, const char *s, } while ((convert_res != XML_CONVERT_COMPLETED) && (convert_res != XML_CONVERT_INPUT_INCOMPLETE)); } else - parser->m_defaultHandler(parser->m_handlerArg, (XML_Char *)s, - (int)((XML_Char *)end - (XML_Char *)s)); + parser->m_defaultHandler( + parser->m_handlerArg, (const XML_Char *)s, + (int)((const XML_Char *)end - (const XML_Char *)s)); } static int @@ -6480,7 +6600,7 @@ getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, name = poolStoreString(&dtd->pool, enc, start, end); if (! name) return NULL; - /* skip quotation mark - its storage will be re-used (like in name[-1]) */ + /* skip quotation mark - its storage will be reused (like in name[-1]) */ ++name; id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, name, sizeof(ATTRIBUTE_ID)); @@ -6630,6 +6750,10 @@ getContext(XML_Parser parser) { static XML_Bool setContext(XML_Parser parser, const XML_Char *context) { + if (context == NULL) { + return XML_FALSE; + } + DTD *const dtd = parser->m_dtd; /* save one level of indirection */ const XML_Char *s = context; @@ -7220,7 +7344,7 @@ poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, return NULL; for (;;) { const enum XML_Convert_Result convert_res = XmlConvert( - enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end); + enc, &ptr, end, (ICHAR **)&(pool->ptr), (const ICHAR *)pool->end); if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; @@ -7651,7 +7775,7 @@ copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) { return result; } -#ifdef XML_DTD +#if XML_GE == 1 static float accountingGetCurrentAmplification(XML_Parser rootParser) { @@ -7672,7 +7796,7 @@ accountingReportStats(XML_Parser originParser, const char *epilog) { const XML_Parser rootParser = getRootParserOf(originParser, NULL); assert(! rootParser->m_parentParser); - if (rootParser->m_accounting.debugLevel < 1) { + if (rootParser->m_accounting.debugLevel == 0u) { return; } @@ -7709,7 +7833,7 @@ accountingReportDiff(XML_Parser rootParser, /* Note: Performance is of no concern here */ const char *walker = before; - if ((rootParser->m_accounting.debugLevel >= 3) + if ((rootParser->m_accounting.debugLevel >= 3u) || (after - before) <= (ptrdiff_t)(contextLength + ellipsisLength + contextLength)) { for (; walker < after; walker++) { @@ -7774,7 +7898,7 @@ accountingDiffTolerated(XML_Parser originParser, int tok, const char *before, || (amplificationFactor <= rootParser->m_accounting.maximumAmplificationFactor); - if (rootParser->m_accounting.debugLevel >= 2) { + if (rootParser->m_accounting.debugLevel >= 2u) { accountingReportStats(rootParser, ""); accountingReportDiff(rootParser, levelsAwayFromRootParser, before, after, bytesMore, source_line, account); @@ -7801,7 +7925,7 @@ static void entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity, const char *action, int sourceLine) { assert(! rootParser->m_parentParser); - if (rootParser->m_entity_stats.debugLevel < 1) + if (rootParser->m_entity_stats.debugLevel == 0u) return; # if defined(XML_UNICODE) @@ -8382,7 +8506,7 @@ unsignedCharToPrintable(unsigned char c) { assert(0); /* never gets here */ } -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ static unsigned long getDebugLevel(const char *variableName, unsigned long defaultDebugLevel) { @@ -8393,9 +8517,9 @@ getDebugLevel(const char *variableName, unsigned long defaultDebugLevel) { const char *const value = valueOrNull; errno = 0; - char *afterValue = (char *)value; + char *afterValue = NULL; unsigned long debugLevel = strtoul(value, &afterValue, 10); - if ((errno != 0) || (afterValue[0] != '\0')) { + if ((errno != 0) || (afterValue == value) || (afterValue[0] != '\0')) { errno = 0; return defaultDebugLevel; } diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index 3f0f5c150c6278e..2c48bf408679538 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -12,10 +12,10 @@ Copyright (c) 2002-2006 Karl Waclawek <karl@waclawek.net> Copyright (c) 2002-2003 Fred L. Drake, Jr. <fdrake@users.sourceforge.net> Copyright (c) 2005-2009 Steven Solie <steven@solie.ca> - Copyright (c) 2016-2021 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2016-2023 Sebastian Pipping <sebastian@pipping.org> Copyright (c) 2017 Rhodri James <rhodri@wildebeest.org.uk> Copyright (c) 2019 David Loffredo <loffredo@steptools.com> - Copyright (c) 2021 Dong-hee Na <donghee.na@python.org> + Copyright (c) 2021 Donghee Na <donghee.na@python.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -38,7 +38,7 @@ USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include <expat_config.h> +#include "expat_config.h" #include <stddef.h> diff --git a/Modules/expat/xmlrole.h b/Modules/expat/xmlrole.h index d6e1fa150a108ac..a7904274c91d4ec 100644 --- a/Modules/expat/xmlrole.h +++ b/Modules/expat/xmlrole.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper <coopercc@users.sourceforge.net> Copyright (c) 2002 Karl Waclawek <karl@waclawek.net> Copyright (c) 2002 Fred L. Drake, Jr. <fdrake@users.sourceforge.net> - Copyright (c) 2017 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2017-2024 Sebastian Pipping <sebastian@pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -127,9 +127,9 @@ typedef struct prolog_state { #endif /* XML_DTD */ } PROLOG_STATE; -void XmlPrologStateInit(PROLOG_STATE *); +void XmlPrologStateInit(PROLOG_STATE *state); #ifdef XML_DTD -void XmlPrologStateInitExternalEntity(PROLOG_STATE *); +void XmlPrologStateInitExternalEntity(PROLOG_STATE *state); #endif /* XML_DTD */ #define XmlTokenRole(state, tok, ptr, end, enc) \ diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index 2b7012a58be4196..29a66d72ceea5e5 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -12,7 +12,7 @@ Copyright (c) 2002 Greg Stein <gstein@users.sourceforge.net> Copyright (c) 2002-2016 Karl Waclawek <karl@waclawek.net> Copyright (c) 2005-2009 Steven Solie <steven@solie.ca> - Copyright (c) 2016-2022 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2016-2024 Sebastian Pipping <sebastian@pipping.org> Copyright (c) 2016 Pascal Cuoq <cuoq@trust-in-soft.com> Copyright (c) 2016 Don Lewis <truckman@apache.org> Copyright (c) 2017 Rhodri James <rhodri@wildebeest.org.uk> @@ -20,8 +20,10 @@ Copyright (c) 2017 Benbuck Nason <bnason@netflix.com> Copyright (c) 2017 José Gutiérrez de la Concha <jose@zeroc.com> Copyright (c) 2019 David Loffredo <loffredo@steptools.com> - Copyright (c) 2021 Dong-hee Na <donghee.na@python.org> + Copyright (c) 2021 Donghee Na <donghee.na@python.org> Copyright (c) 2022 Martin Ettl <ettl.martin78@googlemail.com> + Copyright (c) 2022 Sean McBride <sean@rogue-research.com> + Copyright (c) 2023 Hanno Böck <hanno@gentoo.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -44,7 +46,7 @@ USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include <expat_config.h> +#include "expat_config.h" #include <stddef.h> #include <string.h> /* memcpy */ @@ -76,7 +78,7 @@ #define VTABLE VTABLE1, PREFIX(toUtf8), PREFIX(toUtf16) #define UCS2_GET_NAMING(pages, hi, lo) \ - (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo)&0x1F))) + (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo) & 0x1F))) /* A 2 byte UTF-8 representation splits the characters 11 bits between the bottom 5 and 6 bits of the bytes. We need 8 bits to index into @@ -100,7 +102,7 @@ & (1u << (((byte)[2]) & 0x1F))) /* Detection of invalid UTF-8 sequences is based on Table 3.1B - of Unicode 3.2: http://www.unicode.org/unicode/reports/tr28/ + of Unicode 3.2: https://www.unicode.org/unicode/reports/tr28/ with the additional restriction of not allowing the Unicode code points 0xFFFF and 0xFFFE (sequences EF,BF,BF and EF,BF,BE). Implementation details: @@ -225,7 +227,7 @@ struct normal_encoding { /* isNmstrt2 */ NULL, /* isNmstrt3 */ NULL, /* isNmstrt4 */ NULL, \ /* isInvalid2 */ NULL, /* isInvalid3 */ NULL, /* isInvalid4 */ NULL -static int FASTCALL checkCharRefNumber(int); +static int FASTCALL checkCharRefNumber(int result); #include "xmltok_impl.h" #include "ascii.h" @@ -243,7 +245,7 @@ static int FASTCALL checkCharRefNumber(int); #endif #define SB_BYTE_TYPE(enc, p) \ - (((struct normal_encoding *)(enc))->type[(unsigned char)*(p)]) + (((const struct normal_encoding *)(enc))->type[(unsigned char)*(p)]) #ifdef XML_MIN_SIZE static int PTRFASTCALL @@ -407,7 +409,7 @@ utf8_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short *to = *toP; const char *from = *fromP; while (from < fromLim && to < toLim) { - switch (((struct normal_encoding *)enc)->type[(unsigned char)*from]) { + switch (SB_BYTE_TYPE(enc, from)) { case BT_LEAD2: if (fromLim - from < 2) { res = XML_CONVERT_INPUT_INCOMPLETE; @@ -715,31 +717,26 @@ unicode_byte_type(char hi, char lo) { return res; \ } -#define SET2(ptr, ch) (((ptr)[0] = ((ch)&0xff)), ((ptr)[1] = ((ch) >> 8))) #define GET_LO(ptr) ((unsigned char)(ptr)[0]) #define GET_HI(ptr) ((unsigned char)(ptr)[1]) DEFINE_UTF16_TO_UTF8(little2_) DEFINE_UTF16_TO_UTF16(little2_) -#undef SET2 #undef GET_LO #undef GET_HI -#define SET2(ptr, ch) (((ptr)[0] = ((ch) >> 8)), ((ptr)[1] = ((ch)&0xFF))) #define GET_LO(ptr) ((unsigned char)(ptr)[1]) #define GET_HI(ptr) ((unsigned char)(ptr)[0]) DEFINE_UTF16_TO_UTF8(big2_) DEFINE_UTF16_TO_UTF16(big2_) -#undef SET2 #undef GET_LO #undef GET_HI #define LITTLE2_BYTE_TYPE(enc, p) \ - ((p)[1] == 0 ? ((struct normal_encoding *)(enc))->type[(unsigned char)*(p)] \ - : unicode_byte_type((p)[1], (p)[0])) + ((p)[1] == 0 ? SB_BYTE_TYPE(enc, p) : unicode_byte_type((p)[1], (p)[0])) #define LITTLE2_BYTE_TO_ASCII(p) ((p)[1] == 0 ? (p)[0] : -1) #define LITTLE2_CHAR_MATCHES(p, c) ((p)[1] == 0 && (p)[0] == (c)) #define LITTLE2_IS_NAME_CHAR_MINBPC(p) \ @@ -872,9 +869,7 @@ static const struct normal_encoding internal_little2_encoding #endif #define BIG2_BYTE_TYPE(enc, p) \ - ((p)[0] == 0 \ - ? ((struct normal_encoding *)(enc))->type[(unsigned char)(p)[1]] \ - : unicode_byte_type((p)[0], (p)[1])) + ((p)[0] == 0 ? SB_BYTE_TYPE(enc, p + 1) : unicode_byte_type((p)[0], (p)[1])) #define BIG2_BYTE_TO_ASCII(p) ((p)[0] == 0 ? (p)[1] : -1) #define BIG2_CHAR_MATCHES(p, c) ((p)[0] == 0 && (p)[1] == (c)) #define BIG2_IS_NAME_CHAR_MINBPC(p) \ diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h index 6f630c2f9ba96d5..c51fce1ec1518be 100644 --- a/Modules/expat/xmltok.h +++ b/Modules/expat/xmltok.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper <coopercc@users.sourceforge.net> Copyright (c) 2002 Fred L. Drake, Jr. <fdrake@users.sourceforge.net> Copyright (c) 2002-2005 Karl Waclawek <karl@waclawek.net> - Copyright (c) 2016-2017 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2016-2024 Sebastian Pipping <sebastian@pipping.org> Copyright (c) 2017 Rhodri James <rhodri@wildebeest.org.uk> Licensed under the MIT license: @@ -289,7 +289,8 @@ int XmlParseXmlDecl(int isGeneralTextEntity, const ENCODING *enc, const char **encodingNamePtr, const ENCODING **namedEncodingPtr, int *standalonePtr); -int XmlInitEncoding(INIT_ENCODING *, const ENCODING **, const char *name); +int XmlInitEncoding(INIT_ENCODING *p, const ENCODING **encPtr, + const char *name); const ENCODING *XmlGetUtf8InternalEncoding(void); const ENCODING *XmlGetUtf16InternalEncoding(void); int FASTCALL XmlUtf8Encode(int charNumber, char *buf); @@ -307,7 +308,8 @@ int XmlParseXmlDeclNS(int isGeneralTextEntity, const ENCODING *enc, const char **encodingNamePtr, const ENCODING **namedEncodingPtr, int *standalonePtr); -int XmlInitEncodingNS(INIT_ENCODING *, const ENCODING **, const char *name); +int XmlInitEncodingNS(INIT_ENCODING *p, const ENCODING **encPtr, + const char *name); const ENCODING *XmlGetUtf8InternalEncodingNS(void); const ENCODING *XmlGetUtf16InternalEncodingNS(void); ENCODING *XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert, diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c index 1971d74bf8c91f8..239a2d06c4512ce 100644 --- a/Modules/expat/xmltok_impl.c +++ b/Modules/expat/xmltok_impl.c @@ -126,7 +126,7 @@ # endif # define HAS_CHARS(enc, ptr, end, count) \ - ((end) - (ptr) >= ((count)*MINBPC(enc))) + ((end) - (ptr) >= ((count) * MINBPC(enc))) # define HAS_CHAR(enc, ptr, end) HAS_CHARS(enc, ptr, end, 1) diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index a2e3c2300b3ce85..02e94a211914839 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -4,6 +4,7 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_signal.h" // Py_NSIG #include "pycore_sysmodule.h" // _PySys_GetAttr() +#include "pycore_time.h" // _PyTime_FromSecondsObject() #include "pycore_traceback.h" // _Py_DumpTracebackThreads #ifdef HAVE_UNISTD_H @@ -119,6 +120,13 @@ faulthandler_get_fileno(PyObject **file_ptr) } } else if (PyLong_Check(file)) { + if (PyBool_Check(file)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return -1; + } + } fd = PyLong_AsInt(file); if (fd == -1 && PyErr_Occurred()) return -1; @@ -616,7 +624,7 @@ cancel_dump_traceback_later(void) #define SEC_TO_US (1000 * 1000) static char* -format_timeout(_PyTime_t us) +format_timeout(PyTime_t us) { unsigned long sec, min, hour; char buffer[100]; @@ -649,7 +657,7 @@ faulthandler_dump_traceback_later(PyObject *self, { static char *kwlist[] = {"timeout", "repeat", "file", "exit", NULL}; PyObject *timeout_obj; - _PyTime_t timeout, timeout_us; + PyTime_t timeout, timeout_us; int repeat = 0; PyObject *file = NULL; int fd; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index ffddef34ecce7a5..9807d2e7d48a364 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -201,6 +201,16 @@ gc_get_count_impl(PyObject *module) /*[clinic end generated code: output=354012e67b16398f input=a392794a08251751]*/ { GCState *gcstate = get_gc_state(); + +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + struct _gc_thread_state *gc = &tstate->gc; + + // Flush the local allocation count to the global count + _Py_atomic_add_int(&gcstate->generations[0].count, (int)gc->alloc_count); + gc->alloc_count = 0; +#endif + return Py_BuildValue("(iii)", gcstate->generations[0].count, gcstate->generations[1].count, @@ -235,6 +245,26 @@ referentsvisit(PyObject *obj, void *arg) return PyList_Append(list, obj) < 0; } +static int +append_referrents(PyObject *result, PyObject *args) +{ + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(args); i++) { + PyObject *obj = PyTuple_GET_ITEM(args, i); + if (!_PyObject_IS_GC(obj)) { + continue; + } + + traverseproc traverse = Py_TYPE(obj)->tp_traverse; + if (!traverse) { + continue; + } + if (traverse(obj, referentsvisit, result)) { + return -1; + } + } + return 0; +} + /*[clinic input] gc.get_referents @@ -247,29 +277,24 @@ static PyObject * gc_get_referents_impl(PyObject *module, PyObject *args) /*[clinic end generated code: output=d47dc02cefd06fe8 input=b3ceab0c34038cbf]*/ { - Py_ssize_t i; if (PySys_Audit("gc.get_referents", "(O)", args) < 0) { return NULL; } + PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *result = PyList_New(0); if (result == NULL) return NULL; - for (i = 0; i < PyTuple_GET_SIZE(args); i++) { - traverseproc traverse; - PyObject *obj = PyTuple_GET_ITEM(args, i); + // NOTE: stop the world is a no-op in default build + _PyEval_StopTheWorld(interp); + int err = append_referrents(result, args); + _PyEval_StartTheWorld(interp); - if (!_PyObject_IS_GC(obj)) - continue; - traverse = Py_TYPE(obj)->tp_traverse; - if (! traverse) - continue; - if (traverse(obj, referentsvisit, result)) { - Py_DECREF(result); - return NULL; - } + if (err < 0) { + Py_CLEAR(result); } + return result; } @@ -358,7 +383,7 @@ gc_get_stats_impl(PyObject *module) /*[clinic input] -gc.is_tracked +gc.is_tracked -> bool obj: object / @@ -368,21 +393,15 @@ Returns true if the object is tracked by the garbage collector. Simple atomic objects will return false. [clinic start generated code]*/ -static PyObject * -gc_is_tracked(PyObject *module, PyObject *obj) -/*[clinic end generated code: output=14f0103423b28e31 input=d83057f170ea2723]*/ +static int +gc_is_tracked_impl(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=91c8d086b7f47a33 input=423b98ec680c3126]*/ { - PyObject *result; - - if (_PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)) - result = Py_True; - else - result = Py_False; - return Py_NewRef(result); + return PyObject_GC_IsTracked(obj); } /*[clinic input] -gc.is_finalized +gc.is_finalized -> bool obj: object / @@ -390,14 +409,11 @@ gc.is_finalized Returns true if the object has been already finalized by the GC. [clinic start generated code]*/ -static PyObject * -gc_is_finalized(PyObject *module, PyObject *obj) -/*[clinic end generated code: output=e1516ac119a918ed input=201d0c58f69ae390]*/ +static int +gc_is_finalized_impl(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=401ff5d6fc660429 input=ca4d111c8f8c4e3a]*/ { - if (_PyObject_IS_GC(obj) && _PyGC_FINALIZED(obj)) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + return PyObject_GC_IsFinalized(obj); } /*[clinic input] diff --git a/Modules/getpath.c b/Modules/getpath.c index a3c8fc269d1c3cf..abed139028244ad 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -262,6 +262,10 @@ getpath_joinpath(PyObject *Py_UNUSED(self), PyObject *args) } /* Convert all parts to wchar and accumulate max final length */ wchar_t **parts = (wchar_t **)PyMem_Malloc(n * sizeof(wchar_t *)); + if (parts == NULL) { + PyErr_NoMemory(); + return NULL; + } memset(parts, 0, n * sizeof(wchar_t *)); Py_ssize_t cchFinal = 0; Py_ssize_t first = 0; diff --git a/Modules/makesetup b/Modules/makesetup index f000c9cd67310e3..d41b6640bb5186f 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -87,18 +87,6 @@ esac NL='\ ' -# Setup to link with extra libraries when making shared extensions. -# Currently, only Cygwin needs this baggage. -case `uname -s` in -CYGWIN*) if test $libdir = . - then - ExtraLibDir=. - else - ExtraLibDir='$(LIBPL)' - fi - ExtraLibs="-L$ExtraLibDir -lpython\$(LDVERSION)";; -esac - # Main loop for i in ${*-Setup} do @@ -286,7 +274,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | ;; esac rule="$file: $objs" - rule="$rule; \$(BLDSHARED) $objs $libs $ExtraLibs -o $file" + rule="$rule; \$(BLDSHARED) $objs $libs \$(MODULE_LDFLAGS) -o $file" echo "$rule" >>$rulesf done done diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 0be46b1574c1feb..a877bfcd6afb687 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -2074,11 +2074,6 @@ math_trunc(PyObject *module, PyObject *x) return PyFloat_Type.tp_as_number->nb_int(x); } - if (!_PyType_IsReady(Py_TYPE(x))) { - if (PyType_Ready(Py_TYPE(x)) < 0) - return NULL; - } - math_module_state *state = get_math_module_state(module); trunc = _PyObject_LookupSpecial(x, state->str___trunc__); if (trunc == NULL) { diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 007fc1cb116f84c..fd70b38bddec79e 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -23,6 +23,7 @@ #include "pycore_pylifecycle.h" // _PyOS_URandom() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_signal.h" // Py_NSIG +#include "pycore_time.h" // _PyLong_FromTime_t() #ifdef HAVE_UNISTD_H # include <unistd.h> // symlink() @@ -52,6 +53,12 @@ # define EX_OK EXIT_SUCCESS #endif +#ifdef __APPLE__ + /* Needed for the implementation of os.statvfs */ +# include <sys/param.h> +# include <sys/mount.h> +#endif + /* On android API level 21, 'AT_EACCESS' is not declared although * HAVE_FACCESSAT is defined. */ #ifdef __ANDROID__ @@ -637,6 +644,11 @@ PyOS_AfterFork_Child(void) tstate->native_thread_id = PyThread_get_thread_native_id(); #endif +#ifdef Py_GIL_DISABLED + _Py_brc_after_fork(tstate->interp); + _Py_qsbr_after_fork((_PyThreadStateImpl *)tstate); +#endif + status = _PyEval_ReInitThreads(tstate); if (_PyStatus_EXCEPTION(status)) { goto fatal_error; @@ -969,6 +981,13 @@ _fd_converter(PyObject *o, int *p) int overflow; long long_value; + if (PyBool_Check(o)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return 0; + } + } PyObject *index = _PyNumber_Index(o); if (index == NULL) { return 0; @@ -1620,7 +1639,7 @@ convertenviron(void) Py_DECREF(d); return NULL; } - if (PyDict_SetDefault(d, k, v) == NULL) { + if (PyDict_SetDefaultRef(d, k, v, NULL) < 0) { Py_DECREF(v); Py_DECREF(k); Py_DECREF(d); @@ -7824,7 +7843,7 @@ os_fork1_impl(PyObject *module) PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->finalizing) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_PythonFinalizationError, "can't fork at interpreter shutdown"); return NULL; } @@ -7868,7 +7887,7 @@ os_fork_impl(PyObject *module) pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->finalizing) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_PythonFinalizationError, "can't fork at interpreter shutdown"); return NULL; } @@ -8358,6 +8377,149 @@ os_sched_getaffinity_impl(PyObject *module, pid_t pid) #endif /* HAVE_SCHED_H */ +#ifdef HAVE_POSIX_OPENPT +/*[clinic input] +os.posix_openpt -> int + + oflag: int + / + +Open and return a file descriptor for a master pseudo-terminal device. + +Performs a posix_openpt() C function call. The oflag argument is used to +set file status flags and file access modes as specified in the manual page +of posix_openpt() of your system. +[clinic start generated code]*/ + +static int +os_posix_openpt_impl(PyObject *module, int oflag) +/*[clinic end generated code: output=ee0bc2624305fc79 input=0de33d0e29693caa]*/ +{ + int fd; + +#if defined(O_CLOEXEC) + oflag |= O_CLOEXEC; +#endif + + fd = posix_openpt(oflag); + if (fd == -1) { + posix_error(); + return -1; + } + + // Just in case, likely a no-op given O_CLOEXEC above. + if (_Py_set_inheritable(fd, 0, NULL) < 0) { + close(fd); + return -1; + } + + return fd; +} +#endif /* HAVE_POSIX_OPENPT */ + +#ifdef HAVE_GRANTPT +/*[clinic input] +os.grantpt + + fd: fildes + File descriptor of a master pseudo-terminal device. + / + +Grant access to the slave pseudo-terminal device. + +Performs a grantpt() C function call. +[clinic start generated code]*/ + +static PyObject * +os_grantpt_impl(PyObject *module, int fd) +/*[clinic end generated code: output=dfd580015cf548ab input=0668e3b96760e849]*/ +{ + int ret; + int saved_errno; + PyOS_sighandler_t sig_saved; + + sig_saved = PyOS_setsig(SIGCHLD, SIG_DFL); + + ret = grantpt(fd); + if (ret == -1) + saved_errno = errno; + + PyOS_setsig(SIGCHLD, sig_saved); + + if (ret == -1) { + errno = saved_errno; + return posix_error(); + } + + Py_RETURN_NONE; +} +#endif /* HAVE_GRANTPT */ + +#ifdef HAVE_UNLOCKPT +/*[clinic input] +os.unlockpt + + fd: fildes + File descriptor of a master pseudo-terminal device. + / + +Unlock a pseudo-terminal master/slave pair. + +Performs an unlockpt() C function call. +[clinic start generated code]*/ + +static PyObject * +os_unlockpt_impl(PyObject *module, int fd) +/*[clinic end generated code: output=e08d354dec12d30c input=de7ab1f59f69a2b4]*/ +{ + if (unlockpt(fd) == -1) + return posix_error(); + + Py_RETURN_NONE; +} +#endif /* HAVE_UNLOCKPT */ + +#if defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R) +/*[clinic input] +os.ptsname + + fd: fildes + File descriptor of a master pseudo-terminal device. + / + +Return the name of the slave pseudo-terminal device. + +If the ptsname_r() C function is available, it is called; +otherwise, performs a ptsname() C function call. +[clinic start generated code]*/ + +static PyObject * +os_ptsname_impl(PyObject *module, int fd) +/*[clinic end generated code: output=ef300fadc5675872 input=1369ccc0546f3130]*/ +{ +#ifdef HAVE_PTSNAME_R + int ret; + char name[MAXPATHLEN+1]; + + ret = ptsname_r(fd, name, sizeof(name)); + if (ret != 0) { + errno = ret; + return posix_error(); + } +#else + char *name; + + name = ptsname(fd); + /* POSIX manpage: Upon failure, ptsname() shall return a null pointer and may set errno. + *MAY* set errno? Hmm... */ + if (name == NULL) + return posix_error(); +#endif /* HAVE_PTSNAME_R */ + + return PyUnicode_DecodeFSDefault(name); +} +#endif /* defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R) */ + /* AIX uses /dev/ptc but is otherwise the same as /dev/ptmx */ #if defined(HAVE_DEV_PTC) && !defined(HAVE_DEV_PTMX) # define DEV_PTY_FILE "/dev/ptc" @@ -8558,7 +8720,7 @@ os_forkpty_impl(PyObject *module) PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->finalizing) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_PythonFinalizationError, "can't fork at interpreter shutdown"); return NULL; } @@ -10249,7 +10411,7 @@ build_itimerspec(const struct itimerspec* curr_value) static PyObject * build_itimerspec_ns(const struct itimerspec* curr_value) { - _PyTime_t value, interval; + PyTime_t value, interval; if (_PyTime_FromTimespec(&value, &curr_value->it_value) < 0) { return NULL; } @@ -12732,6 +12894,62 @@ os_WSTOPSIG_impl(PyObject *module, int status) #endif #include <sys/statvfs.h> +#ifdef __APPLE__ +/* On macOS struct statvfs uses 32-bit integers for block counts, + * resulting in overflow when filesystems are larger than 4TB. Therefore + * os.statvfs is implemented in terms of statfs(2). + */ + +static PyObject* +_pystatvfs_fromstructstatfs(PyObject *module, struct statfs st) { + PyObject *StatVFSResultType = get_posix_state(module)->StatVFSResultType; + PyObject *v = PyStructSequence_New((PyTypeObject *)StatVFSResultType); + if (v == NULL) { + return NULL; + } + + long flags = 0; + if (st.f_flags & MNT_RDONLY) { + flags |= ST_RDONLY; + } + if (st.f_flags & MNT_NOSUID) { + flags |= ST_NOSUID; + } + + _Static_assert(sizeof(st.f_blocks) == sizeof(long long), "assuming large file"); + +#define SET_ITEM(SEQ, INDEX, EXPR) \ + do { \ + PyObject *obj = (EXPR); \ + if (obj == NULL) { \ + Py_DECREF((SEQ)); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM((SEQ), (INDEX), obj); \ + } while (0) + + SET_ITEM(v, 0, PyLong_FromLong((long) st.f_iosize)); + SET_ITEM(v, 1, PyLong_FromLong((long) st.f_bsize)); + SET_ITEM(v, 2, PyLong_FromLongLong((long long) st.f_blocks)); + SET_ITEM(v, 3, PyLong_FromLongLong((long long) st.f_bfree)); + SET_ITEM(v, 4, PyLong_FromLongLong((long long) st.f_bavail)); + SET_ITEM(v, 5, PyLong_FromLongLong((long long) st.f_files)); + SET_ITEM(v, 6, PyLong_FromLongLong((long long) st.f_ffree)); + SET_ITEM(v, 7, PyLong_FromLongLong((long long) st.f_ffree)); + SET_ITEM(v, 8, PyLong_FromLong((long) flags)); + + SET_ITEM(v, 9, PyLong_FromLong((long) NAME_MAX)); + SET_ITEM(v, 10, PyLong_FromUnsignedLong(st.f_fsid.val[0])); + +#undef SET_ITEM + + return v; +} + +#else + + + static PyObject* _pystatvfs_fromstructstatvfs(PyObject *module, struct statvfs st) { PyObject *StatVFSResultType = get_posix_state(module)->StatVFSResultType; @@ -12783,6 +13001,8 @@ _pystatvfs_fromstructstatvfs(PyObject *module, struct statvfs st) { return v; } +#endif + /*[clinic input] os.fstatvfs @@ -12800,6 +13020,22 @@ os_fstatvfs_impl(PyObject *module, int fd) { int result; int async_err = 0; +#ifdef __APPLE__ + struct statfs st; + /* On macOS os.fstatvfs is implemented using fstatfs(2) because + * the former uses 32-bit values for block counts. + */ + do { + Py_BEGIN_ALLOW_THREADS + result = fstatfs(fd, &st); + Py_END_ALLOW_THREADS + } while (result != 0 && errno == EINTR && + !(async_err = PyErr_CheckSignals())); + if (result != 0) + return (!async_err) ? posix_error() : NULL; + + return _pystatvfs_fromstructstatfs(module, st); +#else struct statvfs st; do { @@ -12812,6 +13048,7 @@ os_fstatvfs_impl(PyObject *module, int fd) return (!async_err) ? posix_error() : NULL; return _pystatvfs_fromstructstatvfs(module, st); +#endif } #endif /* defined(HAVE_FSTATVFS) && defined(HAVE_SYS_STATVFS_H) */ @@ -12835,6 +13072,28 @@ os_statvfs_impl(PyObject *module, path_t *path) /*[clinic end generated code: output=87106dd1beb8556e input=3f5c35791c669bd9]*/ { int result; + +#ifdef __APPLE__ + /* On macOS os.statvfs is implemented using statfs(2)/fstatfs(2) because + * the former uses 32-bit values for block counts. + */ + struct statfs st; + + Py_BEGIN_ALLOW_THREADS + if (path->fd != -1) { + result = fstatfs(path->fd, &st); + } + else + result = statfs(path->narrow, &st); + Py_END_ALLOW_THREADS + + if (result) { + return path_error(path); + } + + return _pystatvfs_fromstructstatfs(module, st); + +#else struct statvfs st; Py_BEGIN_ALLOW_THREADS @@ -12852,6 +13111,7 @@ os_statvfs_impl(PyObject *module, path_t *path) } return _pystatvfs_fromstructstatvfs(module, st); +#endif } #endif /* defined(HAVE_STATVFS) && defined(HAVE_SYS_STATVFS_H) */ @@ -16275,6 +16535,10 @@ static PyMethodDef posix_methods[] = { OS_SCHED_YIELD_METHODDEF OS_SCHED_SETAFFINITY_METHODDEF OS_SCHED_GETAFFINITY_METHODDEF + OS_POSIX_OPENPT_METHODDEF + OS_GRANTPT_METHODDEF + OS_UNLOCKPT_METHODDEF + OS_PTSNAME_METHODDEF OS_OPENPTY_METHODDEF OS_LOGIN_TTY_METHODDEF OS_FORKPTY_METHODDEF diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 7c08eda83e66b2a..62cd262a7885e93 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1615,7 +1615,8 @@ static int init_handler_descrs(pyexpat_state *state) if (descr == NULL) return -1; - if (PyDict_SetDefault(state->xml_parse_type->tp_dict, PyDescr_NAME(descr), descr) == NULL) { + if (PyDict_SetDefaultRef(state->xml_parse_type->tp_dict, + PyDescr_NAME(descr), descr, NULL) < 0) { Py_DECREF(descr); return -1; } diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 1dbde3e9e6ca5df..f16173aafa7d3c8 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -15,7 +15,7 @@ #include "Python.h" #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_import.h" // _PyImport_GetModuleAttrString() -#include "pycore_time.h" // _PyTime_t +#include "pycore_time.h" // _PyTime_FromSecondsObject() #include <stdbool.h> #include <stddef.h> // offsetof() @@ -297,7 +297,7 @@ select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, struct timeval tv, *tvp; int imax, omax, emax, max; int n; - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; if (timeout_obj == Py_None) tvp = (struct timeval *)NULL; @@ -619,7 +619,7 @@ select_poll_poll_impl(pollObject *self, PyObject *timeout_obj) PyObject *result_list = NULL; int poll_result, i, j; PyObject *value = NULL, *num = NULL; - _PyTime_t timeout = -1, ms = -1, deadline = 0; + PyTime_t timeout = -1, ms = -1, deadline = 0; int async_err = 0; if (timeout_obj != Py_None) { @@ -946,7 +946,7 @@ select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj) PyObject *result_list = NULL; int poll_result, i; PyObject *value, *num1, *num2; - _PyTime_t timeout, ms, deadline = 0; + PyTime_t timeout, ms, deadline = 0; if (self->fd_devpoll < 0) return devpoll_err_closed(); @@ -1559,7 +1559,7 @@ select_epoll_poll_impl(pyEpoll_Object *self, PyObject *timeout_obj, int nfds, i; PyObject *elist = NULL, *etuple = NULL; struct epoll_event *evs = NULL; - _PyTime_t timeout = -1, ms = -1, deadline = 0; + PyTime_t timeout = -1, ms = -1, deadline = 0; if (self->epfd < 0) return pyepoll_err_closed(); @@ -2242,7 +2242,7 @@ select_kqueue_control_impl(kqueue_queue_Object *self, PyObject *changelist, struct kevent *chl = NULL; struct timespec timeoutspec; struct timespec *ptimeoutspec; - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; _selectstate *state = _selectstate_by_type(Py_TYPE(self)); if (self->kqfd < 0) diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 394a997b20c06de..5804e30af1b4265 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -13,6 +13,7 @@ #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_signal.h" // _Py_RestoreSignals() +#include "pycore_time.h" // _PyTime_FromSecondsObject() #ifndef MS_WINDOWS # include "posixmodule.h" // _PyLong_FromUid() @@ -173,7 +174,7 @@ timeval_from_double(PyObject *obj, struct timeval *tv) return 0; } - _PyTime_t t; + PyTime_t t; if (_PyTime_FromSecondsObject(&t, obj, _PyTime_ROUND_CEILING) < 0) { return -1; } @@ -276,11 +277,7 @@ trip_signal(int sig_num) cleared in PyErr_CheckSignals() before .tripped. */ _Py_atomic_store_int(&is_tripped, 1); - /* Signals are always handled by the main interpreter */ - PyInterpreterState *interp = _PyInterpreterState_Main(); - - /* Notify ceval.c */ - _PyEval_SignalReceived(interp); + _PyEval_SignalReceived(); /* And then write to the wakeup fd *after* setting all the globals and doing the _PyEval_SignalReceived. We used to write to the wakeup fd @@ -303,6 +300,7 @@ trip_signal(int sig_num) int fd = wakeup.fd; if (fd != INVALID_FD) { + PyInterpreterState *interp = _PyInterpreterState_Main(); unsigned char byte = (unsigned char)sig_num; #ifdef MS_WINDOWS if (wakeup.use_send) { @@ -1210,7 +1208,7 @@ signal_sigtimedwait_impl(PyObject *module, sigset_t sigset, PyObject *timeout_obj) /*[clinic end generated code: output=59c8971e8ae18a64 input=87fd39237cf0b7ba]*/ { - _PyTime_t timeout; + PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_CEILING) < 0) return NULL; @@ -1220,7 +1218,7 @@ signal_sigtimedwait_impl(PyObject *module, sigset_t sigset, return NULL; } - _PyTime_t deadline = _PyDeadline_Init(timeout); + PyTime_t deadline = _PyDeadline_Init(timeout); siginfo_t si; do { @@ -1770,8 +1768,8 @@ PyErr_CheckSignals(void) Python code to ensure signals are handled. Checking for the GC here allows long running native code to clean cycles created using the C-API even if it doesn't run the evaluation loop */ - if (_Py_eval_breaker_bit_is_set(tstate->interp, _PY_GC_SCHEDULED_BIT)) { - _Py_set_eval_breaker_bit(tstate->interp, _PY_GC_SCHEDULED_BIT, 0); + if (_Py_eval_breaker_bit_is_set(tstate, _PY_GC_SCHEDULED_BIT)) { + _Py_unset_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT); _Py_RunGC(tstate); } diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 0a0e0e78656f76b..cd9a803648be712 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -109,6 +109,7 @@ Local naming conventions: #include "pycore_capsule.h" // _PyCapsule_SetTraverse() #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_moduleobject.h" // _PyModule_GetState +#include "pycore_time.h" // _PyTime_AsMilliseconds() #ifdef _Py_MEMORY_SANITIZER # include <sanitizer/msan_interface.h> @@ -547,7 +548,7 @@ typedef struct _socket_state { PyObject *socket_gaierror; /* Default timeout for new sockets */ - _PyTime_t defaulttimeout; + PyTime_t defaulttimeout; #if defined(HAVE_ACCEPT) || defined(HAVE_ACCEPT4) #if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) @@ -772,13 +773,13 @@ internal_setblocking(PySocketSockObject *s, int block) } static int -internal_select(PySocketSockObject *s, int writing, _PyTime_t interval, +internal_select(PySocketSockObject *s, int writing, PyTime_t interval, int connect) { int n; #ifdef HAVE_POLL struct pollfd pollfd; - _PyTime_t ms; + PyTime_t ms; #else fd_set fds, efds; struct timeval tv, *tvp; @@ -888,10 +889,10 @@ sock_call_ex(PySocketSockObject *s, void *data, int connect, int *err, - _PyTime_t timeout) + PyTime_t timeout) { int has_timeout = (timeout > 0); - _PyTime_t deadline = 0; + PyTime_t deadline = 0; int deadline_initialized = 0; int res; @@ -905,7 +906,7 @@ sock_call_ex(PySocketSockObject *s, runs asynchronously. */ if (has_timeout || connect) { if (has_timeout) { - _PyTime_t interval; + PyTime_t interval; if (deadline_initialized) { /* recompute the timeout */ @@ -3011,13 +3012,13 @@ Returns True if socket is in blocking mode, or False if it\n\ is in non-blocking mode."); static int -socket_parse_timeout(_PyTime_t *timeout, PyObject *timeout_obj) +socket_parse_timeout(PyTime_t *timeout, PyObject *timeout_obj) { #ifdef MS_WINDOWS struct timeval tv; #endif #ifndef HAVE_POLL - _PyTime_t ms; + PyTime_t ms; #endif int overflow = 0; @@ -3060,7 +3061,7 @@ socket_parse_timeout(_PyTime_t *timeout, PyObject *timeout_obj) static PyObject * sock_settimeout(PySocketSockObject *s, PyObject *arg) { - _PyTime_t timeout; + PyTime_t timeout; if (socket_parse_timeout(&timeout, arg) < 0) return NULL; @@ -3112,7 +3113,7 @@ sock_gettimeout(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } else { - double seconds = _PyTime_AsSecondsDouble(s->sock_timeout); + double seconds = PyTime_AsSecondsDouble(s->sock_timeout); return PyFloat_FromDouble(seconds); } } @@ -4382,8 +4383,8 @@ sock_sendall(PySocketSockObject *s, PyObject *args) Py_buffer pbuf; struct sock_send ctx; int has_timeout = (s->sock_timeout > 0); - _PyTime_t timeout = s->sock_timeout; - _PyTime_t deadline = 0; + PyTime_t timeout = s->sock_timeout; + PyTime_t deadline = 0; int deadline_initialized = 0; PyObject *res = NULL; @@ -6916,7 +6917,7 @@ socket_getdefaulttimeout(PyObject *self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } else { - double seconds = _PyTime_AsSecondsDouble(state->defaulttimeout); + double seconds = PyTime_AsSecondsDouble(state->defaulttimeout); return PyFloat_FromDouble(seconds); } } @@ -6931,7 +6932,7 @@ When the socket module is first imported, the default is None."); static PyObject * socket_setdefaulttimeout(PyObject *self, PyObject *arg) { - _PyTime_t timeout; + PyTime_t timeout; if (socket_parse_timeout(&timeout, arg) < 0) return NULL; @@ -7926,6 +7927,9 @@ socket_exec(PyObject *m) #ifdef SO_BINDTODEVICE ADD_INT_MACRO(m, SO_BINDTODEVICE); #endif +#ifdef SO_BINDTOIFINDEX + ADD_INT_MACRO(m, SO_BINDTOIFINDEX); +#endif #ifdef SO_PRIORITY ADD_INT_MACRO(m, SO_PRIORITY); #endif diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index 47146a28e02c8f4..09fd70f351f1d84 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -1,7 +1,5 @@ /* Socket module header file */ -#include "pycore_time.h" // _PyTime_t - /* Includes needed for the sockaddr_* symbols below */ #ifndef MS_WINDOWS #ifdef __VMS @@ -324,7 +322,7 @@ typedef struct { PyObject *(*errorhandler)(void); /* Error handler; checks errno, returns NULL and sets a Python exception */ - _PyTime_t sock_timeout; /* Operation timeout in seconds; + PyTime_t sock_timeout; /* Operation timeout in seconds; 0.0 means non-blocking */ struct _socket_state *state; } PySocketSockObject; diff --git a/Modules/termios.c b/Modules/termios.c index 69dbd88be5fcc20..4635fefb8f3f5ae 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -27,9 +27,7 @@ #include <termios.h> #include <sys/ioctl.h> -#if defined(__sun) && defined(__SVR4) -# include <unistd.h> // ioctl() -#endif +#include <unistd.h> // _POSIX_VDISABLE /* HP-UX requires that this be included to pick up MDCD, MCTS, MDSR, * MDTR, MRI, and MRTS (apparently used internally by some things @@ -1315,6 +1313,9 @@ static struct constant { #ifdef TIOCTTYGSTRUCT {"TIOCTTYGSTRUCT", TIOCTTYGSTRUCT}, #endif +#ifdef _POSIX_VDISABLE + {"_POSIX_VDISABLE", _POSIX_VDISABLE}, +#endif /* sentinel */ {NULL, 0} diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 2b0d3900dbddd63..ed41ffd3662aa8c 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -5,6 +5,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_time.h" // _PyTimeFraction #include <time.h> // clock() #ifdef HAVE_SYS_TIMES_H @@ -70,12 +71,13 @@ module time /* Forward declarations */ -static int pysleep(_PyTime_t timeout); +static int pysleep(PyTime_t timeout); typedef struct { PyTypeObject *struct_time_type; -#ifdef HAVE_TIMES +// gh-115714: Don't use times() on WASI. +#if defined(HAVE_TIMES) && !defined(__wasi__) // times() clock frequency in hertz _PyTimeFraction times_base; #endif @@ -95,26 +97,18 @@ get_time_state(PyObject *module) static PyObject* -_PyFloat_FromPyTime(_PyTime_t t) +_PyFloat_FromPyTime(PyTime_t t) { - double d = _PyTime_AsSecondsDouble(t); + double d = PyTime_AsSecondsDouble(t); return PyFloat_FromDouble(d); } -static int -get_system_time(_PyTime_t *t) -{ - // Avoid _PyTime_GetSystemClock() which silently ignores errors. - return _PyTime_GetSystemClockWithInfo(t, NULL); -} - - static PyObject * time_time(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_system_time(&t) < 0) { + PyTime_t t; + if (PyTime_Time(&t) < 0) { return NULL; } return _PyFloat_FromPyTime(t); @@ -130,11 +124,11 @@ Fractions of a second may be present if the system clock provides them."); static PyObject * time_time_ns(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_system_time(&t) < 0) { + PyTime_t t; + if (PyTime_Time(&t) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(time_ns_doc, @@ -153,7 +147,7 @@ Return the current time in nanoseconds since the Epoch."); #endif static int -py_clock(time_module_state *state, _PyTime_t *tp, _Py_clock_info_t *info) +py_clock(time_module_state *state, PyTime_t *tp, _Py_clock_info_t *info) { _PyTimeFraction *base = &state->clock_base; @@ -171,8 +165,7 @@ py_clock(time_module_state *state, _PyTime_t *tp, _Py_clock_info_t *info) "or its value cannot be represented"); return -1; } - _PyTime_t ns = _PyTimeFraction_Mul(ticks, base); - *tp = _PyTime_FromNanoseconds(ns); + *tp = _PyTimeFraction_Mul(ticks, base); return 0; } #endif /* HAVE_CLOCK */ @@ -262,11 +255,11 @@ time_clock_gettime_ns_impl(PyObject *module, clockid_t clk_id) return NULL; } - _PyTime_t t; + PyTime_t t; if (_PyTime_FromTimespec(&t, &ts) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } #endif /* HAVE_CLOCK_GETTIME */ @@ -276,7 +269,7 @@ time_clock_settime(PyObject *self, PyObject *args) { int clk_id; PyObject *obj; - _PyTime_t t; + PyTime_t t; struct timespec tp; int ret; @@ -307,7 +300,7 @@ time_clock_settime_ns(PyObject *self, PyObject *args) { int clk_id; PyObject *obj; - _PyTime_t t; + PyTime_t t; struct timespec ts; int ret; @@ -315,7 +308,7 @@ time_clock_settime_ns(PyObject *self, PyObject *args) return NULL; } - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } if (_PyTime_AsTimespec(t, &ts) == -1) { @@ -402,7 +395,7 @@ time_sleep(PyObject *self, PyObject *timeout_obj) return NULL; } - _PyTime_t timeout; + PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_TIMEOUT)) return NULL; if (timeout < 0) { @@ -1155,19 +1148,11 @@ should not be relied on."); #endif /* HAVE_WORKING_TZSET */ -static int -get_monotonic(_PyTime_t *t) -{ - // Avoid _PyTime_GetMonotonicClock() which silently ignores errors. - return _PyTime_GetMonotonicClockWithInfo(t, NULL); -} - - static PyObject * time_monotonic(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_monotonic(&t) < 0) { + PyTime_t t; + if (PyTime_Monotonic(&t) < 0) { return NULL; } return _PyFloat_FromPyTime(t); @@ -1181,11 +1166,11 @@ Monotonic clock, cannot go backward."); static PyObject * time_monotonic_ns(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_monotonic(&t) < 0) { + PyTime_t t; + if (PyTime_Monotonic(&t) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(monotonic_ns_doc, @@ -1194,19 +1179,11 @@ PyDoc_STRVAR(monotonic_ns_doc, Monotonic clock, cannot go backward, as nanoseconds."); -static int -get_perf_counter(_PyTime_t *t) -{ - // Avoid _PyTime_GetPerfCounter() which silently ignores errors. - return _PyTime_GetPerfCounterWithInfo(t, NULL); -} - - static PyObject * time_perf_counter(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_perf_counter(&t) < 0) { + PyTime_t t; + if (PyTime_PerfCounter(&t) < 0) { return NULL; } return _PyFloat_FromPyTime(t); @@ -1221,11 +1198,11 @@ Performance counter for benchmarking."); static PyObject * time_perf_counter_ns(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_perf_counter(&t) < 0) { + PyTime_t t; + if (PyTime_PerfCounter(&t) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(perf_counter_ns_doc, @@ -1234,9 +1211,10 @@ PyDoc_STRVAR(perf_counter_ns_doc, Performance counter for benchmarking as nanoseconds."); -#ifdef HAVE_TIMES +// gh-115714: Don't use times() on WASI. +#if defined(HAVE_TIMES) && !defined(__wasi__) static int -process_time_times(time_module_state *state, _PyTime_t *tp, +process_time_times(time_module_state *state, PyTime_t *tp, _Py_clock_info_t *info) { _PyTimeFraction *base = &state->times_base; @@ -1253,24 +1231,24 @@ process_time_times(time_module_state *state, _PyTime_t *tp, info->adjustable = 0; } - _PyTime_t ns; + PyTime_t ns; ns = _PyTimeFraction_Mul(process.tms_utime, base); ns += _PyTimeFraction_Mul(process.tms_stime, base); - *tp = _PyTime_FromNanoseconds(ns); + *tp = ns; return 1; } #endif static int -py_process_time(time_module_state *state, _PyTime_t *tp, +py_process_time(time_module_state *state, PyTime_t *tp, _Py_clock_info_t *info) { #if defined(MS_WINDOWS) HANDLE process; FILETIME creation_time, exit_time, kernel_time, user_time; ULARGE_INTEGER large; - _PyTime_t ktime, utime, t; + PyTime_t ktime, utime; BOOL ok; process = GetCurrentProcess(); @@ -1297,14 +1275,15 @@ py_process_time(time_module_state *state, _PyTime_t *tp, utime = large.QuadPart; /* ktime and utime have a resolution of 100 nanoseconds */ - t = _PyTime_FromNanoseconds((ktime + utime) * 100); - *tp = t; + *tp = (ktime + utime) * 100; return 0; #else /* clock_gettime */ +// gh-115714: Don't use CLOCK_PROCESS_CPUTIME_ID on WASI. #if defined(HAVE_CLOCK_GETTIME) \ - && (defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_PROF)) + && (defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_PROF)) \ + && !defined(__wasi__) struct timespec ts; if (HAVE_CLOCK_GETTIME_RUNTIME) { @@ -1343,7 +1322,7 @@ py_process_time(time_module_state *state, _PyTime_t *tp, struct rusage ru; if (getrusage(RUSAGE_SELF, &ru) == 0) { - _PyTime_t utime, stime; + PyTime_t utime, stime; if (info) { info->implementation = "getrusage(RUSAGE_SELF)"; @@ -1359,14 +1338,15 @@ py_process_time(time_module_state *state, _PyTime_t *tp, return -1; } - _PyTime_t total = utime + stime; + PyTime_t total = utime + stime; *tp = total; return 0; } #endif /* times() */ -#ifdef HAVE_TIMES +// gh-115714: Don't use times() on WASI. +#if defined(HAVE_TIMES) && !defined(__wasi__) int res = process_time_times(state, tp, info); if (res < 0) { return -1; @@ -1386,7 +1366,7 @@ static PyObject * time_process_time(PyObject *module, PyObject *unused) { time_module_state *state = get_time_state(module); - _PyTime_t t; + PyTime_t t; if (py_process_time(state, &t, NULL) < 0) { return NULL; } @@ -1402,11 +1382,11 @@ static PyObject * time_process_time_ns(PyObject *module, PyObject *unused) { time_module_state *state = get_time_state(module); - _PyTime_t t; + PyTime_t t; if (py_process_time(state, &t, NULL) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(process_time_ns_doc, @@ -1419,12 +1399,12 @@ sum of the kernel and user-space CPU time."); #if defined(MS_WINDOWS) #define HAVE_THREAD_TIME static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { HANDLE thread; FILETIME creation_time, exit_time, kernel_time, user_time; ULARGE_INTEGER large; - _PyTime_t ktime, utime, t; + PyTime_t ktime, utime; BOOL ok; thread = GetCurrentThread(); @@ -1451,15 +1431,14 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) utime = large.QuadPart; /* ktime and utime have a resolution of 100 nanoseconds */ - t = _PyTime_FromNanoseconds((ktime + utime) * 100); - *tp = t; + *tp = (ktime + utime) * 100; return 0; } #elif defined(_AIX) #define HAVE_THREAD_TIME static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { /* bpo-40192: On AIX, thread_cputime() is preferred: it has nanosecond resolution, whereas clock_gettime(CLOCK_THREAD_CPUTIME_ID) @@ -1476,14 +1455,14 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) info->adjustable = 0; info->resolution = 1e-9; } - *tp = _PyTime_FromNanoseconds(tc.stime + tc.utime); + *tp = (tc.stime + tc.utime); return 0; } #elif defined(__sun) && defined(__SVR4) #define HAVE_THREAD_TIME static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { /* bpo-35455: On Solaris, CLOCK_THREAD_CPUTIME_ID clock is not always available; use gethrvtime() to substitute this functionality. */ @@ -1493,7 +1472,7 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) info->monotonic = 1; info->adjustable = 0; } - *tp = _PyTime_FromNanoseconds(gethrvtime()); + *tp = gethrvtime(); return 0; } @@ -1504,7 +1483,7 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #if defined(__APPLE__) && defined(__has_attribute) && __has_attribute(availability) static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) __attribute__((availability(macos, introduced=10.12))) __attribute__((availability(ios, introduced=10.0))) __attribute__((availability(tvos, introduced=10.0))) @@ -1512,7 +1491,7 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #endif static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { struct timespec ts; const clockid_t clk_id = CLOCK_THREAD_CPUTIME_ID; @@ -1554,7 +1533,7 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) static PyObject * time_thread_time(PyObject *self, PyObject *unused) { - _PyTime_t t; + PyTime_t t; if (_PyTime_GetThreadTimeWithInfo(&t, NULL) < 0) { return NULL; } @@ -1569,11 +1548,11 @@ Thread time for profiling: sum of the kernel and user-space CPU time."); static PyObject * time_thread_time_ns(PyObject *self, PyObject *unused) { - _PyTime_t t; + PyTime_t t; if (_PyTime_GetThreadTimeWithInfo(&t, NULL) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(thread_time_ns_doc, @@ -1595,7 +1574,7 @@ time_get_clock_info(PyObject *module, PyObject *args) char *name; _Py_clock_info_t info; PyObject *obj = NULL, *dict, *ns; - _PyTime_t t; + PyTime_t t; if (!PyArg_ParseTuple(args, "s:get_clock_info", &name)) { return NULL; @@ -1614,17 +1593,17 @@ time_get_clock_info(PyObject *module, PyObject *args) #endif if (strcmp(name, "time") == 0) { - if (_PyTime_GetSystemClockWithInfo(&t, &info) < 0) { + if (_PyTime_TimeWithInfo(&t, &info) < 0) { return NULL; } } else if (strcmp(name, "monotonic") == 0) { - if (_PyTime_GetMonotonicClockWithInfo(&t, &info) < 0) { + if (_PyTime_MonotonicWithInfo(&t, &info) < 0) { return NULL; } } else if (strcmp(name, "perf_counter") == 0) { - if (_PyTime_GetPerfCounterWithInfo(&t, &info) < 0) { + if (_PyTime_PerfCounterWithInfo(&t, &info) < 0) { return NULL; } } @@ -2094,7 +2073,8 @@ time_exec(PyObject *module) } #endif -#ifdef HAVE_TIMES +// gh-115714: Don't use times() on WASI. +#if defined(HAVE_TIMES) && !defined(__wasi__) long ticks_per_second; if (_Py_GetTicksPerSecond(&ticks_per_second) < 0) { PyErr_SetString(PyExc_RuntimeError, @@ -2174,7 +2154,7 @@ PyInit_time(void) // On error, raise an exception and return -1. // On success, return 0. static int -pysleep(_PyTime_t timeout) +pysleep(PyTime_t timeout) { assert(timeout >= 0); @@ -2186,10 +2166,10 @@ pysleep(_PyTime_t timeout) #else struct timeval timeout_tv; #endif - _PyTime_t deadline, monotonic; + PyTime_t deadline, monotonic; int err = 0; - if (get_monotonic(&monotonic) < 0) { + if (PyTime_Monotonic(&monotonic) < 0) { return -1; } deadline = monotonic + timeout; @@ -2242,7 +2222,7 @@ pysleep(_PyTime_t timeout) } #ifndef HAVE_CLOCK_NANOSLEEP - if (get_monotonic(&monotonic) < 0) { + if (PyTime_Monotonic(&monotonic) < 0) { return -1; } timeout = deadline - monotonic; @@ -2255,7 +2235,7 @@ pysleep(_PyTime_t timeout) return 0; #else // MS_WINDOWS - _PyTime_t timeout_100ns = _PyTime_As100Nanoseconds(timeout, + PyTime_t timeout_100ns = _PyTime_As100Nanoseconds(timeout, _PyTime_ROUND_CEILING); // Maintain Windows Sleep() semantics for time.sleep(0) diff --git a/Objects/abstract.c b/Objects/abstract.c index 1ec5c5b8c3dc2f5..07d4b89fe188c84 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -425,6 +425,12 @@ PyObject_AsWriteBuffer(PyObject *obj, int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) { + if (flags != PyBUF_SIMPLE) { /* fast path */ + if (flags == PyBUF_READ || flags == PyBUF_WRITE) { + PyErr_BadInternalCall(); + return -1; + } + } PyBufferProcs *pb = Py_TYPE(obj)->tp_as_buffer; if (pb == NULL || pb->bf_getbuffer == NULL) { @@ -761,11 +767,17 @@ PyBuffer_FillInfo(Py_buffer *view, PyObject *obj, void *buf, Py_ssize_t len, return -1; } - if (((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE) && - (readonly == 1)) { - PyErr_SetString(PyExc_BufferError, - "Object is not writable."); - return -1; + if (flags != PyBUF_SIMPLE) { /* fast path */ + if (flags == PyBUF_READ || flags == PyBUF_WRITE) { + PyErr_BadInternalCall(); + return -1; + } + if (((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE) && + (readonly == 1)) { + PyErr_SetString(PyExc_BufferError, + "Object is not writable."); + return -1; + } } view->obj = Py_XNewRef(obj); diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index acc59b926448ca4..5e3b3affbc76c53 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1729,6 +1729,10 @@ bytearray_extend(PyByteArrayObject *self, PyObject *iterable_of_ints) while ((item = PyIter_Next(it)) != NULL) { if (! _getbytevalue(item, &value)) { + if (PyErr_ExceptionMatches(PyExc_TypeError) && PyUnicode_Check(iterable_of_ints)) { + PyErr_Format(PyExc_TypeError, + "expected iterable of integers; got: 'str'"); + } Py_DECREF(item); Py_DECREF(it); Py_DECREF(bytearray_obj); diff --git a/Objects/clinic/dictobject.c.h b/Objects/clinic/dictobject.c.h index 8f532f454156dec..fb46c4c64334f91 100644 --- a/Objects/clinic/dictobject.c.h +++ b/Objects/clinic/dictobject.c.h @@ -2,6 +2,7 @@ preserve [clinic start generated code]*/ +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(dict_fromkeys__doc__, @@ -93,7 +94,9 @@ dict_get(PyDictObject *self, PyObject *const *args, Py_ssize_t nargs) } default_value = args[1]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = dict_get_impl(self, key, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -130,7 +133,9 @@ dict_setdefault(PyDictObject *self, PyObject *const *args, Py_ssize_t nargs) } default_value = args[1]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = dict_setdefault_impl(self, key, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -209,7 +214,13 @@ dict_popitem_impl(PyDictObject *self); static PyObject * dict_popitem(PyDictObject *self, PyObject *Py_UNUSED(ignored)) { - return dict_popitem_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = dict_popitem_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(dict___sizeof____doc__, @@ -301,4 +312,4 @@ dict_values(PyDictObject *self, PyObject *Py_UNUSED(ignored)) { return dict_values_impl(self); } -/*[clinic end generated code: output=f3ac47dfbf341b23 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f3dd5f3fb8122aef input=a9049054013a1b77]*/ diff --git a/Objects/clinic/setobject.c.h b/Objects/clinic/setobject.c.h new file mode 100644 index 000000000000000..f3c96995ede60dd --- /dev/null +++ b/Objects/clinic/setobject.c.h @@ -0,0 +1,414 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +PyDoc_STRVAR(set_pop__doc__, +"pop($self, /)\n" +"--\n" +"\n" +"Remove and return an arbitrary set element.\n" +"\n" +"Raises KeyError if the set is empty."); + +#define SET_POP_METHODDEF \ + {"pop", (PyCFunction)set_pop, METH_NOARGS, set_pop__doc__}, + +static PyObject * +set_pop_impl(PySetObject *so); + +static PyObject * +set_pop(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return set_pop_impl(so); +} + +PyDoc_STRVAR(set_update__doc__, +"update($self, /, *others)\n" +"--\n" +"\n" +"Update the set, adding elements from all others."); + +#define SET_UPDATE_METHODDEF \ + {"update", _PyCFunction_CAST(set_update), METH_FASTCALL, set_update__doc__}, + +static PyObject * +set_update_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_update(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("update", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_update_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of a set."); + +#define SET_COPY_METHODDEF \ + {"copy", (PyCFunction)set_copy, METH_NOARGS, set_copy__doc__}, + +static PyObject * +set_copy_impl(PySetObject *so); + +static PyObject * +set_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return set_copy_impl(so); +} + +PyDoc_STRVAR(frozenset_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of a set."); + +#define FROZENSET_COPY_METHODDEF \ + {"copy", (PyCFunction)frozenset_copy, METH_NOARGS, frozenset_copy__doc__}, + +static PyObject * +frozenset_copy_impl(PySetObject *so); + +static PyObject * +frozenset_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return frozenset_copy_impl(so); +} + +PyDoc_STRVAR(set_clear__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all elements from this set."); + +#define SET_CLEAR_METHODDEF \ + {"clear", (PyCFunction)set_clear, METH_NOARGS, set_clear__doc__}, + +static PyObject * +set_clear_impl(PySetObject *so); + +static PyObject * +set_clear(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return set_clear_impl(so); +} + +PyDoc_STRVAR(set_union__doc__, +"union($self, /, *others)\n" +"--\n" +"\n" +"Return a new set with elements from the set and all others."); + +#define SET_UNION_METHODDEF \ + {"union", _PyCFunction_CAST(set_union), METH_FASTCALL, set_union__doc__}, + +static PyObject * +set_union_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_union(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("union", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_union_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_intersection_multi__doc__, +"intersection($self, /, *others)\n" +"--\n" +"\n" +"Return a new set with elements common to the set and all others."); + +#define SET_INTERSECTION_MULTI_METHODDEF \ + {"intersection", _PyCFunction_CAST(set_intersection_multi), METH_FASTCALL, set_intersection_multi__doc__}, + +static PyObject * +set_intersection_multi_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_intersection_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("intersection", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_intersection_multi_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_intersection_update_multi__doc__, +"intersection_update($self, /, *others)\n" +"--\n" +"\n" +"Update the set, keeping only elements found in it and all others."); + +#define SET_INTERSECTION_UPDATE_MULTI_METHODDEF \ + {"intersection_update", _PyCFunction_CAST(set_intersection_update_multi), METH_FASTCALL, set_intersection_update_multi__doc__}, + +static PyObject * +set_intersection_update_multi_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_intersection_update_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("intersection_update", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_intersection_update_multi_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_isdisjoint__doc__, +"isdisjoint($self, other, /)\n" +"--\n" +"\n" +"Return True if two sets have a null intersection."); + +#define SET_ISDISJOINT_METHODDEF \ + {"isdisjoint", (PyCFunction)set_isdisjoint, METH_O, set_isdisjoint__doc__}, + +PyDoc_STRVAR(set_difference_update__doc__, +"difference_update($self, /, *others)\n" +"--\n" +"\n" +"Update the set, removing elements found in others."); + +#define SET_DIFFERENCE_UPDATE_METHODDEF \ + {"difference_update", _PyCFunction_CAST(set_difference_update), METH_FASTCALL, set_difference_update__doc__}, + +static PyObject * +set_difference_update_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_difference_update(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("difference_update", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_difference_update_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_difference_multi__doc__, +"difference($self, /, *others)\n" +"--\n" +"\n" +"Return a new set with elements in the set that are not in the others."); + +#define SET_DIFFERENCE_MULTI_METHODDEF \ + {"difference", _PyCFunction_CAST(set_difference_multi), METH_FASTCALL, set_difference_multi__doc__}, + +static PyObject * +set_difference_multi_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_difference_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("difference", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_difference_multi_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_symmetric_difference_update__doc__, +"symmetric_difference_update($self, other, /)\n" +"--\n" +"\n" +"Update the set, keeping only elements found in either set, but not in both."); + +#define SET_SYMMETRIC_DIFFERENCE_UPDATE_METHODDEF \ + {"symmetric_difference_update", (PyCFunction)set_symmetric_difference_update, METH_O, set_symmetric_difference_update__doc__}, + +PyDoc_STRVAR(set_symmetric_difference__doc__, +"symmetric_difference($self, other, /)\n" +"--\n" +"\n" +"Return a new set with elements in either the set or other but not both."); + +#define SET_SYMMETRIC_DIFFERENCE_METHODDEF \ + {"symmetric_difference", (PyCFunction)set_symmetric_difference, METH_O, set_symmetric_difference__doc__}, + +PyDoc_STRVAR(set_issubset__doc__, +"issubset($self, other, /)\n" +"--\n" +"\n" +"Report whether another set contains this set."); + +#define SET_ISSUBSET_METHODDEF \ + {"issubset", (PyCFunction)set_issubset, METH_O, set_issubset__doc__}, + +PyDoc_STRVAR(set_issuperset__doc__, +"issuperset($self, other, /)\n" +"--\n" +"\n" +"Report whether this set contains another set."); + +#define SET_ISSUPERSET_METHODDEF \ + {"issuperset", (PyCFunction)set_issuperset, METH_O, set_issuperset__doc__}, + +PyDoc_STRVAR(set_add__doc__, +"add($self, object, /)\n" +"--\n" +"\n" +"Add an element to a set.\n" +"\n" +"This has no effect if the element is already present."); + +#define SET_ADD_METHODDEF \ + {"add", (PyCFunction)set_add, METH_O, set_add__doc__}, + +PyDoc_STRVAR(set___contains____doc__, +"__contains__($self, object, /)\n" +"--\n" +"\n" +"x.__contains__(y) <==> y in x."); + +#define SET___CONTAINS___METHODDEF \ + {"__contains__", (PyCFunction)set___contains__, METH_O|METH_COEXIST, set___contains____doc__}, + +PyDoc_STRVAR(set_remove__doc__, +"remove($self, object, /)\n" +"--\n" +"\n" +"Remove an element from a set; it must be a member.\n" +"\n" +"If the element is not a member, raise a KeyError."); + +#define SET_REMOVE_METHODDEF \ + {"remove", (PyCFunction)set_remove, METH_O, set_remove__doc__}, + +PyDoc_STRVAR(set_discard__doc__, +"discard($self, object, /)\n" +"--\n" +"\n" +"Remove an element from a set if it is a member.\n" +"\n" +"Unlike set.remove(), the discard() method does not raise\n" +"an exception when an element is missing from the set."); + +#define SET_DISCARD_METHODDEF \ + {"discard", (PyCFunction)set_discard, METH_O, set_discard__doc__}, + +PyDoc_STRVAR(set___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n" +"Return state information for pickling."); + +#define SET___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)set___reduce__, METH_NOARGS, set___reduce____doc__}, + +static PyObject * +set___reduce___impl(PySetObject *so); + +static PyObject * +set___reduce__(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return set___reduce___impl(so); +} + +PyDoc_STRVAR(set___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"S.__sizeof__() -> size of S in memory, in bytes."); + +#define SET___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)set___sizeof__, METH_NOARGS, set___sizeof____doc__}, + +static PyObject * +set___sizeof___impl(PySetObject *so); + +static PyObject * +set___sizeof__(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + return set___sizeof___impl(so); +} +/*[clinic end generated code: output=34a30591148da884 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index dc46b773c26528a..30336fa86111a71 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1489,27 +1489,19 @@ PyCode_GetFreevars(PyCodeObject *code) static void clear_executors(PyCodeObject *co) { + assert(co->co_executors); for (int i = 0; i < co->co_executors->size; i++) { - Py_CLEAR(co->co_executors->executors[i]); + if (co->co_executors->executors[i]) { + _Py_ExecutorClear(co->co_executors->executors[i]); + } } PyMem_Free(co->co_executors); co->co_executors = NULL; } void -_PyCode_Clear_Executors(PyCodeObject *code) { - int code_len = (int)Py_SIZE(code); - for (int i = 0; i < code_len; i += _PyInstruction_GetLength(code, i)) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - uint8_t opcode = instr->op.code; - uint8_t oparg = instr->op.arg; - if (opcode == ENTER_EXECUTOR) { - _PyExecutorObject *exec = code->co_executors->executors[oparg]; - assert(exec->vm_data.opcode != ENTER_EXECUTOR); - instr->op.code = exec->vm_data.opcode; - instr->op.arg = exec->vm_data.oparg; - } - } +_PyCode_Clear_Executors(PyCodeObject *code) +{ clear_executors(code); } @@ -2360,10 +2352,10 @@ _PyCode_ConstantKey(PyObject *op) void _PyStaticCode_Fini(PyCodeObject *co) { - deopt_code(co, _PyCode_CODE(co)); if (co->co_executors != NULL) { clear_executors(co); } + deopt_code(co, _PyCode_CODE(co)); PyMem_Free(co->co_extra); if (co->_co_cached != NULL) { Py_CLEAR(co->_co_cached->_co_code); diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 8d771adf307dc43..df546a090c28e45 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -393,7 +393,7 @@ method_vectorcall_FASTCALL( if (method_check_args(func, args, nargs, kwnames)) { return NULL; } - _PyCFunctionFast meth = (_PyCFunctionFast) + PyCFunctionFast meth = (PyCFunctionFast) method_enter_call(tstate, func); if (meth == NULL) { return NULL; @@ -412,7 +412,7 @@ method_vectorcall_FASTCALL_KEYWORDS( if (method_check_args(func, args, nargs, NULL)) { return NULL; } - _PyCFunctionFastWithKeywords meth = (_PyCFunctionFastWithKeywords) + PyCFunctionFastWithKeywords meth = (PyCFunctionFastWithKeywords) method_enter_call(tstate, func); if (meth == NULL) { return NULL; @@ -1519,22 +1519,34 @@ class property(object): self.__doc__ = doc except AttributeError: # read-only or dict-less class pass + self.__name = None + + def __set_name__(self, owner, name): + self.__name = name + + @property + def __name__(self): + return self.__name if self.__name is not None else self.fget.__name__ + + @__name__.setter + def __name__(self, value): + self.__name = value def __get__(self, inst, type=None): if inst is None: return self if self.__get is None: - raise AttributeError, "property has no getter" + raise AttributeError("property has no getter") return self.__get(inst) def __set__(self, inst, value): if self.__set is None: - raise AttributeError, "property has no setter" + raise AttributeError("property has no setter") return self.__set(inst, value) def __delete__(self, inst): if self.__del is None: - raise AttributeError, "property has no deleter" + raise AttributeError("property has no deleter") return self.__del(inst) */ @@ -1628,6 +1640,20 @@ property_dealloc(PyObject *self) Py_TYPE(self)->tp_free(self); } +static int +property_name(propertyobject *prop, PyObject **name) +{ + if (prop->prop_name != NULL) { + *name = Py_NewRef(prop->prop_name); + return 1; + } + if (prop->prop_get == NULL) { + *name = NULL; + return 0; + } + return PyObject_GetOptionalAttr(prop->prop_get, &_Py_ID(__name__), name); +} + static PyObject * property_descr_get(PyObject *self, PyObject *obj, PyObject *type) { @@ -1637,11 +1663,15 @@ property_descr_get(PyObject *self, PyObject *obj, PyObject *type) propertyobject *gs = (propertyobject *)self; if (gs->prop_get == NULL) { + PyObject *propname; + if (property_name(gs, &propname) < 0) { + return NULL; + } PyObject *qualname = PyType_GetQualName(Py_TYPE(obj)); - if (gs->prop_name != NULL && qualname != NULL) { + if (propname != NULL && qualname != NULL) { PyErr_Format(PyExc_AttributeError, "property %R of %R object has no getter", - gs->prop_name, + propname, qualname); } else if (qualname != NULL) { @@ -1652,6 +1682,7 @@ property_descr_get(PyObject *self, PyObject *obj, PyObject *type) PyErr_SetString(PyExc_AttributeError, "property has no getter"); } + Py_XDECREF(propname); Py_XDECREF(qualname); return NULL; } @@ -1673,16 +1704,20 @@ property_descr_set(PyObject *self, PyObject *obj, PyObject *value) } if (func == NULL) { + PyObject *propname; + if (property_name(gs, &propname) < 0) { + return -1; + } PyObject *qualname = NULL; if (obj != NULL) { qualname = PyType_GetQualName(Py_TYPE(obj)); } - if (gs->prop_name != NULL && qualname != NULL) { + if (propname != NULL && qualname != NULL) { PyErr_Format(PyExc_AttributeError, value == NULL ? "property %R of %R object has no deleter" : "property %R of %R object has no setter", - gs->prop_name, + propname, qualname); } else if (qualname != NULL) { @@ -1698,6 +1733,7 @@ property_descr_set(PyObject *self, PyObject *obj, PyObject *value) "property has no deleter" : "property has no setter"); } + Py_XDECREF(propname); Py_XDECREF(qualname); return -1; } @@ -1730,15 +1766,12 @@ property_copy(PyObject *old, PyObject *get, PyObject *set, PyObject *del) return NULL; if (get == NULL || get == Py_None) { - Py_XDECREF(get); get = pold->prop_get ? pold->prop_get : Py_None; } if (set == NULL || set == Py_None) { - Py_XDECREF(set); set = pold->prop_set ? pold->prop_set : Py_None; } if (del == NULL || del == Py_None) { - Py_XDECREF(del); del = pold->prop_del ? pold->prop_del : Py_None; } if (pold->getter_doc && get != Py_None) { @@ -1886,6 +1919,28 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset, return 0; } +static PyObject * +property_get__name__(propertyobject *prop, void *Py_UNUSED(ignored)) +{ + PyObject *name; + if (property_name(prop, &name) < 0) { + return NULL; + } + if (name == NULL) { + PyErr_SetString(PyExc_AttributeError, + "'property' object has no attribute '__name__'"); + } + return name; +} + +static int +property_set__name__(propertyobject *prop, PyObject *value, + void *Py_UNUSED(ignored)) +{ + Py_XSETREF(prop->prop_name, Py_XNewRef(value)); + return 0; +} + static PyObject * property_get___isabstractmethod__(propertyobject *prop, void *closure) { @@ -1916,6 +1971,7 @@ property_get___isabstractmethod__(propertyobject *prop, void *closure) } static PyGetSetDef property_getsetlist[] = { + {"__name__", (getter)property_get__name__, (setter)property_set__name__}, {"__isabstractmethod__", (getter)property_get___isabstractmethod__, NULL, NULL, diff --git a/Objects/dictobject.c b/Objects/dictobject.c index e608b91679b5688..5016e255f70ef98 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -80,6 +80,8 @@ DK_ENTRIES(keys)[index] if index >= 0): Active upon key insertion. Dummy slots cannot be made Unused again else the probe sequence in case of collision would have no way to know they were once active. + In free-threaded builds dummy slots are not re-used to allow lock-free + lookups to proceed safely. 4. Pending. index >= 0, key != NULL, and value == NULL (split only) Not yet inserted in split-table. @@ -113,17 +115,20 @@ As a consequence of this, split keys have a maximum size of 16. #define PyDict_MINSIZE 8 #include "Python.h" -#include "pycore_bitutils.h" // _Py_bit_length -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_code.h" // stats -#include "pycore_dict.h" // export _PyDict_SizeOf() -#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() -#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() -#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() -#include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_setobject.h" // _PySet_NextEntry() -#include "stringlib/eq.h" // unicode_eq() +#include "pycore_bitutils.h" // _Py_bit_length +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_code.h" // stats +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION +#include "pycore_dict.h" // export _PyDict_SizeOf() +#include "pycore_freelist.h" // _PyFreeListState_GET() +#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_RELAXED +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_setobject.h" // _PySet_NextEntry() +#include "stringlib/eq.h" // unicode_eq() #include <stdbool.h> @@ -140,6 +145,119 @@ To avoid slowing down lookups on a near-full table, we resize the table when it's USABLE_FRACTION (currently two-thirds) full. */ +#ifdef Py_GIL_DISABLED + +static inline void +ASSERT_DICT_LOCKED(PyObject *op) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); +} +#define ASSERT_DICT_LOCKED(op) ASSERT_DICT_LOCKED(_Py_CAST(PyObject*, op)) +#define IS_DICT_SHARED(mp) _PyObject_GC_IS_SHARED(mp) +#define SET_DICT_SHARED(mp) _PyObject_GC_SET_SHARED(mp) +#define LOAD_INDEX(keys, size, idx) _Py_atomic_load_int##size##_relaxed(&((const int##size##_t*)keys->dk_indices)[idx]); +#define STORE_INDEX(keys, size, idx, value) _Py_atomic_store_int##size##_relaxed(&((int##size##_t*)keys->dk_indices)[idx], (int##size##_t)value); +#define ASSERT_OWNED_OR_SHARED(mp) \ + assert(_Py_IsOwnedByCurrentThread((PyObject *)mp) || IS_DICT_SHARED(mp)); +#define LOAD_KEYS_NENTRIES(d) + +static inline Py_ssize_t +load_keys_nentries(PyDictObject *mp) +{ + PyDictKeysObject *keys = _Py_atomic_load_ptr(&mp->ma_keys); + return _Py_atomic_load_ssize(&keys->dk_nentries); +} + +static inline void +set_keys(PyDictObject *mp, PyDictKeysObject *keys) +{ + ASSERT_OWNED_OR_SHARED(mp); + _Py_atomic_store_ptr_release(&mp->ma_keys, keys); +} + +static inline void +set_values(PyDictObject *mp, PyDictValues *values) +{ + ASSERT_OWNED_OR_SHARED(mp); + _Py_atomic_store_ptr_release(&mp->ma_values, values); +} + +// Defined until we get QSBR +#define _PyMem_FreeQsbr PyMem_Free + +#define LOCK_KEYS(keys) PyMutex_LockFlags(&keys->dk_mutex, _Py_LOCK_DONT_DETACH) +#define UNLOCK_KEYS(keys) PyMutex_Unlock(&keys->dk_mutex) + +#define ASSERT_KEYS_LOCKED(keys) assert(PyMutex_IsLocked(&keys->dk_mutex)) +#define LOAD_SHARED_KEY(key) _Py_atomic_load_ptr_acquire(&key) +#define STORE_SHARED_KEY(key, value) _Py_atomic_store_ptr_release(&key, value) +// Inc refs the keys object, giving the previous value +#define INCREF_KEYS(dk) _Py_atomic_add_ssize(&dk->dk_refcnt, 1) +// Dec refs the keys object, giving the previous value +#define DECREF_KEYS(dk) _Py_atomic_add_ssize(&dk->dk_refcnt, -1) +#define LOAD_KEYS_NENTIRES(keys) _Py_atomic_load_ssize_relaxed(&keys->dk_nentries) + +static inline void split_keys_entry_added(PyDictKeysObject *keys) +{ + ASSERT_KEYS_LOCKED(keys); + + // We increase before we decrease so we never get too small of a value + // when we're racing with reads + _Py_atomic_store_ssize_relaxed(&keys->dk_nentries, keys->dk_nentries + 1); + _Py_atomic_store_ssize_release(&keys->dk_usable, keys->dk_usable - 1); +} + +#else /* Py_GIL_DISABLED */ + +#define ASSERT_DICT_LOCKED(op) +#define LOCK_KEYS(keys) +#define UNLOCK_KEYS(keys) +#define ASSERT_KEYS_LOCKED(keys) +#define LOAD_SHARED_KEY(key) key +#define STORE_SHARED_KEY(key, value) key = value +#define INCREF_KEYS(dk) dk->dk_refcnt++ +#define DECREF_KEYS(dk) dk->dk_refcnt-- +#define LOAD_KEYS_NENTIRES(keys) keys->dk_nentries +#define IS_DICT_SHARED(mp) (false) +#define SET_DICT_SHARED(mp) +#define LOAD_INDEX(keys, size, idx) ((const int##size##_t*)(keys->dk_indices))[idx] +#define STORE_INDEX(keys, size, idx, value) ((int##size##_t*)(keys->dk_indices))[idx] = (int##size##_t)value + +static inline void split_keys_entry_added(PyDictKeysObject *keys) +{ + keys->dk_usable--; + keys->dk_nentries++; +} + +static inline void +set_keys(PyDictObject *mp, PyDictKeysObject *keys) +{ + mp->ma_keys = keys; +} + +static inline void +set_values(PyDictObject *mp, PyDictValues *values) +{ + mp->ma_values = values; +} + +static inline Py_ssize_t +load_keys_nentries(PyDictObject *mp) +{ + return mp->ma_keys->dk_nentries; +} + + +#endif + +#define STORE_KEY(ep, key) FT_ATOMIC_STORE_PTR_RELEASE(ep->me_key, key) +#define STORE_VALUE(ep, value) FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, value) +#define STORE_SPLIT_VALUE(mp, idx, value) FT_ATOMIC_STORE_PTR_RELEASE(mp->ma_values->values[idx], value) +#define STORE_HASH(ep, hash) FT_ATOMIC_STORE_SSIZE_RELAXED(ep->me_hash, hash) +#define STORE_KEYS_USABLE(keys, usable) FT_ATOMIC_STORE_SSIZE_RELAXED(keys->dk_usable, usable) +#define STORE_KEYS_NENTRIES(keys, nentries) FT_ATOMIC_STORE_SSIZE_RELAXED(keys->dk_nentries, nentries) +#define STORE_USED(mp, used) FT_ATOMIC_STORE_SSIZE_RELAXED(mp->ma_used, used) + #define PERTURB_SHIFT 5 /* @@ -239,43 +357,50 @@ static int dictresize(PyInterpreterState *interp, PyDictObject *mp, static PyObject* dict_iter(PyObject *dict); +static int +setitem_lock_held(PyDictObject *mp, PyObject *key, PyObject *value); +static int +dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_value, + PyObject **result, int incref_result); + #include "clinic/dictobject.c.h" -#if PyDict_MAXFREELIST > 0 -static struct _Py_dict_state * -get_dict_state(PyInterpreterState *interp) +#ifdef WITH_FREELISTS +static struct _Py_dict_freelist * +get_dict_freelist(void) { - return &interp->dict_state; + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + return &freelists->dicts; +} + +static struct _Py_dictkeys_freelist * +get_dictkeys_freelist(void) +{ + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + return &freelists->dictkeys; } #endif void -_PyDict_ClearFreeList(PyInterpreterState *interp) +_PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) { -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = &interp->dict_state; - while (state->numfree) { - PyDictObject *op = state->free_list[--state->numfree]; +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *freelist = &freelists->dicts; + while (freelist->numfree > 0) { + PyDictObject *op = freelist->items[--freelist->numfree]; assert(PyDict_CheckExact(op)); PyObject_GC_Del(op); } - while (state->keys_numfree) { - PyObject_Free(state->keys_free_list[--state->keys_numfree]); + struct _Py_dictkeys_freelist *keys_freelist = &freelists->dictkeys; + while (keys_freelist->numfree > 0) { + PyMem_Free(keys_freelist->items[--keys_freelist->numfree]); + } + if (is_finalization) { + freelist->numfree = -1; + keys_freelist->numfree = -1; } -#endif -} - - -void -_PyDict_Fini(PyInterpreterState *interp) -{ - _PyDict_ClearFreeList(interp); -#if defined(Py_DEBUG) && PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = &interp->dict_state; - state->numfree = -1; - state->keys_numfree = -1; #endif } @@ -290,17 +415,19 @@ unicode_get_hash(PyObject *o) void _PyDict_DebugMallocStats(FILE *out) { -#if PyDict_MAXFREELIST > 0 - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _Py_dict_state *state = get_dict_state(interp); +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *dict_freelist = get_dict_freelist(); _PyDebugAllocatorStats(out, "free PyDictObject", - state->numfree, sizeof(PyDictObject)); + dict_freelist->numfree, sizeof(PyDictObject)); + struct _Py_dictkeys_freelist *dictkeys_freelist = get_dictkeys_freelist(); + _PyDebugAllocatorStats(out, "free PyDictKeysObject", + dictkeys_freelist->numfree, sizeof(PyDictKeysObject)); #endif } #define DK_MASK(dk) (DK_SIZE(dk)-1) -static void free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys); +static void free_keys_object(PyDictKeysObject *keys, bool use_qsbr); /* PyDictKeysObject has refcounts like PyObject does, so we have the following two functions to mirror what Py_INCREF() and Py_DECREF() do. @@ -318,11 +445,11 @@ dictkeys_incref(PyDictKeysObject *dk) #ifdef Py_REF_DEBUG _Py_IncRefTotal(_PyInterpreterState_GET()); #endif - dk->dk_refcnt++; + INCREF_KEYS(dk); } static inline void -dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk) +dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk, bool use_qsbr) { if (dk->dk_refcnt == _Py_IMMORTAL_REFCNT) { return; @@ -331,8 +458,24 @@ dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk) #ifdef Py_REF_DEBUG _Py_DecRefTotal(_PyInterpreterState_GET()); #endif - if (--dk->dk_refcnt == 0) { - free_keys_object(interp, dk); + if (DECREF_KEYS(dk) == 1) { + if (DK_IS_UNICODE(dk)) { + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dk); + Py_ssize_t i, n; + for (i = 0, n = dk->dk_nentries; i < n; i++) { + Py_XDECREF(entries[i].me_key); + Py_XDECREF(entries[i].me_value); + } + } + else { + PyDictKeyEntry *entries = DK_ENTRIES(dk); + Py_ssize_t i, n; + for (i = 0, n = dk->dk_nentries; i < n; i++) { + Py_XDECREF(entries[i].me_key); + Py_XDECREF(entries[i].me_value); + } + } + free_keys_object(dk, use_qsbr); } } @@ -344,22 +487,18 @@ dictkeys_get_index(const PyDictKeysObject *keys, Py_ssize_t i) Py_ssize_t ix; if (log2size < 8) { - const int8_t *indices = (const int8_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 8, i); } else if (log2size < 16) { - const int16_t *indices = (const int16_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 16, i); } #if SIZEOF_VOID_P > 4 else if (log2size >= 32) { - const int64_t *indices = (const int64_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 64, i); } #endif else { - const int32_t *indices = (const int32_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 32, i); } assert(ix >= DKIX_DUMMY); return ix; @@ -375,25 +514,21 @@ dictkeys_set_index(PyDictKeysObject *keys, Py_ssize_t i, Py_ssize_t ix) assert(keys->dk_version == 0); if (log2size < 8) { - int8_t *indices = (int8_t*)(keys->dk_indices); assert(ix <= 0x7f); - indices[i] = (char)ix; + STORE_INDEX(keys, 8, i, ix); } else if (log2size < 16) { - int16_t *indices = (int16_t*)(keys->dk_indices); assert(ix <= 0x7fff); - indices[i] = (int16_t)ix; + STORE_INDEX(keys, 16, i, ix); } #if SIZEOF_VOID_P > 4 else if (log2size >= 32) { - int64_t *indices = (int64_t*)(keys->dk_indices); - indices[i] = ix; + STORE_INDEX(keys, 64, i, ix); } #endif else { - int32_t *indices = (int32_t*)(keys->dk_indices); assert(ix <= 0x7fffffff); - indices[i] = (int32_t)ix; + STORE_INDEX(keys, 32, i, ix); } } @@ -466,6 +601,9 @@ static PyDictKeysObject empty_keys_struct = { 0, /* dk_log2_size */ 0, /* dk_log2_index_bytes */ DICT_KEYS_UNICODE, /* dk_kind */ +#ifdef Py_GIL_DISABLED + {0}, /* dk_mutex */ +#endif 1, /* dk_version */ 0, /* dk_usable (immutable) */ 0, /* dk_nentries */ @@ -627,22 +765,18 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) log2_bytes = log2_size + 2; } -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // new_keys_object() must not be called after _PyDict_Fini() - assert(state->keys_numfree != -1); -#endif - if (log2_size == PyDict_LOG_MINSIZE && unicode && state->keys_numfree > 0) { - dk = state->keys_free_list[--state->keys_numfree]; +#ifdef WITH_FREELISTS + struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist(); + if (log2_size == PyDict_LOG_MINSIZE && unicode && freelist->numfree > 0) { + dk = freelist->items[--freelist->numfree]; OBJECT_STAT_INC(from_freelist); } else #endif { - dk = PyObject_Malloc(sizeof(PyDictKeysObject) - + ((size_t)1 << log2_bytes) - + entry_size * usable); + dk = PyMem_Malloc(sizeof(PyDictKeysObject) + + ((size_t)1 << log2_bytes) + + entry_size * usable); if (dk == NULL) { PyErr_NoMemory(); return NULL; @@ -655,6 +789,9 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) dk->dk_log2_size = log2_size; dk->dk_log2_index_bytes = log2_bytes; dk->dk_kind = unicode ? DICT_KEYS_UNICODE : DICT_KEYS_GENERAL; +#ifdef Py_GIL_DISABLED + dk->dk_mutex = (PyMutex){0}; +#endif dk->dk_nentries = 0; dk->dk_usable = usable; dk->dk_version = 0; @@ -664,40 +801,26 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) } static void -free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys) +free_keys_object(PyDictKeysObject *keys, bool use_qsbr) { - assert(keys != Py_EMPTY_KEYS); - if (DK_IS_UNICODE(keys)) { - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - Py_ssize_t i, n; - for (i = 0, n = keys->dk_nentries; i < n; i++) { - Py_XDECREF(entries[i].me_key); - Py_XDECREF(entries[i].me_value); - } - } - else { - PyDictKeyEntry *entries = DK_ENTRIES(keys); - Py_ssize_t i, n; - for (i = 0, n = keys->dk_nentries; i < n; i++) { - Py_XDECREF(entries[i].me_key); - Py_XDECREF(entries[i].me_value); - } +#ifdef Py_GIL_DISABLED + if (use_qsbr) { + _PyMem_FreeQsbr(keys); + return; } -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // free_keys_object() must not be called after _PyDict_Fini() - assert(state->keys_numfree != -1); #endif +#ifdef WITH_FREELISTS + struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist(); if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE - && state->keys_numfree < PyDict_MAXFREELIST + && freelist->numfree < PyDict_MAXFREELIST + && freelist->numfree >= 0 && DK_IS_UNICODE(keys)) { - state->keys_free_list[state->keys_numfree++] = keys; + freelist->items[freelist->numfree++] = keys; OBJECT_STAT_INC(to_freelist); return; } #endif - PyObject_Free(keys); + PyMem_Free(keys); } static inline PyDictValues* @@ -717,9 +840,15 @@ new_values(size_t size) } static inline void -free_values(PyDictValues *values) +free_values(PyDictValues *values, bool use_qsbr) { - int prefix_size = ((uint8_t *)values)[-1]; + int prefix_size = DICT_VALUES_SIZE(values); +#ifdef Py_GIL_DISABLED + if (use_qsbr) { + _PyMem_FreeQsbr(((char *)values)-prefix_size); + return; + } +#endif PyMem_Free(((char *)values)-prefix_size); } @@ -731,14 +860,10 @@ new_dict(PyInterpreterState *interp, { PyDictObject *mp; assert(keys != NULL); -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // new_dict() must not be called after _PyDict_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree) { - mp = state->free_list[--state->numfree]; +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *freelist = get_dict_freelist(); + if (freelist->numfree > 0) { + mp = freelist->items[--freelist->numfree]; assert (mp != NULL); assert (Py_IS_TYPE(mp, &PyDict_Type)); OBJECT_STAT_INC(from_freelist); @@ -749,9 +874,9 @@ new_dict(PyInterpreterState *interp, { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); if (mp == NULL) { - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); if (free_values_on_failure) { - free_values(values); + free_values(values, false); } return NULL; } @@ -767,7 +892,19 @@ new_dict(PyInterpreterState *interp, static inline size_t shared_keys_usable_size(PyDictKeysObject *keys) { +#ifdef Py_GIL_DISABLED + // dk_usable will decrease for each instance that is created and each + // value that is added. dk_nentries will increase for each value that + // is added. We want to always return the right value or larger. + // We therefore increase dk_nentries first and we decrease dk_usable + // second, and conversely here we read dk_usable first and dk_entries + // second (to avoid the case where we read entries before the increment + // and read usable after the decrement) + return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) + + _Py_atomic_load_ssize_acquire(&keys->dk_nentries)); +#else return (size_t)keys->dk_nentries + (size_t)keys->dk_usable; +#endif } /* Consumes a reference to the keys object */ @@ -777,7 +914,7 @@ new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); if (values == NULL) { - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); return PyErr_NoMemory(); } ((char *)values)[-2] = 0; @@ -797,8 +934,10 @@ clone_combined_dict_keys(PyDictObject *orig) assert(orig->ma_keys != Py_EMPTY_KEYS); assert(orig->ma_keys->dk_refcnt == 1); + ASSERT_DICT_LOCKED(orig); + size_t keys_size = _PyDict_KeysSize(orig->ma_keys); - PyDictKeysObject *keys = PyObject_Malloc(keys_size); + PyDictKeysObject *keys = PyMem_Malloc(keys_size); if (keys == NULL) { PyErr_NoMemory(); return NULL; @@ -875,11 +1014,11 @@ lookdict_index(PyDictKeysObject *k, Py_hash_t hash, Py_ssize_t index) Py_UNREACHABLE(); } -// Search non-Unicode key from Unicode table -static Py_ssize_t -unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +static inline Py_ALWAYS_INLINE Py_ssize_t +do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, + Py_ssize_t (*check_lookup)(PyDictObject *, PyDictKeysObject *, void *, Py_ssize_t ix, PyObject *key, Py_hash_t)) { - PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(dk); + void *ep0 = _DK_ENTRIES(dk); size_t mask = DK_MASK(dk); size_t perturb = hash; size_t i = (size_t)hash & mask; @@ -887,73 +1026,26 @@ unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key for (;;) { ix = dictkeys_get_index(dk, i); if (ix >= 0) { - PyDictUnicodeEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key) { + Py_ssize_t cmp = check_lookup(mp, dk, ep0, ix, key, hash); + if (cmp < 0) { + return cmp; + } else if (cmp) { return ix; } - if (unicode_get_hash(ep->me_key) == hash) { - PyObject *startkey = ep->me_key; - Py_INCREF(startkey); - int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); - Py_DECREF(startkey); - if (cmp < 0) { - return DKIX_ERROR; - } - if (dk == mp->ma_keys && ep->me_key == startkey) { - if (cmp > 0) { - return ix; - } - } - else { - /* The dict was mutated, restart */ - return DKIX_KEY_CHANGED; - } - } } else if (ix == DKIX_EMPTY) { return DKIX_EMPTY; } perturb >>= PERTURB_SHIFT; i = mask & (i*5 + perturb + 1); - } - Py_UNREACHABLE(); -} -// Search Unicode key from Unicode table. -static Py_ssize_t _Py_HOT_FUNCTION -unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) -{ - PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(dk); - size_t mask = DK_MASK(dk); - size_t perturb = hash; - size_t i = (size_t)hash & mask; - Py_ssize_t ix; - for (;;) { - ix = dictkeys_get_index(dk, i); - if (ix >= 0) { - PyDictUnicodeEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key || - (unicode_get_hash(ep->me_key) == hash && unicode_eq(ep->me_key, key))) { - return ix; - } - } - else if (ix == DKIX_EMPTY) { - return DKIX_EMPTY; - } - perturb >>= PERTURB_SHIFT; - i = mask & (i*5 + perturb + 1); // Manual loop unrolling ix = dictkeys_get_index(dk, i); if (ix >= 0) { - PyDictUnicodeEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key || - (unicode_get_hash(ep->me_key) == hash && unicode_eq(ep->me_key, key))) { + Py_ssize_t cmp = check_lookup(mp, dk, ep0, ix, key, hash); + if (cmp < 0) { + return cmp; + } else if (cmp) { return ix; } } @@ -966,49 +1058,94 @@ unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) Py_UNREACHABLE(); } -// Search key from Generic table. +static inline Py_ALWAYS_INLINE Py_ssize_t +compare_unicode_generic(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + assert(ep->me_key != NULL); + assert(PyUnicode_CheckExact(ep->me_key)); + assert(!PyUnicode_CheckExact(key)); + // TODO: Thread safety + + if (unicode_get_hash(ep->me_key) == hash) { + PyObject *startkey = ep->me_key; + Py_INCREF(startkey); + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; + } + if (dk == mp->ma_keys && ep->me_key == startkey) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; + } + } + return 0; +} + +// Search non-Unicode key from Unicode table static Py_ssize_t -dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) { - PyDictKeyEntry *ep0 = DK_ENTRIES(dk); - size_t mask = DK_MASK(dk); - size_t perturb = hash; - size_t i = (size_t)hash & mask; - Py_ssize_t ix; - for (;;) { - ix = dictkeys_get_index(dk, i); - if (ix >= 0) { - PyDictKeyEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - if (ep->me_key == key) { - return ix; - } - if (ep->me_hash == hash) { - PyObject *startkey = ep->me_key; - Py_INCREF(startkey); - int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); - Py_DECREF(startkey); - if (cmp < 0) { - return DKIX_ERROR; - } - if (dk == mp->ma_keys && ep->me_key == startkey) { - if (cmp > 0) { - return ix; - } - } - else { - /* The dict was mutated, restart */ - return DKIX_KEY_CHANGED; - } - } + return do_lookup(mp, dk, key, hash, compare_unicode_generic); +} + +static inline Py_ALWAYS_INLINE Py_ssize_t +compare_unicode_unicode(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + assert(ep->me_key != NULL); + assert(PyUnicode_CheckExact(ep->me_key)); + if (ep->me_key == key || + (unicode_get_hash(ep->me_key) == hash && unicode_eq(ep->me_key, key))) { + return 1; + } + return 0; +} + +static Py_ssize_t _Py_HOT_FUNCTION +unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(NULL, dk, key, hash, compare_unicode_unicode); +} + +static inline Py_ALWAYS_INLINE Py_ssize_t +compare_generic(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictKeyEntry *ep = &((PyDictKeyEntry *)ep0)[ix]; + assert(ep->me_key != NULL); + if (ep->me_key == key) { + return 1; + } + if (ep->me_hash == hash) { + PyObject *startkey = ep->me_key; + Py_INCREF(startkey); + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; } - else if (ix == DKIX_EMPTY) { - return DKIX_EMPTY; + if (dk == mp->ma_keys && ep->me_key == startkey) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; } - perturb >>= PERTURB_SHIFT; - i = mask & (i*5 + perturb + 1); } - Py_UNREACHABLE(); + return 0; +} + +static Py_ssize_t +dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(mp, dk, key, hash, compare_generic); } /* Lookup a string in a (all unicode) dict keys. @@ -1056,7 +1193,7 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu PyDictKeysObject *dk; DictKeysKind kind; Py_ssize_t ix; - + // TODO: Thread safety start: dk = mp->ma_keys; kind = dk->dk_kind; @@ -1100,6 +1237,260 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu return ix; } +static inline void +ensure_shared_on_read(PyDictObject *mp) +{ +#ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread((PyObject *)mp) && !IS_DICT_SHARED(mp)) { + // The first time we access a dict from a non-owning thread we mark it + // as shared. This ensures that a concurrent resize operation will + // delay freeing the old keys or values using QSBR, which is necessary + // to safely allow concurrent reads without locking... + Py_BEGIN_CRITICAL_SECTION(mp); + if (!IS_DICT_SHARED(mp)) { + SET_DICT_SHARED(mp); + } + Py_END_CRITICAL_SECTION(); + } +#endif +} + +static inline void +ensure_shared_on_resize(PyDictObject *mp) +{ +#ifdef Py_GIL_DISABLED + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp); + + if (!_Py_IsOwnedByCurrentThread((PyObject *)mp) && !IS_DICT_SHARED(mp)) { + // We are writing to the dict from another thread that owns + // it and we haven't marked it as shared which will ensure + // that when we re-size ma_keys or ma_values that we will + // free using QSBR. We need to lock the dictionary to + // contend with writes from the owning thread, mark it as + // shared, and then we can continue with lock-free reads. + // Technically this is a little heavy handed, we could just + // free the individual old keys / old-values using qsbr + SET_DICT_SHARED(mp); + } +#endif +} + +#ifdef Py_GIL_DISABLED + +static inline Py_ALWAYS_INLINE +Py_ssize_t compare_unicode_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + assert(startkey == NULL || PyUnicode_CheckExact(ep->me_key)); + assert(!PyUnicode_CheckExact(key)); + + if (startkey != NULL) { + if (!_Py_TryIncref(&ep->me_key, startkey)) { + return DKIX_KEY_CHANGED; + } + + if (unicode_get_hash(startkey) == hash) { + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; + } + if (dk == _Py_atomic_load_ptr_relaxed(&mp->ma_keys) && + startkey == _Py_atomic_load_ptr_relaxed(&ep->me_key)) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; + } + } + else { + Py_DECREF(startkey); + } + } + return 0; +} + +// Search non-Unicode key from Unicode table +static Py_ssize_t +unicodekeys_lookup_generic_threadsafe(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(mp, dk, key, hash, compare_unicode_generic_threadsafe); +} + +static inline Py_ALWAYS_INLINE Py_ssize_t +compare_unicode_unicode_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + assert(startkey == NULL || PyUnicode_CheckExact(startkey)); + if (startkey == key) { + return 1; + } + if (startkey != NULL) { + if (_Py_IsImmortal(startkey)) { + return unicode_get_hash(startkey) == hash && unicode_eq(startkey, key); + } + else { + if (!_Py_TryIncref(&ep->me_key, startkey)) { + return DKIX_KEY_CHANGED; + } + if (unicode_get_hash(startkey) == hash && unicode_eq(startkey, key)) { + Py_DECREF(startkey); + return 1; + } + Py_DECREF(startkey); + } + } + return 0; +} + +static Py_ssize_t _Py_HOT_FUNCTION +unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(NULL, dk, key, hash, compare_unicode_unicode_threadsafe); +} + +static inline Py_ALWAYS_INLINE +Py_ssize_t compare_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictKeyEntry *ep = &((PyDictKeyEntry *)ep0)[ix]; + PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + if (startkey == key) { + return 1; + } + Py_ssize_t ep_hash = _Py_atomic_load_ssize_relaxed(&ep->me_hash); + if (ep_hash == hash) { + if (startkey == NULL || !_Py_TryIncref(&ep->me_key, startkey)) { + return DKIX_KEY_CHANGED; + } + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; + } + if (dk == _Py_atomic_load_ptr_relaxed(&mp->ma_keys) && + startkey == _Py_atomic_load_ptr_relaxed(&ep->me_key)) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; + } + } + return 0; +} + +static Py_ssize_t +dictkeys_generic_lookup_threadsafe(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(mp, dk, key, hash, compare_generic_threadsafe); +} + +static Py_ssize_t +_Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr) +{ + PyDictKeysObject *dk; + DictKeysKind kind; + Py_ssize_t ix; + PyObject *value; + + ensure_shared_on_read(mp); + + dk = _Py_atomic_load_ptr(&mp->ma_keys); + kind = dk->dk_kind; + + if (kind != DICT_KEYS_GENERAL) { + if (PyUnicode_CheckExact(key)) { + ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); + } + else { + ix = unicodekeys_lookup_generic_threadsafe(mp, dk, key, hash); + } + if (ix == DKIX_KEY_CHANGED) { + goto read_failed; + } + + if (ix >= 0) { + if (kind == DICT_KEYS_SPLIT) { + PyDictValues *values = _Py_atomic_load_ptr(&mp->ma_values); + if (values == NULL) + goto read_failed; + + uint8_t capacity = _Py_atomic_load_uint8_relaxed(&DICT_VALUES_SIZE(values)); + if (ix >= (Py_ssize_t)capacity) + goto read_failed; + + value = _Py_TryXGetRef(&values->values[ix]); + if (value == NULL) + goto read_failed; + + if (values != _Py_atomic_load_ptr(&mp->ma_values)) { + Py_DECREF(value); + goto read_failed; + } + } + else { + value = _Py_TryXGetRef(&DK_UNICODE_ENTRIES(dk)[ix].me_value); + if (value == NULL) { + goto read_failed; + } + + if (dk != _Py_atomic_load_ptr(&mp->ma_keys)) { + Py_DECREF(value); + goto read_failed; + } + } + } + else { + value = NULL; + } + } + else { + ix = dictkeys_generic_lookup_threadsafe(mp, dk, key, hash); + if (ix == DKIX_KEY_CHANGED) { + goto read_failed; + } + if (ix >= 0) { + value = _Py_TryXGetRef(&DK_ENTRIES(dk)[ix].me_value); + if (value == NULL) + goto read_failed; + + if (dk != _Py_atomic_load_ptr(&mp->ma_keys)) { + Py_DECREF(value); + goto read_failed; + } + } + else { + value = NULL; + } + } + + *value_addr = value; + return ix; + +read_failed: + // In addition to the normal races of the dict being modified the _Py_TryXGetRef + // can all fail if they don't yet have a shared ref count. That can happen here + // or in the *_lookup_* helper. In that case we need to take the lock to avoid + // mutation and do a normal incref which will make them shared. + Py_BEGIN_CRITICAL_SECTION(mp); + ix = _Py_dict_lookup(mp, key, hash, &value); + *value_addr = value; + if (value != NULL) { + assert(ix >= 0); + Py_INCREF(value); + } + Py_END_CRITICAL_SECTION(); + return ix; +} + +#endif + int _PyDict_HasOnlyStringKeys(PyObject *dict) { @@ -1170,10 +1561,19 @@ _PyDict_MaybeUntrack(PyObject *op) _PyObject_GC_UNTRACK(op); } +static inline int +is_unusable_slot(Py_ssize_t ix) +{ +#ifdef Py_GIL_DISABLED + return ix >= 0 || ix == DKIX_DUMMY; +#else + return ix >= 0; +#endif +} + /* Internal function to find slot for an item from its hash when it is known that the key is not present in the dict. - - The dict must be combined. */ + */ static Py_ssize_t find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash) { @@ -1182,7 +1582,7 @@ find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash) const size_t mask = DK_MASK(keys); size_t i = hash & mask; Py_ssize_t ix = dictkeys_get_index(keys, i); - for (size_t perturb = hash; ix >= 0;) { + for (size_t perturb = hash; is_unusable_slot(ix);) { perturb >>= PERTURB_SHIFT; i = (i*5 + perturb + 1) & mask; ix = dictkeys_get_index(keys, i); @@ -1197,9 +1597,11 @@ insertion_resize(PyInterpreterState *interp, PyDictObject *mp, int unicode) } static Py_ssize_t -insert_into_dictkeys(PyDictKeysObject *keys, PyObject *name) +insert_into_splitdictkeys(PyDictKeysObject *keys, PyObject *name) { assert(PyUnicode_CheckExact(name)); + ASSERT_KEYS_LOCKED(keys); + Py_hash_t hash = unicode_get_hash(name); if (hash == -1) { hash = PyUnicode_Type.tp_hash(name); @@ -1213,19 +1615,86 @@ insert_into_dictkeys(PyDictKeysObject *keys, PyObject *name) if (keys->dk_usable <= 0) { return DKIX_EMPTY; } - /* Insert into new slot. */ - keys->dk_version = 0; - Py_ssize_t hashpos = find_empty_slot(keys, hash); - ix = keys->dk_nentries; - PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(keys)[ix]; - dictkeys_set_index(keys, hashpos, ix); - assert(ep->me_key == NULL); - ep->me_key = Py_NewRef(name); - keys->dk_usable--; - keys->dk_nentries++; + /* Insert into new slot. */ + keys->dk_version = 0; + Py_ssize_t hashpos = find_empty_slot(keys, hash); + ix = keys->dk_nentries; + PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(keys)[ix]; + dictkeys_set_index(keys, hashpos, ix); + assert(ep->me_key == NULL); + ep->me_key = Py_NewRef(name); + split_keys_entry_added(keys); + } + assert (ix < SHARED_KEYS_MAX_SIZE); + return ix; +} + +static inline int +insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, + Py_hash_t hash, PyObject *key, PyObject *value) +{ + if (mp->ma_keys->dk_usable <= 0) { + /* Need to resize. */ + if (insertion_resize(interp, mp, 1) < 0) { + return -1; + } + } + + Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); + dictkeys_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); + + if (DK_IS_UNICODE(mp->ma_keys)) { + PyDictUnicodeEntry *ep; + ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; + STORE_KEY(ep, key); + STORE_VALUE(ep, value); + } + else { + PyDictKeyEntry *ep; + ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; + STORE_KEY(ep, key); + STORE_VALUE(ep, value); + STORE_HASH(ep, hash); + } + STORE_KEYS_USABLE(mp->ma_keys, mp->ma_keys->dk_usable - 1); + STORE_KEYS_NENTRIES(mp->ma_keys, mp->ma_keys->dk_nentries + 1); + assert(mp->ma_keys->dk_usable >= 0); + return 0; +} + +static int +insert_split_dict(PyInterpreterState *interp, PyDictObject *mp, + Py_hash_t hash, PyObject *key, PyObject *value) +{ + PyDictKeysObject *keys = mp->ma_keys; + LOCK_KEYS(keys); + if (keys->dk_usable <= 0) { + /* Need to resize. */ + UNLOCK_KEYS(keys); + int ins = insertion_resize(interp, mp, 1); + if (ins < 0) { + return -1; + } + assert(!_PyDict_HasSplitTable(mp)); + return insert_combined_dict(interp, mp, hash, key, value); } - assert (ix < SHARED_KEYS_MAX_SIZE); - return ix; + + Py_ssize_t hashpos = find_empty_slot(keys, hash); + dictkeys_set_index(keys, hashpos, keys->dk_nentries); + + PyDictUnicodeEntry *ep; + ep = &DK_UNICODE_ENTRIES(keys)[keys->dk_nentries]; + STORE_SHARED_KEY(ep->me_key, key); + + Py_ssize_t index = keys->dk_nentries; + _PyDictValues_AddToInsertionOrder(mp->ma_values, index); + assert (mp->ma_values->values[index] == NULL); + STORE_SPLIT_VALUE(mp, index, value); + + split_keys_entry_added(keys); + assert(keys->dk_usable >= 0); + UNLOCK_KEYS(keys); + return 0; } /* @@ -1240,6 +1709,8 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, { PyObject *old_value; + ASSERT_DICT_LOCKED(mp); + if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { if (insertion_resize(interp, mp, 0) < 0) goto Fail; @@ -1258,41 +1729,19 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, /* Insert into new slot. */ mp->ma_keys->dk_version = 0; assert(old_value == NULL); - if (mp->ma_keys->dk_usable <= 0) { - /* Need to resize. */ - if (insertion_resize(interp, mp, 1) < 0) - goto Fail; - } - - Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); - dictkeys_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); - if (DK_IS_UNICODE(mp->ma_keys)) { - PyDictUnicodeEntry *ep; - ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = key; - if (mp->ma_values) { - Py_ssize_t index = mp->ma_keys->dk_nentries; - _PyDictValues_AddToInsertionOrder(mp->ma_values, index); - assert (mp->ma_values->values[index] == NULL); - mp->ma_values->values[index] = value; - } - else { - ep->me_value = value; + if (!_PyDict_HasSplitTable(mp)) { + if (insert_combined_dict(interp, mp, hash, key, value) < 0) { + goto Fail; } } else { - PyDictKeyEntry *ep; - ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = key; - ep->me_hash = hash; - ep->me_value = value; + if (insert_split_dict(interp, mp, hash, key, value) < 0) + goto Fail; } + mp->ma_used++; mp->ma_version_tag = new_version; - mp->ma_keys->dk_usable--; - mp->ma_keys->dk_nentries++; - assert(mp->ma_keys->dk_usable >= 0); ASSERT_CONSISTENT(mp); return 0; } @@ -1329,13 +1778,14 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, return -1; } -// Same to insertdict but specialized for ma_keys = Py_EMPTY_KEYS. +// Same as insertdict but specialized for ma_keys == Py_EMPTY_KEYS. // Consumes key and value references. static int insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) { assert(mp->ma_keys == Py_EMPTY_KEYS); + ASSERT_DICT_LOCKED(mp); uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_ADDED, mp, key, value); @@ -1349,28 +1799,37 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, return -1; } /* We don't decref Py_EMPTY_KEYS here because it is immortal. */ - mp->ma_keys = newkeys; - mp->ma_values = NULL; + assert(mp->ma_values == NULL); MAINTAIN_TRACKING(mp, key, value); size_t hashpos = (size_t)hash & (PyDict_MINSIZE-1); - dictkeys_set_index(mp->ma_keys, hashpos, 0); + dictkeys_set_index(newkeys, hashpos, 0); if (unicode) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mp->ma_keys); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(newkeys); ep->me_key = key; ep->me_value = value; } else { - PyDictKeyEntry *ep = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *ep = DK_ENTRIES(newkeys); ep->me_key = key; ep->me_hash = hash; ep->me_value = value; } mp->ma_used++; mp->ma_version_tag = new_version; - mp->ma_keys->dk_usable--; - mp->ma_keys->dk_nentries++; + newkeys->dk_usable--; + newkeys->dk_nentries++; + // We store the keys last so no one can see them in a partially inconsistent + // state so that we don't need to switch the keys to being shared yet for + // the case where we're inserting from the non-owner thread. We don't use + // set_keys here because the transition from empty to non-empty is safe + // as the empty keys will never be freed. +#ifdef Py_GIL_DISABLED + _Py_atomic_store_ptr_release(&mp->ma_keys, newkeys); +#else + mp->ma_keys = newkeys; +#endif return 0; } @@ -1426,9 +1885,11 @@ static int dictresize(PyInterpreterState *interp, PyDictObject *mp, uint8_t log2_newsize, int unicode) { - PyDictKeysObject *oldkeys; + PyDictKeysObject *oldkeys, *newkeys; PyDictValues *oldvalues; + ASSERT_DICT_LOCKED(mp); + if (log2_newsize >= SIZEOF_SIZE_T*8) { PyErr_NoMemory(); return -1; @@ -1442,30 +1903,31 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, unicode = 0; } + ensure_shared_on_resize(mp); /* NOTE: Current odict checks mp->ma_keys to detect resize happen. * So we can't reuse oldkeys even if oldkeys->dk_size == newsize. * TODO: Try reusing oldkeys when reimplement odict. */ /* Allocate a new table. */ - mp->ma_keys = new_keys_object(interp, log2_newsize, unicode); - if (mp->ma_keys == NULL) { - mp->ma_keys = oldkeys; + newkeys = new_keys_object(interp, log2_newsize, unicode); + if (newkeys == NULL) { return -1; } // New table must be large enough. - assert(mp->ma_keys->dk_usable >= mp->ma_used); + assert(newkeys->dk_usable >= mp->ma_used); Py_ssize_t numentries = mp->ma_used; if (oldvalues != NULL) { - PyDictUnicodeEntry *oldentries = DK_UNICODE_ENTRIES(oldkeys); + LOCK_KEYS(oldkeys); + PyDictUnicodeEntry *oldentries = DK_UNICODE_ENTRIES(oldkeys); /* Convert split table into new combined table. * We must incref keys; we can transfer values. */ - if (mp->ma_keys->dk_kind == DICT_KEYS_GENERAL) { + if (newkeys->dk_kind == DICT_KEYS_GENERAL) { // split -> generic - PyDictKeyEntry *newentries = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *newentries = DK_ENTRIES(newkeys); for (Py_ssize_t i = 0; i < numentries; i++) { int index = get_index_from_order(mp, i); @@ -1475,10 +1937,10 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i].me_hash = unicode_get_hash(ep->me_key); newentries[i].me_value = oldvalues->values[index]; } - build_indices_generic(mp->ma_keys, newentries, numentries); + build_indices_generic(newkeys, newentries, numentries); } else { // split -> combined unicode - PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(mp->ma_keys); + PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(newkeys); for (Py_ssize_t i = 0; i < numentries; i++) { int index = get_index_from_order(mp, i); @@ -1487,18 +1949,20 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i].me_key = Py_NewRef(ep->me_key); newentries[i].me_value = oldvalues->values[index]; } - build_indices_unicode(mp->ma_keys, newentries, numentries); + build_indices_unicode(newkeys, newentries, numentries); } - dictkeys_decref(interp, oldkeys); - mp->ma_values = NULL; - free_values(oldvalues); + UNLOCK_KEYS(oldkeys); + set_keys(mp, newkeys); + dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + set_values(mp, NULL); + free_values(oldvalues, IS_DICT_SHARED(mp)); } else { // oldkeys is combined. if (oldkeys->dk_kind == DICT_KEYS_GENERAL) { // generic -> generic - assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); + assert(newkeys->dk_kind == DICT_KEYS_GENERAL); PyDictKeyEntry *oldentries = DK_ENTRIES(oldkeys); - PyDictKeyEntry *newentries = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *newentries = DK_ENTRIES(newkeys); if (oldkeys->dk_nentries == numentries) { memcpy(newentries, oldentries, numentries * sizeof(PyDictKeyEntry)); } @@ -1510,12 +1974,12 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i] = *ep++; } } - build_indices_generic(mp->ma_keys, newentries, numentries); + build_indices_generic(newkeys, newentries, numentries); } else { // oldkeys is combined unicode PyDictUnicodeEntry *oldentries = DK_UNICODE_ENTRIES(oldkeys); if (unicode) { // combined unicode -> combined unicode - PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(mp->ma_keys); + PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(newkeys); if (oldkeys->dk_nentries == numentries && mp->ma_keys->dk_kind == DICT_KEYS_UNICODE) { memcpy(newentries, oldentries, numentries * sizeof(PyDictUnicodeEntry)); } @@ -1527,10 +1991,10 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i] = *ep++; } } - build_indices_unicode(mp->ma_keys, newentries, numentries); + build_indices_unicode(newkeys, newentries, numentries); } else { // combined unicode -> generic - PyDictKeyEntry *newentries = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *newentries = DK_ENTRIES(newkeys); PyDictUnicodeEntry *ep = oldentries; for (Py_ssize_t i = 0; i < numentries; i++) { while (ep->me_value == NULL) @@ -1540,41 +2004,24 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i].me_value = ep->me_value; ep++; } - build_indices_generic(mp->ma_keys, newentries, numentries); + build_indices_generic(newkeys, newentries, numentries); } } - // We can not use free_keys_object here because key's reference - // are moved already. + set_keys(mp, newkeys); + if (oldkeys != Py_EMPTY_KEYS) { #ifdef Py_REF_DEBUG _Py_DecRefTotal(_PyInterpreterState_GET()); #endif assert(oldkeys->dk_kind != DICT_KEYS_SPLIT); assert(oldkeys->dk_refcnt == 1); -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // dictresize() must not be called after _PyDict_Fini() - assert(state->keys_numfree != -1); -#endif - if (DK_LOG_SIZE(oldkeys) == PyDict_LOG_MINSIZE && - DK_IS_UNICODE(oldkeys) && - state->keys_numfree < PyDict_MAXFREELIST) - { - state->keys_free_list[state->keys_numfree++] = oldkeys; - OBJECT_STAT_INC(to_freelist); - } - else -#endif - { - PyObject_Free(oldkeys); - } + free_keys_object(oldkeys, IS_DICT_SHARED(mp)); } } - mp->ma_keys->dk_usable -= numentries; - mp->ma_keys->dk_nentries = numentries; + STORE_KEYS_USABLE(mp->ma_keys, mp->ma_keys->dk_usable - numentries); + STORE_KEYS_NENTRIES(mp->ma_keys, numentries); ASSERT_CONSISTENT(mp); return 0; } @@ -1642,7 +2089,7 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, for (Py_ssize_t i = 0; i < length; i++) { PyObject *key = *ks; PyObject *value = *vs; - if (PyDict_SetItem(dict, key, value) < 0) { + if (setitem_lock_held((PyDictObject *)dict, key, value) < 0) { Py_DECREF(dict); return NULL; } @@ -1691,9 +2138,13 @@ dict_getitem(PyObject *op, PyObject *key, const char *warnmsg) PyObject *value; Py_ssize_t ix; (void)ix; - PyObject *exc = _PyErr_GetRaisedException(tstate); +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif /* Ignore any exception raised by the lookup */ PyObject *exc2 = _PyErr_Occurred(tstate); @@ -1717,6 +2168,7 @@ PyDict_GetItem(PyObject *op, PyObject *key) Py_ssize_t _PyDict_LookupIndex(PyDictObject *mp, PyObject *key) { + // TODO: Thread safety PyObject *value; assert(PyDict_CheckExact((PyObject*)mp)); assert(PyUnicode_CheckExact(key)); @@ -1748,11 +2200,47 @@ _PyDict_GetItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return NULL; } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif assert(ix >= 0 || value == NULL); return value; // borrowed reference } +/* Gets an item and provides a new reference if the value is present. + * Returns 1 if the key is present, 0 if the key is missing, and -1 if an + * exception occurred. +*/ +int +_PyDict_GetItemRef_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash, PyObject **result) +{ + PyDictObject*mp = (PyDictObject *)op; + + PyObject *value; +#ifdef Py_GIL_DISABLED + Py_ssize_t ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); +#else + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); +#endif + assert(ix >= 0 || value == NULL); + if (ix == DKIX_ERROR) { + *result = NULL; + return -1; + } + if (value == NULL) { + *result = NULL; + return 0; // missing key + } +#ifdef Py_GIL_DISABLED + *result = value; +#else + *result = Py_NewRef(value); +#endif + return 1; // key is present +} int PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) @@ -1762,7 +2250,6 @@ PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) *result = NULL; return -1; } - PyDictObject*mp = (PyDictObject *)op; Py_hash_t hash; if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) @@ -1774,19 +2261,7 @@ PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) } } - PyObject *value; - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); - assert(ix >= 0 || value == NULL); - if (ix == DKIX_ERROR) { - *result = NULL; - return -1; - } - if (value == NULL) { - *result = NULL; - return 0; // missing key - } - *result = Py_NewRef(value); - return 1; // key is present + return _PyDict_GetItemRef_KnownHash(op, key, hash, result); } @@ -1814,7 +2289,12 @@ PyDict_GetItemWithError(PyObject *op, PyObject *key) } } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif assert(ix >= 0 || value == NULL); return value; // borrowed reference } @@ -1855,7 +2335,6 @@ _PyDict_GetItemStringWithError(PyObject *v, const char *key) return rv; } - /* Fast version of global value lookup (LOAD_GLOBAL). * Lookup in globals, then builtins. * @@ -1869,6 +2348,7 @@ _PyDict_GetItemStringWithError(PyObject *v, const char *key) PyObject * _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) { + // TODO: Thread safety Py_ssize_t ix; Py_hash_t hash; PyObject *value; @@ -1893,9 +2373,11 @@ _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) } /* Consumes references to key and value */ -int -_PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) +static int +setitem_take2_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) { + ASSERT_DICT_LOCKED(mp); + assert(key); assert(value); assert(PyDict_Check(mp)); @@ -1908,7 +2390,9 @@ _PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) return -1; } } + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (mp->ma_keys == Py_EMPTY_KEYS) { return insert_to_emptydict(interp, mp, key, hash, value); } @@ -1916,6 +2400,16 @@ _PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) return insertdict(interp, mp, key, hash, value); } +int +_PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(mp); + res = setitem_take2_lock_held(mp, key, value); + Py_END_CRITICAL_SECTION(); + return res; +} + /* CAUTION: PyDict_SetItem() must guarantee that it won't resize the * dictionary if it's merely replacing the value for an existing key. * This means that it's safe to loop over a dictionary with PyDict_Next() @@ -1935,6 +2429,16 @@ PyDict_SetItem(PyObject *op, PyObject *key, PyObject *value) Py_NewRef(key), Py_NewRef(value)); } +static int +setitem_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) +{ + assert(key); + assert(value); + return setitem_take2_lock_held(mp, + Py_NewRef(key), Py_NewRef(value)); +} + + int _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, Py_hash_t hash) @@ -1950,12 +2454,21 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, assert(hash != -1); mp = (PyDictObject *)op; + int res; PyInterpreterState *interp = _PyInterpreterState_GET(); + + Py_BEGIN_CRITICAL_SECTION(mp); + if (mp->ma_keys == Py_EMPTY_KEYS) { - return insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); + res = insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); } - /* insertdict() handles any resizing that might be necessary */ - return insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); + else { + /* insertdict() handles any resizing that might be necessary */ + res = insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); + } + + Py_END_CRITICAL_SECTION(); + return res; } static void @@ -1980,12 +2493,14 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, { PyObject *old_key; + ASSERT_DICT_LOCKED(mp); + Py_ssize_t hashpos = lookdict_index(mp->ma_keys, hash, ix); assert(hashpos >= 0); mp->ma_used--; mp->ma_version_tag = new_version; - if (mp->ma_values) { + if (_PyDict_HasSplitTable(mp)) { assert(old_value == mp->ma_values->values[ix]); mp->ma_values->values[ix] = NULL; assert(ix < SHARED_KEYS_MAX_SIZE); @@ -1999,15 +2514,15 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, if (DK_IS_UNICODE(mp->ma_keys)) { PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; old_key = ep->me_key; - ep->me_key = NULL; - ep->me_value = NULL; + STORE_KEY(ep, NULL); + STORE_VALUE(ep, NULL); } else { PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; old_key = ep->me_key; - ep->me_key = NULL; - ep->me_value = NULL; - ep->me_hash = 0; + STORE_KEY(ep, NULL); + STORE_VALUE(ep, NULL); + STORE_HASH(ep, 0); } Py_DECREF(old_key); } @@ -2031,8 +2546,8 @@ PyDict_DelItem(PyObject *op, PyObject *key) return _PyDict_DelItem_KnownHash(op, key, hash); } -int -_PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) +static int +delitem_knownhash_lock_held(PyObject *op, PyObject *key, Py_hash_t hash) { Py_ssize_t ix; PyDictObject *mp; @@ -2042,6 +2557,9 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) PyErr_BadInternalCall(); return -1; } + + ASSERT_DICT_LOCKED(op); + assert(key); assert(hash != -1); mp = (PyDictObject *)op; @@ -2059,13 +2577,19 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return delitem_common(mp, hash, ix, old_value, new_version); } -/* This function promises that the predicate -> deletion sequence is atomic - * (i.e. protected by the GIL), assuming the predicate itself doesn't - * release the GIL. - */ int -_PyDict_DelItemIf(PyObject *op, PyObject *key, - int (*predicate)(PyObject *value)) +_PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(op); + res = delitem_knownhash_lock_held(op, key, hash); + Py_END_CRITICAL_SECTION(); + return res; +} + +static int +delitemif_lock_held(PyObject *op, PyObject *key, + int (*predicate)(PyObject *value)) { Py_ssize_t hashpos, ix; PyDictObject *mp; @@ -2073,6 +2597,8 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, PyObject *old_value; int res; + ASSERT_DICT_LOCKED(op); + if (!PyDict_Check(op)) { PyErr_BadInternalCall(); return -1; @@ -2106,16 +2632,32 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, return 0; } } +/* This function promises that the predicate -> deletion sequence is atomic + * (i.e. protected by the GIL or the per-dict mutex in free threaded builds), + * assuming the predicate itself doesn't release the GIL (or cause re-entrancy + * which would release the per-dict mutex) + */ +int +_PyDict_DelItemIf(PyObject *op, PyObject *key, + int (*predicate)(PyObject *value)) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(op); + res = delitemif_lock_held(op, key, predicate); + Py_END_CRITICAL_SECTION(); + return res; +} - -void -PyDict_Clear(PyObject *op) +static void +clear_lock_held(PyObject *op) { PyDictObject *mp; PyDictKeysObject *oldkeys; PyDictValues *oldvalues; Py_ssize_t i, n; + ASSERT_DICT_LOCKED(op); + if (!PyDict_Check(op)) return; mp = ((PyDictObject *)op); @@ -2128,9 +2670,11 @@ PyDict_Clear(PyObject *op) PyInterpreterState *interp = _PyInterpreterState_GET(); uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_CLEARED, mp, NULL, NULL); - dictkeys_incref(Py_EMPTY_KEYS); - mp->ma_keys = Py_EMPTY_KEYS; - mp->ma_values = NULL; + // We don't inc ref empty keys because they're immortal + ensure_shared_on_resize(mp); + + set_keys(mp, Py_EMPTY_KEYS); + set_values(mp, NULL); mp->ma_used = 0; mp->ma_version_tag = new_version; /* ...then clear the keys and values */ @@ -2138,16 +2682,24 @@ PyDict_Clear(PyObject *op) n = oldkeys->dk_nentries; for (i = 0; i < n; i++) Py_CLEAR(oldvalues->values[i]); - free_values(oldvalues); - dictkeys_decref(interp, oldkeys); + free_values(oldvalues, IS_DICT_SHARED(mp)); + dictkeys_decref(interp, oldkeys, false); } else { assert(oldkeys->dk_refcnt == 1); - dictkeys_decref(interp, oldkeys); + dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); } ASSERT_CONSISTENT(mp); } +void +PyDict_Clear(PyObject *op) +{ + Py_BEGIN_CRITICAL_SECTION(op); + clear_lock_held(op); + Py_END_CRITICAL_SECTION(); +} + /* Internal version of PyDict_Next that returns a hash value in addition * to the key and value. * Return 1 on success, return 0 when the reached the end of the dictionary @@ -2164,16 +2716,18 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, if (!PyDict_Check(op)) return 0; + + ASSERT_DICT_LOCKED(op); + mp = (PyDictObject *)op; i = *ppos; - if (mp->ma_values) { + if (_PyDict_HasSplitTable(mp)) { assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE); if (i < 0 || i >= mp->ma_used) return 0; int index = get_index_from_order(mp, i); value = mp->ma_values->values[index]; - - key = DK_UNICODE_ENTRIES(mp->ma_keys)[index].me_key; + key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(mp->ma_keys)[index].me_key); hash = unicode_get_hash(key); assert(value != NULL); } @@ -2237,7 +2791,11 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, int PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) { - return _PyDict_Next(op, ppos, pkey, pvalue, NULL); + int res; + Py_BEGIN_CRITICAL_SECTION(op); + res = _PyDict_Next(op, ppos, pkey, pvalue, NULL); + Py_END_CRITICAL_SECTION(); + return res; } @@ -2248,6 +2806,8 @@ _PyDict_Pop_KnownHash(PyDictObject *mp, PyObject *key, Py_hash_t hash, { assert(PyDict_Check(mp)); + ASSERT_DICT_LOCKED(mp); + if (mp->ma_used == 0) { if (result) { *result = NULL; @@ -2287,10 +2847,11 @@ _PyDict_Pop_KnownHash(PyDictObject *mp, PyObject *key, Py_hash_t hash, return 1; } - -int -PyDict_Pop(PyObject *op, PyObject *key, PyObject **result) +static int +pop_lock_held(PyObject *op, PyObject *key, PyObject **result) { + ASSERT_DICT_LOCKED(op); + if (!PyDict_Check(op)) { if (result) { *result = NULL; @@ -2320,6 +2881,17 @@ PyDict_Pop(PyObject *op, PyObject *key, PyObject **result) return _PyDict_Pop_KnownHash(dict, key, hash, result); } +int +PyDict_Pop(PyObject *op, PyObject *key, PyObject **result) +{ + int err; + Py_BEGIN_CRITICAL_SECTION(op); + err = pop_lock_held(op, key, result); + Py_END_CRITICAL_SECTION(); + + return err; +} + int PyDict_PopString(PyObject *op, const char *key, PyObject **result) @@ -2352,6 +2924,55 @@ _PyDict_Pop(PyObject *dict, PyObject *key, PyObject *default_value) return result; } +static PyDictObject * +dict_dict_fromkeys(PyInterpreterState *interp, PyDictObject *mp, + PyObject *iterable, PyObject *value) +{ + PyObject *oldvalue; + Py_ssize_t pos = 0; + PyObject *key; + Py_hash_t hash; + int unicode = DK_IS_UNICODE(((PyDictObject*)iterable)->ma_keys); + uint8_t new_size = Py_MAX( + estimate_log2_keysize(PyDict_GET_SIZE(iterable)), + DK_LOG_SIZE(mp->ma_keys)); + if (dictresize(interp, mp, new_size, unicode)) { + Py_DECREF(mp); + return NULL; + } + + while (_PyDict_Next(iterable, &pos, &key, &oldvalue, &hash)) { + if (insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value))) { + Py_DECREF(mp); + return NULL; + } + } + return mp; +} + +static PyDictObject * +dict_set_fromkeys(PyInterpreterState *interp, PyDictObject *mp, + PyObject *iterable, PyObject *value) +{ + Py_ssize_t pos = 0; + PyObject *key; + Py_hash_t hash; + + if (dictresize(interp, mp, + estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { + Py_DECREF(mp); + return NULL; + } + + while (_PySet_NextEntry(iterable, &pos, &key, &hash)) { + if (insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value))) { + Py_DECREF(mp); + return NULL; + } + } + return mp; +} /* Internal version of dict.from_keys(). It is subclass-friendly. */ PyObject * @@ -2367,49 +2988,22 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) if (d == NULL) return NULL; - if (PyDict_CheckExact(d) && ((PyDictObject *)d)->ma_used == 0) { + + if (PyDict_CheckExact(d)) { if (PyDict_CheckExact(iterable)) { PyDictObject *mp = (PyDictObject *)d; - PyObject *oldvalue; - Py_ssize_t pos = 0; - PyObject *key; - Py_hash_t hash; - - int unicode = DK_IS_UNICODE(((PyDictObject*)iterable)->ma_keys); - if (dictresize(interp, mp, - estimate_log2_keysize(PyDict_GET_SIZE(iterable)), - unicode)) { - Py_DECREF(d); - return NULL; - } - while (_PyDict_Next(iterable, &pos, &key, &oldvalue, &hash)) { - if (insertdict(interp, mp, - Py_NewRef(key), hash, Py_NewRef(value))) { - Py_DECREF(d); - return NULL; - } - } + Py_BEGIN_CRITICAL_SECTION2(d, iterable); + d = (PyObject *)dict_dict_fromkeys(interp, mp, iterable, value); + Py_END_CRITICAL_SECTION2(); return d; } - if (PyAnySet_CheckExact(iterable)) { + else if (PyAnySet_CheckExact(iterable)) { PyDictObject *mp = (PyDictObject *)d; - Py_ssize_t pos = 0; - PyObject *key; - Py_hash_t hash; - - if (dictresize(interp, mp, - estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { - Py_DECREF(d); - return NULL; - } - while (_PySet_NextEntry(iterable, &pos, &key, &hash)) { - if (insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value))) { - Py_DECREF(d); - return NULL; - } - } + Py_BEGIN_CRITICAL_SECTION2(d, iterable); + d = (PyObject *)dict_set_fromkeys(interp, mp, iterable, value); + Py_END_CRITICAL_SECTION2(); return d; } } @@ -2421,12 +3015,17 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) } if (PyDict_CheckExact(d)) { + Py_BEGIN_CRITICAL_SECTION(d); while ((key = PyIter_Next(it)) != NULL) { - status = PyDict_SetItem(d, key, value); + status = setitem_lock_held((PyDictObject *)d, key, value); Py_DECREF(key); - if (status < 0) - goto Fail; + if (status < 0) { + assert(PyErr_Occurred()); + goto dict_iter_exit; + } } +dict_iter_exit: + Py_END_CRITICAL_SECTION(); } else { while ((key = PyIter_Next(it)) != NULL) { status = PyObject_SetItem(d, key, value); @@ -2473,21 +3072,18 @@ dict_dealloc(PyObject *self) for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) { Py_XDECREF(values->values[i]); } - free_values(values); - dictkeys_decref(interp, keys); + free_values(values, false); + dictkeys_decref(interp, keys, false); } else if (keys != NULL) { assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); } -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // new_dict() must not be called after _PyDict_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree < PyDict_MAXFREELIST && Py_IS_TYPE(mp, &PyDict_Type)) { - state->free_list[state->numfree++] = mp; +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *freelist = get_dict_freelist(); + if (freelist->numfree < PyDict_MAXFREELIST && freelist->numfree >=0 && + Py_IS_TYPE(mp, &PyDict_Type)) { + freelist->items[freelist->numfree++] = mp; OBJECT_STAT_INC(to_freelist); } else @@ -2500,7 +3096,7 @@ dict_dealloc(PyObject *self) static PyObject * -dict_repr(PyObject *self) +dict_repr_lock_held(PyObject *self) { PyDictObject *mp = (PyDictObject *)self; Py_ssize_t i; @@ -2530,7 +3126,7 @@ dict_repr(PyObject *self) Note that repr may mutate the dict. */ i = 0; first = 1; - while (PyDict_Next((PyObject *)mp, &i, &key, &value)) { + while (_PyDict_Next((PyObject *)mp, &i, &key, &value, NULL)) { PyObject *s; int res; @@ -2583,11 +3179,21 @@ dict_repr(PyObject *self) return NULL; } +static PyObject * +dict_repr(PyObject *self) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(self); + res = dict_repr_lock_held(self); + Py_END_CRITICAL_SECTION(); + return res; +} + static Py_ssize_t dict_length(PyObject *self) { PyDictObject *mp = (PyDictObject *)self; - return mp->ma_used; + return _Py_atomic_load_ssize_relaxed(&mp->ma_used); } static PyObject * @@ -2603,7 +3209,11 @@ dict_subscript(PyObject *self, PyObject *key) if (hash == -1) return NULL; } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif if (ix == DKIX_ERROR) return NULL; if (ix == DKIX_EMPTY || value == NULL) { @@ -2623,7 +3233,11 @@ dict_subscript(PyObject *self, PyObject *key) _PyErr_SetKeyError(key); return NULL; } +#ifdef Py_GIL_DISABLED + return value; +#else return Py_NewRef(value); +#endif } static int @@ -2641,9 +3255,11 @@ static PyMappingMethods dict_as_mapping = { dict_ass_sub, /*mp_ass_subscript*/ }; -PyObject * -PyDict_Keys(PyObject *dict) +static PyObject * +keys_lock_held(PyObject *dict) { + ASSERT_DICT_LOCKED(dict); + if (dict == NULL || !PyDict_Check(dict)) { PyErr_BadInternalCall(); return NULL; @@ -2678,8 +3294,21 @@ PyDict_Keys(PyObject *dict) } PyObject * -PyDict_Values(PyObject *dict) +PyDict_Keys(PyObject *dict) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(dict); + res = keys_lock_held(dict); + Py_END_CRITICAL_SECTION(); + + return res; +} + +static PyObject * +values_lock_held(PyObject *dict) { + ASSERT_DICT_LOCKED(dict); + if (dict == NULL || !PyDict_Check(dict)) { PyErr_BadInternalCall(); return NULL; @@ -2714,8 +3343,20 @@ PyDict_Values(PyObject *dict) } PyObject * -PyDict_Items(PyObject *dict) +PyDict_Values(PyObject *dict) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(dict); + res = values_lock_held(dict); + Py_END_CRITICAL_SECTION(); + return res; +} + +static PyObject * +items_lock_held(PyObject *dict) { + ASSERT_DICT_LOCKED(dict); + if (dict == NULL || !PyDict_Check(dict)) { PyErr_BadInternalCall(); return NULL; @@ -2764,6 +3405,17 @@ PyDict_Items(PyObject *dict) return v; } +PyObject * +PyDict_Items(PyObject *dict) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(dict); + res = items_lock_held(dict); + Py_END_CRITICAL_SECTION(); + + return res; +} + /*[clinic input] @classmethod dict.fromkeys @@ -2842,8 +3494,8 @@ dict_update(PyObject *self, PyObject *args, PyObject *kwds) producing iterable objects of length 2. */ -int -PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) +static int +merge_from_seq2_lock_held(PyObject *d, PyObject *seq2, int override) { PyObject *it; /* iter(seq2) */ Py_ssize_t i; /* index into seq2 of current element */ @@ -2895,14 +3547,14 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) Py_INCREF(key); Py_INCREF(value); if (override) { - if (PyDict_SetItem(d, key, value) < 0) { + if (setitem_lock_held((PyDictObject *)d, key, value) < 0) { Py_DECREF(key); Py_DECREF(value); goto Fail; } } else { - if (PyDict_SetDefault(d, key, value) == NULL) { + if (dict_setdefault_ref_lock_held(d, key, value, NULL, 0) < 0) { Py_DECREF(key); Py_DECREF(value); goto Fail; @@ -2927,6 +3579,118 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) return Py_SAFE_DOWNCAST(i, Py_ssize_t, int); } +int +PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(d); + res = merge_from_seq2_lock_held(d, seq2, override); + Py_END_CRITICAL_SECTION(); + + return res; +} + +static int +dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *other, int override) +{ + if (other == mp || other->ma_used == 0) + /* a.update(a) or a.update({}); nothing to do */ + return 0; + if (mp->ma_used == 0) { + /* Since the target dict is empty, PyDict_GetItem() + * always returns NULL. Setting override to 1 + * skips the unnecessary test. + */ + override = 1; + PyDictKeysObject *okeys = other->ma_keys; + + // If other is clean, combined, and just allocated, just clone it. + if (other->ma_values == NULL && + other->ma_used == okeys->dk_nentries && + (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || + USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL); + PyDictKeysObject *keys = clone_combined_dict_keys(other); + if (keys == NULL) + return -1; + + ensure_shared_on_resize(mp); + dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp)); + mp->ma_keys = keys; + if (_PyDict_HasSplitTable(mp)) { + free_values(mp->ma_values, IS_DICT_SHARED(mp)); + mp->ma_values = NULL; + } + + mp->ma_used = other->ma_used; + mp->ma_version_tag = new_version; + ASSERT_CONSISTENT(mp); + + if (_PyObject_GC_IS_TRACKED(other) && !_PyObject_GC_IS_TRACKED(mp)) { + /* Maintain tracking. */ + _PyObject_GC_TRACK(mp); + } + + return 0; + } + } + /* Do one big resize at the start, rather than + * incrementally resizing as we insert new items. Expect + * that there will be no (or few) overlapping keys. + */ + if (USABLE_FRACTION(DK_SIZE(mp->ma_keys)) < other->ma_used) { + int unicode = DK_IS_UNICODE(other->ma_keys); + if (dictresize(interp, mp, + estimate_log2_keysize(mp->ma_used + other->ma_used), + unicode)) { + return -1; + } + } + + Py_ssize_t orig_size = other->ma_keys->dk_nentries; + Py_ssize_t pos = 0; + Py_hash_t hash; + PyObject *key, *value; + + while (_PyDict_Next((PyObject*)other, &pos, &key, &value, &hash)) { + int err = 0; + Py_INCREF(key); + Py_INCREF(value); + if (override == 1) { + err = insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value)); + } + else { + err = _PyDict_Contains_KnownHash((PyObject *)mp, key, hash); + if (err == 0) { + err = insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value)); + } + else if (err > 0) { + if (override != 0) { + _PyErr_SetKeyError(key); + Py_DECREF(value); + Py_DECREF(key); + return -1; + } + err = 0; + } + } + Py_DECREF(value); + Py_DECREF(key); + if (err != 0) + return -1; + + if (orig_size != other->ma_keys->dk_nentries) { + PyErr_SetString(PyExc_RuntimeError, + "dict mutated during update"); + return -1; + } + } + return 0; +} + static int dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) { @@ -2944,123 +3708,40 @@ dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) return -1; } mp = (PyDictObject*)a; + int res = 0; if (PyDict_Check(b) && (Py_TYPE(b)->tp_iter == dict_iter)) { other = (PyDictObject*)b; - if (other == mp || other->ma_used == 0) - /* a.update(a) or a.update({}); nothing to do */ - return 0; - if (mp->ma_used == 0) { - /* Since the target dict is empty, PyDict_GetItem() - * always returns NULL. Setting override to 1 - * skips the unnecessary test. - */ - override = 1; - PyDictKeysObject *okeys = other->ma_keys; - - // If other is clean, combined, and just allocated, just clone it. - if (other->ma_values == NULL && - other->ma_used == okeys->dk_nentries && - (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || - USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_CLONED, mp, b, NULL); - PyDictKeysObject *keys = clone_combined_dict_keys(other); - if (keys == NULL) { - return -1; - } - - dictkeys_decref(interp, mp->ma_keys); - mp->ma_keys = keys; - if (mp->ma_values != NULL) { - free_values(mp->ma_values); - mp->ma_values = NULL; - } - - mp->ma_used = other->ma_used; - mp->ma_version_tag = new_version; - ASSERT_CONSISTENT(mp); - - if (_PyObject_GC_IS_TRACKED(other) && !_PyObject_GC_IS_TRACKED(mp)) { - /* Maintain tracking. */ - _PyObject_GC_TRACK(mp); - } - - return 0; - } - } - /* Do one big resize at the start, rather than - * incrementally resizing as we insert new items. Expect - * that there will be no (or few) overlapping keys. - */ - if (USABLE_FRACTION(DK_SIZE(mp->ma_keys)) < other->ma_used) { - int unicode = DK_IS_UNICODE(other->ma_keys); - if (dictresize(interp, mp, - estimate_log2_keysize(mp->ma_used + other->ma_used), - unicode)) { - return -1; - } - } - - Py_ssize_t orig_size = other->ma_keys->dk_nentries; - Py_ssize_t pos = 0; - Py_hash_t hash; - PyObject *key, *value; - - while (_PyDict_Next((PyObject*)other, &pos, &key, &value, &hash)) { - int err = 0; - Py_INCREF(key); - Py_INCREF(value); - if (override == 1) { - err = insertdict(interp, mp, - Py_NewRef(key), hash, Py_NewRef(value)); - } - else { - err = _PyDict_Contains_KnownHash(a, key, hash); - if (err == 0) { - err = insertdict(interp, mp, - Py_NewRef(key), hash, Py_NewRef(value)); - } - else if (err > 0) { - if (override != 0) { - _PyErr_SetKeyError(key); - Py_DECREF(value); - Py_DECREF(key); - return -1; - } - err = 0; - } - } - Py_DECREF(value); - Py_DECREF(key); - if (err != 0) - return -1; - - if (orig_size != other->ma_keys->dk_nentries) { - PyErr_SetString(PyExc_RuntimeError, - "dict mutated during update"); - return -1; - } - } + int res; + Py_BEGIN_CRITICAL_SECTION2(a, b); + res = dict_dict_merge(interp, (PyDictObject *)a, other, override); + ASSERT_CONSISTENT(a); + Py_END_CRITICAL_SECTION2(); + return res; } else { /* Do it the generic, slower way */ + Py_BEGIN_CRITICAL_SECTION(a); PyObject *keys = PyMapping_Keys(b); PyObject *iter; PyObject *key, *value; int status; - if (keys == NULL) + if (keys == NULL) { /* Docstring says this is equivalent to E.keys() so * if E doesn't have a .keys() method we want * AttributeError to percolate up. Might as well * do the same for any other error. */ - return -1; + res = -1; + goto slow_exit; + } iter = PyObject_GetIter(keys); Py_DECREF(keys); - if (iter == NULL) - return -1; + if (iter == NULL) { + res = -1; + goto slow_exit; + } for (key = PyIter_Next(iter); key; key = PyIter_Next(iter)) { if (override != 1) { @@ -3075,30 +3756,39 @@ dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) } Py_DECREF(key); Py_DECREF(iter); - return -1; + res = -1; + goto slow_exit; } } value = PyObject_GetItem(b, key); if (value == NULL) { Py_DECREF(iter); Py_DECREF(key); - return -1; + res = -1; + goto slow_exit; } - status = PyDict_SetItem(a, key, value); + status = setitem_lock_held(mp, key, value); Py_DECREF(key); Py_DECREF(value); if (status < 0) { Py_DECREF(iter); + res = -1; + goto slow_exit; return -1; } } Py_DECREF(iter); - if (PyErr_Occurred()) + if (PyErr_Occurred()) { /* Iterator completed, via error */ - return -1; + res = -1; + goto slow_exit; + } + +slow_exit: + ASSERT_CONSISTENT(a); + Py_END_CRITICAL_SECTION(); + return res; } - ASSERT_CONSISTENT(a); - return 0; } int @@ -3136,17 +3826,14 @@ dict_copy_impl(PyDictObject *self) return PyDict_Copy((PyObject *)self); } -PyObject * -PyDict_Copy(PyObject *o) +static PyObject * +copy_lock_held(PyObject *o) { PyObject *copy; PyDictObject *mp; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (o == NULL || !PyDict_Check(o)) { - PyErr_BadInternalCall(); - return NULL; - } + ASSERT_DICT_LOCKED(o); mp = (PyDictObject *)o; if (mp->ma_used == 0) { @@ -3162,7 +3849,7 @@ PyDict_Copy(PyObject *o) return PyErr_NoMemory(); split_copy = PyObject_GC_New(PyDictObject, &PyDict_Type); if (split_copy == NULL) { - free_values(newvalues); + free_values(newvalues, false); return NULL; } size_t prefix_size = ((uint8_t *)newvalues)[-1]; @@ -3229,6 +3916,23 @@ PyDict_Copy(PyObject *o) return NULL; } +PyObject * +PyDict_Copy(PyObject *o) +{ + if (o == NULL || !PyDict_Check(o)) { + PyErr_BadInternalCall(); + return NULL; + } + + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(o); + + res = copy_lock_held(o); + + Py_END_CRITICAL_SECTION(); + return res; +} + Py_ssize_t PyDict_Size(PyObject *mp) { @@ -3244,15 +3948,18 @@ PyDict_Size(PyObject *mp) * Uses only Py_EQ comparison. */ static int -dict_equal(PyDictObject *a, PyDictObject *b) +dict_equal_lock_held(PyDictObject *a, PyDictObject *b) { Py_ssize_t i; + ASSERT_DICT_LOCKED(a); + ASSERT_DICT_LOCKED(b); + if (a->ma_used != b->ma_used) /* can't be equal if # of entries differ */ return 0; /* Same # of entries -- check all of 'em. Exit early on any diff. */ - for (i = 0; i < a->ma_keys->dk_nentries; i++) { + for (i = 0; i < LOAD_KEYS_NENTIRES(a->ma_keys); i++) { PyObject *key, *aval; Py_hash_t hash; if (DK_IS_UNICODE(a->ma_keys)) { @@ -3262,7 +3969,7 @@ dict_equal(PyDictObject *a, PyDictObject *b) continue; } hash = unicode_get_hash(key); - if (a->ma_values) + if (_PyDict_HasSplitTable(a)) aval = a->ma_values->values[i]; else aval = ep->me_value; @@ -3302,6 +4009,17 @@ dict_equal(PyDictObject *a, PyDictObject *b) return 1; } +static int +dict_equal(PyDictObject *a, PyDictObject *b) +{ + int res; + Py_BEGIN_CRITICAL_SECTION2(a, b); + res = dict_equal_lock_held(a, b); + Py_END_CRITICAL_SECTION2(); + + return res; +} + static PyObject * dict_richcompare(PyObject *v, PyObject *w, int op) { @@ -3337,25 +4055,18 @@ static PyObject * dict___contains__(PyDictObject *self, PyObject *key) /*[clinic end generated code: output=a3d03db709ed6e6b input=fe1cb42ad831e820]*/ { - register PyDictObject *mp = self; - Py_hash_t hash; - Py_ssize_t ix; - PyObject *value; - - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return NULL; - } - ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) + int contains = PyDict_Contains((PyObject *)self, key); + if (contains < 0) { return NULL; - if (ix == DKIX_EMPTY || value == NULL) - Py_RETURN_FALSE; - Py_RETURN_TRUE; + } + if (contains) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; } /*[clinic input] +@critical_section dict.get key: object @@ -3367,7 +4078,7 @@ Return the value for key if key is in the dictionary, else default. static PyObject * dict_get_impl(PyDictObject *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=bba707729dee05bf input=279ddb5790b6b107]*/ +/*[clinic end generated code: output=bba707729dee05bf input=a631d3f18f584c60]*/ { PyObject *val = NULL; Py_hash_t hash; @@ -3378,96 +4089,127 @@ dict_get_impl(PyDictObject *self, PyObject *key, PyObject *default_value) if (hash == -1) return NULL; } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(self, key, hash, &val); +#else ix = _Py_dict_lookup(self, key, hash, &val); +#endif if (ix == DKIX_ERROR) return NULL; +#ifdef Py_GIL_DISABLED + if (ix == DKIX_EMPTY || val == NULL) { + val = Py_NewRef(default_value); + } + return val; +#else if (ix == DKIX_EMPTY || val == NULL) { val = default_value; } return Py_NewRef(val); +#endif } -PyObject * -PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) +static int +dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_value, + PyObject **result, int incref_result) { PyDictObject *mp = (PyDictObject *)d; PyObject *value; Py_hash_t hash; PyInterpreterState *interp = _PyInterpreterState_GET(); + ASSERT_DICT_LOCKED(d); + if (!PyDict_Check(d)) { PyErr_BadInternalCall(); - return NULL; + if (result) { + *result = NULL; + } + return -1; } if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { hash = PyObject_Hash(key); - if (hash == -1) - return NULL; + if (hash == -1) { + if (result) { + *result = NULL; + } + return -1; + } } if (mp->ma_keys == Py_EMPTY_KEYS) { if (insert_to_emptydict(interp, mp, Py_NewRef(key), hash, - Py_NewRef(defaultobj)) < 0) { - return NULL; + Py_NewRef(default_value)) < 0) { + if (result) { + *result = NULL; + } + return -1; + } + if (result) { + *result = incref_result ? Py_NewRef(default_value) : default_value; } - return defaultobj; + return 0; } if (!PyUnicode_CheckExact(key) && DK_IS_UNICODE(mp->ma_keys)) { if (insertion_resize(interp, mp, 0) < 0) { - return NULL; + if (result) { + *result = NULL; + } + return -1; } } Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) - return NULL; + if (ix == DKIX_ERROR) { + if (result) { + *result = NULL; + } + return -1; + } if (ix == DKIX_EMPTY) { uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, defaultobj); + interp, PyDict_EVENT_ADDED, mp, key, default_value); mp->ma_keys->dk_version = 0; - value = defaultobj; - if (mp->ma_keys->dk_usable <= 0) { - if (insertion_resize(interp, mp, 1) < 0) { - return NULL; - } - } - Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); - dictkeys_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); - if (DK_IS_UNICODE(mp->ma_keys)) { - assert(PyUnicode_CheckExact(key)); - PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = Py_NewRef(key); - if (_PyDict_HasSplitTable(mp)) { - Py_ssize_t index = (int)mp->ma_keys->dk_nentries; - assert(index < SHARED_KEYS_MAX_SIZE); - assert(mp->ma_values->values[index] == NULL); - mp->ma_values->values[index] = Py_NewRef(value); - _PyDictValues_AddToInsertionOrder(mp->ma_values, index); - } - else { - ep->me_value = Py_NewRef(value); + value = default_value; + + if (!_PyDict_HasSplitTable(mp)) { + if (insert_combined_dict(interp, mp, hash, Py_NewRef(key), Py_NewRef(value)) < 0) { + Py_DECREF(key); + Py_DECREF(value); + if (result) { + *result = NULL; + } + return -1; } } else { - PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = Py_NewRef(key); - ep->me_hash = hash; - ep->me_value = Py_NewRef(value); + if (insert_split_dict(interp, mp, hash, Py_NewRef(key), Py_NewRef(value)) < 0) { + Py_DECREF(key); + Py_DECREF(value); + if (result) { + *result = NULL; + } + return -1; + } } + MAINTAIN_TRACKING(mp, key, value); mp->ma_used++; mp->ma_version_tag = new_version; - mp->ma_keys->dk_usable--; - mp->ma_keys->dk_nentries++; assert(mp->ma_keys->dk_usable >= 0); + ASSERT_CONSISTENT(mp); + if (result) { + *result = incref_result ? Py_NewRef(value) : value; + } + return 0; } else if (value == NULL) { uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, defaultobj); - value = defaultobj; + interp, PyDict_EVENT_ADDED, mp, key, default_value); + value = default_value; assert(_PyDict_HasSplitTable(mp)); assert(mp->ma_values->values[ix] == NULL); MAINTAIN_TRACKING(mp, key, value); @@ -3475,13 +4217,43 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); mp->ma_used++; mp->ma_version_tag = new_version; + ASSERT_CONSISTENT(mp); + if (result) { + *result = incref_result ? Py_NewRef(value) : value; + } + return 0; } ASSERT_CONSISTENT(mp); - return value; + if (result) { + *result = incref_result ? Py_NewRef(value) : value; + } + return 1; +} + +int +PyDict_SetDefaultRef(PyObject *d, PyObject *key, PyObject *default_value, + PyObject **result) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(d); + res = dict_setdefault_ref_lock_held(d, key, default_value, result, 1); + Py_END_CRITICAL_SECTION(); + return res; +} + +PyObject * +PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(d); + dict_setdefault_ref_lock_held(d, key, defaultobj, &result, 0); + Py_END_CRITICAL_SECTION(); + return result; } /*[clinic input] +@critical_section dict.setdefault key: object @@ -3496,12 +4268,11 @@ Return the value for key if key is in the dictionary, else default. static PyObject * dict_setdefault_impl(PyDictObject *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=f8c1101ebf69e220 input=0f063756e815fd9d]*/ +/*[clinic end generated code: output=f8c1101ebf69e220 input=9237af9a0a224302]*/ { PyObject *val; - - val = PyDict_SetDefault((PyObject *)self, key, default_value); - return Py_XNewRef(val); + dict_setdefault_ref_lock_held((PyObject *)self, key, default_value, &val, 1); + return val; } @@ -3540,6 +4311,7 @@ dict_pop_impl(PyDictObject *self, PyObject *key, PyObject *default_value) } /*[clinic input] +@critical_section dict.popitem Remove and return a (key, value) pair as a 2-tuple. @@ -3550,7 +4322,7 @@ Raises KeyError if the dict is empty. static PyObject * dict_popitem_impl(PyDictObject *self) -/*[clinic end generated code: output=e65fcb04420d230d input=1c38a49f21f64941]*/ +/*[clinic end generated code: output=e65fcb04420d230d input=ef28b4da5f0f762e]*/ { Py_ssize_t i, j; PyObject *res; @@ -3575,8 +4347,8 @@ dict_popitem_impl(PyDictObject *self) return NULL; } /* Convert split table to combined table */ - if (self->ma_keys->dk_kind == DICT_KEYS_SPLIT) { - if (dictresize(interp, self, DK_LOG_SIZE(self->ma_keys), 1)) { + if (_PyDict_HasSplitTable(self)) { + if (dictresize(interp, self, DK_LOG_SIZE(self->ma_keys), 1) < 0) { Py_DECREF(res); return NULL; } @@ -3628,8 +4400,8 @@ dict_popitem_impl(PyDictObject *self) PyTuple_SET_ITEM(res, 0, key); PyTuple_SET_ITEM(res, 1, value); /* We can't dk_usable++ since there is DKIX_DUMMY in indices */ - self->ma_keys->dk_nentries = i; - self->ma_used--; + STORE_KEYS_NENTRIES(self->ma_keys, i); + STORE_USED(self, self->ma_used - 1); self->ma_version_tag = new_version; ASSERT_CONSISTENT(self); return res; @@ -3643,7 +4415,7 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) Py_ssize_t i, n = keys->dk_nentries; if (DK_IS_UNICODE(keys)) { - if (mp->ma_values != NULL) { + if (_PyDict_HasSplitTable(mp)) { for (i = 0; i < n; i++) { Py_VISIT(mp->ma_values->values[i]); } @@ -3676,11 +4448,11 @@ dict_tp_clear(PyObject *op) static PyObject *dictiter_new(PyDictObject *, PyTypeObject *); -Py_ssize_t -_PyDict_SizeOf(PyDictObject *mp) +static Py_ssize_t +sizeof_lock_held(PyDictObject *mp) { size_t res = _PyObject_SIZE(Py_TYPE(mp)); - if (mp->ma_values) { + if (_PyDict_HasSplitTable(mp)) { res += shared_keys_usable_size(mp->ma_keys) * sizeof(PyObject*); } /* If the dictionary is split, the keys portion is accounted-for @@ -3692,6 +4464,17 @@ _PyDict_SizeOf(PyDictObject *mp) return (Py_ssize_t)res; } +Py_ssize_t +_PyDict_SizeOf(PyDictObject *mp) +{ + Py_ssize_t res; + Py_BEGIN_CRITICAL_SECTION(mp); + res = sizeof_lock_held(mp); + Py_END_CRITICAL_SECTION(); + + return res; +} + size_t _PyDict_KeysSize(PyDictKeysObject *keys) { @@ -3780,19 +4563,14 @@ int PyDict_Contains(PyObject *op, PyObject *key) { Py_hash_t hash; - Py_ssize_t ix; - PyDictObject *mp = (PyDictObject *)op; - PyObject *value; if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { hash = PyObject_Hash(key); if (hash == -1) return -1; } - ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) - return -1; - return (ix != DKIX_EMPTY && value != NULL); + + return _PyDict_Contains_KnownHash(op, key, hash); } int @@ -3815,10 +4593,20 @@ _PyDict_Contains_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) PyObject *value; Py_ssize_t ix; +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif if (ix == DKIX_ERROR) return -1; - return (ix != DKIX_EMPTY && value != NULL); + if (ix != DKIX_EMPTY && value != NULL) { +#ifdef Py_GIL_DISABLED + Py_DECREF(value); +#endif + return 1; + } + return 0; } int @@ -4089,11 +4877,11 @@ dictiter_new(PyDictObject *dict, PyTypeObject *itertype) if (itertype == &PyDictRevIterKey_Type || itertype == &PyDictRevIterItem_Type || itertype == &PyDictRevIterValue_Type) { - if (dict->ma_values) { + if (_PyDict_HasSplitTable(dict)) { di->di_pos = dict->ma_used - 1; } else { - di->di_pos = dict->ma_keys->dk_nentries - 1; + di->di_pos = load_keys_nentries(dict) - 1; } } else { @@ -4160,18 +4948,24 @@ static PyMethodDef dictiter_methods[] = { {NULL, NULL} /* sentinel */ }; +#ifdef Py_GIL_DISABLED + +static int +dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, + PyObject **out_key, PyObject **out_value); + +#else /* Py_GIL_DISABLED */ + static PyObject* -dictiter_iternextkey(PyObject *self) +dictiter_iternextkey_lock_held(PyDictObject *d, PyObject *self) { dictiterobject *di = (dictiterobject *)self; PyObject *key; Py_ssize_t i; PyDictKeysObject *k; - PyDictObject *d = di->di_dict; - if (d == NULL) - return NULL; assert (PyDict_Check(d)); + ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, @@ -4183,11 +4977,11 @@ dictiter_iternextkey(PyObject *self) i = di->di_pos; k = d->ma_keys; assert(i >= 0); - if (d->ma_values) { + if (_PyDict_HasSplitTable(d)) { if (i >= d->ma_used) goto fail; int index = get_index_from_order(d, i); - key = DK_UNICODE_ENTRIES(k)[index].me_key; + key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(k)[index].me_key); assert(d->ma_values->values[index] != NULL); } else { @@ -4229,6 +5023,29 @@ dictiter_iternextkey(PyObject *self) return NULL; } +#endif /* Py_GIL_DISABLED */ + +static PyObject* +dictiter_iternextkey(PyObject *self) +{ + dictiterobject *di = (dictiterobject *)self; + PyDictObject *d = di->di_dict; + + if (d == NULL) + return NULL; + + PyObject *value; +#ifdef Py_GIL_DISABLED + if (dictiter_iternext_threadsafe(d, self, &value, NULL) < 0) { + value = NULL; + } +#else + value = dictiter_iternextkey_lock_held(d, self); +#endif + + return value; +} + PyTypeObject PyDictIterKey_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_keyiterator", /* tp_name */ @@ -4262,17 +5079,17 @@ PyTypeObject PyDictIterKey_Type = { 0, }; +#ifndef Py_GIL_DISABLED + static PyObject * -dictiter_iternextvalue(PyObject *self) +dictiter_iternextvalue_lock_held(PyDictObject *d, PyObject *self) { dictiterobject *di = (dictiterobject *)self; PyObject *value; Py_ssize_t i; - PyDictObject *d = di->di_dict; - if (d == NULL) - return NULL; assert (PyDict_Check(d)); + ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, @@ -4283,7 +5100,7 @@ dictiter_iternextvalue(PyObject *self) i = di->di_pos; assert(i >= 0); - if (d->ma_values) { + if (_PyDict_HasSplitTable(d)) { if (i >= d->ma_used) goto fail; int index = get_index_from_order(d, i); @@ -4329,6 +5146,29 @@ dictiter_iternextvalue(PyObject *self) return NULL; } +#endif /* Py_GIL_DISABLED */ + +static PyObject * +dictiter_iternextvalue(PyObject *self) +{ + dictiterobject *di = (dictiterobject *)self; + PyDictObject *d = di->di_dict; + + if (d == NULL) + return NULL; + + PyObject *value; +#ifdef Py_GIL_DISABLED + if (dictiter_iternext_threadsafe(d, self, NULL, &value) < 0) { + value = NULL; + } +#else + value = dictiter_iternextvalue_lock_held(d, self); +#endif + + return value; +} + PyTypeObject PyDictIterValue_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_valueiterator", /* tp_name */ @@ -4362,32 +5202,31 @@ PyTypeObject PyDictIterValue_Type = { 0, }; -static PyObject * -dictiter_iternextitem(PyObject *self) +static int +dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self, + PyObject **out_key, PyObject **out_value) { dictiterobject *di = (dictiterobject *)self; - PyObject *key, *value, *result; + PyObject *key, *value; Py_ssize_t i; - PyDictObject *d = di->di_dict; - if (d == NULL) - return NULL; assert (PyDict_Check(d)); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, "dictionary changed size during iteration"); di->di_used = -1; /* Make this state sticky */ - return NULL; + return -1; } - i = di->di_pos; + i = FT_ATOMIC_LOAD_SSIZE_RELAXED(di->di_pos); + assert(i >= 0); - if (d->ma_values) { + if (_PyDict_HasSplitTable(d)) { if (i >= d->ma_used) goto fail; int index = get_index_from_order(d, i); - key = DK_UNICODE_ENTRIES(d->ma_keys)[index].me_key; + key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(d->ma_keys)[index].me_key); value = d->ma_values->values[index]; assert(value != NULL); } @@ -4424,33 +5263,225 @@ dictiter_iternextitem(PyObject *self) } di->di_pos = i+1; di->len--; - result = di->di_result; - if (Py_REFCNT(result) == 1) { - PyObject *oldkey = PyTuple_GET_ITEM(result, 0); - PyObject *oldvalue = PyTuple_GET_ITEM(result, 1); - PyTuple_SET_ITEM(result, 0, Py_NewRef(key)); - PyTuple_SET_ITEM(result, 1, Py_NewRef(value)); - Py_INCREF(result); - Py_DECREF(oldkey); - Py_DECREF(oldvalue); - // bpo-42536: The GC may have untracked this result tuple. Since we're - // recycling it, make sure it's tracked again: - if (!_PyObject_GC_IS_TRACKED(result)) { - _PyObject_GC_TRACK(result); + if (out_key != NULL) { + *out_key = Py_NewRef(key); + } + if (out_value != NULL) { + *out_value = Py_NewRef(value); + } + return 0; + +fail: + di->di_dict = NULL; + Py_DECREF(d); + return -1; +} + +#ifdef Py_GIL_DISABLED + +// Grabs the key and/or value from the provided locations and if successful +// returns them with an increased reference count. If either one is unsucessful +// nothing is incref'd and returns -1. +static int +acquire_key_value(PyObject **key_loc, PyObject *value, PyObject **value_loc, + PyObject **out_key, PyObject **out_value) +{ + if (out_key) { + *out_key = _Py_TryXGetRef(key_loc); + if (*out_key == NULL) { + return -1; + } + } + + if (out_value) { + if (!_Py_TryIncref(value_loc, value)) { + if (out_key) { + Py_DECREF(*out_key); + } + return -1; + } + *out_value = value; + } + + return 0; +} + +static Py_ssize_t +load_values_used_size(PyDictValues *values) +{ + return (Py_ssize_t)_Py_atomic_load_uint8(&DICT_VALUES_USED_SIZE(values)); +} + +static int +dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, + PyObject **out_key, PyObject **out_value) +{ + dictiterobject *di = (dictiterobject *)self; + Py_ssize_t i; + PyDictKeysObject *k; + + assert (PyDict_Check(d)); + + if (di->di_used != _Py_atomic_load_ssize_relaxed(&d->ma_used)) { + PyErr_SetString(PyExc_RuntimeError, + "dictionary changed size during iteration"); + di->di_used = -1; /* Make this state sticky */ + return -1; + } + + ensure_shared_on_read(d); + + i = _Py_atomic_load_ssize_relaxed(&di->di_pos); + k = _Py_atomic_load_ptr_relaxed(&d->ma_keys); + assert(i >= 0); + if (_PyDict_HasSplitTable(d)) { + PyDictValues *values = _Py_atomic_load_ptr_relaxed(&d->ma_values); + if (values == NULL) { + goto concurrent_modification; + } + + Py_ssize_t used = load_values_used_size(values); + if (i >= used) { + goto fail; + } + + // We're racing against writes to the order from delete_index_from_values, but + // single threaded can suffer from concurrent modification to those as well and + // can have either duplicated or skipped attributes, so we strive to do no better + // here. + int index = get_index_from_order(d, i); + PyObject *value = _Py_atomic_load_ptr(&values->values[index]); + if (acquire_key_value(&DK_UNICODE_ENTRIES(k)[index].me_key, value, + &values->values[index], out_key, out_value) < 0) { + goto try_locked; } } else { - result = PyTuple_New(2); - if (result == NULL) - return NULL; - PyTuple_SET_ITEM(result, 0, Py_NewRef(key)); - PyTuple_SET_ITEM(result, 1, Py_NewRef(value)); + Py_ssize_t n = _Py_atomic_load_ssize_relaxed(&k->dk_nentries); + if (DK_IS_UNICODE(k)) { + PyDictUnicodeEntry *entry_ptr = &DK_UNICODE_ENTRIES(k)[i]; + PyObject *value; + while (i < n && + (value = _Py_atomic_load_ptr(&entry_ptr->me_value)) == NULL) { + entry_ptr++; + i++; + } + if (i >= n) + goto fail; + + if (acquire_key_value(&entry_ptr->me_key, value, + &entry_ptr->me_value, out_key, out_value) < 0) { + goto try_locked; + } + } + else { + PyDictKeyEntry *entry_ptr = &DK_ENTRIES(k)[i]; + PyObject *value; + while (i < n && + (value = _Py_atomic_load_ptr(&entry_ptr->me_value)) == NULL) { + entry_ptr++; + i++; + } + + if (i >= n) + goto fail; + + if (acquire_key_value(&entry_ptr->me_key, value, + &entry_ptr->me_value, out_key, out_value) < 0) { + goto try_locked; + } + } } - return result; + // We found an element (key), but did not expect it + Py_ssize_t len; + if ((len = _Py_atomic_load_ssize_relaxed(&di->len)) == 0) { + goto concurrent_modification; + } + + _Py_atomic_store_ssize_relaxed(&di->di_pos, i + 1); + _Py_atomic_store_ssize_relaxed(&di->len, len - 1); + return 0; + +concurrent_modification: + PyErr_SetString(PyExc_RuntimeError, + "dictionary keys changed during iteration"); fail: di->di_dict = NULL; Py_DECREF(d); + return -1; + + int res; +try_locked: + Py_BEGIN_CRITICAL_SECTION(d); + res = dictiter_iternextitem_lock_held(d, self, out_key, out_value); + Py_END_CRITICAL_SECTION(); + return res; +} + +#endif + +static bool +has_unique_reference(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + return (_Py_IsOwnedByCurrentThread(op) && + op->ob_ref_local == 1 && + _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared) == 0); +#else + return Py_REFCNT(op) == 1; +#endif +} + +static bool +acquire_iter_result(PyObject *result) +{ + if (has_unique_reference(result)) { + Py_INCREF(result); + return true; + } + return false; +} + +static PyObject * +dictiter_iternextitem(PyObject *self) +{ + dictiterobject *di = (dictiterobject *)self; + PyDictObject *d = di->di_dict; + + if (d == NULL) + return NULL; + + PyObject *key, *value; +#ifdef Py_GIL_DISABLED + if (dictiter_iternext_threadsafe(d, self, &key, &value) == 0) { +#else + if (dictiter_iternextitem_lock_held(d, self, &key, &value) == 0) { + +#endif + PyObject *result = di->di_result; + if (acquire_iter_result(result)) { + PyObject *oldkey = PyTuple_GET_ITEM(result, 0); + PyObject *oldvalue = PyTuple_GET_ITEM(result, 1); + PyTuple_SET_ITEM(result, 0, key); + PyTuple_SET_ITEM(result, 1, value); + Py_DECREF(oldkey); + Py_DECREF(oldvalue); + // bpo-42536: The GC may have untracked this result tuple. Since we're + // recycling it, make sure it's tracked again: + if (!_PyObject_GC_IS_TRACKED(result)) { + _PyObject_GC_TRACK(result); + } + } + else { + result = PyTuple_New(2); + if (result == NULL) + return NULL; + PyTuple_SET_ITEM(result, 0, key); + PyTuple_SET_ITEM(result, 1, value); + } + return result; + } return NULL; } @@ -4491,15 +5522,12 @@ PyTypeObject PyDictIterItem_Type = { /* dictreviter */ static PyObject * -dictreviter_iternext(PyObject *self) +dictreviter_iter_lock_held(PyDictObject *d, PyObject *self) { dictiterobject *di = (dictiterobject *)self; - PyDictObject *d = di->di_dict; - if (d == NULL) { - return NULL; - } assert (PyDict_Check(d)); + ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, @@ -4515,9 +5543,9 @@ dictreviter_iternext(PyObject *self) if (i < 0) { goto fail; } - if (d->ma_values) { + if (_PyDict_HasSplitTable(d)) { int index = get_index_from_order(d, i); - key = DK_UNICODE_ENTRIES(k)[index].me_key; + key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(k)[index].me_key); value = d->ma_values->values[index]; assert (value != NULL); } @@ -4590,6 +5618,23 @@ dictreviter_iternext(PyObject *self) return NULL; } +static PyObject * +dictreviter_iternext(PyObject *self) +{ + dictiterobject *di = (dictiterobject *)self; + PyDictObject *d = di->di_dict; + + if (d == NULL) + return NULL; + + PyObject *value; + Py_BEGIN_CRITICAL_SECTION(d); + value = dictreviter_iter_lock_held(d, self); + Py_END_CRITICAL_SECTION(); + + return value; +} + PyTypeObject PyDictRevIterKey_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_reversekeyiterator", @@ -5018,14 +6063,12 @@ dictviews_or(PyObject* self, PyObject *other) } static PyObject * -dictitems_xor(PyObject *self, PyObject *other) +dictitems_xor_lock_held(PyObject *d1, PyObject *d2) { - assert(PyDictItems_Check(self)); - assert(PyDictItems_Check(other)); - PyObject *d1 = (PyObject *)((_PyDictViewObject *)self)->dv_dict; - PyObject *d2 = (PyObject *)((_PyDictViewObject *)other)->dv_dict; + ASSERT_DICT_LOCKED(d1); + ASSERT_DICT_LOCKED(d2); - PyObject *temp_dict = PyDict_Copy(d1); + PyObject *temp_dict = copy_lock_held(d1); if (temp_dict == NULL) { return NULL; } @@ -5103,6 +6146,22 @@ dictitems_xor(PyObject *self, PyObject *other) return NULL; } +static PyObject * +dictitems_xor(PyObject *self, PyObject *other) +{ + assert(PyDictItems_Check(self)); + assert(PyDictItems_Check(other)); + PyObject *d1 = (PyObject *)((_PyDictViewObject *)self)->dv_dict; + PyObject *d2 = (PyObject *)((_PyDictViewObject *)other)->dv_dict; + + PyObject *res; + Py_BEGIN_CRITICAL_SECTION2(d1, d2); + res = dictitems_xor_lock_held(d1, d2); + Py_END_CRITICAL_SECTION2(); + + return res; +} + static PyObject* dictviews_xor(PyObject* self, PyObject *other) { @@ -5499,9 +6558,20 @@ _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictKeysObject *keys = CACHED_KEYS(tp); assert(keys != NULL); +#ifdef Py_GIL_DISABLED + Py_ssize_t usable = _Py_atomic_load_ssize_relaxed(&keys->dk_usable); + if (usable > 1) { + LOCK_KEYS(keys); + if (keys->dk_usable > 1) { + _Py_atomic_store_ssize(&keys->dk_usable, keys->dk_usable - 1); + } + UNLOCK_KEYS(keys); + } +#else if (keys->dk_usable > 1) { keys->dk_usable--; } +#endif size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); if (values == NULL) { @@ -5592,9 +6662,13 @@ _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) return false; } assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE)); - if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || Py_REFCNT(dict) != 1) { + if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || + !has_unique_reference((PyObject *)dict)) + { return false; } + ensure_shared_on_resize(dict); + assert(dict->ma_values); // We have an opportunity to do something *really* cool: dematerialize it! _PyDictKeys_DecRef(dict->ma_keys); @@ -5618,22 +6692,26 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); Py_ssize_t ix = DKIX_EMPTY; if (PyUnicode_CheckExact(name)) { - ix = insert_into_dictkeys(keys, name); - } - if (ix == DKIX_EMPTY) { + LOCK_KEYS(keys); + ix = insert_into_splitdictkeys(keys, name); #ifdef Py_STATS - if (PyUnicode_CheckExact(name)) { - if (shared_keys_usable_size(keys) == SHARED_KEYS_MAX_SIZE) { - OBJECT_STAT_INC(dict_materialized_too_big); + if (ix == DKIX_EMPTY) { + if (PyUnicode_CheckExact(name)) { + if (shared_keys_usable_size(keys) == SHARED_KEYS_MAX_SIZE) { + OBJECT_STAT_INC(dict_materialized_too_big); + } + else { + OBJECT_STAT_INC(dict_materialized_new_key); + } } else { - OBJECT_STAT_INC(dict_materialized_new_key); + OBJECT_STAT_INC(dict_materialized_str_subclass); } } - else { - OBJECT_STAT_INC(dict_materialized_str_subclass); - } #endif + UNLOCK_KEYS(keys); + } + if (ix == DKIX_EMPTY) { PyObject *dict = make_dict_from_instance_attributes( interp, keys, values); if (dict == NULL) { @@ -5758,7 +6836,7 @@ _PyObject_FreeInstanceAttributes(PyObject *self) for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { Py_XDECREF(values->values[i]); } - free_values(values); + free_values(values, IS_DICT_SHARED((PyDictObject*)self)); } int @@ -5799,7 +6877,7 @@ PyObject_ClearManagedDict(PyObject *obj) Py_CLEAR(values->values[i]); } dorv_ptr->dict = NULL; - free_values(values); + free_values(values, IS_DICT_SHARED((PyDictObject*)obj)); } else { PyObject *dict = dorv_ptr->dict; @@ -5909,7 +6987,7 @@ void _PyDictKeys_DecRef(PyDictKeysObject *keys) { PyInterpreterState *interp = _PyInterpreterState_GET(); - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); } uint32_t _PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp, @@ -5975,7 +7053,8 @@ PyDict_AddWatcher(PyDict_WatchCallback callback) { PyInterpreterState *interp = _PyInterpreterState_GET(); - for (int i = 0; i < DICT_MAX_WATCHERS; i++) { + /* Start at 2, as 0 and 1 are reserved for CPython */ + for (int i = 2; i < DICT_MAX_WATCHERS; i++) { if (!interp->dict_state.watchers[i]) { interp->dict_state.watchers[i] = callback; return i; diff --git a/Objects/exceptions.c b/Objects/exceptions.c index a685ed803cd02db..63c461d34fb4ff2 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -2177,6 +2177,10 @@ SimpleExtendsException(PyExc_Exception, RuntimeError, SimpleExtendsException(PyExc_RuntimeError, RecursionError, "Recursion limit exceeded."); +// PythonFinalizationError extends RuntimeError +SimpleExtendsException(PyExc_RuntimeError, PythonFinalizationError, + "Operation blocked during Python finalization."); + /* * NotImplementedError extends RuntimeError */ @@ -2566,6 +2570,11 @@ MiddlingExtendsException(PyExc_SyntaxError, IndentationError, SyntaxError, MiddlingExtendsException(PyExc_IndentationError, TabError, SyntaxError, "Improper mixture of spaces and tabs."); +/* + * IncompleteInputError extends SyntaxError + */ +MiddlingExtendsException(PyExc_SyntaxError, IncompleteInputError, SyntaxError, + "incomplete input."); /* * LookupError extends Exception @@ -3534,7 +3543,6 @@ SimpleExtendsException(PyExc_Warning, ResourceWarning, #undef EOPNOTSUPP #undef EPROTONOSUPPORT #undef EPROTOTYPE -#undef ETIMEDOUT #undef EWOULDBLOCK #if defined(WSAEALREADY) && !defined(EALREADY) @@ -3555,9 +3563,6 @@ SimpleExtendsException(PyExc_Warning, ResourceWarning, #if defined(WSAESHUTDOWN) && !defined(ESHUTDOWN) #define ESHUTDOWN WSAESHUTDOWN #endif -#if defined(WSAETIMEDOUT) && !defined(ETIMEDOUT) -#define ETIMEDOUT WSAETIMEDOUT -#endif #if defined(WSAEWOULDBLOCK) && !defined(EWOULDBLOCK) #define EWOULDBLOCK WSAEWOULDBLOCK #endif @@ -3635,10 +3640,12 @@ static struct static_exception static_exceptions[] = { // Level 4: Other subclasses ITEM(IndentationError), // base: SyntaxError(Exception) + ITEM(IncompleteInputError), // base: SyntaxError(Exception) ITEM(IndexError), // base: LookupError(Exception) ITEM(KeyError), // base: LookupError(Exception) ITEM(ModuleNotFoundError), // base: ImportError(Exception) ITEM(NotImplementedError), // base: RuntimeError(Exception) + ITEM(PythonFinalizationError), // base: RuntimeError(Exception) ITEM(RecursionError), // base: RuntimeError(Exception) ITEM(UnboundLocalError), // base: NameError(Exception) ITEM(UnicodeError), // base: ValueError(Exception) @@ -3741,6 +3748,9 @@ _PyExc_InitState(PyInterpreterState *interp) #endif ADD_ERRNO(ProcessLookupError, ESRCH); ADD_ERRNO(TimeoutError, ETIMEDOUT); +#ifdef WSAETIMEDOUT + ADD_ERRNO(TimeoutError, WSAETIMEDOUT); +#endif return _PyStatus_OK(); diff --git a/Objects/fileobject.c b/Objects/fileobject.c index 5522eba34eace9b..e30ab952dff571b 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -174,6 +174,13 @@ PyObject_AsFileDescriptor(PyObject *o) PyObject *meth; if (PyLong_Check(o)) { + if (PyBool_Check(o)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return -1; + } + } fd = PyLong_AsInt(o); } else if (PyObject_GetOptionalAttr(o, &_Py_ID(fileno), &meth) < 0) { diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 3512a9b9bed6db1..57180c2686210c4 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -8,7 +8,7 @@ #include "pycore_dtoa.h" // _Py_dg_dtoa() #include "pycore_floatobject.h" // _PyFloat_FormatAdvancedWriter() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_interp.h" // _PyInterpreterState.float_state +#include "pycore_interp.h" // _Py_float_freelist #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_Init(), _PyDebugAllocatorStats() @@ -27,12 +27,12 @@ class float "PyObject *" "&PyFloat_Type" #include "clinic/floatobject.c.h" #ifdef WITH_FREELISTS -static struct _Py_float_state * -get_float_state(void) +static struct _Py_float_freelist * +get_float_freelist(void) { - _PyFreeListState *state = _PyFreeListState_GET(); - assert(state != NULL); - return &state->floats; + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + assert(freelists != NULL); + return &freelists->floats; } #endif @@ -129,11 +129,11 @@ PyFloat_FromDouble(double fval) { PyFloatObject *op; #ifdef WITH_FREELISTS - struct _Py_float_state *state = get_float_state(); - op = state->free_list; + struct _Py_float_freelist *float_freelist = get_float_freelist(); + op = float_freelist->items; if (op != NULL) { - state->free_list = (PyFloatObject *) Py_TYPE(op); - state->numfree--; + float_freelist->items = (PyFloatObject *) Py_TYPE(op); + float_freelist->numfree--; OBJECT_STAT_INC(from_freelist); } else @@ -245,14 +245,14 @@ _PyFloat_ExactDealloc(PyObject *obj) assert(PyFloat_CheckExact(obj)); PyFloatObject *op = (PyFloatObject *)obj; #ifdef WITH_FREELISTS - struct _Py_float_state *state = get_float_state(); - if (state->numfree >= PyFloat_MAXFREELIST || state->numfree < 0) { + struct _Py_float_freelist *float_freelist = get_float_freelist(); + if (float_freelist->numfree >= PyFloat_MAXFREELIST || float_freelist->numfree < 0) { PyObject_Free(op); return; } - state->numfree++; - Py_SET_TYPE(op, (PyTypeObject *)state->free_list); - state->free_list = op; + float_freelist->numfree++; + Py_SET_TYPE(op, (PyTypeObject *)float_freelist->items); + float_freelist->items = op; OBJECT_STAT_INC(to_freelist); #else PyObject_Free(op); @@ -2022,17 +2022,17 @@ _PyFloat_InitTypes(PyInterpreterState *interp) } void -_PyFloat_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) +_PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) { #ifdef WITH_FREELISTS - struct _Py_float_state *state = &freelist_state->floats; - PyFloatObject *f = state->free_list; + struct _Py_float_freelist *state = &freelists->floats; + PyFloatObject *f = state->items; while (f != NULL) { PyFloatObject *next = (PyFloatObject*) Py_TYPE(f); PyObject_Free(f); f = next; } - state->free_list = NULL; + state->items = NULL; if (is_finalization) { state->numfree = -1; } @@ -2042,12 +2042,6 @@ _PyFloat_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) #endif } -void -_PyFloat_Fini(_PyFreeListState *state) -{ - _PyFloat_ClearFreeList(state, 1); -} - void _PyFloat_FiniType(PyInterpreterState *interp) { @@ -2059,10 +2053,10 @@ void _PyFloat_DebugMallocStats(FILE *out) { #ifdef WITH_FREELISTS - struct _Py_float_state *state = get_float_state(); + struct _Py_float_freelist *float_freelist = get_float_freelist(); _PyDebugAllocatorStats(out, "free PyFloatObject", - state->numfree, sizeof(PyFloatObject)); + float_freelist->numfree, sizeof(PyFloatObject)); #endif } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index cafe4ef6141d9af..a914c61aac2fd51 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -926,6 +926,7 @@ frame_tp_clear(PyFrameObject *f) Py_CLEAR(locals[i]); } f->f_frame->stacktop = 0; + Py_CLEAR(f->f_frame->f_locals); return 0; } diff --git a/Objects/genobject.c b/Objects/genobject.c index f47197330fdd801..8d1dbb72ba9ec21 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -7,7 +7,7 @@ #include "pycore_ceval.h" // _PyEval_EvalFrame() #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_gc.h" // _PyGC_CLEAR_FINALIZED() -#include "pycore_genobject.h" // struct _Py_async_gen_state +#include "pycore_genobject.h" // struct _Py_async_gen_freelist #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_opcode_utils.h" // RESUME_AFTER_YIELD_FROM @@ -1629,11 +1629,18 @@ PyTypeObject PyAsyncGen_Type = { #ifdef WITH_FREELISTS -static struct _Py_async_gen_state * -get_async_gen_state(void) +static struct _Py_async_gen_freelist * +get_async_gen_freelist(void) { - _PyFreeListState *state = _PyFreeListState_GET(); - return &state->async_gens; + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + return &freelists->async_gens; +} + +static struct _Py_async_gen_asend_freelist * +get_async_gen_asend_freelist(void) +{ + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + return &freelists->async_gen_asends; } #endif @@ -1656,39 +1663,34 @@ PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname) void -_PyAsyncGen_ClearFreeLists(_PyFreeListState *freelist_state, int is_finalization) +_PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelist_state, int is_finalization) { #ifdef WITH_FREELISTS - struct _Py_async_gen_state *state = &freelist_state->async_gens; + struct _Py_async_gen_freelist *freelist = &freelist_state->async_gens; - while (state->value_numfree > 0) { + while (freelist->numfree > 0) { _PyAsyncGenWrappedValue *o; - o = state->value_freelist[--state->value_numfree]; + o = freelist->items[--freelist->numfree]; assert(_PyAsyncGenWrappedValue_CheckExact(o)); PyObject_GC_Del(o); } - while (state->asend_numfree > 0) { + struct _Py_async_gen_asend_freelist *asend_freelist = &freelist_state->async_gen_asends; + + while (asend_freelist->numfree > 0) { PyAsyncGenASend *o; - o = state->asend_freelist[--state->asend_numfree]; + o = asend_freelist->items[--asend_freelist->numfree]; assert(Py_IS_TYPE(o, &_PyAsyncGenASend_Type)); PyObject_GC_Del(o); } if (is_finalization) { - state->value_numfree = -1; - state->asend_numfree = -1; + freelist->numfree = -1; + asend_freelist->numfree = -1; } #endif } -void -_PyAsyncGen_Fini(_PyFreeListState *state) -{ - _PyAsyncGen_ClearFreeLists(state, 1); -} - - static PyObject * async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result) { @@ -1733,11 +1735,11 @@ async_gen_asend_dealloc(PyAsyncGenASend *o) Py_CLEAR(o->ags_gen); Py_CLEAR(o->ags_sendval); #ifdef WITH_FREELISTS - struct _Py_async_gen_state *state = get_async_gen_state(); - if (state->asend_numfree >= 0 && state->asend_numfree < _PyAsyncGen_MAXFREELIST) { + struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist(); + if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) { assert(PyAsyncGenASend_CheckExact(o)); _PyGC_CLEAR_FINALIZED((PyObject *)o); - state->asend_freelist[state->asend_numfree++] = o; + freelist->items[freelist->numfree++] = o; } else #endif @@ -1903,10 +1905,10 @@ async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) { PyAsyncGenASend *o; #ifdef WITH_FREELISTS - struct _Py_async_gen_state *state = get_async_gen_state(); - if (state->asend_numfree > 0) { - state->asend_numfree--; - o = state->asend_freelist[state->asend_numfree]; + struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist(); + if (freelist->numfree > 0) { + freelist->numfree--; + o = freelist->items[freelist->numfree]; _Py_NewReference((PyObject *)o); } else @@ -1938,10 +1940,10 @@ async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o) _PyObject_GC_UNTRACK((PyObject *)o); Py_CLEAR(o->agw_val); #ifdef WITH_FREELISTS - struct _Py_async_gen_state *state = get_async_gen_state(); - if (state->value_numfree >= 0 && state->value_numfree < _PyAsyncGen_MAXFREELIST) { + struct _Py_async_gen_freelist *freelist = get_async_gen_freelist(); + if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) { assert(_PyAsyncGenWrappedValue_CheckExact(o)); - state->value_freelist[state->value_numfree++] = o; + freelist->items[freelist->numfree++] = o; OBJECT_STAT_INC(to_freelist); } else @@ -2011,10 +2013,10 @@ _PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val) assert(val); #ifdef WITH_FREELISTS - struct _Py_async_gen_state *state = get_async_gen_state(); - if (state->value_numfree > 0) { - state->value_numfree--; - o = state->value_freelist[state->value_numfree]; + struct _Py_async_gen_freelist *freelist = get_async_gen_freelist(); + if (freelist->numfree > 0) { + freelist->numfree--; + o = freelist->items[freelist->numfree]; OBJECT_STAT_INC(from_freelist); assert(_PyAsyncGenWrappedValue_CheckExact(o)); _Py_NewReference((PyObject*)o); diff --git a/Objects/listobject.c b/Objects/listobject.c index 56785e5f37a450f..2bb7d4ec3424510 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3,8 +3,9 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_interp.h" // PyInterpreterState.list -#include "pycore_list.h" // struct _Py_list_state, _PyListIterObject +#include "pycore_list.h" // struct _Py_list_freelist, _PyListIterObject #include "pycore_long.h" // _PyLong_DigitCount #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() @@ -21,16 +22,15 @@ class list "PyListObject *" "&PyList_Type" _Py_DECLARE_STR(list_err, "list index out of range"); #ifdef WITH_FREELISTS -static struct _Py_list_state * -get_list_state(void) +static struct _Py_list_freelist * +get_list_freelist(void) { - _PyFreeListState *state = _PyFreeListState_GET(); - assert(state != NULL); - return &state->lists; + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + assert(freelists != NULL); + return &freelists->lists; } #endif - /* Ensure ob_item has room for at least newsize elements, and set * ob_size to newsize. If newsize > ob_size on entry, the content * of the new slots at exit is undefined heap trash; it's the caller's @@ -121,12 +121,12 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size) } void -_PyList_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) +_PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) { #ifdef WITH_FREELISTS - struct _Py_list_state *state = &freelist_state->lists; + struct _Py_list_freelist *state = &freelists->lists; while (state->numfree > 0) { - PyListObject *op = state->free_list[--state->numfree]; + PyListObject *op = state->items[--state->numfree]; assert(PyList_CheckExact(op)); PyObject_GC_Del(op); } @@ -136,21 +136,15 @@ _PyList_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) #endif } -void -_PyList_Fini(_PyFreeListState *state) -{ - _PyList_ClearFreeList(state, 1); -} - /* Print summary info about the state of the optimized allocator */ void _PyList_DebugMallocStats(FILE *out) { #ifdef WITH_FREELISTS - struct _Py_list_state *state = get_list_state(); + struct _Py_list_freelist *list_freelist = get_list_freelist(); _PyDebugAllocatorStats(out, "free PyListObject", - state->numfree, sizeof(PyListObject)); + list_freelist->numfree, sizeof(PyListObject)); #endif } @@ -165,10 +159,10 @@ PyList_New(Py_ssize_t size) } #ifdef WITH_FREELISTS - struct _Py_list_state *state = get_list_state(); - if (PyList_MAXFREELIST && state->numfree > 0) { - state->numfree--; - op = state->free_list[state->numfree]; + struct _Py_list_freelist *list_freelist = get_list_freelist(); + if (PyList_MAXFREELIST && list_freelist->numfree > 0) { + list_freelist->numfree--; + op = list_freelist->items[list_freelist->numfree]; OBJECT_STAT_INC(from_freelist); _Py_NewReference((PyObject *)op); } @@ -221,8 +215,9 @@ PyList_Size(PyObject *op) PyErr_BadInternalCall(); return -1; } - else - return Py_SIZE(op); + else { + return PyList_GET_SIZE(op); + } } static inline int @@ -253,6 +248,30 @@ PyList_GetItem(PyObject *op, Py_ssize_t i) return ((PyListObject *)op) -> ob_item[i]; } +PyObject * +PyList_GetItemRef(PyObject *op, Py_ssize_t i) +{ + if (!PyList_Check(op)) { + PyErr_SetString(PyExc_TypeError, "expected a list"); + return NULL; + } + if (!valid_index(i, Py_SIZE(op))) { + _Py_DECLARE_STR(list_err, "list index out of range"); + PyErr_SetObject(PyExc_IndexError, &_Py_STR(list_err)); + return NULL; + } + return Py_NewRef(PyList_GET_ITEM(op, i)); +} + +static inline PyObject* +list_get_item_ref(PyListObject *op, Py_ssize_t i) +{ + if (!valid_index(i, Py_SIZE(op))) { + return NULL; + } + return Py_NewRef(PyList_GET_ITEM(op, i)); +} + int PyList_SetItem(PyObject *op, Py_ssize_t i, PyObject *newitem) @@ -328,7 +347,7 @@ PyList_Insert(PyObject *op, Py_ssize_t where, PyObject *newitem) int _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem) { - Py_ssize_t len = PyList_GET_SIZE(self); + Py_ssize_t len = Py_SIZE(self); assert(self->allocated == -1 || self->allocated == len); if (list_resize(self, len + 1) < 0) { Py_DECREF(newitem); @@ -342,7 +361,11 @@ int PyList_Append(PyObject *op, PyObject *newitem) { if (PyList_Check(op) && (newitem != NULL)) { - return _PyList_AppendTakeRef((PyListObject *)op, Py_NewRef(newitem)); + int ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = _PyList_AppendTakeRef((PyListObject *)op, Py_NewRef(newitem)); + Py_END_CRITICAL_SECTION(); + return ret; } PyErr_BadInternalCall(); return -1; @@ -369,9 +392,9 @@ list_dealloc(PyObject *self) PyMem_Free(op->ob_item); } #ifdef WITH_FREELISTS - struct _Py_list_state *state = get_list_state(); - if (state->numfree < PyList_MAXFREELIST && state->numfree >= 0 && PyList_CheckExact(op)) { - state->free_list[state->numfree++] = op; + struct _Py_list_freelist *list_freelist = get_list_freelist(); + if (list_freelist->numfree < PyList_MAXFREELIST && list_freelist->numfree >= 0 && PyList_CheckExact(op)) { + list_freelist->items[list_freelist->numfree++] = op; OBJECT_STAT_INC(to_freelist); } else @@ -455,25 +478,27 @@ list_length(PyObject *a) static int list_contains(PyObject *aa, PyObject *el) { - PyListObject *a = (PyListObject *)aa; - PyObject *item; - Py_ssize_t i; - int cmp; - for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(a); ++i) { - item = PyList_GET_ITEM(a, i); - Py_INCREF(item); - cmp = PyObject_RichCompareBool(item, el, Py_EQ); + for (Py_ssize_t i = 0; ; i++) { + PyObject *item = list_get_item_ref((PyListObject *)aa, i); + if (item == NULL) { + // out-of-bounds + return 0; + } + int cmp = PyObject_RichCompareBool(item, el, Py_EQ); Py_DECREF(item); + if (cmp != 0) { + return cmp; + } } - return cmp; + return 0; } static PyObject * list_item(PyObject *aa, Py_ssize_t i) { PyListObject *a = (PyListObject *)aa; - if (!valid_index(i, Py_SIZE(a))) { + if (!valid_index(i, PyList_GET_SIZE(a))) { PyErr_SetObject(PyExc_IndexError, &_Py_STR(list_err)); return NULL; } @@ -511,6 +536,8 @@ PyList_GetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) PyErr_BadInternalCall(); return NULL; } + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(a); if (ilow < 0) { ilow = 0; } @@ -523,24 +550,18 @@ PyList_GetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) else if (ihigh > Py_SIZE(a)) { ihigh = Py_SIZE(a); } - return list_slice((PyListObject *)a, ilow, ihigh); + ret = list_slice((PyListObject *)a, ilow, ihigh); + Py_END_CRITICAL_SECTION(); + return ret; } static PyObject * -list_concat(PyObject *aa, PyObject *bb) +list_concat_lock_held(PyListObject *a, PyListObject *b) { - PyListObject *a = (PyListObject *)aa; Py_ssize_t size; Py_ssize_t i; PyObject **src, **dest; PyListObject *np; - if (!PyList_Check(bb)) { - PyErr_Format(PyExc_TypeError, - "can only concatenate list (not \"%.200s\") to list", - Py_TYPE(bb)->tp_name); - return NULL; - } -#define b ((PyListObject *)bb) assert((size_t)Py_SIZE(a) + (size_t)Py_SIZE(b) < PY_SSIZE_T_MAX); size = Py_SIZE(a) + Py_SIZE(b); if (size == 0) { @@ -554,23 +575,39 @@ list_concat(PyObject *aa, PyObject *bb) dest = np->ob_item; for (i = 0; i < Py_SIZE(a); i++) { PyObject *v = src[i]; - dest[i] = Py_NewRef(v); + FT_ATOMIC_STORE_PTR_RELAXED(dest[i], Py_NewRef(v)); } src = b->ob_item; dest = np->ob_item + Py_SIZE(a); for (i = 0; i < Py_SIZE(b); i++) { PyObject *v = src[i]; - dest[i] = Py_NewRef(v); + FT_ATOMIC_STORE_PTR_RELAXED(dest[i], Py_NewRef(v)); } Py_SET_SIZE(np, size); return (PyObject *)np; -#undef b } static PyObject * -list_repeat(PyObject *aa, Py_ssize_t n) +list_concat(PyObject *aa, PyObject *bb) { + if (!PyList_Check(bb)) { + PyErr_Format(PyExc_TypeError, + "can only concatenate list (not \"%.200s\") to list", + Py_TYPE(bb)->tp_name); + return NULL; + } PyListObject *a = (PyListObject *)aa; + PyListObject *b = (PyListObject *)bb; + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION2(a, b); + ret = list_concat_lock_held(a, b); + Py_END_CRITICAL_SECTION2(); + return ret; +} + +static PyObject * +list_repeat_lock_held(PyListObject *a, Py_ssize_t n) +{ const Py_ssize_t input_size = Py_SIZE(a); if (input_size == 0 || n <= 0) return PyList_New(0); @@ -590,7 +627,7 @@ list_repeat(PyObject *aa, Py_ssize_t n) _Py_RefcntAdd(elem, n); PyObject **dest_end = dest + output_size; while (dest < dest_end) { - *dest++ = elem; + FT_ATOMIC_STORE_PTR_RELAXED(*dest++, elem); } } else { @@ -598,7 +635,7 @@ list_repeat(PyObject *aa, Py_ssize_t n) PyObject **src_end = src + input_size; while (src < src_end) { _Py_RefcntAdd(*src, n); - *dest++ = *src++; + FT_ATOMIC_STORE_PTR_RELAXED(*dest++, *src++); } _Py_memory_repeat((char *)np->ob_item, sizeof(PyObject *)*output_size, @@ -609,6 +646,17 @@ list_repeat(PyObject *aa, Py_ssize_t n) return (PyObject *) np; } +static PyObject * +list_repeat(PyObject *aa, Py_ssize_t n) +{ + PyObject *ret; + PyListObject *a = (PyListObject *)aa; + Py_BEGIN_CRITICAL_SECTION(a); + ret = list_repeat_lock_held(a, n); + Py_END_CRITICAL_SECTION(); + return ret; +} + static void list_clear(PyListObject *a) { @@ -621,11 +669,12 @@ list_clear(PyListObject *a) this list, we make it empty first. */ Py_ssize_t i = Py_SIZE(a); Py_SET_SIZE(a, 0); - a->ob_item = NULL; + FT_ATOMIC_STORE_PTR_RELEASE(a->ob_item, NULL); a->allocated = 0; while (--i >= 0) { Py_XDECREF(items[i]); } + // TODO: Use QSBR technique, if the list is shared between threads, PyMem_Free(items); // Note that there is no guarantee that the list is actually empty @@ -762,9 +811,8 @@ PyList_SetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) } static PyObject * -list_inplace_repeat(PyObject *_self, Py_ssize_t n) +list_inplace_repeat_lock_held(PyListObject *self, Py_ssize_t n) { - PyListObject *self = (PyListObject *)_self; Py_ssize_t input_size = PyList_GET_SIZE(self); if (input_size == 0 || n == 1) { return Py_NewRef(self); @@ -793,21 +841,51 @@ list_inplace_repeat(PyObject *_self, Py_ssize_t n) return Py_NewRef(self); } +static PyObject * +list_inplace_repeat(PyObject *_self, Py_ssize_t n) +{ + PyObject *ret; + PyListObject *self = (PyListObject *) _self; + Py_BEGIN_CRITICAL_SECTION(self); + ret = list_inplace_repeat_lock_held(self, n); + Py_END_CRITICAL_SECTION(); + return ret; +} + static int -list_ass_item(PyObject *aa, Py_ssize_t i, PyObject *v) +list_ass_item_lock_held(PyListObject *a, Py_ssize_t i, PyObject *v) { - PyListObject *a = (PyListObject *)aa; if (!valid_index(i, Py_SIZE(a))) { PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); return -1; } - if (v == NULL) - return list_ass_slice(a, i, i+1, v); - Py_SETREF(a->ob_item[i], Py_NewRef(v)); + PyObject *tmp = a->ob_item[i]; + if (v == NULL) { + Py_ssize_t size = Py_SIZE(a); + for (Py_ssize_t idx = i; idx < size - 1; idx++) { + FT_ATOMIC_STORE_PTR_RELAXED(a->ob_item[idx], a->ob_item[idx + 1]); + } + Py_SET_SIZE(a, size - 1); + } + else { + FT_ATOMIC_STORE_PTR_RELEASE(a->ob_item[i], Py_NewRef(v)); + } + Py_DECREF(tmp); return 0; } +static int +list_ass_item(PyObject *aa, Py_ssize_t i, PyObject *v) +{ + int ret; + PyListObject *a = (PyListObject *)aa; + Py_BEGIN_CRITICAL_SECTION(a); + ret = list_ass_item_lock_held(a, i, v); + Py_END_CRITICAL_SECTION(); + return ret; +} + /*[clinic input] @critical_section list.insert @@ -2697,8 +2775,6 @@ list_index_impl(PyListObject *self, PyObject *value, Py_ssize_t start, Py_ssize_t stop) /*[clinic end generated code: output=ec51b88787e4e481 input=40ec5826303a0eb1]*/ { - Py_ssize_t i; - if (start < 0) { start += Py_SIZE(self); if (start < 0) @@ -2709,9 +2785,12 @@ list_index_impl(PyListObject *self, PyObject *value, Py_ssize_t start, if (stop < 0) stop = 0; } - for (i = start; i < stop && i < Py_SIZE(self); i++) { - PyObject *obj = self->ob_item[i]; - Py_INCREF(obj); + for (Py_ssize_t i = start; i < stop; i++) { + PyObject *obj = list_get_item_ref(self, i); + if (obj == NULL) { + // out-of-bounds + break; + } int cmp = PyObject_RichCompareBool(obj, value, Py_EQ); Py_DECREF(obj); if (cmp > 0) @@ -2737,15 +2816,17 @@ list_count(PyListObject *self, PyObject *value) /*[clinic end generated code: output=b1f5d284205ae714 input=3bdc3a5e6f749565]*/ { Py_ssize_t count = 0; - Py_ssize_t i; - - for (i = 0; i < Py_SIZE(self); i++) { - PyObject *obj = self->ob_item[i]; + for (Py_ssize_t i = 0; ; i++) { + PyObject *obj = list_get_item_ref(self, i); + if (obj == NULL) { + // out-of-bounds + break; + } if (obj == value) { count++; + Py_DECREF(obj); continue; } - Py_INCREF(obj); int cmp = PyObject_RichCompareBool(obj, value, Py_EQ); Py_DECREF(obj); if (cmp > 0) @@ -2940,7 +3021,8 @@ list___sizeof___impl(PyListObject *self) /*[clinic end generated code: output=3417541f95f9a53e input=b8030a5d5ce8a187]*/ { size_t res = _PyObject_SIZE(Py_TYPE(self)); - res += (size_t)self->allocated * sizeof(void*); + Py_ssize_t allocated = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->allocated); + res += (size_t)allocated * sizeof(void*); return PyLong_FromSize_t(res); } @@ -3342,33 +3424,34 @@ static PyObject * listiter_next(PyObject *self) { _PyListIterObject *it = (_PyListIterObject *)self; - PyListObject *seq; - PyObject *item; - - assert(it != NULL); - seq = it->it_seq; - if (seq == NULL) + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); + if (index < 0) { return NULL; - assert(PyList_Check(seq)); - - if (it->it_index < PyList_GET_SIZE(seq)) { - item = PyList_GET_ITEM(seq, it->it_index); - ++it->it_index; - return Py_NewRef(item); } - it->it_seq = NULL; - Py_DECREF(seq); - return NULL; + PyObject *item = list_get_item_ref(it->it_seq, index); + if (item == NULL) { + // out-of-bounds + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, -1); +#ifndef Py_GIL_DISABLED + PyListObject *seq = it->it_seq; + it->it_seq = NULL; + Py_DECREF(seq); +#endif + return NULL; + } + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, index + 1); + return item; } static PyObject * listiter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { + assert(self != NULL); _PyListIterObject *it = (_PyListIterObject *)self; - Py_ssize_t len; - if (it->it_seq) { - len = PyList_GET_SIZE(it->it_seq) - it->it_index; + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); + if (index >= 0) { + Py_ssize_t len = PyList_GET_SIZE(it->it_seq) - index; if (len >= 0) return PyLong_FromSsize_t(len); } @@ -3389,8 +3472,8 @@ listiter_setstate(PyObject *self, PyObject *state) if (index == -1 && PyErr_Occurred()) return NULL; if (it->it_seq != NULL) { - if (index < 0) - index = 0; + if (index < -1) + index = -1; else if (index > PyList_GET_SIZE(it->it_seq)) index = PyList_GET_SIZE(it->it_seq); /* iterator exhausted */ it->it_index = index; @@ -3495,26 +3578,24 @@ static PyObject * listreviter_next(PyObject *self) { listreviterobject *it = (listreviterobject *)self; - PyObject *item; - Py_ssize_t index; - PyListObject *seq; - assert(it != NULL); - seq = it->it_seq; - if (seq == NULL) { + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); + if (index < 0) { return NULL; } - assert(PyList_Check(seq)); - index = it->it_index; - if (index>=0 && index < PyList_GET_SIZE(seq)) { - item = PyList_GET_ITEM(seq, index); - it->it_index--; - return Py_NewRef(item); + PyListObject *seq = it->it_seq; + assert(PyList_Check(seq)); + PyObject *item = list_get_item_ref(seq, index); + if (item != NULL) { + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, index - 1); + return item; } - it->it_index = -1; + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, -1); +#ifndef Py_GIL_DISABLED it->it_seq = NULL; Py_DECREF(seq); +#endif return NULL; } @@ -3522,7 +3603,8 @@ static PyObject * listreviter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { listreviterobject *it = (listreviterobject *)self; - Py_ssize_t len = it->it_index + 1; + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); + Py_ssize_t len = index + 1; if (it->it_seq == NULL || PyList_GET_SIZE(it->it_seq) < len) len = 0; return PyLong_FromSsize_t(len); @@ -3557,6 +3639,7 @@ static PyObject * listiter_reduce_general(void *_it, int forward) { PyObject *list; + PyObject *iter; /* _PyEval_GetBuiltin can invoke arbitrary code, * call must be before access of iterator pointers. @@ -3564,29 +3647,21 @@ listiter_reduce_general(void *_it, int forward) /* the objects are not the same, index is of different types! */ if (forward) { - PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter)); - if (!iter) { - return NULL; - } + iter = _PyEval_GetBuiltin(&_Py_ID(iter)); _PyListIterObject *it = (_PyListIterObject *)_it; - if (it->it_seq) { + if (it->it_index >= 0) { return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); } - Py_DECREF(iter); } else { - PyObject *reversed = _PyEval_GetBuiltin(&_Py_ID(reversed)); - if (!reversed) { - return NULL; - } + iter = _PyEval_GetBuiltin(&_Py_ID(reversed)); listreviterobject *it = (listreviterobject *)_it; - if (it->it_seq) { - return Py_BuildValue("N(O)n", reversed, it->it_seq, it->it_index); + if (it->it_index >= 0) { + return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); } - Py_DECREF(reversed); } /* empty iterator, create an empty list */ list = PyList_New(0); if (list == NULL) return NULL; - return Py_BuildValue("N(N)", _PyEval_GetBuiltin(&_Py_ID(iter)), list); + return Py_BuildValue("N(N)", iter, list); } diff --git a/Objects/longobject.c b/Objects/longobject.c index e655ba19e8f1c12..2d1c6ad788e281e 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -928,7 +928,8 @@ _PyLong_FromByteArray(const unsigned char* bytes, size_t n, int _PyLong_AsByteArray(PyLongObject* v, unsigned char* bytes, size_t n, - int little_endian, int is_signed) + int little_endian, int is_signed, + int with_exceptions) { Py_ssize_t i; /* index into v->long_value.ob_digit */ Py_ssize_t ndigits; /* number of digits */ @@ -945,8 +946,10 @@ _PyLong_AsByteArray(PyLongObject* v, ndigits = _PyLong_DigitCount(v); if (_PyLong_IsNegative(v)) { if (!is_signed) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative int to unsigned"); + if (with_exceptions) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative int to unsigned"); + } return -1; } do_twos_comp = 1; @@ -967,7 +970,12 @@ _PyLong_AsByteArray(PyLongObject* v, /* Copy over all the Python digits. It's crucial that every Python digit except for the MSD contribute exactly PyLong_SHIFT bits to the total, so first assert that the int is - normalized. */ + normalized. + NOTE: PyLong_AsNativeBytes() assumes that this function will fill in 'n' + bytes even if it eventually fails to convert the whole number. Make sure + you account for that if you are changing this algorithm to return without + doing that. + */ assert(ndigits == 0 || v->long_value.ob_digit[ndigits - 1] != 0); j = 0; accum = 0; @@ -1052,11 +1060,203 @@ _PyLong_AsByteArray(PyLongObject* v, return 0; Overflow: - PyErr_SetString(PyExc_OverflowError, "int too big to convert"); + if (with_exceptions) { + PyErr_SetString(PyExc_OverflowError, "int too big to convert"); + } return -1; } +// Refactored out for readability, not reuse +static inline int +_fits_in_n_bits(Py_ssize_t v, Py_ssize_t n) +{ + if (n >= (Py_ssize_t)sizeof(Py_ssize_t) * 8) { + return 1; + } + // If all bits above n are the same, we fit. + // (Use n-1 if we require the sign bit to be consistent.) + Py_ssize_t v_extended = v >> ((int)n - 1); + return v_extended == 0 || v_extended == -1; +} + +static inline int +_resolve_endianness(int *endianness) +{ + if (*endianness < 0) { + *endianness = PY_LITTLE_ENDIAN; + } + if (*endianness != 0 && *endianness != 1) { + PyErr_SetString(PyExc_SystemError, "invalid 'endianness' value"); + return -1; + } + return 0; +} + +Py_ssize_t +PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int endianness) +{ + PyLongObject *v; + union { + Py_ssize_t v; + unsigned char b[sizeof(Py_ssize_t)]; + } cv; + int do_decref = 0; + Py_ssize_t res = 0; + + if (vv == NULL || n < 0) { + PyErr_BadInternalCall(); + return -1; + } + + int little_endian = endianness; + if (_resolve_endianness(&little_endian) < 0) { + return -1; + } + + if (PyLong_Check(vv)) { + v = (PyLongObject *)vv; + } + else { + v = (PyLongObject *)_PyNumber_Index(vv); + if (v == NULL) { + return -1; + } + do_decref = 1; + } + + if (_PyLong_IsCompact(v)) { + res = 0; + cv.v = _PyLong_CompactValue(v); + /* Most paths result in res = sizeof(compact value). Only the case + * where 0 < n < sizeof(compact value) do we need to check and adjust + * our return value. */ + res = sizeof(cv.b); + if (n <= 0) { + // nothing to do! + } + else if (n <= (Py_ssize_t)sizeof(cv.b)) { +#if PY_LITTLE_ENDIAN + if (little_endian) { + memcpy(buffer, cv.b, n); + } + else { + for (Py_ssize_t i = 0; i < n; ++i) { + ((unsigned char*)buffer)[n - i - 1] = cv.b[i]; + } + } +#else + if (little_endian) { + for (Py_ssize_t i = 0; i < n; ++i) { + ((unsigned char*)buffer)[i] = cv.b[sizeof(cv.b) - i - 1]; + } + } + else { + memcpy(buffer, &cv.b[sizeof(cv.b) - n], n); + } +#endif + + /* If we fit, return the requested number of bytes */ + if (_fits_in_n_bits(cv.v, n * 8)) { + res = n; + } + } + else { + unsigned char fill = cv.v < 0 ? 0xFF : 0x00; +#if PY_LITTLE_ENDIAN + if (little_endian) { + memcpy(buffer, cv.b, sizeof(cv.b)); + memset((char *)buffer + sizeof(cv.b), fill, n - sizeof(cv.b)); + } + else { + unsigned char *b = (unsigned char *)buffer; + for (Py_ssize_t i = 0; i < n - (int)sizeof(cv.b); ++i) { + *b++ = fill; + } + for (Py_ssize_t i = sizeof(cv.b); i > 0; --i) { + *b++ = cv.b[i - 1]; + } + } +#else + if (little_endian) { + unsigned char *b = (unsigned char *)buffer; + for (Py_ssize_t i = sizeof(cv.b); i > 0; --i) { + *b++ = cv.b[i - 1]; + } + for (Py_ssize_t i = 0; i < n - (int)sizeof(cv.b); ++i) { + *b++ = fill; + } + } + else { + memset(buffer, fill, n - sizeof(cv.b)); + memcpy((char *)buffer + n - sizeof(cv.b), cv.b, sizeof(cv.b)); + } +#endif + } + } + else { + if (n > 0) { + _PyLong_AsByteArray(v, buffer, (size_t)n, little_endian, 1, 0); + } + + // More efficient calculation for number of bytes required? + size_t nb = _PyLong_NumBits((PyObject *)v); + /* Normally this would be((nb - 1) / 8) + 1 to avoid rounding up + * multiples of 8 to the next byte, but we add an implied bit for + * the sign and it cancels out. */ + size_t n_needed = (nb / 8) + 1; + res = (Py_ssize_t)n_needed; + if ((size_t)res != n_needed) { + PyErr_SetString(PyExc_OverflowError, + "value too large to convert"); + res = -1; + } + } + + if (do_decref) { + Py_DECREF(v); + } + + return res; +} + + +PyObject * +PyLong_FromNativeBytes(const void* buffer, size_t n, int endianness) +{ + if (!buffer) { + PyErr_BadInternalCall(); + return NULL; + } + + int little_endian = endianness; + if (_resolve_endianness(&little_endian) < 0) { + return NULL; + } + + return _PyLong_FromByteArray((const unsigned char *)buffer, n, + little_endian, 1); +} + + +PyObject * +PyLong_FromUnsignedNativeBytes(const void* buffer, size_t n, int endianness) +{ + if (!buffer) { + PyErr_BadInternalCall(); + return NULL; + } + + int little_endian = endianness; + if (_resolve_endianness(&little_endian) < 0) { + return NULL; + } + + return _PyLong_FromByteArray((const unsigned char *)buffer, n, + little_endian, 0); +} + + /* Create a new int object from a C pointer */ PyObject * @@ -1231,7 +1431,7 @@ PyLong_AsLongLong(PyObject *vv) } else { res = _PyLong_AsByteArray((PyLongObject *)v, (unsigned char *)&bytes, - SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 1); + SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 1, 1); } if (do_decref) { Py_DECREF(v); @@ -1270,7 +1470,7 @@ PyLong_AsUnsignedLongLong(PyObject *vv) } else { res = _PyLong_AsByteArray((PyLongObject *)vv, (unsigned char *)&bytes, - SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 0); + SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 0, 1); } /* Plan 9 can't handle long long in ? : expressions */ @@ -6068,7 +6268,7 @@ int_to_bytes_impl(PyObject *self, Py_ssize_t length, PyObject *byteorder, if (_PyLong_AsByteArray((PyLongObject *)self, (unsigned char *)PyBytes_AS_STRING(bytes), - length, little_endian, is_signed) < 0) { + length, little_endian, is_signed, 1) < 0) { Py_DECREF(bytes); return NULL; } diff --git a/Objects/methodobject.c b/Objects/methodobject.c index b40b2821c3880d2..599fb05cb5874f2 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -417,7 +417,7 @@ cfunction_vectorcall_FASTCALL( return NULL; } Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - _PyCFunctionFast meth = (_PyCFunctionFast) + PyCFunctionFast meth = (PyCFunctionFast) cfunction_enter_call(tstate, func); if (meth == NULL) { return NULL; @@ -433,7 +433,7 @@ cfunction_vectorcall_FASTCALL_KEYWORDS( { PyThreadState *tstate = _PyThreadState_GET(); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - _PyCFunctionFastWithKeywords meth = (_PyCFunctionFastWithKeywords) + PyCFunctionFastWithKeywords meth = (PyCFunctionFastWithKeywords) cfunction_enter_call(tstate, func); if (meth == NULL) { return NULL; @@ -552,4 +552,3 @@ cfunction_call(PyObject *func, PyObject *args, PyObject *kwargs) } return _Py_CheckFunctionResult(tstate, func, result, NULL); } - diff --git a/Objects/mimalloc/alloc.c b/Objects/mimalloc/alloc.c index f96c6f0b37f873b..b369a5ebcb23a76 100644 --- a/Objects/mimalloc/alloc.c +++ b/Objects/mimalloc/alloc.c @@ -26,6 +26,15 @@ terms of the MIT license. A copy of the license can be found in the file // Allocation // ------------------------------------------------------ +#if (MI_DEBUG>0) +static void mi_debug_fill(mi_page_t* page, mi_block_t* block, int c, size_t size) { + size_t offset = (size_t)page->debug_offset; + if (offset < size) { + memset((char*)block + offset, c, size - offset); + } +} +#endif + // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept { @@ -65,7 +74,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN if (!zero && !mi_page_is_huge(page)) { - memset(block, MI_DEBUG_UNINIT, mi_page_usable_block_size(page)); + mi_debug_fill(page, block, MI_DEBUG_UNINIT, mi_page_usable_block_size(page)); } #elif (MI_SECURE!=0) if (!zero) { block->next = 0; } // don't leak internal data @@ -426,7 +435,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading if (segment->kind != MI_SEGMENT_HUGE) { // not for huge segments as we just reset the content - memset(block, MI_DEBUG_FREED, mi_usable_size(block)); + mi_debug_fill(page, block, MI_DEBUG_FREED, mi_usable_size(block)); } #endif @@ -480,7 +489,7 @@ static inline void _mi_free_block(mi_page_t* page, bool local, mi_block_t* block mi_check_padding(page, block); #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN if (!mi_page_is_huge(page)) { // huge page content may be already decommitted - memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); + mi_debug_fill(page, block, MI_DEBUG_FREED, mi_page_block_size(page)); } #endif mi_block_set_next(page, block, page->local_free); @@ -575,7 +584,7 @@ void mi_free(void* p) mi_attr_noexcept mi_check_padding(page, block); mi_stat_free(page, block); #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN - memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); + mi_debug_fill(page, block, MI_DEBUG_FREED, mi_page_block_size(page)); #endif mi_track_free_size(p, mi_page_usable_size_of(page,block)); // faster then mi_usable_size as we already know the page and that p is unaligned mi_block_set_next(page, block, page->local_free); diff --git a/Objects/mimalloc/heap.c b/Objects/mimalloc/heap.c index 164b28f0fab2402..154dad0b1284805 100644 --- a/Objects/mimalloc/heap.c +++ b/Objects/mimalloc/heap.c @@ -538,7 +538,6 @@ bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t *page, mi_ mi_assert(page != NULL); if (page == NULL) return true; - _mi_page_free_collect(page,true); mi_assert_internal(page->local_free == NULL); if (page->used == 0) return true; @@ -635,6 +634,7 @@ bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t *page, mi_ typedef bool (mi_heap_area_visit_fun)(const mi_heap_t* heap, const mi_heap_area_ex_t* area, void* arg); void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) { + _mi_page_free_collect(page,true); const size_t bsize = mi_page_block_size(page); const size_t ubsize = mi_page_usable_block_size(page); area->reserved = page->reserved * bsize; diff --git a/Objects/mimalloc/init.c b/Objects/mimalloc/init.c index 5897f0512f8ef9d..cb0ef6642803ccc 100644 --- a/Objects/mimalloc/init.c +++ b/Objects/mimalloc/init.c @@ -13,27 +13,7 @@ terms of the MIT license. A copy of the license can be found in the file // Empty page used to initialize the small free pages array -const mi_page_t _mi_page_empty = { - 0, false, false, false, 0, - 0, // capacity - 0, // reserved capacity - { 0 }, // flags - false, // is_zero - 0, // retire_expire - NULL, // free - 0, // used - 0, // xblock_size - NULL, // local_free - #if (MI_PADDING || MI_ENCODE_FREELIST) - { 0, 0 }, - #endif - MI_ATOMIC_VAR_INIT(0), // xthread_free - MI_ATOMIC_VAR_INIT(0), // xheap - NULL, NULL - #if MI_INTPTR_SIZE==8 - , { 0 } // padding - #endif -}; +const mi_page_t _mi_page_empty; #define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty) @@ -122,6 +102,7 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = { MI_BIN_FULL, 0, // page retired min/max NULL, // next false, + 0, 0 }; diff --git a/Objects/mimalloc/page.c b/Objects/mimalloc/page.c index 8f0ce920156e04f..63db893e49405c7 100644 --- a/Objects/mimalloc/page.c +++ b/Objects/mimalloc/page.c @@ -661,6 +661,7 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi // set fields mi_page_set_heap(page, heap); page->tag = heap->tag; + page->debug_offset = heap->debug_offset; page->xblock_size = (block_size < MI_HUGE_BLOCK_SIZE ? (uint32_t)block_size : MI_HUGE_BLOCK_SIZE); // initialize before _mi_segment_page_start size_t page_size; const void* page_start = _mi_segment_page_start(segment, page, &page_size); diff --git a/Objects/object.c b/Objects/object.c index 587c5528c01345b..df14fe0c6fbfec1 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2,6 +2,7 @@ /* Generic object operations; and implementation of None */ #include "Python.h" +#include "pycore_brc.h" // _Py_brc_queue_object() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate() #include "pycore_context.h" // _PyContextTokenMissing_Type @@ -298,13 +299,13 @@ _Py_DecRef(PyObject *o) #ifdef Py_GIL_DISABLED # ifdef Py_REF_DEBUG -static inline int -is_shared_refcnt_dead(Py_ssize_t shared) +static int +is_dead(PyObject *o) { # if SIZEOF_SIZE_T == 8 - return shared == (Py_ssize_t)0xDDDDDDDDDDDDDDDD; + return (uintptr_t)o->ob_type == 0xDDDDDDDDDDDDDDDD; # else - return shared == (Py_ssize_t)0xDDDDDDDD; + return (uintptr_t)o->ob_type == 0xDDDDDDDD; # endif } # endif @@ -334,8 +335,8 @@ _Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) } #ifdef Py_REF_DEBUG - if ((_Py_REF_IS_MERGED(new_shared) && new_shared < 0) || - is_shared_refcnt_dead(shared)) + if ((new_shared < 0 && _Py_REF_IS_MERGED(new_shared)) || + (should_queue && is_dead(o))) { _Py_NegativeRefcount(filename, lineno, o); } @@ -344,12 +345,10 @@ _Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) &shared, new_shared)); if (should_queue) { - // TODO: the inter-thread queue is not yet implemented. For now, - // we just merge the refcount here. - Py_ssize_t refcount = _Py_ExplicitMergeRefcount(o, -1); - if (refcount == 0) { - _Py_Dealloc(o); - } +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyInterpreterState_GET()); +#endif + _Py_brc_queue_object(o); } else if (new_shared == _Py_REF_MERGED) { // refcount is zero AND merged @@ -399,10 +398,6 @@ _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra) Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); do { refcnt = Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); - if (_Py_REF_IS_MERGED(shared)) { - return refcnt; - } - refcnt += (Py_ssize_t)op->ob_ref_local; refcnt += extra; @@ -410,6 +405,10 @@ _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra) } while (!_Py_atomic_compare_exchange_ssize(&op->ob_ref_shared, &shared, new_shared)); +#ifdef Py_REF_DEBUG + _Py_AddRefTotal(_PyInterpreterState_GET(), extra); +#endif + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); return refcnt; @@ -794,6 +793,21 @@ PyObject_Bytes(PyObject *v) return PyBytes_FromObject(v); } +void +_PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization) +{ + // In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear() + // In the default build, freelists are per-interpreter and cleared in finalize_interp_types() + _PyFloat_ClearFreeList(freelists, is_finalization); + _PyTuple_ClearFreeList(freelists, is_finalization); + _PyList_ClearFreeList(freelists, is_finalization); + _PyDict_ClearFreeList(freelists, is_finalization); + _PyContext_ClearFreeList(freelists, is_finalization); + _PyAsyncGen_ClearFreeLists(freelists, is_finalization); + // Only be cleared if is_finalization is true. + _PyObjectStackChunk_ClearFreeList(freelists, is_finalization); + _PySlice_ClearFreeList(freelists, is_finalization); +} /* def _PyObject_FunctionStr(x): @@ -2671,7 +2685,12 @@ _PyTrash_thread_deposit_object(struct _py_trashcan *trash, PyObject *op) _PyObject_ASSERT(op, _PyObject_IS_GC(op)); _PyObject_ASSERT(op, !_PyObject_GC_IS_TRACKED(op)); _PyObject_ASSERT(op, Py_REFCNT(op) == 0); +#ifdef Py_GIL_DISABLED + _PyObject_ASSERT(op, op->ob_tid == 0); + op->ob_tid = (uintptr_t)trash->delete_later; +#else _PyGCHead_SET_PREV(_Py_AS_GC(op), (PyGC_Head*)trash->delete_later); +#endif trash->delete_later = op; } @@ -2697,8 +2716,12 @@ _PyTrash_thread_destroy_chain(struct _py_trashcan *trash) PyObject *op = trash->delete_later; destructor dealloc = Py_TYPE(op)->tp_dealloc; - trash->delete_later = - (PyObject*) _PyGCHead_PREV(_Py_AS_GC(op)); +#ifdef Py_GIL_DISABLED + trash->delete_later = (PyObject*) op->ob_tid; + op->ob_tid = 0; +#else + trash->delete_later = (PyObject*) _PyGCHead_PREV(_Py_AS_GC(op)); +#endif /* Call the deallocator directly. This used to try to * fool Py_DECREF into calling it indirectly, but diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index bea4ea85332bdda..b2a2286ef22b662 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -948,6 +948,196 @@ _PyMem_Strdup(const char *str) return copy; } +/***********************************************/ +/* Delayed freeing support for Py_GIL_DISABLED */ +/***********************************************/ + +// So that sizeof(struct _mem_work_chunk) is 4096 bytes on 64-bit platforms. +#define WORK_ITEMS_PER_CHUNK 254 + +// A pointer to be freed once the QSBR read sequence reaches qsbr_goal. +struct _mem_work_item { + void *ptr; + uint64_t qsbr_goal; +}; + +// A fixed-size buffer of pointers to be freed +struct _mem_work_chunk { + // Linked list node of chunks in queue + struct llist_node node; + + Py_ssize_t rd_idx; // index of next item to read + Py_ssize_t wr_idx; // index of next item to write + struct _mem_work_item array[WORK_ITEMS_PER_CHUNK]; +}; + +void +_PyMem_FreeDelayed(void *ptr) +{ +#ifndef Py_GIL_DISABLED + PyMem_Free(ptr); +#else + if (_PyRuntime.stoptheworld.world_stopped) { + // Free immediately if the world is stopped, including during + // interpreter shutdown. + PyMem_Free(ptr); + return; + } + + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + struct llist_node *head = &tstate->mem_free_queue; + + struct _mem_work_chunk *buf = NULL; + if (!llist_empty(head)) { + // Try to re-use the last buffer + buf = llist_data(head->prev, struct _mem_work_chunk, node); + if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) { + // already full + buf = NULL; + } + } + + if (buf == NULL) { + buf = PyMem_Calloc(1, sizeof(*buf)); + if (buf != NULL) { + llist_insert_tail(head, &buf->node); + } + } + + if (buf == NULL) { + // failed to allocate a buffer, free immediately + _PyEval_StopTheWorld(tstate->base.interp); + PyMem_Free(ptr); + _PyEval_StartTheWorld(tstate->base.interp); + return; + } + + assert(buf != NULL && buf->wr_idx < WORK_ITEMS_PER_CHUNK); + uint64_t seq = _Py_qsbr_deferred_advance(tstate->qsbr); + buf->array[buf->wr_idx].ptr = ptr; + buf->array[buf->wr_idx].qsbr_goal = seq; + buf->wr_idx++; + + if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) { + _PyMem_ProcessDelayed((PyThreadState *)tstate); + } +#endif +} + +static struct _mem_work_chunk * +work_queue_first(struct llist_node *head) +{ + return llist_data(head->next, struct _mem_work_chunk, node); +} + +static void +process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, + bool keep_empty) +{ + while (!llist_empty(head)) { + struct _mem_work_chunk *buf = work_queue_first(head); + + while (buf->rd_idx < buf->wr_idx) { + struct _mem_work_item *item = &buf->array[buf->rd_idx]; + if (!_Py_qsbr_poll(qsbr, item->qsbr_goal)) { + return; + } + + PyMem_Free(item->ptr); + buf->rd_idx++; + } + + assert(buf->rd_idx == buf->wr_idx); + if (keep_empty && buf->node.next == head) { + // Keep the last buffer in the queue to reduce re-allocations + buf->rd_idx = buf->wr_idx = 0; + return; + } + + llist_remove(&buf->node); + PyMem_Free(buf); + } +} + +static void +process_interp_queue(struct _Py_mem_interp_free_queue *queue, + struct _qsbr_thread_state *qsbr) +{ + if (!_Py_atomic_load_int_relaxed(&queue->has_work)) { + return; + } + + // Try to acquire the lock, but don't block if it's already held. + if (_PyMutex_LockTimed(&queue->mutex, 0, 0) == PY_LOCK_ACQUIRED) { + process_queue(&queue->head, qsbr, false); + + int more_work = !llist_empty(&queue->head); + _Py_atomic_store_int_relaxed(&queue->has_work, more_work); + + PyMutex_Unlock(&queue->mutex); + } +} + +void +_PyMem_ProcessDelayed(PyThreadState *tstate) +{ + PyInterpreterState *interp = tstate->interp; + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + + // Process thread-local work + process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr, true); + + // Process shared interpreter work + process_interp_queue(&interp->mem_free_queue, tstate_impl->qsbr); +} + +void +_PyMem_AbandonDelayed(PyThreadState *tstate) +{ + PyInterpreterState *interp = tstate->interp; + struct llist_node *queue = &((_PyThreadStateImpl *)tstate)->mem_free_queue; + + if (llist_empty(queue)) { + return; + } + + // Check if the queue contains one empty buffer + struct _mem_work_chunk *buf = work_queue_first(queue); + if (buf->rd_idx == buf->wr_idx) { + llist_remove(&buf->node); + PyMem_Free(buf); + assert(llist_empty(queue)); + return; + } + + // Merge the thread's work queue into the interpreter's work queue. + PyMutex_Lock(&interp->mem_free_queue.mutex); + llist_concat(&interp->mem_free_queue.head, queue); + _Py_atomic_store_int_relaxed(&interp->mem_free_queue.has_work, 1); + PyMutex_Unlock(&interp->mem_free_queue.mutex); + + assert(llist_empty(queue)); // the thread's queue is now empty +} + +void +_PyMem_FiniDelayed(PyInterpreterState *interp) +{ + struct llist_node *head = &interp->mem_free_queue.head; + while (!llist_empty(head)) { + struct _mem_work_chunk *buf = work_queue_first(head); + + while (buf->rd_idx < buf->wr_idx) { + // Free the remaining items immediately. There should be no other + // threads accessing the memory at this point during shutdown. + struct _mem_work_item *item = &buf->array[buf->rd_idx]; + PyMem_Free(item->ptr); + buf->rd_idx++; + } + + llist_remove(&buf->node); + PyMem_Free(buf); + } +} /**************************/ /* the "object" allocator */ @@ -1073,7 +1263,12 @@ get_mimalloc_allocated_blocks(PyInterpreterState *interp) mi_heap_visit_blocks(heap, false, &count_blocks, &allocated_blocks); } } - // TODO(sgross): count blocks in abandoned segments. + + mi_abandoned_pool_t *pool = &interp->mimalloc.abandoned_pool; + for (uint8_t tag = 0; tag < _Py_MIMALLOC_HEAP_COUNT; tag++) { + _mi_abandoned_pool_visit_blocks(pool, tag, false, &count_blocks, + &allocated_blocks); + } #else // TODO(sgross): this only counts the current thread's blocks. mi_heap_t *heap = mi_heap_get_default(); @@ -1189,6 +1384,7 @@ get_num_global_allocated_blocks(_PyRuntimeState *runtime) } } else { + _PyEval_StopTheWorldAll(&_PyRuntime); HEAD_LOCK(runtime); PyInterpreterState *interp = PyInterpreterState_Head(); assert(interp != NULL); @@ -1208,6 +1404,7 @@ get_num_global_allocated_blocks(_PyRuntimeState *runtime) } } HEAD_UNLOCK(runtime); + _PyEval_StartTheWorldAll(&_PyRuntime); #ifdef Py_DEBUG assert(got_main); #endif @@ -2262,6 +2459,33 @@ write_size_t(void *p, size_t n) } } +static void +fill_mem_debug(debug_alloc_api_t *api, void *data, int c, size_t nbytes, + bool is_alloc) +{ +#ifdef Py_GIL_DISABLED + if (api->api_id == 'o') { + // Don't overwrite the first few bytes of a PyObject allocation in the + // free-threaded build + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + size_t debug_offset; + if (is_alloc) { + debug_offset = tstate->mimalloc.current_object_heap->debug_offset; + } + else { + char *alloc = (char *)data - 2*SST; // start of the allocation + debug_offset = _mi_ptr_page(alloc)->debug_offset; + } + debug_offset -= 2*SST; // account for pymalloc extra bytes + if (debug_offset < nbytes) { + memset((char *)data + debug_offset, c, nbytes - debug_offset); + } + return; + } +#endif + memset(data, c, nbytes); +} + /* Let S = sizeof(size_t). The debug malloc asks for 4 * S extra bytes and fills them with useful stuff, here calling the underlying malloc's result p: @@ -2338,7 +2562,7 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes) memset(p + SST + 1, PYMEM_FORBIDDENBYTE, SST-1); if (nbytes > 0 && !use_calloc) { - memset(data, PYMEM_CLEANBYTE, nbytes); + fill_mem_debug(api, data, PYMEM_CLEANBYTE, nbytes, true); } /* at tail, write pad (SST bytes) and serialno (SST bytes) */ @@ -2386,8 +2610,9 @@ _PyMem_DebugRawFree(void *ctx, void *p) _PyMem_DebugCheckAddress(__func__, api->api_id, p); nbytes = read_size_t(q); - nbytes += PYMEM_DEBUG_EXTRA_BYTES; - memset(q, PYMEM_DEADBYTE, nbytes); + nbytes += PYMEM_DEBUG_EXTRA_BYTES - 2*SST; + memset(q, PYMEM_DEADBYTE, 2*SST); + fill_mem_debug(api, p, PYMEM_DEADBYTE, nbytes, false); api->alloc.free(api->alloc.ctx, q); } @@ -2407,7 +2632,6 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) size_t total; /* 2 * SST + nbytes + 2 * SST */ size_t original_nbytes; #define ERASED_SIZE 64 - uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */ _PyMem_DebugCheckAddress(__func__, api->api_id, p); @@ -2424,9 +2648,11 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) #ifdef PYMEM_DEBUG_SERIALNO size_t block_serialno = read_size_t(tail + SST); #endif +#ifndef Py_GIL_DISABLED /* Mark the header, the trailer, ERASED_SIZE bytes at the begin and ERASED_SIZE bytes at the end as dead and save the copy of erased bytes. */ + uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */ if (original_nbytes <= sizeof(save)) { memcpy(save, data, original_nbytes); memset(data - 2 * SST, PYMEM_DEADBYTE, @@ -2439,6 +2665,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) memset(tail - ERASED_SIZE, PYMEM_DEADBYTE, ERASED_SIZE + PYMEM_DEBUG_EXTRA_BYTES - 2 * SST); } +#endif /* Resize and add decorations. */ r = (uint8_t *)api->alloc.realloc(api->alloc.ctx, head, total); @@ -2466,6 +2693,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) write_size_t(tail + SST, block_serialno); #endif +#ifndef Py_GIL_DISABLED /* Restore saved bytes. */ if (original_nbytes <= sizeof(save)) { memcpy(data, save, Py_MIN(nbytes, original_nbytes)); @@ -2478,6 +2706,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) Py_MIN(nbytes - i, ERASED_SIZE)); } } +#endif if (r == NULL) { return NULL; diff --git a/Objects/odictobject.c b/Objects/odictobject.c index b5280c39e1be542..421bc52992d7354 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -465,12 +465,13 @@ Potential Optimizations */ #include "Python.h" -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_dict.h" // _Py_dict_lookup() -#include "pycore_object.h" // _PyObject_GC_UNTRACK() -#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include <stddef.h> // offsetof() +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_critical_section.h" //_Py_BEGIN_CRITICAL_SECTION +#include "pycore_dict.h" // _Py_dict_lookup() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include <stddef.h> // offsetof() #include "clinic/odictobject.c.h" @@ -1039,6 +1040,8 @@ _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, { PyObject *value = NULL; + Py_BEGIN_CRITICAL_SECTION(od); + _ODictNode *node = _odict_find_node_hash((PyODictObject *)od, key, hash); if (node != NULL) { /* Pop the node first to avoid a possible dict resize (due to @@ -1046,7 +1049,7 @@ _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, resolution. */ int res = _odict_clear_node((PyODictObject *)od, node, key, hash); if (res < 0) { - return NULL; + goto done; } /* Now delete the value from the dict. */ if (_PyDict_Pop_KnownHash((PyDictObject *)od, key, hash, @@ -1063,6 +1066,8 @@ _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, PyErr_SetObject(PyExc_KeyError, key); } } + Py_END_CRITICAL_SECTION(); +done: return value; } diff --git a/Objects/setobject.c b/Objects/setobject.c index 93de8e84f2ddf99..6a4c8c45f0836d1 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -32,13 +32,27 @@ */ #include "Python.h" -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_dict.h" // _PyDict_Contains_KnownHash() -#include "pycore_modsupport.h" // _PyArg_NoKwnames() -#include "pycore_object.h" // _PyObject_GC_UNTRACK() -#include "pycore_pyerrors.h" // _PyErr_SetKeyError() -#include "pycore_setobject.h" // _PySet_NextEntry() definition -#include <stddef.h> // offsetof() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION +#include "pycore_dict.h" // _PyDict_Contains_KnownHash() +#include "pycore_modsupport.h" // _PyArg_NoKwnames() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyerrors.h" // _PyErr_SetKeyError() +#include "pycore_setobject.h" // _PySet_NextEntry() definition +#include <stddef.h> // offsetof() +#include "clinic/setobject.c.h" + +/*[clinic input] +class set "PySetObject *" "&PySet_Type" +class frozenset "PySetObject *" "&PyFrozenSet_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=97ad1d3e9f117079]*/ + +/*[python input] +class setobject_converter(self_converter): + type = "PySetObject *" +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=33a44506d4d57793]*/ /* Object used as dummy key to fill deleted entries */ static PyObject _dummy_struct; @@ -630,8 +644,18 @@ set_merge(PySetObject *so, PyObject *otherset) return 0; } +/*[clinic input] +set.pop + so: setobject + +Remove and return an arbitrary set element. + +Raises KeyError if the set is empty. +[clinic start generated code]*/ + static PyObject * -set_pop(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set_pop_impl(PySetObject *so) +/*[clinic end generated code: output=4d65180f1271871b input=4a3f5552e660a260]*/ { /* Make sure the search finger is in bounds */ setentry *entry = so->table + (so->finger & so->mask); @@ -655,9 +679,6 @@ set_pop(PySetObject *so, PyObject *Py_UNUSED(ignored)) return key; } -PyDoc_STRVAR(pop_doc, "Remove and return an arbitrary set element.\n\ -Raises KeyError if the set is empty."); - static int set_traverse(PySetObject *so, visitproc visit, void *arg) { @@ -903,11 +924,17 @@ set_update_internal(PySetObject *so, PyObject *other) if (set_table_resize(so, (so->used + dictsize)*2) != 0) return -1; } + int err = 0; + Py_BEGIN_CRITICAL_SECTION(other); while (_PyDict_Next(other, &pos, &key, &value, &hash)) { - if (set_add_entry(so, key, hash)) - return -1; + if (set_add_entry(so, key, hash)) { + err = -1; + goto exit; + } } - return 0; +exit: + Py_END_CRITICAL_SECTION(); + return err; } it = PyObject_GetIter(other); @@ -928,8 +955,18 @@ set_update_internal(PySetObject *so, PyObject *other) return 0; } +/*[clinic input] +set.update + so: setobject + *others as args: object + / + +Update the set, adding elements from all others. +[clinic start generated code]*/ + static PyObject * -set_update(PySetObject *so, PyObject *args) +set_update_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=34f6371704974c8a input=eb47c4fbaeb3286e]*/ { Py_ssize_t i; @@ -941,12 +978,6 @@ set_update(PySetObject *so, PyObject *args) Py_RETURN_NONE; } -PyDoc_STRVAR(update_doc, -"update($self, /, *others)\n\ ---\n\ -\n\ -Update the set, adding elements from all others."); - /* XXX Todo: If aligned memory allocations become available, make the set object 64 byte aligned so that most of the fields @@ -1094,14 +1125,30 @@ set_swap_bodies(PySetObject *a, PySetObject *b) } } +/*[clinic input] +set.copy + so: setobject + +Return a shallow copy of a set. +[clinic start generated code]*/ + static PyObject * -set_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set_copy_impl(PySetObject *so) +/*[clinic end generated code: output=c9223a1e1cc6b041 input=2b80b288d47b8cf1]*/ { return make_new_set_basetype(Py_TYPE(so), (PyObject *)so); } +/*[clinic input] +frozenset.copy + so: setobject + +Return a shallow copy of a set. +[clinic start generated code]*/ + static PyObject * -frozenset_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +frozenset_copy_impl(PySetObject *so) +/*[clinic end generated code: output=b356263526af9e70 input=3dc65577d344eff7]*/ { if (PyFrozenSet_CheckExact(so)) { return Py_NewRef(so); @@ -1109,19 +1156,33 @@ frozenset_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) return set_copy(so, NULL); } -PyDoc_STRVAR(copy_doc, "Return a shallow copy of a set."); +/*[clinic input] +set.clear + so: setobject + +Remove all elements from this set. +[clinic start generated code]*/ static PyObject * -set_clear(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set_clear_impl(PySetObject *so) +/*[clinic end generated code: output=4e71d5a83904161a input=74ac19794da81a39]*/ { set_clear_internal(so); Py_RETURN_NONE; } -PyDoc_STRVAR(clear_doc, "Remove all elements from this set."); +/*[clinic input] +set.union + so: setobject + *others as args: object + / + +Return a new set with elements from the set and all others. +[clinic start generated code]*/ static PyObject * -set_union(PySetObject *so, PyObject *args) +set_union_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=2c83d05a446a1477 input=2e2024fa1e40ac84]*/ { PySetObject *result; PyObject *other; @@ -1143,12 +1204,6 @@ set_union(PySetObject *so, PyObject *args) return (PyObject *)result; } -PyDoc_STRVAR(union_doc, -"union($self, /, *others)\n\ ---\n\ -\n\ -Return a new set with elements from the set and all others."); - static PyObject * set_or(PySetObject *so, PyObject *other) { @@ -1263,8 +1318,18 @@ set_intersection(PySetObject *so, PyObject *other) return NULL; } +/*[clinic input] +set.intersection as set_intersection_multi + so: setobject + *others as args: object + / + +Return a new set with elements common to the set and all others. +[clinic start generated code]*/ + static PyObject * -set_intersection_multi(PySetObject *so, PyObject *args) +set_intersection_multi_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=2406ef3387adbe2f input=04108ea6d7f0532b]*/ { Py_ssize_t i; @@ -1284,12 +1349,6 @@ set_intersection_multi(PySetObject *so, PyObject *args) return result; } -PyDoc_STRVAR(intersection_doc, -"intersection($self, /, *others)\n\ ---\n\ -\n\ -Return a new set with elements common to the set and all others."); - static PyObject * set_intersection_update(PySetObject *so, PyObject *other) { @@ -1303,12 +1362,22 @@ set_intersection_update(PySetObject *so, PyObject *other) Py_RETURN_NONE; } +/*[clinic input] +set.intersection_update as set_intersection_update_multi + so: setobject + *others as args: object + / + +Update the set, keeping only elements found in it and all others. +[clinic start generated code]*/ + static PyObject * -set_intersection_update_multi(PySetObject *so, PyObject *args) +set_intersection_update_multi_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=251c1f729063609d input=ff8f119f97458d16]*/ { PyObject *tmp; - tmp = set_intersection_multi(so, args); + tmp = set_intersection_multi_impl(so, args); if (tmp == NULL) return NULL; set_swap_bodies(so, (PySetObject *)tmp); @@ -1316,12 +1385,6 @@ set_intersection_update_multi(PySetObject *so, PyObject *args) Py_RETURN_NONE; } -PyDoc_STRVAR(intersection_update_doc, -"intersection_update($self, /, *others)\n\ ---\n\ -\n\ -Update the set, keeping only elements found in it and all others."); - static PyObject * set_and(PySetObject *so, PyObject *other) { @@ -1344,8 +1407,18 @@ set_iand(PySetObject *so, PyObject *other) return Py_NewRef(so); } +/*[clinic input] +set.isdisjoint + so: setobject + other: object + / + +Return True if two sets have a null intersection. +[clinic start generated code]*/ + static PyObject * set_isdisjoint(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=a92bbf9a2db6a3da input=c254ddec8a2326e3]*/ { PyObject *key, *it, *tmp; int rv; @@ -1403,9 +1476,6 @@ set_isdisjoint(PySetObject *so, PyObject *other) Py_RETURN_TRUE; } -PyDoc_STRVAR(isdisjoint_doc, -"Return True if two sets have a null intersection."); - static int set_difference_update_internal(PySetObject *so, PyObject *other) { @@ -1464,8 +1534,18 @@ set_difference_update_internal(PySetObject *so, PyObject *other) return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); } +/*[clinic input] +set.difference_update + so: setobject + *others as args: object + / + +Update the set, removing elements found in others. +[clinic start generated code]*/ + static PyObject * -set_difference_update(PySetObject *so, PyObject *args) +set_difference_update_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=28685b2fc63e41c4 input=e7abb43c9f2c5a73]*/ { Py_ssize_t i; @@ -1477,12 +1557,6 @@ set_difference_update(PySetObject *so, PyObject *args) Py_RETURN_NONE; } -PyDoc_STRVAR(difference_update_doc, -"difference_update($self, /, *others)\n\ ---\n\ -\n\ -Update the set, removing elements found in others."); - static PyObject * set_copy_and_difference(PySetObject *so, PyObject *other) { @@ -1573,8 +1647,18 @@ set_difference(PySetObject *so, PyObject *other) return result; } +/*[clinic input] +set.difference as set_difference_multi + so: setobject + *others as args: object + / + +Return a new set with elements in the set that are not in the others. +[clinic start generated code]*/ + static PyObject * -set_difference_multi(PySetObject *so, PyObject *args) +set_difference_multi_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=3130c3bb3cac873d input=d8ae9bb6d518ab95]*/ { Py_ssize_t i; PyObject *result, *other; @@ -1597,11 +1681,6 @@ set_difference_multi(PySetObject *so, PyObject *args) return result; } -PyDoc_STRVAR(difference_doc, -"difference($self, /, *others)\n\ ---\n\ -\n\ -Return a new set with elements in the set that are not in the others."); static PyObject * set_sub(PySetObject *so, PyObject *other) { @@ -1620,8 +1699,45 @@ set_isub(PySetObject *so, PyObject *other) return Py_NewRef(so); } +static PyObject * +set_symmetric_difference_update_dict(PySetObject *so, PyObject *other) +{ + PyObject *key; + Py_ssize_t pos = 0; + Py_hash_t hash; + PyObject *value; + int rv; + + while (_PyDict_Next(other, &pos, &key, &value, &hash)) { + Py_INCREF(key); + rv = set_discard_entry(so, key, hash); + if (rv < 0) { + Py_DECREF(key); + return NULL; + } + if (rv == DISCARD_NOTFOUND) { + if (set_add_entry(so, key, hash)) { + Py_DECREF(key); + return NULL; + } + } + Py_DECREF(key); + } + Py_RETURN_NONE; +} + +/*[clinic input] +set.symmetric_difference_update + so: setobject + other: object + / + +Update the set, keeping only elements found in either set, but not in both. +[clinic start generated code]*/ + static PyObject * set_symmetric_difference_update(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=fbb049c0806028de input=a50acf0365e1f0a5]*/ { PySetObject *otherset; PyObject *key; @@ -1634,23 +1750,13 @@ set_symmetric_difference_update(PySetObject *so, PyObject *other) return set_clear(so, NULL); if (PyDict_CheckExact(other)) { - PyObject *value; - while (_PyDict_Next(other, &pos, &key, &value, &hash)) { - Py_INCREF(key); - rv = set_discard_entry(so, key, hash); - if (rv < 0) { - Py_DECREF(key); - return NULL; - } - if (rv == DISCARD_NOTFOUND) { - if (set_add_entry(so, key, hash)) { - Py_DECREF(key); - return NULL; - } - } - Py_DECREF(key); - } - Py_RETURN_NONE; + PyObject *res; + + Py_BEGIN_CRITICAL_SECTION(other); + res = set_symmetric_difference_update_dict(so, other); + Py_END_CRITICAL_SECTION(); + + return res; } if (PyAnySet_Check(other)) { @@ -1684,14 +1790,18 @@ set_symmetric_difference_update(PySetObject *so, PyObject *other) Py_RETURN_NONE; } -PyDoc_STRVAR(symmetric_difference_update_doc, -"symmetric_difference_update($self, other, /)\n\ ---\n\ -\n\ -Update the set, keeping only elements found in either set, but not in both."); +/*[clinic input] +set.symmetric_difference + so: setobject + other: object + / + +Return a new set with elements in either the set or other but not both. +[clinic start generated code]*/ static PyObject * set_symmetric_difference(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=f95364211b88775a input=f18af370ad72ebac]*/ { PyObject *rv; PySetObject *otherset; @@ -1708,12 +1818,6 @@ set_symmetric_difference(PySetObject *so, PyObject *other) return (PyObject *)otherset; } -PyDoc_STRVAR(symmetric_difference_doc, -"symmetric_difference($self, other, /)\n\ ---\n\ -\n\ -Return a new set with elements in either the set or other but not both."); - static PyObject * set_xor(PySetObject *so, PyObject *other) { @@ -1736,8 +1840,18 @@ set_ixor(PySetObject *so, PyObject *other) return Py_NewRef(so); } +/*[clinic input] +set.issubset + so: setobject + other: object + / + +Report whether another set contains this set. +[clinic start generated code]*/ + static PyObject * set_issubset(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=78aef1f377aedef1 input=37fbc579b609db0c]*/ { setentry *entry; Py_ssize_t pos = 0; @@ -1770,14 +1884,18 @@ set_issubset(PySetObject *so, PyObject *other) Py_RETURN_TRUE; } -PyDoc_STRVAR(issubset_doc, -"issubset($self, other, /)\n\ ---\n\ -\n\ -Test whether every element in the set is in other."); +/*[clinic input] +set.issuperset + so: setobject + other: object + / + +Report whether this set contains another set. +[clinic start generated code]*/ static PyObject * set_issuperset(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=7d2b71dd714a7ec7 input=fd5dab052f2e9bb3]*/ { if (PyAnySet_Check(other)) { return set_issubset((PySetObject *)other, (PyObject *)so); @@ -1806,12 +1924,6 @@ set_issuperset(PySetObject *so, PyObject *other) Py_RETURN_TRUE; } -PyDoc_STRVAR(issuperset_doc, -"issuperset($self, other, /)\n\ ---\n\ -\n\ -Test whether every element in other is in the set."); - static PyObject * set_richcompare(PySetObject *v, PyObject *w, int op) { @@ -1855,19 +1967,26 @@ set_richcompare(PySetObject *v, PyObject *w, int op) Py_RETURN_NOTIMPLEMENTED; } +/*[clinic input] +set.add + so: setobject + object as key: object + / + +Add an element to a set. + +This has no effect if the element is already present. +[clinic start generated code]*/ + static PyObject * set_add(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=cd9c2d5c2069c2ba input=96f1efe029e47972]*/ { if (set_add_key(so, key)) return NULL; Py_RETURN_NONE; } -PyDoc_STRVAR(add_doc, -"Add an element to a set.\n\ -\n\ -This has no effect if the element is already present."); - static int set_contains(PySetObject *so, PyObject *key) { @@ -1888,8 +2007,19 @@ set_contains(PySetObject *so, PyObject *key) return rv; } +/*[clinic input] +@coexist +set.__contains__ + so: setobject + object as key: object + / + +x.__contains__(y) <==> y in x. +[clinic start generated code]*/ + static PyObject * -set_direct_contains(PySetObject *so, PyObject *key) +set___contains__(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=b5948bc5c590d3ca input=cf4c72db704e4cf0]*/ { long result; @@ -1899,10 +2029,20 @@ set_direct_contains(PySetObject *so, PyObject *key) return PyBool_FromLong(result); } -PyDoc_STRVAR(contains_doc, "x.__contains__(y) <==> y in x."); +/*[clinic input] +set.remove + so: setobject + object as key: object + / + +Remove an element from a set; it must be a member. + +If the element is not a member, raise a KeyError. +[clinic start generated code]*/ static PyObject * set_remove(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=08ae496d0cd2b8c1 input=10132515dfe8ebd7]*/ { PyObject *tmpkey; int rv; @@ -1928,13 +2068,21 @@ set_remove(PySetObject *so, PyObject *key) Py_RETURN_NONE; } -PyDoc_STRVAR(remove_doc, -"Remove an element from a set; it must be a member.\n\ -\n\ -If the element is not a member, raise a KeyError."); +/*[clinic input] +set.discard + so: setobject + object as key: object + / + +Remove an element from a set if it is a member. + +Unlike set.remove(), the discard() method does not raise +an exception when an element is missing from the set. +[clinic start generated code]*/ static PyObject * set_discard(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=9181b60d7bb7d480 input=82a689eba94d5ad9]*/ { PyObject *tmpkey; int rv; @@ -1955,14 +2103,16 @@ set_discard(PySetObject *so, PyObject *key) Py_RETURN_NONE; } -PyDoc_STRVAR(discard_doc, -"Remove an element from a set if it is a member.\n\ -\n\ -Unlike set.remove(), the discard() method does not raise\n\ -an exception when an element is missing from the set."); +/*[clinic input] +set.__reduce__ + so: setobject + +Return state information for pickling. +[clinic start generated code]*/ static PyObject * -set_reduce(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set___reduce___impl(PySetObject *so) +/*[clinic end generated code: output=9af7d0e029df87ee input=531375e87a24a449]*/ { PyObject *keys=NULL, *args=NULL, *result=NULL, *state=NULL; @@ -1983,8 +2133,16 @@ set_reduce(PySetObject *so, PyObject *Py_UNUSED(ignored)) return result; } +/*[clinic input] +set.__sizeof__ + so: setobject + +S.__sizeof__() -> size of S in memory, in bytes. +[clinic start generated code]*/ + static PyObject * -set_sizeof(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set___sizeof___impl(PySetObject *so) +/*[clinic end generated code: output=4bfa3df7bd38ed88 input=0f214fc2225319fc]*/ { size_t res = _PyObject_SIZE(Py_TYPE(so)); if (so->table != so->smalltable) { @@ -1993,7 +2151,6 @@ set_sizeof(PySetObject *so, PyObject *Py_UNUSED(ignored)) return PyLong_FromSize_t(res); } -PyDoc_STRVAR(sizeof_doc, "S.__sizeof__() -> size of S in memory, in bytes"); static int set_init(PySetObject *self, PyObject *args, PyObject *kwds) { @@ -2047,46 +2204,26 @@ static PySequenceMethods set_as_sequence = { /* set object ********************************************************/ static PyMethodDef set_methods[] = { - {"add", (PyCFunction)set_add, METH_O, - add_doc}, - {"clear", (PyCFunction)set_clear, METH_NOARGS, - clear_doc}, - {"__contains__",(PyCFunction)set_direct_contains, METH_O | METH_COEXIST, - contains_doc}, - {"copy", (PyCFunction)set_copy, METH_NOARGS, - copy_doc}, - {"discard", (PyCFunction)set_discard, METH_O, - discard_doc}, - {"difference", (PyCFunction)set_difference_multi, METH_VARARGS, - difference_doc}, - {"difference_update", (PyCFunction)set_difference_update, METH_VARARGS, - difference_update_doc}, - {"intersection",(PyCFunction)set_intersection_multi, METH_VARARGS, - intersection_doc}, - {"intersection_update",(PyCFunction)set_intersection_update_multi, METH_VARARGS, - intersection_update_doc}, - {"isdisjoint", (PyCFunction)set_isdisjoint, METH_O, - isdisjoint_doc}, - {"issubset", (PyCFunction)set_issubset, METH_O, - issubset_doc}, - {"issuperset", (PyCFunction)set_issuperset, METH_O, - issuperset_doc}, - {"pop", (PyCFunction)set_pop, METH_NOARGS, - pop_doc}, - {"__reduce__", (PyCFunction)set_reduce, METH_NOARGS, - reduce_doc}, - {"remove", (PyCFunction)set_remove, METH_O, - remove_doc}, - {"__sizeof__", (PyCFunction)set_sizeof, METH_NOARGS, - sizeof_doc}, - {"symmetric_difference",(PyCFunction)set_symmetric_difference, METH_O, - symmetric_difference_doc}, - {"symmetric_difference_update",(PyCFunction)set_symmetric_difference_update, METH_O, - symmetric_difference_update_doc}, - {"union", (PyCFunction)set_union, METH_VARARGS, - union_doc}, - {"update", (PyCFunction)set_update, METH_VARARGS, - update_doc}, + SET_ADD_METHODDEF + SET_CLEAR_METHODDEF + SET___CONTAINS___METHODDEF + SET_COPY_METHODDEF + SET_DISCARD_METHODDEF + SET_DIFFERENCE_MULTI_METHODDEF + SET_DIFFERENCE_UPDATE_METHODDEF + SET_INTERSECTION_MULTI_METHODDEF + SET_INTERSECTION_UPDATE_MULTI_METHODDEF + SET_ISDISJOINT_METHODDEF + SET_ISSUBSET_METHODDEF + SET_ISSUPERSET_METHODDEF + SET_POP_METHODDEF + SET___REDUCE___METHODDEF + SET_REMOVE_METHODDEF + SET___SIZEOF___METHODDEF + SET_SYMMETRIC_DIFFERENCE_METHODDEF + SET_SYMMETRIC_DIFFERENCE_UPDATE_METHODDEF + SET_UNION_METHODDEF + SET_UPDATE_METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; @@ -2179,28 +2316,17 @@ PyTypeObject PySet_Type = { static PyMethodDef frozenset_methods[] = { - {"__contains__",(PyCFunction)set_direct_contains, METH_O | METH_COEXIST, - contains_doc}, - {"copy", (PyCFunction)frozenset_copy, METH_NOARGS, - copy_doc}, - {"difference", (PyCFunction)set_difference_multi, METH_VARARGS, - difference_doc}, - {"intersection", (PyCFunction)set_intersection_multi, METH_VARARGS, - intersection_doc}, - {"isdisjoint", (PyCFunction)set_isdisjoint, METH_O, - isdisjoint_doc}, - {"issubset", (PyCFunction)set_issubset, METH_O, - issubset_doc}, - {"issuperset", (PyCFunction)set_issuperset, METH_O, - issuperset_doc}, - {"__reduce__", (PyCFunction)set_reduce, METH_NOARGS, - reduce_doc}, - {"__sizeof__", (PyCFunction)set_sizeof, METH_NOARGS, - sizeof_doc}, - {"symmetric_difference",(PyCFunction)set_symmetric_difference, METH_O, - symmetric_difference_doc}, - {"union", (PyCFunction)set_union, METH_VARARGS, - union_doc}, + SET___CONTAINS___METHODDEF + FROZENSET_COPY_METHODDEF + SET_DIFFERENCE_MULTI_METHODDEF + SET_INTERSECTION_MULTI_METHODDEF + SET_ISDISJOINT_METHODDEF + SET_ISSUBSET_METHODDEF + SET_ISSUPERSET_METHODDEF + SET___REDUCE___METHODDEF + SET___SIZEOF___METHODDEF + SET_SYMMETRIC_DIFFERENCE_METHODDEF + SET_UNION_METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 8b9d6bbfd858b75..7333aea91e5648a 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -103,24 +103,20 @@ PyObject _Py_EllipsisObject = _PyObject_HEAD_INIT(&PyEllipsis_Type); /* Slice object implementation */ -void _PySlice_ClearCache(_PyFreeListState *state) +void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) { + if (!is_finalization) { + return; + } #ifdef WITH_FREELISTS - PySliceObject *obj = state->slices.slice_cache; + PySliceObject *obj = freelists->slices.slice_cache; if (obj != NULL) { - state->slices.slice_cache = NULL; + freelists->slices.slice_cache = NULL; PyObject_GC_Del(obj); } #endif } -void _PySlice_Fini(_PyFreeListState *state) -{ -#ifdef WITH_FREELISTS - _PySlice_ClearCache(state); -#endif -} - /* start, stop, and step are python objects with None indicating no index is present. */ @@ -131,10 +127,10 @@ _PyBuildSlice_Consume2(PyObject *start, PyObject *stop, PyObject *step) assert(start != NULL && stop != NULL && step != NULL); PySliceObject *obj; #ifdef WITH_FREELISTS - _PyFreeListState *state = _PyFreeListState_GET(); - if (state->slices.slice_cache != NULL) { - obj = state->slices.slice_cache; - state->slices.slice_cache = NULL; + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + if (freelists->slices.slice_cache != NULL) { + obj = freelists->slices.slice_cache; + freelists->slices.slice_cache = NULL; _Py_NewReference((PyObject *)obj); } else @@ -369,9 +365,9 @@ slice_dealloc(PySliceObject *r) Py_DECREF(r->start); Py_DECREF(r->stop); #ifdef WITH_FREELISTS - _PyFreeListState *state = _PyFreeListState_GET(); - if (state->slices.slice_cache == NULL) { - state->slices.slice_cache = r; + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + if (freelists->slices.slice_cache == NULL) { + freelists->slices.slice_cache = r; } else #endif diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index b9bf6cd48f61292..d9dc00da368a84c 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -962,18 +962,13 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) } -static void maybe_freelist_clear(_PyFreeListState *, int); +static void maybe_freelist_clear(struct _Py_object_freelists *, int); -void -_PyTuple_Fini(_PyFreeListState *state) -{ - maybe_freelist_clear(state, 1); -} void -_PyTuple_ClearFreeList(_PyFreeListState *state, int is_finalization) +_PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) { - maybe_freelist_clear(state, is_finalization); + maybe_freelist_clear(freelists, is_finalization); } /*********************** Tuple Iterator **************************/ @@ -1125,26 +1120,26 @@ tuple_iter(PyObject *seq) * freelists * *************/ -#define STATE (state->tuples) -#define FREELIST_FINALIZED (STATE.numfree[0] < 0) +#define TUPLE_FREELIST (freelists->tuples) +#define FREELIST_FINALIZED (TUPLE_FREELIST.numfree[0] < 0) static inline PyTupleObject * maybe_freelist_pop(Py_ssize_t size) { #ifdef WITH_FREELISTS - _PyFreeListState *state = _PyFreeListState_GET(); + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); if (size == 0) { return NULL; } assert(size > 0); if (size < PyTuple_MAXSAVESIZE) { Py_ssize_t index = size - 1; - PyTupleObject *op = STATE.free_list[index]; + PyTupleObject *op = TUPLE_FREELIST.items[index]; if (op != NULL) { /* op is the head of a linked list, with the first item pointing to the next node. Here we pop off the old head. */ - STATE.free_list[index] = (PyTupleObject *) op->ob_item[0]; - STATE.numfree[index]--; + TUPLE_FREELIST.items[index] = (PyTupleObject *) op->ob_item[0]; + TUPLE_FREELIST.numfree[index]--; /* Inlined _PyObject_InitVar() without _PyType_HasFeature() test */ #ifdef Py_TRACE_REFS /* maybe_freelist_push() ensures these were already set. */ @@ -1166,21 +1161,21 @@ static inline int maybe_freelist_push(PyTupleObject *op) { #ifdef WITH_FREELISTS - _PyFreeListState *state = _PyFreeListState_GET(); + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); if (Py_SIZE(op) == 0) { return 0; } Py_ssize_t index = Py_SIZE(op) - 1; if (index < PyTuple_NFREELISTS - && STATE.numfree[index] < PyTuple_MAXFREELIST - && STATE.numfree[index] >= 0 + && TUPLE_FREELIST.numfree[index] < PyTuple_MAXFREELIST + && TUPLE_FREELIST.numfree[index] >= 0 && Py_IS_TYPE(op, &PyTuple_Type)) { /* op is the head of a linked list, with the first item pointing to the next node. Here we set op as the new head. */ - op->ob_item[0] = (PyObject *) STATE.free_list[index]; - STATE.free_list[index] = op; - STATE.numfree[index]++; + op->ob_item[0] = (PyObject *) TUPLE_FREELIST.items[index]; + TUPLE_FREELIST.items[index] = op; + TUPLE_FREELIST.numfree[index]++; OBJECT_STAT_INC(to_freelist); return 1; } @@ -1189,13 +1184,13 @@ maybe_freelist_push(PyTupleObject *op) } static void -maybe_freelist_clear(_PyFreeListState *state, int fini) +maybe_freelist_clear(struct _Py_object_freelists *freelists, int fini) { #ifdef WITH_FREELISTS for (Py_ssize_t i = 0; i < PyTuple_NFREELISTS; i++) { - PyTupleObject *p = STATE.free_list[i]; - STATE.free_list[i] = NULL; - STATE.numfree[i] = fini ? -1 : 0; + PyTupleObject *p = TUPLE_FREELIST.items[i]; + TUPLE_FREELIST.items[i] = NULL; + TUPLE_FREELIST.numfree[i] = fini ? -1 : 0; while (p) { PyTupleObject *q = p; p = (PyTupleObject *)(p->ob_item[0]); @@ -1210,13 +1205,13 @@ void _PyTuple_DebugMallocStats(FILE *out) { #ifdef WITH_FREELISTS - _PyFreeListState *state = _PyFreeListState_GET(); + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); for (int i = 0; i < PyTuple_NFREELISTS; i++) { int len = i + 1; char buf[128]; PyOS_snprintf(buf, sizeof(buf), "free %d-sized PyTupleObject", len); - _PyDebugAllocatorStats(out, buf, STATE.numfree[i], + _PyDebugAllocatorStats(out, buf, TUPLE_FREELIST.numfree[i], _PyObject_VAR_SIZE(&PyTuple_Type, len)); } #endif diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 114cf21f95e744b..fe3b7b87c8b4b68 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6,6 +6,7 @@ #include "pycore_code.h" // CO_FAST_FREE #include "pycore_dict.h" // _PyDict_KeysSize() #include "pycore_frame.h" // _PyInterpreterFrame +#include "pycore_lock.h" // _PySeqLock_* #include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_memoryobject.h" // _PyMemoryView_FromBufferProc() #include "pycore_modsupport.h" // _PyArg_NoKwnames() @@ -52,6 +53,35 @@ class object "PyObject *" "&PyBaseObject_Type" #define NEXT_VERSION_TAG(interp) \ (interp)->types.next_version_tag +#ifdef Py_GIL_DISABLED + +// There's a global lock for mutation of types. This avoids having to take +// additonal locks while doing various subclass processing which may result +// in odd behaviors w.r.t. running with the GIL as the outer type lock could +// be released and reacquired during a subclass update if there's contention +// on the subclass lock. +#define TYPE_LOCK &PyInterpreterState_Get()->types.mutex +#define BEGIN_TYPE_LOCK() \ + { \ + _PyCriticalSection _cs; \ + _PyCriticalSection_Begin(&_cs, TYPE_LOCK); \ + +#define END_TYPE_LOCK() \ + _PyCriticalSection_End(&_cs); \ + } + +#define ASSERT_TYPE_LOCK_HELD() \ + _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(TYPE_LOCK) + +#else + +#define BEGIN_TYPE_LOCK() +#define END_TYPE_LOCK() +#define ASSERT_TYPE_LOCK_HELD() + +#endif + + typedef struct PySlot_Offset { short subslot_offset; short slot_offset; @@ -279,8 +309,14 @@ lookup_tp_bases(PyTypeObject *self) PyObject * _PyType_GetBases(PyTypeObject *self) { - /* It returns a borrowed reference. */ - return lookup_tp_bases(self); + PyObject *res; + + BEGIN_TYPE_LOCK(); + res = lookup_tp_bases(self); + Py_INCREF(res); + END_TYPE_LOCK() + + return res; } static inline void @@ -330,14 +366,19 @@ clear_tp_bases(PyTypeObject *self) static inline PyObject * lookup_tp_mro(PyTypeObject *self) { + ASSERT_TYPE_LOCK_HELD(); return self->tp_mro; } PyObject * _PyType_GetMRO(PyTypeObject *self) { - /* It returns a borrowed reference. */ - return lookup_tp_mro(self); + PyObject *mro; + BEGIN_TYPE_LOCK(); + mro = lookup_tp_mro(self); + Py_INCREF(mro); + END_TYPE_LOCK() + return mro; } static inline void @@ -646,9 +687,15 @@ type_cache_clear(struct type_cache *cache, PyObject *value) { for (Py_ssize_t i = 0; i < (1 << MCACHE_SIZE_EXP); i++) { struct type_cache_entry *entry = &cache->hashtable[i]; +#ifdef Py_GIL_DISABLED + _PySeqLock_LockWrite(&entry->sequence); +#endif entry->version = 0; Py_XSETREF(entry->name, _Py_XNewRef(value)); entry->value = NULL; +#ifdef Py_GIL_DISABLED + _PySeqLock_UnlockWrite(&entry->sequence); +#endif } } @@ -760,8 +807,10 @@ PyType_Watch(int watcher_id, PyObject* obj) return -1; } // ensure we will get a callback on the next modification + BEGIN_TYPE_LOCK() assign_version_tag(interp, type); type->tp_watched |= (1 << watcher_id); + END_TYPE_LOCK() return 0; } @@ -781,8 +830,8 @@ PyType_Unwatch(int watcher_id, PyObject* obj) return 0; } -void -PyType_Modified(PyTypeObject *type) +static void +type_modified_unlocked(PyTypeObject *type) { /* Invalidate any cached data for the specified type and all subclasses. This function is called after the base @@ -848,6 +897,22 @@ PyType_Modified(PyTypeObject *type) } } +void +PyType_Modified(PyTypeObject *type) +{ + // Quick check without the lock held + if (!_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) { + return; + } + + BEGIN_TYPE_LOCK() + type_modified_unlocked(type); + END_TYPE_LOCK() +} + +static int +is_subtype_unlocked(PyTypeObject *a, PyTypeObject *b); + static void type_mro_modified(PyTypeObject *type, PyObject *bases) { /* @@ -866,6 +931,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { int custom = !Py_IS_TYPE(type, &PyType_Type); int unbound; + ASSERT_TYPE_LOCK_HELD(); if (custom) { PyObject *mro_meth, *type_mro_meth; mro_meth = lookup_maybe_method( @@ -891,7 +957,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { PyObject *b = PyTuple_GET_ITEM(bases, i); PyTypeObject *cls = _PyType_CAST(b); - if (!PyType_IsSubtype(type, cls)) { + if (!is_subtype_unlocked(type, cls)) { goto clear; } } @@ -908,9 +974,13 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { } } +#define MAX_VERSIONS_PER_CLASS 1000 + static int assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) { + ASSERT_TYPE_LOCK_HELD(); + /* Ensure that the tp_version_tag is valid and set Py_TPFLAGS_VALID_VERSION_TAG. To respect the invariant, this must first be done on all super classes. Return 0 if this @@ -922,7 +992,10 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) if (!_PyType_HasFeature(type, Py_TPFLAGS_READY)) { return 0; } - + if (type->tp_versions_used >= MAX_VERSIONS_PER_CLASS) { + return 0; + } + type->tp_versions_used++; if (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { /* static types */ if (NEXT_GLOBAL_VERSION_TAG > _Py_MAX_GLOBAL_TYPE_VERSION_TAG) { @@ -956,7 +1029,11 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) { PyInterpreterState *interp = _PyInterpreterState_GET(); - return assign_version_tag(interp, type); + int assigned; + BEGIN_TYPE_LOCK() + assigned = assign_version_tag(interp, type); + END_TYPE_LOCK() + return assigned; } @@ -1178,21 +1255,28 @@ type_set_abstractmethods(PyTypeObject *type, PyObject *value, void *context) static PyObject * type_get_bases(PyTypeObject *type, void *context) { - PyObject *bases = lookup_tp_bases(type); + PyObject *bases = _PyType_GetBases(type); if (bases == NULL) { Py_RETURN_NONE; } - return Py_NewRef(bases); + return bases; } static PyObject * type_get_mro(PyTypeObject *type, void *context) { - PyObject *mro = lookup_tp_mro(type); + PyObject *mro; + + BEGIN_TYPE_LOCK() + mro = lookup_tp_mro(type); if (mro == NULL) { - Py_RETURN_NONE; + mro = Py_None; + } else { + Py_INCREF(mro); } - return Py_NewRef(mro); + + END_TYPE_LOCK() + return mro; } static PyTypeObject *best_base(PyObject *); @@ -1214,6 +1298,8 @@ static int recurse_down_subclasses(PyTypeObject *type, PyObject *name, static int mro_hierarchy(PyTypeObject *type, PyObject *temp) { + ASSERT_TYPE_LOCK_HELD(); + PyObject *old_mro; int res = mro_internal(type, &old_mro); if (res <= 0) { @@ -1277,7 +1363,7 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) } static int -type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) +type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases, void *context) { // Check arguments if (!check_set_special_type_attr(type, new_bases, "__bases__")) { @@ -1308,7 +1394,7 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) } PyTypeObject *base = (PyTypeObject*)ob; - if (PyType_IsSubtype(base, type) || + if (is_subtype_unlocked(base, type) || /* In case of reentering here again through a custom mro() the above check is not enough since it relies on base->tp_mro which would gonna be updated inside @@ -1413,6 +1499,16 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) return -1; } +static int +type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) +{ + int res; + BEGIN_TYPE_LOCK(); + res = type_set_bases_unlocked(type, new_bases, context); + END_TYPE_LOCK(); + return res; +} + static PyObject * type_dict(PyTypeObject *type, void *context) { @@ -1739,6 +1835,8 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) if (presize) { ((PyObject **)alloc)[0] = NULL; ((PyObject **)alloc)[1] = NULL; + } + if (PyType_IS_GC(type)) { _PyObject_GC_Link(obj); } memset(obj, '\0', size); @@ -2151,11 +2249,12 @@ type_is_subtype_base_chain(PyTypeObject *a, PyTypeObject *b) return (b == &PyBaseObject_Type); } -int -PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) +static int +is_subtype_unlocked(PyTypeObject *a, PyTypeObject *b) { PyObject *mro; + ASSERT_TYPE_LOCK_HELD(); mro = lookup_tp_mro(a); if (mro != NULL) { /* Deal with multiple inheritance without recursion @@ -2174,6 +2273,16 @@ PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) return type_is_subtype_base_chain(a, b); } +int +PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) +{ + int res; + BEGIN_TYPE_LOCK(); + res = is_subtype_unlocked(a, b); + END_TYPE_LOCK() + return res; +} + /* Routines to do a method lookup in the type without looking in the instance dictionary (so we can't use PyObject_GetAttr) but still binding it to the instance. @@ -2533,8 +2642,10 @@ pmerge(PyObject *acc, PyObject **to_merge, Py_ssize_t to_merge_size) } static PyObject * -mro_implementation(PyTypeObject *type) +mro_implementation_unlocked(PyTypeObject *type) { + ASSERT_TYPE_LOCK_HELD(); + if (!_PyType_IsReady(type)) { if (PyType_Ready(type) < 0) return NULL; @@ -2614,6 +2725,16 @@ mro_implementation(PyTypeObject *type) return result; } +static PyObject * +mro_implementation(PyTypeObject *type) +{ + PyObject *mro; + BEGIN_TYPE_LOCK() + mro = mro_implementation_unlocked(type); + END_TYPE_LOCK() + return mro; +} + /*[clinic input] type.mro @@ -2652,7 +2773,7 @@ mro_check(PyTypeObject *type, PyObject *mro) } PyTypeObject *base = (PyTypeObject*)obj; - if (!PyType_IsSubtype(solid, solid_base(base))) { + if (!is_subtype_unlocked(solid, solid_base(base))) { PyErr_Format( PyExc_TypeError, "mro() returned base with unsuitable layout ('%.500s')", @@ -2683,6 +2804,9 @@ mro_invoke(PyTypeObject *type) { PyObject *mro_result; PyObject *new_mro; + + ASSERT_TYPE_LOCK_HELD(); + const int custom = !Py_IS_TYPE(type, &PyType_Type); if (custom) { @@ -2695,7 +2819,7 @@ mro_invoke(PyTypeObject *type) Py_DECREF(mro_meth); } else { - mro_result = mro_implementation(type); + mro_result = mro_implementation_unlocked(type); } if (mro_result == NULL) return NULL; @@ -2742,8 +2866,10 @@ mro_invoke(PyTypeObject *type) - Returns -1 in case of an error. */ static int -mro_internal(PyTypeObject *type, PyObject **p_old_mro) +mro_internal_unlocked(PyTypeObject *type, PyObject **p_old_mro) { + ASSERT_TYPE_LOCK_HELD(); + PyObject *new_mro, *old_mro; int reent; @@ -2788,6 +2914,16 @@ mro_internal(PyTypeObject *type, PyObject **p_old_mro) return 1; } +static int +mro_internal(PyTypeObject *type, PyObject **p_old_mro) +{ + int res; + BEGIN_TYPE_LOCK() + res = mro_internal_unlocked(type, p_old_mro); + END_TYPE_LOCK() + return res; +} + /* Calculate the best base amongst multiple base classes. This is the first one that's on the path to the "solid base". */ @@ -3828,6 +3964,17 @@ type_new_impl(type_new_ctx *ctx) // Put the proper slots in place fixup_slot_dispatchers(type); + if (!_PyDict_HasOnlyStringKeys(type->tp_dict)) { + if (PyErr_WarnFormat( + PyExc_RuntimeWarning, + 1, + "non-string key in the __dict__ of class %.200s", + type->tp_name) == -1) + { + goto error; + } + } + if (type_new_set_names(type) < 0) { goto error; } @@ -4615,6 +4762,9 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) { assert(PyType_Check(type)); + PyObject *res = NULL; + BEGIN_TYPE_LOCK() + PyObject *mro = lookup_tp_mro(type); // The type must be ready assert(mro != NULL); @@ -4634,15 +4784,19 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) PyHeapTypeObject *ht = (PyHeapTypeObject*)super; PyObject *module = ht->ht_module; if (module && _PyModule_GetDef(module) == def) { - return module; + res = module; + break; } } + END_TYPE_LOCK() - PyErr_Format( - PyExc_TypeError, - "PyType_GetModuleByDef: No superclass of '%s' has the given module", - type->tp_name); - return NULL; + if (res == NULL) { + PyErr_Format( + PyExc_TypeError, + "PyType_GetModuleByDef: No superclass of '%s' has the given module", + type->tp_name); + } + return res; } void * @@ -4680,6 +4834,8 @@ PyObject_GetItemData(PyObject *obj) static PyObject * find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) { + ASSERT_TYPE_LOCK_HELD(); + Py_hash_t hash; if (!PyUnicode_CheckExact(name) || (hash = _PyASCIIObject_CAST(name)->hash) == -1) @@ -4749,6 +4905,62 @@ is_dunder_name(PyObject *name) return 0; } +static void +update_cache(struct type_cache_entry *entry, PyObject *name, unsigned int version_tag, PyObject *value) +{ + entry->version = version_tag; + entry->value = value; /* borrowed */ + assert(_PyASCIIObject_CAST(name)->hash != -1); + OBJECT_STAT_INC_COND(type_cache_collisions, entry->name != Py_None && entry->name != name); + // We're releasing this under the lock for simplicity sake because it's always a + // exact unicode object or Py_None so it's safe to do so. + Py_SETREF(entry->name, Py_NewRef(name)); +} + +#if Py_GIL_DISABLED + +#define TYPE_CACHE_IS_UPDATING(sequence) (sequence & 0x01) + +static void +update_cache_gil_disabled(struct type_cache_entry *entry, PyObject *name, + unsigned int version_tag, PyObject *value) +{ + _PySeqLock_LockWrite(&entry->sequence); + + // update the entry + if (entry->name == name && + entry->value == value && + entry->version == version_tag) { + // We raced with another update, bail and restore previous sequence. + _PySeqLock_AbandonWrite(&entry->sequence); + return; + } + + update_cache(entry, name, version_tag, value); + + // Then update sequence to the next valid value + _PySeqLock_UnlockWrite(&entry->sequence); +} + +#endif + +void +_PyTypes_AfterFork(void) +{ +#ifdef Py_GIL_DISABLED + struct type_cache *cache = get_type_cache(); + for (Py_ssize_t i = 0; i < (1 << MCACHE_SIZE_EXP); i++) { + struct type_cache_entry *entry = &cache->hashtable[i]; + if (_PySeqLock_AfterFork(&entry->sequence)) { + // Entry was in the process of updating while forking, clear it... + entry->value = NULL; + Py_SETREF(entry->name, Py_None); + entry->version = 0; + } + } +#endif +} + /* Internal API to look for a name through the MRO. This returns a borrowed reference, and doesn't set an exception! */ PyObject * @@ -4761,6 +4973,27 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) unsigned int h = MCACHE_HASH_METHOD(type, name); struct type_cache *cache = get_type_cache(); struct type_cache_entry *entry = &cache->hashtable[h]; +#ifdef Py_GIL_DISABLED + // synchronize-with other writing threads by doing an acquire load on the sequence + while (1) { + int sequence = _PySeqLock_BeginRead(&entry->sequence); + if (_Py_atomic_load_uint32_relaxed(&entry->version) == type->tp_version_tag && + _Py_atomic_load_ptr_relaxed(&entry->name) == name) { + assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); + OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); + OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); + PyObject *value = _Py_atomic_load_ptr_relaxed(&entry->value); + + // If the sequence is still valid then we're done + if (_PySeqLock_EndRead(&entry->sequence, sequence)) { + return value; + } + } else { + // cache miss + break; + } + } +#else if (entry->version == type->tp_version_tag && entry->name == name) { assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); @@ -4768,13 +5001,27 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); return entry->value; } +#endif OBJECT_STAT_INC_COND(type_cache_misses, !is_dunder_name(name)); OBJECT_STAT_INC_COND(type_cache_dunder_misses, is_dunder_name(name)); /* We may end up clearing live exceptions below, so make sure it's ours. */ assert(!PyErr_Occurred()); + // We need to atomically do the lookup and capture the version before + // anyone else can modify our mro or mutate the type. + + int has_version = 0; + int version = 0; + BEGIN_TYPE_LOCK() res = find_name_in_mro(type, name, &error); + if (MCACHE_CACHEABLE_NAME(name)) { + has_version = assign_version_tag(interp, type); + version = type->tp_version_tag; + assert(!has_version || _PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); + } + END_TYPE_LOCK() + /* Only put NULL results into cache if there was no error. */ if (error) { /* It's not ideal to clear the error condition, @@ -4791,15 +5038,12 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) return NULL; } - if (MCACHE_CACHEABLE_NAME(name) && assign_version_tag(interp, type)) { - h = MCACHE_HASH_METHOD(type, name); - struct type_cache_entry *entry = &cache->hashtable[h]; - entry->version = type->tp_version_tag; - entry->value = res; /* borrowed */ - assert(_PyASCIIObject_CAST(name)->hash != -1); - OBJECT_STAT_INC_COND(type_cache_collisions, entry->name != Py_None && entry->name != name); - assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); - Py_SETREF(entry->name, Py_NewRef(name)); + if (has_version) { +#if Py_GIL_DISABLED + update_cache_gil_disabled(entry, name, version, res); +#else + update_cache(entry, name, version, res); +#endif } return res; } @@ -4962,6 +5206,8 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) /* Will fail in _PyObject_GenericSetAttrWithDict. */ Py_INCREF(name); } + + BEGIN_TYPE_LOCK() res = _PyObject_GenericSetAttrWithDict((PyObject *)type, name, value, NULL); if (res == 0) { /* Clear the VALID_VERSION flag of 'type' and all its @@ -4969,13 +5215,15 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) update_subclasses() recursion in update_slot(), but carefully: they each have their own conditions on which to stop recursing into subclasses. */ - PyType_Modified(type); + type_modified_unlocked(type); if (is_dunder_name(name)) { res = update_slot(type, name); } assert(_PyType_CheckConsistency(type)); } + END_TYPE_LOCK() + Py_DECREF(name); return res; } @@ -6667,7 +6915,7 @@ type_add_method(PyTypeObject *type, PyMethodDef *meth) int err; PyObject *dict = lookup_tp_dict(type); if (!(meth->ml_flags & METH_COEXIST)) { - err = PyDict_SetDefault(dict, name, descr) == NULL; + err = PyDict_SetDefaultRef(dict, name, descr, NULL) < 0; } else { err = PyDict_SetItem(dict, name, descr) < 0; @@ -6715,7 +6963,7 @@ type_add_members(PyTypeObject *type) if (descr == NULL) return -1; - if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { + if (PyDict_SetDefaultRef(dict, PyDescr_NAME(descr), descr, NULL) < 0) { Py_DECREF(descr); return -1; } @@ -6740,7 +6988,7 @@ type_add_getset(PyTypeObject *type) return -1; } - if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { + if (PyDict_SetDefaultRef(dict, PyDescr_NAME(descr), descr, NULL) < 0) { Py_DECREF(descr); return -1; } @@ -6780,28 +7028,28 @@ inherit_special(PyTypeObject *type, PyTypeObject *base) #undef COPYVAL /* Setup fast subclass flags */ - if (PyType_IsSubtype(base, (PyTypeObject*)PyExc_BaseException)) { + if (is_subtype_unlocked(base, (PyTypeObject*)PyExc_BaseException)) { type->tp_flags |= Py_TPFLAGS_BASE_EXC_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyType_Type)) { + else if (is_subtype_unlocked(base, &PyType_Type)) { type->tp_flags |= Py_TPFLAGS_TYPE_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyLong_Type)) { + else if (is_subtype_unlocked(base, &PyLong_Type)) { type->tp_flags |= Py_TPFLAGS_LONG_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyBytes_Type)) { + else if (is_subtype_unlocked(base, &PyBytes_Type)) { type->tp_flags |= Py_TPFLAGS_BYTES_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyUnicode_Type)) { + else if (is_subtype_unlocked(base, &PyUnicode_Type)) { type->tp_flags |= Py_TPFLAGS_UNICODE_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyTuple_Type)) { + else if (is_subtype_unlocked(base, &PyTuple_Type)) { type->tp_flags |= Py_TPFLAGS_TUPLE_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyList_Type)) { + else if (is_subtype_unlocked(base, &PyList_Type)) { type->tp_flags |= Py_TPFLAGS_LIST_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyDict_Type)) { + else if (is_subtype_unlocked(base, &PyDict_Type)) { type->tp_flags |= Py_TPFLAGS_DICT_SUBCLASS; } @@ -7240,6 +7488,8 @@ type_ready_preheader(PyTypeObject *type) static int type_ready_mro(PyTypeObject *type) { + ASSERT_TYPE_LOCK_HELD(); + if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { if (!_Py_IsMainInterpreter(_PyInterpreterState_GET())) { assert(lookup_tp_mro(type) != NULL); @@ -7249,7 +7499,7 @@ type_ready_mro(PyTypeObject *type) } /* Calculate method resolution order */ - if (mro_internal(type, NULL) < 0) { + if (mro_internal_unlocked(type, NULL) < 0) { return -1; } PyObject *mro = lookup_tp_mro(type); @@ -7313,6 +7563,8 @@ inherit_patma_flags(PyTypeObject *type, PyTypeObject *base) { static int type_ready_inherit(PyTypeObject *type) { + ASSERT_TYPE_LOCK_HELD(); + /* Inherit special flags from dominant base */ PyTypeObject *base = type->tp_base; if (base != NULL) { @@ -7505,6 +7757,8 @@ type_ready_post_checks(PyTypeObject *type) static int type_ready(PyTypeObject *type, int rerunbuiltin) { + ASSERT_TYPE_LOCK_HELD(); + _PyObject_ASSERT((PyObject *)type, !is_readying(type)); start_readying(type); @@ -7592,7 +7846,16 @@ PyType_Ready(PyTypeObject *type) type->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; } - return type_ready(type, 0); + int res; + BEGIN_TYPE_LOCK() + if (!(type->tp_flags & Py_TPFLAGS_READY)) { + res = type_ready(type, 0); + } else { + res = 0; + assert(_PyType_CheckConsistency(type)); + } + END_TYPE_LOCK() + return res; } int @@ -7622,7 +7885,10 @@ _PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) static_builtin_state_init(interp, self); - int res = type_ready(self, !ismain); + int res; + BEGIN_TYPE_LOCK(); + res = type_ready(self, !ismain); + END_TYPE_LOCK() if (res < 0) { static_builtin_state_clear(interp, self); } @@ -8024,9 +8290,12 @@ wrap_delitem(PyObject *self, PyObject *args, void *wrapped) https://mail.python.org/pipermail/python-dev/2003-April/034535.html */ static int -hackcheck(PyObject *self, setattrofunc func, const char *what) +hackcheck_unlocked(PyObject *self, setattrofunc func, const char *what) { PyTypeObject *type = Py_TYPE(self); + + ASSERT_TYPE_LOCK_HELD(); + PyObject *mro = lookup_tp_mro(type); if (!mro) { /* Probably ok not to check the call in this case. */ @@ -8068,6 +8337,20 @@ hackcheck(PyObject *self, setattrofunc func, const char *what) return 1; } +static int +hackcheck(PyObject *self, setattrofunc func, const char *what) +{ + if (!PyType_Check(self)) { + return 1; + } + + int res; + BEGIN_TYPE_LOCK(); + res = hackcheck_unlocked(self, func, what); + END_TYPE_LOCK() + return res; +} + static PyObject * wrap_setattr(PyObject *self, PyObject *args, void *wrapped) { @@ -9236,13 +9519,13 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) return -1; } -static int -releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) +static releasebufferproc +releasebuffer_maybe_call_super_unlocked(PyObject *self, Py_buffer *buffer) { PyTypeObject *self_type = Py_TYPE(self); PyObject *mro = lookup_tp_mro(self_type); if (mro == NULL) { - return -1; + return NULL; } assert(PyTuple_Check(mro)); @@ -9256,9 +9539,8 @@ releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) } i++; /* skip self_type */ if (i >= n) - return -1; + return NULL; - releasebufferproc base_releasebuffer = NULL; for (; i < n; i++) { PyObject *obj = PyTuple_GET_ITEM(mro, i); if (!PyType_Check(obj)) { @@ -9268,15 +9550,25 @@ releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) if (base_type->tp_as_buffer != NULL && base_type->tp_as_buffer->bf_releasebuffer != NULL && base_type->tp_as_buffer->bf_releasebuffer != slot_bf_releasebuffer) { - base_releasebuffer = base_type->tp_as_buffer->bf_releasebuffer; - break; + return base_type->tp_as_buffer->bf_releasebuffer; } } + return NULL; +} + +static void +releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) +{ + releasebufferproc base_releasebuffer; + + BEGIN_TYPE_LOCK(); + base_releasebuffer = releasebuffer_maybe_call_super_unlocked(self, buffer); + END_TYPE_LOCK(); + if (base_releasebuffer != NULL) { base_releasebuffer(self, buffer); } - return 0; } static void @@ -9353,11 +9645,7 @@ static void slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) { releasebuffer_call_python(self, buffer); - if (releasebuffer_maybe_call_super(self, buffer) < 0) { - if (PyErr_Occurred()) { - PyErr_WriteUnraisable(self); - } - } + releasebuffer_maybe_call_super(self, buffer); } static PyObject * @@ -9812,6 +10100,8 @@ resolve_slotdups(PyTypeObject *type, PyObject *name) static pytype_slotdef * update_one_slot(PyTypeObject *type, pytype_slotdef *p) { + ASSERT_TYPE_LOCK_HELD(); + PyObject *descr; PyWrapperDescrObject *d; @@ -9860,7 +10150,7 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) d = (PyWrapperDescrObject *)descr; if ((specific == NULL || specific == d->d_wrapped) && d->d_base->wrapper == p->wrapper && - PyType_IsSubtype(type, PyDescr_TYPE(d))) + is_subtype_unlocked(type, PyDescr_TYPE(d))) { specific = d->d_wrapped; } @@ -9925,6 +10215,8 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) static int update_slots_callback(PyTypeObject *type, void *data) { + ASSERT_TYPE_LOCK_HELD(); + pytype_slotdef **pp = (pytype_slotdef **)data; for (; *pp; pp++) { update_one_slot(type, *pp); @@ -9941,6 +10233,7 @@ update_slot(PyTypeObject *type, PyObject *name) pytype_slotdef **pp; int offset; + ASSERT_TYPE_LOCK_HELD(); assert(PyUnicode_CheckExact(name)); assert(PyUnicode_CHECK_INTERNED(name)); @@ -9974,10 +10267,17 @@ update_slot(PyTypeObject *type, PyObject *name) static void fixup_slot_dispatchers(PyTypeObject *type) { + // This lock isn't strictly necessary because the type has not been + // exposed to anyone else yet, but update_ont_slot calls find_name_in_mro + // where we'd like to assert that the tyep is locked. + BEGIN_TYPE_LOCK() + assert(!PyErr_Occurred()); for (pytype_slotdef *p = slotdefs; p->name; ) { p = update_one_slot(type, p); } + + END_TYPE_LOCK() } static void @@ -9985,6 +10285,8 @@ update_all_slots(PyTypeObject* type) { pytype_slotdef *p; + ASSERT_TYPE_LOCK_HELD(); + /* Clear the VALID_VERSION flag of 'type' and all its subclasses. */ PyType_Modified(type); @@ -10257,7 +10559,15 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * PyObject *mro, *res; Py_ssize_t i, n; + BEGIN_TYPE_LOCK(); mro = lookup_tp_mro(su_obj_type); + /* keep a strong reference to mro because su_obj_type->tp_mro can be + replaced during PyDict_GetItemRef(dict, name, &res) and because + another thread can modify it after we end the critical section + below */ + Py_XINCREF(mro); + END_TYPE_LOCK() + if (mro == NULL) return NULL; @@ -10270,12 +10580,11 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * break; } i++; /* skip su->type (if any) */ - if (i >= n) + if (i >= n) { + Py_DECREF(mro); return NULL; + } - /* keep a strong reference to mro because su_obj_type->tp_mro can be - replaced during PyDict_GetItemRef(dict, name, &res) */ - Py_INCREF(mro); do { PyObject *obj = PyTuple_GET_ITEM(mro, i); PyObject *dict = lookup_tp_dict(_PyType_CAST(obj)); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 4b03cc3f4da5fab..0a569a950e88e29 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -996,7 +996,7 @@ resize_compact(PyObject *unicode, Py_ssize_t length) new_size = (struct_size + (length + 1) * char_size); if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { - PyObject_Free(_PyUnicode_UTF8(unicode)); + PyMem_Free(_PyUnicode_UTF8(unicode)); _PyUnicode_UTF8(unicode) = NULL; _PyUnicode_UTF8_LENGTH(unicode) = 0; } @@ -1049,7 +1049,7 @@ resize_inplace(PyObject *unicode, Py_ssize_t length) if (!share_utf8 && _PyUnicode_HAS_UTF8_MEMORY(unicode)) { - PyObject_Free(_PyUnicode_UTF8(unicode)); + PyMem_Free(_PyUnicode_UTF8(unicode)); _PyUnicode_UTF8(unicode) = NULL; _PyUnicode_UTF8_LENGTH(unicode) = 0; } @@ -1590,10 +1590,10 @@ unicode_dealloc(PyObject *unicode) return; } if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { - PyObject_Free(_PyUnicode_UTF8(unicode)); + PyMem_Free(_PyUnicode_UTF8(unicode)); } if (!PyUnicode_IS_COMPACT(unicode) && _PyUnicode_DATA_ANY(unicode)) { - PyObject_Free(_PyUnicode_DATA_ANY(unicode)); + PyMem_Free(_PyUnicode_DATA_ANY(unicode)); } Py_TYPE(unicode)->tp_free(unicode); @@ -5203,7 +5203,7 @@ unicode_fill_utf8(PyObject *unicode) PyBytes_AS_STRING(writer.buffer); Py_ssize_t len = end - start; - char *cache = PyObject_Malloc(len + 1); + char *cache = PyMem_Malloc(len + 1); if (cache == NULL) { _PyBytesWriter_Dealloc(&writer); PyErr_NoMemory(); @@ -14674,7 +14674,7 @@ unicode_subtype_new(PyTypeObject *type, PyObject *unicode) PyErr_NoMemory(); goto onError; } - data = PyObject_Malloc((length + 1) * char_size); + data = PyMem_Malloc((length + 1) * char_size); if (data == NULL) { PyErr_NoMemory(); goto onError; @@ -14894,16 +14894,18 @@ _PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p) PyObject *interned = get_interned_dict(interp); assert(interned != NULL); - PyObject *t = PyDict_SetDefault(interned, s, s); - if (t == NULL) { + PyObject *t; + int res = PyDict_SetDefaultRef(interned, s, s, &t); + if (res < 0) { PyErr_Clear(); return; } - - if (t != s) { - Py_SETREF(*p, Py_NewRef(t)); + else if (res == 1) { + // value was already present (not inserted) + Py_SETREF(*p, t); return; } + Py_DECREF(t); if (_Py_IsImmortal(s)) { // XXX Restrict this to the main interpreter? diff --git a/PC/errmap.h b/PC/errmap.h index a7489ab75c65618..a064ecb80b1ed93 100644 --- a/PC/errmap.h +++ b/PC/errmap.h @@ -129,6 +129,9 @@ winerror_to_errno(int winerror) case ERROR_NO_UNICODE_TRANSLATION: // 1113 return EILSEQ; + case WAIT_TIMEOUT: // 258 + return ETIMEDOUT; + case ERROR_INVALID_FUNCTION: // 1 case ERROR_INVALID_ACCESS: // 12 case ERROR_INVALID_DATA: // 13 diff --git a/PC/launcher2.c b/PC/launcher2.c index e426eccd7000447..139aa61bbe5cc2a 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -1594,6 +1594,7 @@ _registryReadLegacyEnvironment(const SearchInfo *search, HKEY root, EnvironmentI int count = swprintf_s(realTag, tagLength + 4, L"%s-32", env->tag); if (count == -1) { + debug(L"# Failed to generate 32bit tag\n"); free(realTag); return RC_INTERNAL_ERROR; } @@ -1749,10 +1750,18 @@ appxSearch(const SearchInfo *search, EnvironmentInfo **result, const wchar_t *pa exeName = search->windowed ? L"pythonw.exe" : L"python.exe"; } - if (FAILED(SHGetFolderPathW(NULL, CSIDL_LOCAL_APPDATA, NULL, 0, buffer)) || - !join(buffer, MAXLEN, L"Microsoft\\WindowsApps") || + // Failure to get LocalAppData may just mean we're running as a user who + // doesn't have a profile directory. + // In this case, return "not found", but don't fail. + // Chances are they can't launch Store installs anyway. + if (FAILED(SHGetFolderPathW(NULL, CSIDL_LOCAL_APPDATA, NULL, 0, buffer))) { + return RC_NO_PYTHON; + } + + if (!join(buffer, MAXLEN, L"Microsoft\\WindowsApps") || !join(buffer, MAXLEN, packageFamilyName) || !join(buffer, MAXLEN, exeName)) { + debug(L"# Failed to construct App Execution Alias path\n"); return RC_INTERNAL_ERROR; } @@ -1953,6 +1962,7 @@ struct AppxSearchInfo { struct AppxSearchInfo APPX_SEARCH[] = { // Releases made through the Store + { L"PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0", L"3.13", 10 }, { L"PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0", L"3.12", 10 }, { L"PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0", L"3.11", 10 }, { L"PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0", L"3.10", 10 }, @@ -1962,6 +1972,7 @@ struct AppxSearchInfo APPX_SEARCH[] = { // Side-loadable releases. Note that the publisher ID changes whenever we // renew our code-signing certificate, so the newer ID has a higher // priority (lower sortKey) + { L"PythonSoftwareFoundation.Python.3.13_3847v3x7pw1km", L"3.13", 11 }, { L"PythonSoftwareFoundation.Python.3.12_3847v3x7pw1km", L"3.12", 11 }, { L"PythonSoftwareFoundation.Python.3.11_3847v3x7pw1km", L"3.11", 11 }, { L"PythonSoftwareFoundation.Python.3.11_hd69rhyc2wevp", L"3.11", 12 }, @@ -1982,6 +1993,7 @@ collectEnvironments(const SearchInfo *search, EnvironmentInfo **result) EnvironmentInfo *env = NULL; if (!result) { + debug(L"# collectEnvironments() was passed a NULL result\n"); return RC_INTERNAL_ERROR; } *result = NULL; @@ -2042,7 +2054,8 @@ struct StoreSearchInfo { struct StoreSearchInfo STORE_SEARCH[] = { - { L"3", /* 3.11 */ L"9NRWMJP3717K" }, + { L"3", /* 3.12 */ L"9NCVDN91XZQP" }, + { L"3.13", L"9PNRBTZXMB4Z" }, { L"3.12", L"9NCVDN91XZQP" }, { L"3.11", L"9NRWMJP3717K" }, { L"3.10", L"9PJPW5LDXLZ5" }, @@ -2276,6 +2289,7 @@ int selectEnvironment(const SearchInfo *search, EnvironmentInfo *root, EnvironmentInfo **best) { if (!best) { + debug(L"# selectEnvironment() was passed a NULL best\n"); return RC_INTERNAL_ERROR; } if (!root) { diff --git a/PC/pyconfig.h.in b/PC/pyconfig.h.in index 8bbf877a5bb5edc..d72d6282c2806f1 100644 --- a/PC/pyconfig.h.in +++ b/PC/pyconfig.h.in @@ -95,7 +95,12 @@ WIN32 is still required for the locale module. #endif /* Py_BUILD_CORE || Py_BUILD_CORE_BUILTIN || Py_BUILD_CORE_MODULE */ /* Define to 1 if you want to disable the GIL */ -#undef Py_GIL_DISABLED +/* Uncomment the definition for free-threaded builds, or define it manually + * when compiling extension modules. Note that we test with #ifdef, so + * defining as 0 will still disable the GIL. */ +#ifndef Py_GIL_DISABLED +/* #define Py_GIL_DISABLED 1 */ +#endif /* Compiler specific defines */ diff --git a/PC/python3dll.c b/PC/python3dll.c index 07aa84c91f9fc7f..aa6bfe2c4022db0 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -324,6 +324,7 @@ EXPORT_FUNC(PyIter_Send) EXPORT_FUNC(PyList_Append) EXPORT_FUNC(PyList_AsTuple) EXPORT_FUNC(PyList_GetItem) +EXPORT_FUNC(PyList_GetItemRef) EXPORT_FUNC(PyList_GetSlice) EXPORT_FUNC(PyList_Insert) EXPORT_FUNC(PyList_New) @@ -830,6 +831,7 @@ EXPORT_DATA(PyExc_FutureWarning) EXPORT_DATA(PyExc_GeneratorExit) EXPORT_DATA(PyExc_ImportError) EXPORT_DATA(PyExc_ImportWarning) +EXPORT_DATA(PyExc_IncompleteInputError) EXPORT_DATA(PyExc_IndentationError) EXPORT_DATA(PyExc_IndexError) EXPORT_DATA(PyExc_InterruptedError) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index dde801fc0fd525e..3a8a417a6bf47a0 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -191,6 +191,7 @@ <ClCompile Include="..\Python\ast_opt.c" /> <ClCompile Include="..\Python\ast_unparse.c" /> <ClCompile Include="..\Python\bltinmodule.c" /> + <ClCompile Include="..\Python\brc.c" /> <ClCompile Include="..\Python\bootstrap_hash.c" /> <ClCompile Include="..\Python\ceval.c" /> <ClCompile Include="..\Python\codecs.c" /> @@ -224,6 +225,7 @@ <ClCompile Include="..\Python\initconfig.c" /> <ClCompile Include="..\Python\intrinsics.c" /> <ClCompile Include="..\Python\instrumentation.c" /> + <ClCompile Include="..\Python\jit.c" /> <ClCompile Include="..\Python\legacy_tracing.c" /> <ClCompile Include="..\Python\lock.c" /> <ClCompile Include="..\Python\marshal.c" /> @@ -233,6 +235,7 @@ <ClCompile Include="..\Python\object_stack.c" /> <ClCompile Include="..\Python\optimizer.c" /> <ClCompile Include="..\Python\optimizer_analysis.c" /> + <ClCompile Include="..\Python\optimizer_symbols.c" /> <ClCompile Include="..\Python\parking_lot.c" /> <ClCompile Include="..\Python\pathconfig.c" /> <ClCompile Include="..\Python\perf_trampoline.c" /> @@ -251,6 +254,7 @@ <ClCompile Include="..\Python\pythonrun.c" /> <ClCompile Include="..\Python\Python-tokenize.c" /> <ClCompile Include="..\Python\pytime.c" /> + <ClCompile Include="..\Python\qsbr.c" /> <ClCompile Include="..\Python\specialize.c" /> <ClCompile Include="..\Python\structmember.c" /> <ClCompile Include="..\Python\suggestions.c" /> diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 90ccb954b424bc1..5b34440af9322b5 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -46,6 +46,9 @@ <ClCompile Include="..\Python\bltinmodule.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Python\brc.c"> + <Filter>Python</Filter> + </ClCompile> <ClCompile Include="..\Objects\boolobject.c"> <Filter>Source Files</Filter> </ClCompile> @@ -250,6 +253,9 @@ <ClCompile Include="..\Objects\iterobject.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Python\jit.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Objects\listobject.c"> <Filter>Source Files</Filter> </ClCompile> @@ -304,6 +310,9 @@ <ClCompile Include="..\Python\optimizer_analysis.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Python\optimizer_symbols.c"> + <Filter>Python</Filter> + </ClCompile> <ClCompile Include="..\Parser\parser.c"> <Filter>Source Files</Filter> </ClCompile> @@ -370,6 +379,9 @@ <ClCompile Include="..\Python\pytime.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Python\qsbr.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Objects\rangeobject.c"> <Filter>Source Files</Filter> </ClCompile> diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 6911aacab29b979..66df0a61b5b5a66 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -125,6 +125,7 @@ <ClCompile Include="..\Modules\_testcapi\codec.c" /> <ClCompile Include="..\Modules\_testcapi\sys.c" /> <ClCompile Include="..\Modules\_testcapi\hash.c" /> + <ClCompile Include="..\Modules\_testcapi\time.c" /> <ClCompile Include="..\Modules\_testcapi\immortal.c" /> <ClCompile Include="..\Modules\_testcapi\gc.c" /> </ItemGroup> diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 6059959bb9a040f..651eb1d6ba0b7f3 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -105,6 +105,9 @@ <ClCompile Include="..\Modules\_testcapi\hash.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Modules\_testcapi\time.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Modules\_testcapi\gc.c"> <Filter>Source Files</Filter> </ClCompile> diff --git a/PCbuild/build.bat b/PCbuild/build.bat index e61267b5852a8f2..83b50db44670337 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -36,6 +36,7 @@ echo. overrides -c and -d echo. --disable-gil Enable experimental support for running without the GIL. echo. --test-marker Enable the test marker within the build. echo. --regen Regenerate all opcodes, grammar and tokens. +echo. --experimental-jit Enable the experimental just-in-time compiler. echo. echo.Available flags to avoid building certain modules. echo.These flags have no effect if '-e' is not given: @@ -85,6 +86,7 @@ if "%~1"=="--disable-gil" (set UseDisableGil=true) & shift & goto CheckOpts if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts if "%~1"=="-V" shift & goto Version if "%~1"=="--regen" (set Regen=true) & shift & goto CheckOpts +if "%~1"=="--experimental-jit" (set UseJIT=true) & shift & goto CheckOpts rem These use the actual property names used by MSBuild. We could just let rem them in through the environment, but we specify them on the command line rem anyway for visibility so set defaults after this @@ -176,6 +178,7 @@ echo on /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ /p:DisableGil=%UseDisableGil%^ /p:UseTestMarker=%UseTestMarker% %GITProperty%^ + /p:UseJIT=%UseJIT%^ %1 %2 %3 %4 %5 %6 %7 %8 %9 @echo off diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 3919c0592ec00d0..60ce12b725e2331 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -53,12 +53,12 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.11 -set libraries=%libraries% sqlite-3.44.2.0 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.13 +set libraries=%libraries% sqlite-3.45.1.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.1 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.1 set libraries=%libraries% xz-5.2.5 -set libraries=%libraries% zlib-1.2.13 +set libraries=%libraries% zlib-1.3.1 for %%e in (%libraries%) do ( if exist "%EXTERNALS_DIR%\%%e" ( @@ -76,7 +76,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.11 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.13 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.13.1 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index fd5fbc9e910eee4..9c85e5efa4af4ad 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -250,7 +250,7 @@ public override bool Execute() { <VCRuntimeDLL Include="$(VCRuntimeDLL)" /> </ItemGroup> <ItemGroup Condition="$(VCInstallDir) != '' and $(VCRuntimeDLL) == ''"> - <VCRuntimeDLL Include="$(VCRedistDir)\Microsoft.VC*.CRT\vcruntime*.dll" /> + <VCRuntimeDLL Include="$(VCRedistDir)\Microsoft.VC*.CRT\vcruntime*.dll" Exclude="$(VCRedistDir)\Microsoft.VC*.CRT\vcruntime*_threads.dll" /> </ItemGroup> <Warning Text="vcruntime*.dll not found under $(VCRedistDir)." Condition="@(VCRuntimeDLL) == ''" /> diff --git a/PCbuild/python.props b/PCbuild/python.props index e8796081c4eaf37..e21f1f60464bc8c 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -68,17 +68,17 @@ <Import Project="$(ExternalProps)" Condition="$(ExternalProps) != '' and Exists('$(ExternalProps)')" /> <PropertyGroup> - <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.44.2.0\</sqlite3Dir> + <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.45.1.0\</sqlite3Dir> <bz2Dir Condition="$(bz2Dir) == ''">$(ExternalsDir)bzip2-1.0.8\</bz2Dir> <lzmaDir Condition="$(lzmaDir) == ''">$(ExternalsDir)xz-5.2.5\</lzmaDir> <libffiDir Condition="$(libffiDir) == ''">$(ExternalsDir)libffi-3.4.4\</libffiDir> <libffiOutDir Condition="$(libffiOutDir) == ''">$(libffiDir)$(ArchName)\</libffiOutDir> <libffiIncludeDir Condition="$(libffiIncludeDir) == ''">$(libffiOutDir)include</libffiIncludeDir> - <opensslDir Condition="$(opensslDir) == ''">$(ExternalsDir)openssl-3.0.11\</opensslDir> - <opensslOutDir Condition="$(opensslOutDir) == ''">$(ExternalsDir)openssl-bin-3.0.11\$(ArchName)\</opensslOutDir> + <opensslDir Condition="$(opensslDir) == ''">$(ExternalsDir)openssl-3.0.13\</opensslDir> + <opensslOutDir Condition="$(opensslOutDir) == ''">$(ExternalsDir)openssl-bin-3.0.13\$(ArchName)\</opensslOutDir> <opensslIncludeDir Condition="$(opensslIncludeDir) == ''">$(opensslOutDir)include</opensslIncludeDir> <nasmDir Condition="$(nasmDir) == ''">$(ExternalsDir)\nasm-2.11.06\</nasmDir> - <zlibDir Condition="$(zlibDir) == ''">$(ExternalsDir)\zlib-1.2.13\</zlibDir> + <zlibDir Condition="$(zlibDir) == ''">$(ExternalsDir)\zlib-1.3.1\</zlibDir> </PropertyGroup> <PropertyGroup> diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index e0b9fc137457a08..88a4a7c95643098 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -104,6 +104,7 @@ <AdditionalIncludeDirectories Condition="$(IncludeExternals)">$(zlibDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <PreprocessorDefinitions>_USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions Condition="$(IncludeExternals)">_Py_HAVE_ZLIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions Condition="'$(UseJIT)' == 'true'">_Py_JIT;%(PreprocessorDefinitions)</PreprocessorDefinitions> </ClCompile> <Link> <AdditionalDependencies>version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies)</AdditionalDependencies> @@ -205,6 +206,7 @@ <ClInclude Include="..\Include\internal\pycore_ast_state.h" /> <ClInclude Include="..\Include\internal\pycore_atexit.h" /> <ClInclude Include="..\Include\internal\pycore_bitutils.h" /> + <ClInclude Include="..\Include\internal\pycore_brc.h" /> <ClInclude Include="..\Include\internal\pycore_bytes_methods.h" /> <ClInclude Include="..\Include\internal\pycore_bytesobject.h" /> <ClInclude Include="..\Include\internal\pycore_call.h" /> @@ -247,6 +249,7 @@ <ClInclude Include="..\Include\internal\pycore_initconfig.h" /> <ClInclude Include="..\Include\internal\pycore_interp.h" /> <ClInclude Include="..\Include\internal\pycore_intrinsics.h" /> + <ClInclude Include="..\Include\internal\pycore_jit.h" /> <ClInclude Include="..\Include\internal\pycore_list.h" /> <ClInclude Include="..\Include\internal\pycore_llist.h" /> <ClInclude Include="..\Include\internal\pycore_lock.h" /> @@ -263,6 +266,7 @@ <ClInclude Include="..\Include\internal\pycore_parking_lot.h" /> <ClInclude Include="..\Include\internal\pycore_pathconfig.h" /> <ClInclude Include="..\Include\internal\pycore_pyarena.h" /> + <ClInclude Include="..\Include\internal\pycore_pyatomic_ft_wrappers.h" /> <ClInclude Include="..\Include\internal\pycore_pyerrors.h" /> <ClInclude Include="..\Include\internal\pycore_pyhash.h" /> <ClInclude Include="..\Include\internal\pycore_pylifecycle.h" /> @@ -272,6 +276,7 @@ <ClInclude Include="..\Include\internal\pycore_pystats.h" /> <ClInclude Include="..\Include\internal\pycore_pythonrun.h" /> <ClInclude Include="..\Include\internal\pycore_pythread.h" /> + <ClInclude Include="..\Include\internal\pycore_qsbr.h" /> <ClInclude Include="..\Include\internal\pycore_range.h" /> <ClInclude Include="..\Include\internal\pycore_runtime.h" /> <ClInclude Include="..\Include\internal\pycore_runtime_init.h" /> @@ -551,6 +556,7 @@ <ClCompile Include="..\Python\ast_unparse.c" /> <ClCompile Include="..\Python\bltinmodule.c" /> <ClCompile Include="..\Python\bootstrap_hash.c" /> + <ClCompile Include="..\Python\brc.c" /> <ClCompile Include="..\Python\ceval.c" /> <ClCompile Include="..\Python\codecs.c" /> <ClCompile Include="..\Python\compile.c" /> @@ -585,6 +591,7 @@ <ClCompile Include="..\Python\initconfig.c" /> <ClCompile Include="..\Python\intrinsics.c" /> <ClCompile Include="..\Python\instrumentation.c" /> + <ClCompile Include="..\Python\jit.c" /> <ClCompile Include="..\Python\legacy_tracing.c" /> <ClCompile Include="..\Python\lock.c" /> <ClCompile Include="..\Python\marshal.c" /> @@ -594,6 +601,7 @@ <ClCompile Include="..\Python\object_stack.c" /> <ClCompile Include="..\Python\optimizer.c" /> <ClCompile Include="..\Python\optimizer_analysis.c" /> + <ClCompile Include="..\Python\optimizer_symbols.c" /> <ClCompile Include="..\Python\parking_lot.c" /> <ClCompile Include="..\Python\pathconfig.c" /> <ClCompile Include="..\Python\perf_trampoline.c" /> @@ -608,6 +616,7 @@ <ClCompile Include="..\Python\pystrcmp.c" /> <ClCompile Include="..\Python\pystrhex.c" /> <ClCompile Include="..\Python\pystrtod.c" /> + <ClCompile Include="..\Python\qsbr.c" /> <ClCompile Include="..\Python\dtoa.c" /> <ClCompile Include="..\Python\Python-ast.c" /> <ClCompile Include="..\Python\Python-tokenize.c" /> @@ -669,7 +678,7 @@ <OldPyConfigH Condition="Exists('$(IntDir)pyconfig.h')">$([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h'))</OldPyConfigH> </PropertyGroup> <PropertyGroup Condition="$(DisableGil) == 'true'"> - <PyConfigHText>$(PyConfigHText.Replace('#undef Py_GIL_DISABLED', '#define Py_GIL_DISABLED 1'))</PyConfigHText> + <PyConfigHText>$(PyConfigHText.Replace('/* #define Py_GIL_DISABLED 1 */', '#define Py_GIL_DISABLED 1'))</PyConfigHText> </PropertyGroup> <Message Text="Updating pyconfig.h" Condition="$(PyConfigHText.TrimEnd()) != $(OldPyConfigH.TrimEnd())" /> <WriteLinesToFile File="$(IntDir)pyconfig.h" diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index fd79436f5add970..27bd1121663398f 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -546,6 +546,9 @@ <ClInclude Include="..\Include\internal\pycore_bitutils.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_brc.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_bytes_methods.h"> <Filter>Include\internal</Filter> </ClInclude> @@ -669,6 +672,9 @@ <ClInclude Include="..\Include\internal\pycore_intrinsics.h"> <Filter>Include\cpython</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_jit.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_list.h"> <Filter>Include\internal</Filter> </ClInclude> @@ -717,6 +723,9 @@ <ClInclude Include="..\Include\internal\pycore_pyarena.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_pyatomic_ft_wrappers.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_pyerrors.h"> <Filter>Include\internal</Filter> </ClInclude> @@ -744,6 +753,9 @@ <ClInclude Include="..\Include\internal\pycore_pythread.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_qsbr.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_range.h"> <Filter>Include\internal</Filter> </ClInclude> @@ -1250,6 +1262,9 @@ <ClCompile Include="..\Python\bltinmodule.c"> <Filter>Python</Filter> </ClCompile> + <ClCompile Include="..\Python\brc.c"> + <Filter>Python</Filter> + </ClCompile> <ClCompile Include="..\Python\ceval.c"> <Filter>Python</Filter> </ClCompile> @@ -1337,6 +1352,9 @@ <ClCompile Include="..\Python\instrumentation.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Python\jit.c"> + <Filter>Python</Filter> + </ClCompile> <ClCompile Include="..\Python\legacy_tracing.c"> <Filter>Source Files</Filter> </ClCompile> @@ -1364,6 +1382,9 @@ <ClCompile Include="..\Python\optimizer_analysis.c"> <Filter>Python</Filter> </ClCompile> + <ClCompile Include="..\Python\optimizer_symbols.c"> + <Filter>Python</Filter> + </ClCompile> <ClCompile Include="..\Python\parking_lot.c"> <Filter>Python</Filter> </ClCompile> @@ -1406,6 +1427,9 @@ <ClCompile Include="..\Python\pystrtod.c"> <Filter>Python</Filter> </ClCompile> + <ClCompile Include="..\Python\qsbr.c"> + <Filter>Python</Filter> + </ClCompile> <ClCompile Include="..\Python\dtoa.c"> <Filter>Python</Filter> </ClCompile> diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index b9d76515c383f77..387565515fa0b02 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -189,7 +189,7 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.44.2, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.45.1, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ _tkinter diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index cc9469c7ddd726b..f36387231a0d3a9 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -28,11 +28,16 @@ </_TokenOutputs> <_KeywordSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\Tokens" /> <_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" /> + <!-- Taken from _Target._compute_digest in Tools\jit\_targets.py: --> + <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/> + <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/> + <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/> + <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> </ItemGroup> <Target Name="_TouchRegenSources" Condition="$(ForceRegen) == 'true'"> <Message Text="Touching source files to force regeneration" Importance="high" /> - <Touch Files="@(_PegenSources);@(_ASTSources);@(_TokenSources);@(_KeywordOutputs)" + <Touch Files="@(_PegenSources);@(_ASTSources);@(_TokenSources);@(_KeywordOutputs);@(_CasesSources)" AlwaysCreate="False" /> </Target> @@ -77,9 +82,51 @@ WorkingDirectory="$(PySourcePath)" /> </Target> - <Target Name="Regen" + <Target Name="_RegenCases" + Inputs="@(_CasesSources)" Outputs="@(_CasesOutputs)" + DependsOnTargets="FindPythonForBuild"> + <Message Text="Regenerate cases" Importance="high" /> + <Exec Command="$(PythonForBuild) $(PySourcePath)Tools\cases_generator\opcode_id_generator.py $(PySourcePath)Python\bytecodes.c" + WorkingDirectory="$(PySourcePath)" /> + <Exec Command="$(PythonForBuild) $(PySourcePath)Tools\cases_generator\target_generator.py $(PySourcePath)Python\bytecodes.c" + WorkingDirectory="$(PySourcePath)" /> + <Exec Command="$(PythonForBuild) $(PySourcePath)Tools\cases_generator\uop_id_generator.py $(PySourcePath)Python\bytecodes.c" + WorkingDirectory="$(PySourcePath)" /> + <Exec Command="$(PythonForBuild) $(PySourcePath)Tools\cases_generator\py_metadata_generator.py $(PySourcePath)Python\bytecodes.c" + WorkingDirectory="$(PySourcePath)" /> + <Exec Command="$(PythonForBuild) $(PySourcePath)Tools\cases_generator\tier1_generator.py $(PySourcePath)Python\bytecodes.c" + WorkingDirectory="$(PySourcePath)" /> + <Exec Command="$(PythonForBuild) $(PySourcePath)Tools\cases_generator\tier2_generator.py $(PySourcePath)Python\bytecodes.c" + WorkingDirectory="$(PySourcePath)" /> + <Exec Command="$(PythonForBuild) $(PySourcePath)Tools\cases_generator\optimizer_generator.py $(PySourcePath)Python\optimizer_bytecodes.c $(PySourcePath)Python\bytecodes.c" + WorkingDirectory="$(PySourcePath)" /> + <Exec Command="$(PythonForBuild) $(PySourcePath)Tools\cases_generator\opcode_metadata_generator.py $(PySourcePath)Python\bytecodes.c" + WorkingDirectory="$(PySourcePath)" /> + <Exec Command="$(PythonForBuild) $(PySourcePath)Tools\cases_generator\uop_metadata_generator.py $(PySourcePath)Python\bytecodes.c" + WorkingDirectory="$(PySourcePath)" /> + </Target> + + <Target Name="_RegenJIT" + Condition="'$(UseJIT)' == 'true'" + DependsOnTargets="_UpdatePyconfig;FindPythonForBuild" + Inputs="@(_JITSources)" + Outputs="@(_JITOutputs)"> + <PropertyGroup> + <JITArgs Condition="$(Platform) == 'ARM64'">aarch64-pc-windows-msvc</JITArgs> + <JITArgs Condition="$(Platform) == 'Win32'">i686-pc-windows-msvc</JITArgs> + <JITArgs Condition="$(Platform) == 'x64'">x86_64-pc-windows-msvc</JITArgs> + <JITArgs Condition="$(Configuration) == 'Debug'">$(JITArgs) --debug</JITArgs> + </PropertyGroup> + <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\jit\build.py" $(JITArgs)' + WorkingDirectory="$(GeneratedPyConfigDir)"/> + </Target> + + <Target Name="_RegenNoPGUpdate" Condition="$(Configuration) != 'PGUpdate'" DependsOnTargets="_TouchRegenSources;_RegenPegen;_RegenAST_H;_RegenTokens;_RegenKeywords;_RegenGlobalObjects"> + </Target> + + <Target Name="Regen" DependsOnTargets="_RegenNoPGUpdate;_RegenJIT;_RegenCases"> <Message Text="Generated sources are up to date" Importance="high" /> </Target> diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 293f99ae135faac..ac530a5206271f4 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -38,18 +38,18 @@ set regrtestargs=--fast-ci set exe= :CheckOpts -if "%1"=="-O" (set dashO=-O) & shift & goto CheckOpts -if "%1"=="-q" (set qmode=yes) & shift & goto CheckOpts -if "%1"=="-d" (set suffix=_d) & shift & goto CheckOpts +if "%~1"=="-O" (set dashO=-O) & shift & goto CheckOpts +if "%~1"=="-q" (set qmode=yes) & shift & goto CheckOpts +if "%~1"=="-d" (set suffix=_d) & shift & goto CheckOpts rem HACK: Need some way to infer the version number in this script -if "%1"=="--disable-gil" (set pyname=python3.13t) & shift & goto CheckOpts -if "%1"=="-win32" (set prefix=%pcbuild%win32) & shift & goto CheckOpts -if "%1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts -if "%1"=="-amd64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts -if "%1"=="-arm64" (set prefix=%pcbuild%arm64) & shift & goto CheckOpts -if "%1"=="-arm32" (set prefix=%pcbuild%arm32) & shift & goto CheckOpts -if "%1"=="-p" (call :SetPlatform %~2) & shift & shift & goto CheckOpts -if NOT "%1"=="" (set regrtestargs=%regrtestargs% %1) & shift & goto CheckOpts +if "%~1"=="--disable-gil" (set pyname=python3.13t) & shift & goto CheckOpts +if "%~1"=="-win32" (set prefix=%pcbuild%win32) & shift & goto CheckOpts +if "%~1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts +if "%~1"=="-amd64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts +if "%~1"=="-arm64" (set prefix=%pcbuild%arm64) & shift & goto CheckOpts +if "%~1"=="-arm32" (set prefix=%pcbuild%arm32) & shift & goto CheckOpts +if "%~1"=="-p" (call :SetPlatform %~2) & shift & shift & goto CheckOpts +if NOT "%~1"=="" (set regrtestargs=%regrtestargs% %~1) & shift & goto CheckOpts if not defined prefix set prefix=%pcbuild%amd64 set exe=%prefix%\%pyname%%suffix%.exe diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index ce92672bf00776a..865fd76acf697d8 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -15,6 +15,13 @@ MAX_COL = 80 AUTOGEN_MESSAGE = "// File automatically generated by {}.\n\n" +builtin_type_to_c_type = { + "identifier": "PyUnicode_Type", + "string": "PyUnicode_Type", + "int": "PyLong_Type", + "constant": "PyBaseObject_Type", +} + def get_c_type(name): """Return a string for the C name of the type. @@ -764,6 +771,67 @@ def visitConstructor(self, cons, name): self.emit("};",0) +class AnnotationsVisitor(PickleVisitor): + def visitModule(self, mod): + self.file.write(textwrap.dedent(''' + static int + add_ast_annotations(struct ast_state *state) + { + bool cond; + ''')) + for dfn in mod.dfns: + self.visit(dfn) + self.file.write(textwrap.dedent(''' + return 1; + } + ''')) + + def visitProduct(self, prod, name): + self.emit_annotations(name, prod.fields) + + def visitSum(self, sum, name): + for t in sum.types: + self.visitConstructor(t, name) + + def visitConstructor(self, cons, name): + self.emit_annotations(cons.name, cons.fields) + + def emit_annotations(self, name, fields): + self.emit(f"PyObject *{name}_annotations = PyDict_New();", 1) + self.emit(f"if (!{name}_annotations) return 0;", 1) + for field in fields: + self.emit("{", 1) + if field.type in builtin_type_to_c_type: + self.emit(f"PyObject *type = (PyObject *)&{builtin_type_to_c_type[field.type]};", 2) + else: + self.emit(f"PyObject *type = state->{field.type}_type;", 2) + if field.opt: + self.emit("type = _Py_union_type_or(type, Py_None);", 2) + self.emit("cond = type != NULL;", 2) + self.emit_annotations_error(name, 2) + elif field.seq: + self.emit("type = Py_GenericAlias((PyObject *)&PyList_Type, type);", 2) + self.emit("cond = type != NULL;", 2) + self.emit_annotations_error(name, 2) + else: + self.emit("Py_INCREF(type);", 2) + self.emit(f"cond = PyDict_SetItemString({name}_annotations, \"{field.name}\", type) == 0;", 2) + self.emit("Py_DECREF(type);", 2) + self.emit_annotations_error(name, 2) + self.emit("}", 1) + self.emit(f'cond = PyObject_SetAttrString(state->{name}_type, "_field_types", {name}_annotations) == 0;', 1) + self.emit_annotations_error(name, 1) + self.emit(f'cond = PyObject_SetAttrString(state->{name}_type, "__annotations__", {name}_annotations) == 0;', 1) + self.emit_annotations_error(name, 1) + self.emit(f"Py_DECREF({name}_annotations);", 1) + + def emit_annotations_error(self, name, depth): + self.emit("if (!cond) {", depth) + self.emit(f"Py_DECREF({name}_annotations);", depth + 1) + self.emit("return 0;", depth + 1) + self.emit("}", depth) + + class PyTypesVisitor(PickleVisitor): def visitModule(self, mod): @@ -812,7 +880,7 @@ def visitModule(self, mod): Py_ssize_t i, numfields = 0; int res = -1; - PyObject *key, *value, *fields; + PyObject *key, *value, *fields, *remaining_fields = NULL; if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { goto cleanup; } @@ -821,6 +889,13 @@ def visitModule(self, mod): if (numfields == -1) { goto cleanup; } + remaining_fields = PySet_New(fields); + } + else { + remaining_fields = PySet_New(NULL); + } + if (remaining_fields == NULL) { + goto cleanup; } res = 0; /* if no error occurs, this stays 0 to the end */ @@ -840,6 +915,11 @@ def visitModule(self, mod): goto cleanup; } res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i)); + if (PySet_Discard(remaining_fields, name) < 0) { + res = -1; + Py_DECREF(name); + goto cleanup; + } Py_DECREF(name); if (res < 0) { goto cleanup; @@ -852,13 +932,14 @@ def visitModule(self, mod): if (contains == -1) { res = -1; goto cleanup; - } else if (contains == 1) { - Py_ssize_t p = PySequence_Index(fields, key); + } + else if (contains == 1) { + int p = PySet_Discard(remaining_fields, key); if (p == -1) { res = -1; goto cleanup; } - if (p < PyTuple_GET_SIZE(args)) { + if (p == 0) { PyErr_Format(PyExc_TypeError, "%.400s got multiple values for argument '%U'", Py_TYPE(self)->tp_name, key); @@ -866,15 +947,91 @@ def visitModule(self, mod): goto cleanup; } } + else if ( + PyUnicode_CompareWithASCIIString(key, "lineno") != 0 && + PyUnicode_CompareWithASCIIString(key, "col_offset") != 0 && + PyUnicode_CompareWithASCIIString(key, "end_lineno") != 0 && + PyUnicode_CompareWithASCIIString(key, "end_col_offset") != 0 + ) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ got an unexpected keyword argument '%U'. " + "Support for arbitrary keyword arguments is deprecated " + "and will be removed in Python 3.15.", + Py_TYPE(self)->tp_name, key + ) < 0) { + res = -1; + goto cleanup; + } + } res = PyObject_SetAttr(self, key, value); if (res < 0) { goto cleanup; } } } + Py_ssize_t size = PySet_Size(remaining_fields); + PyObject *field_types = NULL, *remaining_list = NULL; + if (size > 0) { + if (!PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), &_Py_ID(_field_types), + &field_types)) { + res = -1; + goto cleanup; + } + remaining_list = PySequence_List(remaining_fields); + if (!remaining_list) { + goto set_remaining_cleanup; + } + for (Py_ssize_t i = 0; i < size; i++) { + PyObject *name = PyList_GET_ITEM(remaining_list, i); + PyObject *type = PyDict_GetItemWithError(field_types, name); + if (!type) { + if (!PyErr_Occurred()) { + PyErr_SetObject(PyExc_KeyError, name); + } + goto set_remaining_cleanup; + } + if (_PyUnion_Check(type)) { + // optional field + // do nothing, we'll have set a None default on the class + } + else if (Py_IS_TYPE(type, &Py_GenericAliasType)) { + // list field + PyObject *empty = PyList_New(0); + if (!empty) { + goto set_remaining_cleanup; + } + res = PyObject_SetAttr(self, name, empty); + Py_DECREF(empty); + if (res < 0) { + goto set_remaining_cleanup; + } + } + else { + // simple field (e.g., identifier) + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ missing 1 required positional argument: '%U'. " + "This will become an error in Python 3.15.", + Py_TYPE(self)->tp_name, name + ) < 0) { + res = -1; + goto cleanup; + } + } + } + Py_DECREF(remaining_list); + Py_DECREF(field_types); + } cleanup: Py_XDECREF(fields); + Py_XDECREF(remaining_fields); return res; + set_remaining_cleanup: + Py_XDECREF(remaining_list); + Py_XDECREF(field_types); + res = -1; + goto cleanup; } /* Pickling support */ @@ -886,14 +1043,75 @@ def visitModule(self, mod): return NULL; } - PyObject *dict; + PyObject *dict = NULL, *fields = NULL, *remaining_fields = NULL, + *remaining_dict = NULL, *positional_args = NULL; if (PyObject_GetOptionalAttr(self, state->__dict__, &dict) < 0) { return NULL; } + PyObject *result = NULL; if (dict) { - return Py_BuildValue("O()N", Py_TYPE(self), dict); + // Serialize the fields as positional args if possible, because if we + // serialize them as a dict, during unpickling they are set only *after* + // the object is constructed, which will now trigger a DeprecationWarning + // if the AST type has required fields. + if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { + goto cleanup; + } + if (fields) { + Py_ssize_t numfields = PySequence_Size(fields); + if (numfields == -1) { + Py_DECREF(dict); + goto cleanup; + } + remaining_dict = PyDict_Copy(dict); + Py_DECREF(dict); + if (!remaining_dict) { + goto cleanup; + } + positional_args = PyList_New(0); + if (!positional_args) { + goto cleanup; + } + for (Py_ssize_t i = 0; i < numfields; i++) { + PyObject *name = PySequence_GetItem(fields, i); + if (!name) { + goto cleanup; + } + PyObject *value = PyDict_GetItemWithError(remaining_dict, name); + if (!value) { + if (PyErr_Occurred()) { + goto cleanup; + } + break; + } + if (PyList_Append(positional_args, value) < 0) { + goto cleanup; + } + if (PyDict_DelItem(remaining_dict, name) < 0) { + goto cleanup; + } + Py_DECREF(name); + } + PyObject *args_tuple = PyList_AsTuple(positional_args); + if (!args_tuple) { + goto cleanup; + } + result = Py_BuildValue("ONO", Py_TYPE(self), args_tuple, + remaining_dict); + } + else { + result = Py_BuildValue("O()N", Py_TYPE(self), dict); + } + } + else { + result = Py_BuildValue("O()", Py_TYPE(self)); } - return Py_BuildValue("O()", Py_TYPE(self)); +cleanup: + Py_XDECREF(fields); + Py_XDECREF(remaining_fields); + Py_XDECREF(remaining_dict); + Py_XDECREF(positional_args); + return result; } static PyMemberDef ast_type_members[] = { @@ -1117,6 +1335,9 @@ def visitModule(self, mod): for dfn in mod.dfns: self.visit(dfn) self.file.write(textwrap.dedent(''' + if (!add_ast_annotations(state)) { + return -1; + } return 0; } ''')) @@ -1534,6 +1755,8 @@ def generate_module_def(mod, metadata, f, internal_h): #include "pycore_lock.h" // _PyOnceFlag #include "pycore_interp.h" // _PyInterpreterState.ast #include "pycore_pystate.h" // _PyInterpreterState_GET() + #include "pycore_unionobject.h" // _Py_union_type_or + #include "structmember.h" #include <stddef.h> struct validator { @@ -1651,6 +1874,7 @@ def write_source(mod, metadata, f, internal_h_file): v = ChainOfVisitors( SequenceConstructorVisitor(f), PyTypesDeclareVisitor(f), + AnnotationsVisitor(f), PyTypesVisitor(f), Obj2ModPrototypeVisitor(f), FunctionVisitor(f), diff --git a/Parser/parser.c b/Parser/parser.c index 779b18e9650e9f5..f1170c261974520 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -6603,7 +6603,7 @@ with_stmt_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = CHECK_VERSION ( stmt_ty , 9 , "Parenthesized context managers are" , _PyAST_With ( a , b , NEW_TYPE_COMMENT ( p , tc ) , EXTRA ) ); + _res = _PyAST_With ( a , b , NEW_TYPE_COMMENT ( p , tc ) , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; diff --git a/Parser/pegen.c b/Parser/pegen.c index 7766253a76066f0..3d3e64559403b16 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -844,7 +844,7 @@ _PyPegen_run_parser(Parser *p) if (res == NULL) { if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) && _is_end_of_source(p)) { PyErr_Clear(); - return RAISE_SYNTAX_ERROR("incomplete input"); + return _PyPegen_raise_error(p, PyExc_IncompleteInputError, 0, "incomplete input"); } if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_SyntaxError)) { return NULL; diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index e15673d02dd3b05..e8f11a67e50fa0a 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -369,20 +369,18 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, Py_ssize_t col_number = col_offset; Py_ssize_t end_col_number = end_col_offset; - if (p->tok->encoding != NULL) { - col_number = _PyPegen_byte_offset_to_character_offset(error_line, col_offset); - if (col_number < 0) { + col_number = _PyPegen_byte_offset_to_character_offset(error_line, col_offset); + if (col_number < 0) { + goto error; + } + + if (end_col_offset > 0) { + end_col_number = _PyPegen_byte_offset_to_character_offset(error_line, end_col_offset); + if (end_col_number < 0) { goto error; } - if (end_col_number > 0) { - Py_ssize_t end_col_offset = _PyPegen_byte_offset_to_character_offset(error_line, end_col_number); - if (end_col_offset < 0) { - goto error; - } else { - end_col_number = end_col_offset; - } - } } + tmp = Py_BuildValue("(OnnNnn)", p->tok->filename, lineno, col_number, error_line, end_lineno, end_col_number); if (!tmp) { goto error; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index d77e986ba067a3e..46387493214829f 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -7,6 +7,8 @@ #include "pycore_lock.h" // _PyOnceFlag #include "pycore_interp.h" // _PyInterpreterState.ast #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_unionobject.h" // _Py_union_type_or +#include "structmember.h" #include <stddef.h> struct validator { @@ -816,6 +818,4170 @@ static const char * const TypeVarTuple_fields[]={ }; +static int +add_ast_annotations(struct ast_state *state) +{ + bool cond; + PyObject *Module_annotations = PyDict_New(); + if (!Module_annotations) return 0; + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + cond = PyDict_SetItemString(Module_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + } + { + PyObject *type = state->type_ignore_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + cond = PyDict_SetItemString(Module_annotations, "type_ignores", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Module_type, "_field_types", + Module_annotations) == 0; + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Module_type, "__annotations__", + Module_annotations) == 0; + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + Py_DECREF(Module_annotations); + PyObject *Interactive_annotations = PyDict_New(); + if (!Interactive_annotations) return 0; + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Interactive_annotations); + return 0; + } + cond = PyDict_SetItemString(Interactive_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Interactive_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Interactive_type, "_field_types", + Interactive_annotations) == 0; + if (!cond) { + Py_DECREF(Interactive_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Interactive_type, "__annotations__", + Interactive_annotations) == 0; + if (!cond) { + Py_DECREF(Interactive_annotations); + return 0; + } + Py_DECREF(Interactive_annotations); + PyObject *Expression_annotations = PyDict_New(); + if (!Expression_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Expression_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Expression_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Expression_type, "_field_types", + Expression_annotations) == 0; + if (!cond) { + Py_DECREF(Expression_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Expression_type, "__annotations__", + Expression_annotations) == 0; + if (!cond) { + Py_DECREF(Expression_annotations); + return 0; + } + Py_DECREF(Expression_annotations); + PyObject *FunctionType_annotations = PyDict_New(); + if (!FunctionType_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionType_annotations, "argtypes", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(FunctionType_annotations, "returns", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->FunctionType_type, "_field_types", + FunctionType_annotations) == 0; + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->FunctionType_type, "__annotations__", + FunctionType_annotations) == 0; + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + Py_DECREF(FunctionType_annotations); + PyObject *FunctionDef_annotations = PyDict_New(); + if (!FunctionDef_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(FunctionDef_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->arguments_type; + Py_INCREF(type); + cond = PyDict_SetItemString(FunctionDef_annotations, "args", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "decorator_list", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "returns", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "type_comment", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->type_param_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "type_params", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->FunctionDef_type, "_field_types", + FunctionDef_annotations) == 0; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->FunctionDef_type, "__annotations__", + FunctionDef_annotations) == 0; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + Py_DECREF(FunctionDef_annotations); + PyObject *AsyncFunctionDef_annotations = PyDict_New(); + if (!AsyncFunctionDef_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, "name", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->arguments_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, "args", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, "body", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, + "decorator_list", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, "returns", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, + "type_comment", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->type_param_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, + "type_params", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AsyncFunctionDef_type, "_field_types", + AsyncFunctionDef_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AsyncFunctionDef_type, + "__annotations__", + AsyncFunctionDef_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + Py_DECREF(AsyncFunctionDef_annotations); + PyObject *ClassDef_annotations = PyDict_New(); + if (!ClassDef_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(ClassDef_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "bases", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->keyword_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "keywords", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "decorator_list", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->type_param_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "type_params", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ClassDef_type, "_field_types", + ClassDef_annotations) == 0; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ClassDef_type, "__annotations__", + ClassDef_annotations) == 0; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + Py_DECREF(ClassDef_annotations); + PyObject *Return_annotations = PyDict_New(); + if (!Return_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Return_annotations); + return 0; + } + cond = PyDict_SetItemString(Return_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Return_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Return_type, "_field_types", + Return_annotations) == 0; + if (!cond) { + Py_DECREF(Return_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Return_type, "__annotations__", + Return_annotations) == 0; + if (!cond) { + Py_DECREF(Return_annotations); + return 0; + } + Py_DECREF(Return_annotations); + PyObject *Delete_annotations = PyDict_New(); + if (!Delete_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Delete_annotations); + return 0; + } + cond = PyDict_SetItemString(Delete_annotations, "targets", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Delete_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Delete_type, "_field_types", + Delete_annotations) == 0; + if (!cond) { + Py_DECREF(Delete_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Delete_type, "__annotations__", + Delete_annotations) == 0; + if (!cond) { + Py_DECREF(Delete_annotations); + return 0; + } + Py_DECREF(Delete_annotations); + PyObject *Assign_annotations = PyDict_New(); + if (!Assign_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + cond = PyDict_SetItemString(Assign_annotations, "targets", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Assign_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + cond = PyDict_SetItemString(Assign_annotations, "type_comment", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Assign_type, "_field_types", + Assign_annotations) == 0; + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Assign_type, "__annotations__", + Assign_annotations) == 0; + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + Py_DECREF(Assign_annotations); + PyObject *TypeAlias_annotations = PyDict_New(); + if (!TypeAlias_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeAlias_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + } + { + PyObject *type = state->type_param_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + cond = PyDict_SetItemString(TypeAlias_annotations, "type_params", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeAlias_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TypeAlias_type, "_field_types", + TypeAlias_annotations) == 0; + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TypeAlias_type, "__annotations__", + TypeAlias_annotations) == 0; + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + Py_DECREF(TypeAlias_annotations); + PyObject *AugAssign_annotations = PyDict_New(); + if (!AugAssign_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AugAssign_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + } + { + PyObject *type = state->operator_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AugAssign_annotations, "op", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AugAssign_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AugAssign_type, "_field_types", + AugAssign_annotations) == 0; + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AugAssign_type, "__annotations__", + AugAssign_annotations) == 0; + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + Py_DECREF(AugAssign_annotations); + PyObject *AnnAssign_annotations = PyDict_New(); + if (!AnnAssign_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AnnAssign_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AnnAssign_annotations, "annotation", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + cond = PyDict_SetItemString(AnnAssign_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyLong_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(AnnAssign_annotations, "simple", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AnnAssign_type, "_field_types", + AnnAssign_annotations) == 0; + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AnnAssign_type, "__annotations__", + AnnAssign_annotations) == 0; + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + Py_DECREF(AnnAssign_annotations); + PyObject *For_annotations = PyDict_New(); + if (!For_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(For_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(For_annotations, "iter", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + cond = PyDict_SetItemString(For_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + cond = PyDict_SetItemString(For_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + cond = PyDict_SetItemString(For_annotations, "type_comment", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->For_type, "_field_types", + For_annotations) == 0; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->For_type, "__annotations__", + For_annotations) == 0; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + Py_DECREF(For_annotations); + PyObject *AsyncFor_annotations = PyDict_New(); + if (!AsyncFor_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AsyncFor_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AsyncFor_annotations, "iter", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFor_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFor_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFor_annotations, "type_comment", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AsyncFor_type, "_field_types", + AsyncFor_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AsyncFor_type, "__annotations__", + AsyncFor_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + Py_DECREF(AsyncFor_annotations); + PyObject *While_annotations = PyDict_New(); + if (!While_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(While_annotations, "test", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + cond = PyDict_SetItemString(While_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + cond = PyDict_SetItemString(While_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->While_type, "_field_types", + While_annotations) == 0; + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->While_type, "__annotations__", + While_annotations) == 0; + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + Py_DECREF(While_annotations); + PyObject *If_annotations = PyDict_New(); + if (!If_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(If_annotations, "test", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + cond = PyDict_SetItemString(If_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + cond = PyDict_SetItemString(If_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->If_type, "_field_types", + If_annotations) == 0; + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->If_type, "__annotations__", + If_annotations) == 0; + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + Py_DECREF(If_annotations); + PyObject *With_annotations = PyDict_New(); + if (!With_annotations) return 0; + { + PyObject *type = state->withitem_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + cond = PyDict_SetItemString(With_annotations, "items", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + cond = PyDict_SetItemString(With_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + cond = PyDict_SetItemString(With_annotations, "type_comment", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->With_type, "_field_types", + With_annotations) == 0; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->With_type, "__annotations__", + With_annotations) == 0; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + Py_DECREF(With_annotations); + PyObject *AsyncWith_annotations = PyDict_New(); + if (!AsyncWith_annotations) return 0; + { + PyObject *type = state->withitem_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncWith_annotations, "items", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncWith_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncWith_annotations, "type_comment", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AsyncWith_type, "_field_types", + AsyncWith_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AsyncWith_type, "__annotations__", + AsyncWith_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + Py_DECREF(AsyncWith_annotations); + PyObject *Match_annotations = PyDict_New(); + if (!Match_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Match_annotations, "subject", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + } + { + PyObject *type = state->match_case_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + cond = PyDict_SetItemString(Match_annotations, "cases", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Match_type, "_field_types", + Match_annotations) == 0; + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Match_type, "__annotations__", + Match_annotations) == 0; + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + Py_DECREF(Match_annotations); + PyObject *Raise_annotations = PyDict_New(); + if (!Raise_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + cond = PyDict_SetItemString(Raise_annotations, "exc", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + cond = PyDict_SetItemString(Raise_annotations, "cause", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Raise_type, "_field_types", + Raise_annotations) == 0; + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Raise_type, "__annotations__", + Raise_annotations) == 0; + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + Py_DECREF(Raise_annotations); + PyObject *Try_annotations = PyDict_New(); + if (!Try_annotations) return 0; + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyDict_SetItemString(Try_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + } + { + PyObject *type = state->excepthandler_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyDict_SetItemString(Try_annotations, "handlers", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyDict_SetItemString(Try_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyDict_SetItemString(Try_annotations, "finalbody", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Try_type, "_field_types", + Try_annotations) == 0; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Try_type, "__annotations__", + Try_annotations) == 0; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + Py_DECREF(Try_annotations); + PyObject *TryStar_annotations = PyDict_New(); + if (!TryStar_annotations) return 0; + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyDict_SetItemString(TryStar_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + } + { + PyObject *type = state->excepthandler_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyDict_SetItemString(TryStar_annotations, "handlers", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyDict_SetItemString(TryStar_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyDict_SetItemString(TryStar_annotations, "finalbody", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TryStar_type, "_field_types", + TryStar_annotations) == 0; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TryStar_type, "__annotations__", + TryStar_annotations) == 0; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + Py_DECREF(TryStar_annotations); + PyObject *Assert_annotations = PyDict_New(); + if (!Assert_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Assert_annotations, "test", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + cond = PyDict_SetItemString(Assert_annotations, "msg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Assert_type, "_field_types", + Assert_annotations) == 0; + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Assert_type, "__annotations__", + Assert_annotations) == 0; + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + Py_DECREF(Assert_annotations); + PyObject *Import_annotations = PyDict_New(); + if (!Import_annotations) return 0; + { + PyObject *type = state->alias_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + cond = PyDict_SetItemString(Import_annotations, "names", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Import_type, "_field_types", + Import_annotations) == 0; + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Import_type, "__annotations__", + Import_annotations) == 0; + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + Py_DECREF(Import_annotations); + PyObject *ImportFrom_annotations = PyDict_New(); + if (!ImportFrom_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + cond = PyDict_SetItemString(ImportFrom_annotations, "module", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + } + { + PyObject *type = state->alias_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + cond = PyDict_SetItemString(ImportFrom_annotations, "names", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyLong_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + cond = PyDict_SetItemString(ImportFrom_annotations, "level", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ImportFrom_type, "_field_types", + ImportFrom_annotations) == 0; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ImportFrom_type, "__annotations__", + ImportFrom_annotations) == 0; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + Py_DECREF(ImportFrom_annotations); + PyObject *Global_annotations = PyDict_New(); + if (!Global_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Global_annotations); + return 0; + } + cond = PyDict_SetItemString(Global_annotations, "names", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Global_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Global_type, "_field_types", + Global_annotations) == 0; + if (!cond) { + Py_DECREF(Global_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Global_type, "__annotations__", + Global_annotations) == 0; + if (!cond) { + Py_DECREF(Global_annotations); + return 0; + } + Py_DECREF(Global_annotations); + PyObject *Nonlocal_annotations = PyDict_New(); + if (!Nonlocal_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Nonlocal_annotations); + return 0; + } + cond = PyDict_SetItemString(Nonlocal_annotations, "names", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Nonlocal_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Nonlocal_type, "_field_types", + Nonlocal_annotations) == 0; + if (!cond) { + Py_DECREF(Nonlocal_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Nonlocal_type, "__annotations__", + Nonlocal_annotations) == 0; + if (!cond) { + Py_DECREF(Nonlocal_annotations); + return 0; + } + Py_DECREF(Nonlocal_annotations); + PyObject *Expr_annotations = PyDict_New(); + if (!Expr_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Expr_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Expr_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Expr_type, "_field_types", + Expr_annotations) == 0; + if (!cond) { + Py_DECREF(Expr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Expr_type, "__annotations__", + Expr_annotations) == 0; + if (!cond) { + Py_DECREF(Expr_annotations); + return 0; + } + Py_DECREF(Expr_annotations); + PyObject *Pass_annotations = PyDict_New(); + if (!Pass_annotations) return 0; + cond = PyObject_SetAttrString(state->Pass_type, "_field_types", + Pass_annotations) == 0; + if (!cond) { + Py_DECREF(Pass_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Pass_type, "__annotations__", + Pass_annotations) == 0; + if (!cond) { + Py_DECREF(Pass_annotations); + return 0; + } + Py_DECREF(Pass_annotations); + PyObject *Break_annotations = PyDict_New(); + if (!Break_annotations) return 0; + cond = PyObject_SetAttrString(state->Break_type, "_field_types", + Break_annotations) == 0; + if (!cond) { + Py_DECREF(Break_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Break_type, "__annotations__", + Break_annotations) == 0; + if (!cond) { + Py_DECREF(Break_annotations); + return 0; + } + Py_DECREF(Break_annotations); + PyObject *Continue_annotations = PyDict_New(); + if (!Continue_annotations) return 0; + cond = PyObject_SetAttrString(state->Continue_type, "_field_types", + Continue_annotations) == 0; + if (!cond) { + Py_DECREF(Continue_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Continue_type, "__annotations__", + Continue_annotations) == 0; + if (!cond) { + Py_DECREF(Continue_annotations); + return 0; + } + Py_DECREF(Continue_annotations); + PyObject *BoolOp_annotations = PyDict_New(); + if (!BoolOp_annotations) return 0; + { + PyObject *type = state->boolop_type; + Py_INCREF(type); + cond = PyDict_SetItemString(BoolOp_annotations, "op", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + cond = PyDict_SetItemString(BoolOp_annotations, "values", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->BoolOp_type, "_field_types", + BoolOp_annotations) == 0; + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BoolOp_type, "__annotations__", + BoolOp_annotations) == 0; + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + Py_DECREF(BoolOp_annotations); + PyObject *NamedExpr_annotations = PyDict_New(); + if (!NamedExpr_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(NamedExpr_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(NamedExpr_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(NamedExpr_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(NamedExpr_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->NamedExpr_type, "_field_types", + NamedExpr_annotations) == 0; + if (!cond) { + Py_DECREF(NamedExpr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->NamedExpr_type, "__annotations__", + NamedExpr_annotations) == 0; + if (!cond) { + Py_DECREF(NamedExpr_annotations); + return 0; + } + Py_DECREF(NamedExpr_annotations); + PyObject *BinOp_annotations = PyDict_New(); + if (!BinOp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(BinOp_annotations, "left", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + } + { + PyObject *type = state->operator_type; + Py_INCREF(type); + cond = PyDict_SetItemString(BinOp_annotations, "op", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(BinOp_annotations, "right", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->BinOp_type, "_field_types", + BinOp_annotations) == 0; + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BinOp_type, "__annotations__", + BinOp_annotations) == 0; + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + Py_DECREF(BinOp_annotations); + PyObject *UnaryOp_annotations = PyDict_New(); + if (!UnaryOp_annotations) return 0; + { + PyObject *type = state->unaryop_type; + Py_INCREF(type); + cond = PyDict_SetItemString(UnaryOp_annotations, "op", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(UnaryOp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(UnaryOp_annotations, "operand", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(UnaryOp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->UnaryOp_type, "_field_types", + UnaryOp_annotations) == 0; + if (!cond) { + Py_DECREF(UnaryOp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->UnaryOp_type, "__annotations__", + UnaryOp_annotations) == 0; + if (!cond) { + Py_DECREF(UnaryOp_annotations); + return 0; + } + Py_DECREF(UnaryOp_annotations); + PyObject *Lambda_annotations = PyDict_New(); + if (!Lambda_annotations) return 0; + { + PyObject *type = state->arguments_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Lambda_annotations, "args", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Lambda_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Lambda_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Lambda_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Lambda_type, "_field_types", + Lambda_annotations) == 0; + if (!cond) { + Py_DECREF(Lambda_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Lambda_type, "__annotations__", + Lambda_annotations) == 0; + if (!cond) { + Py_DECREF(Lambda_annotations); + return 0; + } + Py_DECREF(Lambda_annotations); + PyObject *IfExp_annotations = PyDict_New(); + if (!IfExp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(IfExp_annotations, "test", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(IfExp_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(IfExp_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->IfExp_type, "_field_types", + IfExp_annotations) == 0; + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->IfExp_type, "__annotations__", + IfExp_annotations) == 0; + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + Py_DECREF(IfExp_annotations); + PyObject *Dict_annotations = PyDict_New(); + if (!Dict_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + cond = PyDict_SetItemString(Dict_annotations, "keys", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + cond = PyDict_SetItemString(Dict_annotations, "values", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Dict_type, "_field_types", + Dict_annotations) == 0; + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Dict_type, "__annotations__", + Dict_annotations) == 0; + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + Py_DECREF(Dict_annotations); + PyObject *Set_annotations = PyDict_New(); + if (!Set_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Set_annotations); + return 0; + } + cond = PyDict_SetItemString(Set_annotations, "elts", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Set_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Set_type, "_field_types", + Set_annotations) == 0; + if (!cond) { + Py_DECREF(Set_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Set_type, "__annotations__", + Set_annotations) == 0; + if (!cond) { + Py_DECREF(Set_annotations); + return 0; + } + Py_DECREF(Set_annotations); + PyObject *ListComp_annotations = PyDict_New(); + if (!ListComp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(ListComp_annotations, "elt", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + } + { + PyObject *type = state->comprehension_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + cond = PyDict_SetItemString(ListComp_annotations, "generators", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ListComp_type, "_field_types", + ListComp_annotations) == 0; + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ListComp_type, "__annotations__", + ListComp_annotations) == 0; + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + Py_DECREF(ListComp_annotations); + PyObject *SetComp_annotations = PyDict_New(); + if (!SetComp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(SetComp_annotations, "elt", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + } + { + PyObject *type = state->comprehension_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + cond = PyDict_SetItemString(SetComp_annotations, "generators", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->SetComp_type, "_field_types", + SetComp_annotations) == 0; + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->SetComp_type, "__annotations__", + SetComp_annotations) == 0; + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + Py_DECREF(SetComp_annotations); + PyObject *DictComp_annotations = PyDict_New(); + if (!DictComp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(DictComp_annotations, "key", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(DictComp_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + } + { + PyObject *type = state->comprehension_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + cond = PyDict_SetItemString(DictComp_annotations, "generators", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->DictComp_type, "_field_types", + DictComp_annotations) == 0; + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->DictComp_type, "__annotations__", + DictComp_annotations) == 0; + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + Py_DECREF(DictComp_annotations); + PyObject *GeneratorExp_annotations = PyDict_New(); + if (!GeneratorExp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(GeneratorExp_annotations, "elt", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + } + { + PyObject *type = state->comprehension_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + cond = PyDict_SetItemString(GeneratorExp_annotations, "generators", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->GeneratorExp_type, "_field_types", + GeneratorExp_annotations) == 0; + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->GeneratorExp_type, "__annotations__", + GeneratorExp_annotations) == 0; + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + Py_DECREF(GeneratorExp_annotations); + PyObject *Await_annotations = PyDict_New(); + if (!Await_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Await_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Await_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Await_type, "_field_types", + Await_annotations) == 0; + if (!cond) { + Py_DECREF(Await_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Await_type, "__annotations__", + Await_annotations) == 0; + if (!cond) { + Py_DECREF(Await_annotations); + return 0; + } + Py_DECREF(Await_annotations); + PyObject *Yield_annotations = PyDict_New(); + if (!Yield_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Yield_annotations); + return 0; + } + cond = PyDict_SetItemString(Yield_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Yield_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Yield_type, "_field_types", + Yield_annotations) == 0; + if (!cond) { + Py_DECREF(Yield_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Yield_type, "__annotations__", + Yield_annotations) == 0; + if (!cond) { + Py_DECREF(Yield_annotations); + return 0; + } + Py_DECREF(Yield_annotations); + PyObject *YieldFrom_annotations = PyDict_New(); + if (!YieldFrom_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(YieldFrom_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(YieldFrom_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->YieldFrom_type, "_field_types", + YieldFrom_annotations) == 0; + if (!cond) { + Py_DECREF(YieldFrom_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->YieldFrom_type, "__annotations__", + YieldFrom_annotations) == 0; + if (!cond) { + Py_DECREF(YieldFrom_annotations); + return 0; + } + Py_DECREF(YieldFrom_annotations); + PyObject *Compare_annotations = PyDict_New(); + if (!Compare_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Compare_annotations, "left", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + } + { + PyObject *type = state->cmpop_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + cond = PyDict_SetItemString(Compare_annotations, "ops", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + cond = PyDict_SetItemString(Compare_annotations, "comparators", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Compare_type, "_field_types", + Compare_annotations) == 0; + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Compare_type, "__annotations__", + Compare_annotations) == 0; + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + Py_DECREF(Compare_annotations); + PyObject *Call_annotations = PyDict_New(); + if (!Call_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Call_annotations, "func", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + cond = PyDict_SetItemString(Call_annotations, "args", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + } + { + PyObject *type = state->keyword_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + cond = PyDict_SetItemString(Call_annotations, "keywords", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Call_type, "_field_types", + Call_annotations) == 0; + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Call_type, "__annotations__", + Call_annotations) == 0; + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + Py_DECREF(Call_annotations); + PyObject *FormattedValue_annotations = PyDict_New(); + if (!FormattedValue_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(FormattedValue_annotations, "value", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyLong_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(FormattedValue_annotations, "conversion", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + cond = PyDict_SetItemString(FormattedValue_annotations, "format_spec", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->FormattedValue_type, "_field_types", + FormattedValue_annotations) == 0; + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->FormattedValue_type, + "__annotations__", + FormattedValue_annotations) == 0; + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + Py_DECREF(FormattedValue_annotations); + PyObject *JoinedStr_annotations = PyDict_New(); + if (!JoinedStr_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(JoinedStr_annotations); + return 0; + } + cond = PyDict_SetItemString(JoinedStr_annotations, "values", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(JoinedStr_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->JoinedStr_type, "_field_types", + JoinedStr_annotations) == 0; + if (!cond) { + Py_DECREF(JoinedStr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->JoinedStr_type, "__annotations__", + JoinedStr_annotations) == 0; + if (!cond) { + Py_DECREF(JoinedStr_annotations); + return 0; + } + Py_DECREF(JoinedStr_annotations); + PyObject *Constant_annotations = PyDict_New(); + if (!Constant_annotations) return 0; + { + PyObject *type = (PyObject *)&PyBaseObject_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(Constant_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + cond = PyDict_SetItemString(Constant_annotations, "kind", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Constant_type, "_field_types", + Constant_annotations) == 0; + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Constant_type, "__annotations__", + Constant_annotations) == 0; + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + Py_DECREF(Constant_annotations); + PyObject *Attribute_annotations = PyDict_New(); + if (!Attribute_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Attribute_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(Attribute_annotations, "attr", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Attribute_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Attribute_type, "_field_types", + Attribute_annotations) == 0; + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Attribute_type, "__annotations__", + Attribute_annotations) == 0; + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + Py_DECREF(Attribute_annotations); + PyObject *Subscript_annotations = PyDict_New(); + if (!Subscript_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Subscript_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Subscript_annotations, "slice", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Subscript_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Subscript_type, "_field_types", + Subscript_annotations) == 0; + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Subscript_type, "__annotations__", + Subscript_annotations) == 0; + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + Py_DECREF(Subscript_annotations); + PyObject *Starred_annotations = PyDict_New(); + if (!Starred_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Starred_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Starred_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Starred_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Starred_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Starred_type, "_field_types", + Starred_annotations) == 0; + if (!cond) { + Py_DECREF(Starred_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Starred_type, "__annotations__", + Starred_annotations) == 0; + if (!cond) { + Py_DECREF(Starred_annotations); + return 0; + } + Py_DECREF(Starred_annotations); + PyObject *Name_annotations = PyDict_New(); + if (!Name_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(Name_annotations, "id", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Name_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Name_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Name_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Name_type, "_field_types", + Name_annotations) == 0; + if (!cond) { + Py_DECREF(Name_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Name_type, "__annotations__", + Name_annotations) == 0; + if (!cond) { + Py_DECREF(Name_annotations); + return 0; + } + Py_DECREF(Name_annotations); + PyObject *List_annotations = PyDict_New(); + if (!List_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + cond = PyDict_SetItemString(List_annotations, "elts", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(List_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->List_type, "_field_types", + List_annotations) == 0; + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->List_type, "__annotations__", + List_annotations) == 0; + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + Py_DECREF(List_annotations); + PyObject *Tuple_annotations = PyDict_New(); + if (!Tuple_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + cond = PyDict_SetItemString(Tuple_annotations, "elts", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Tuple_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Tuple_type, "_field_types", + Tuple_annotations) == 0; + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Tuple_type, "__annotations__", + Tuple_annotations) == 0; + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + Py_DECREF(Tuple_annotations); + PyObject *Slice_annotations = PyDict_New(); + if (!Slice_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + cond = PyDict_SetItemString(Slice_annotations, "lower", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + cond = PyDict_SetItemString(Slice_annotations, "upper", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + cond = PyDict_SetItemString(Slice_annotations, "step", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Slice_type, "_field_types", + Slice_annotations) == 0; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Slice_type, "__annotations__", + Slice_annotations) == 0; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + Py_DECREF(Slice_annotations); + PyObject *Load_annotations = PyDict_New(); + if (!Load_annotations) return 0; + cond = PyObject_SetAttrString(state->Load_type, "_field_types", + Load_annotations) == 0; + if (!cond) { + Py_DECREF(Load_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Load_type, "__annotations__", + Load_annotations) == 0; + if (!cond) { + Py_DECREF(Load_annotations); + return 0; + } + Py_DECREF(Load_annotations); + PyObject *Store_annotations = PyDict_New(); + if (!Store_annotations) return 0; + cond = PyObject_SetAttrString(state->Store_type, "_field_types", + Store_annotations) == 0; + if (!cond) { + Py_DECREF(Store_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Store_type, "__annotations__", + Store_annotations) == 0; + if (!cond) { + Py_DECREF(Store_annotations); + return 0; + } + Py_DECREF(Store_annotations); + PyObject *Del_annotations = PyDict_New(); + if (!Del_annotations) return 0; + cond = PyObject_SetAttrString(state->Del_type, "_field_types", + Del_annotations) == 0; + if (!cond) { + Py_DECREF(Del_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Del_type, "__annotations__", + Del_annotations) == 0; + if (!cond) { + Py_DECREF(Del_annotations); + return 0; + } + Py_DECREF(Del_annotations); + PyObject *And_annotations = PyDict_New(); + if (!And_annotations) return 0; + cond = PyObject_SetAttrString(state->And_type, "_field_types", + And_annotations) == 0; + if (!cond) { + Py_DECREF(And_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->And_type, "__annotations__", + And_annotations) == 0; + if (!cond) { + Py_DECREF(And_annotations); + return 0; + } + Py_DECREF(And_annotations); + PyObject *Or_annotations = PyDict_New(); + if (!Or_annotations) return 0; + cond = PyObject_SetAttrString(state->Or_type, "_field_types", + Or_annotations) == 0; + if (!cond) { + Py_DECREF(Or_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Or_type, "__annotations__", + Or_annotations) == 0; + if (!cond) { + Py_DECREF(Or_annotations); + return 0; + } + Py_DECREF(Or_annotations); + PyObject *Add_annotations = PyDict_New(); + if (!Add_annotations) return 0; + cond = PyObject_SetAttrString(state->Add_type, "_field_types", + Add_annotations) == 0; + if (!cond) { + Py_DECREF(Add_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Add_type, "__annotations__", + Add_annotations) == 0; + if (!cond) { + Py_DECREF(Add_annotations); + return 0; + } + Py_DECREF(Add_annotations); + PyObject *Sub_annotations = PyDict_New(); + if (!Sub_annotations) return 0; + cond = PyObject_SetAttrString(state->Sub_type, "_field_types", + Sub_annotations) == 0; + if (!cond) { + Py_DECREF(Sub_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Sub_type, "__annotations__", + Sub_annotations) == 0; + if (!cond) { + Py_DECREF(Sub_annotations); + return 0; + } + Py_DECREF(Sub_annotations); + PyObject *Mult_annotations = PyDict_New(); + if (!Mult_annotations) return 0; + cond = PyObject_SetAttrString(state->Mult_type, "_field_types", + Mult_annotations) == 0; + if (!cond) { + Py_DECREF(Mult_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Mult_type, "__annotations__", + Mult_annotations) == 0; + if (!cond) { + Py_DECREF(Mult_annotations); + return 0; + } + Py_DECREF(Mult_annotations); + PyObject *MatMult_annotations = PyDict_New(); + if (!MatMult_annotations) return 0; + cond = PyObject_SetAttrString(state->MatMult_type, "_field_types", + MatMult_annotations) == 0; + if (!cond) { + Py_DECREF(MatMult_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatMult_type, "__annotations__", + MatMult_annotations) == 0; + if (!cond) { + Py_DECREF(MatMult_annotations); + return 0; + } + Py_DECREF(MatMult_annotations); + PyObject *Div_annotations = PyDict_New(); + if (!Div_annotations) return 0; + cond = PyObject_SetAttrString(state->Div_type, "_field_types", + Div_annotations) == 0; + if (!cond) { + Py_DECREF(Div_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Div_type, "__annotations__", + Div_annotations) == 0; + if (!cond) { + Py_DECREF(Div_annotations); + return 0; + } + Py_DECREF(Div_annotations); + PyObject *Mod_annotations = PyDict_New(); + if (!Mod_annotations) return 0; + cond = PyObject_SetAttrString(state->Mod_type, "_field_types", + Mod_annotations) == 0; + if (!cond) { + Py_DECREF(Mod_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Mod_type, "__annotations__", + Mod_annotations) == 0; + if (!cond) { + Py_DECREF(Mod_annotations); + return 0; + } + Py_DECREF(Mod_annotations); + PyObject *Pow_annotations = PyDict_New(); + if (!Pow_annotations) return 0; + cond = PyObject_SetAttrString(state->Pow_type, "_field_types", + Pow_annotations) == 0; + if (!cond) { + Py_DECREF(Pow_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Pow_type, "__annotations__", + Pow_annotations) == 0; + if (!cond) { + Py_DECREF(Pow_annotations); + return 0; + } + Py_DECREF(Pow_annotations); + PyObject *LShift_annotations = PyDict_New(); + if (!LShift_annotations) return 0; + cond = PyObject_SetAttrString(state->LShift_type, "_field_types", + LShift_annotations) == 0; + if (!cond) { + Py_DECREF(LShift_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->LShift_type, "__annotations__", + LShift_annotations) == 0; + if (!cond) { + Py_DECREF(LShift_annotations); + return 0; + } + Py_DECREF(LShift_annotations); + PyObject *RShift_annotations = PyDict_New(); + if (!RShift_annotations) return 0; + cond = PyObject_SetAttrString(state->RShift_type, "_field_types", + RShift_annotations) == 0; + if (!cond) { + Py_DECREF(RShift_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->RShift_type, "__annotations__", + RShift_annotations) == 0; + if (!cond) { + Py_DECREF(RShift_annotations); + return 0; + } + Py_DECREF(RShift_annotations); + PyObject *BitOr_annotations = PyDict_New(); + if (!BitOr_annotations) return 0; + cond = PyObject_SetAttrString(state->BitOr_type, "_field_types", + BitOr_annotations) == 0; + if (!cond) { + Py_DECREF(BitOr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BitOr_type, "__annotations__", + BitOr_annotations) == 0; + if (!cond) { + Py_DECREF(BitOr_annotations); + return 0; + } + Py_DECREF(BitOr_annotations); + PyObject *BitXor_annotations = PyDict_New(); + if (!BitXor_annotations) return 0; + cond = PyObject_SetAttrString(state->BitXor_type, "_field_types", + BitXor_annotations) == 0; + if (!cond) { + Py_DECREF(BitXor_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BitXor_type, "__annotations__", + BitXor_annotations) == 0; + if (!cond) { + Py_DECREF(BitXor_annotations); + return 0; + } + Py_DECREF(BitXor_annotations); + PyObject *BitAnd_annotations = PyDict_New(); + if (!BitAnd_annotations) return 0; + cond = PyObject_SetAttrString(state->BitAnd_type, "_field_types", + BitAnd_annotations) == 0; + if (!cond) { + Py_DECREF(BitAnd_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BitAnd_type, "__annotations__", + BitAnd_annotations) == 0; + if (!cond) { + Py_DECREF(BitAnd_annotations); + return 0; + } + Py_DECREF(BitAnd_annotations); + PyObject *FloorDiv_annotations = PyDict_New(); + if (!FloorDiv_annotations) return 0; + cond = PyObject_SetAttrString(state->FloorDiv_type, "_field_types", + FloorDiv_annotations) == 0; + if (!cond) { + Py_DECREF(FloorDiv_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->FloorDiv_type, "__annotations__", + FloorDiv_annotations) == 0; + if (!cond) { + Py_DECREF(FloorDiv_annotations); + return 0; + } + Py_DECREF(FloorDiv_annotations); + PyObject *Invert_annotations = PyDict_New(); + if (!Invert_annotations) return 0; + cond = PyObject_SetAttrString(state->Invert_type, "_field_types", + Invert_annotations) == 0; + if (!cond) { + Py_DECREF(Invert_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Invert_type, "__annotations__", + Invert_annotations) == 0; + if (!cond) { + Py_DECREF(Invert_annotations); + return 0; + } + Py_DECREF(Invert_annotations); + PyObject *Not_annotations = PyDict_New(); + if (!Not_annotations) return 0; + cond = PyObject_SetAttrString(state->Not_type, "_field_types", + Not_annotations) == 0; + if (!cond) { + Py_DECREF(Not_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Not_type, "__annotations__", + Not_annotations) == 0; + if (!cond) { + Py_DECREF(Not_annotations); + return 0; + } + Py_DECREF(Not_annotations); + PyObject *UAdd_annotations = PyDict_New(); + if (!UAdd_annotations) return 0; + cond = PyObject_SetAttrString(state->UAdd_type, "_field_types", + UAdd_annotations) == 0; + if (!cond) { + Py_DECREF(UAdd_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->UAdd_type, "__annotations__", + UAdd_annotations) == 0; + if (!cond) { + Py_DECREF(UAdd_annotations); + return 0; + } + Py_DECREF(UAdd_annotations); + PyObject *USub_annotations = PyDict_New(); + if (!USub_annotations) return 0; + cond = PyObject_SetAttrString(state->USub_type, "_field_types", + USub_annotations) == 0; + if (!cond) { + Py_DECREF(USub_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->USub_type, "__annotations__", + USub_annotations) == 0; + if (!cond) { + Py_DECREF(USub_annotations); + return 0; + } + Py_DECREF(USub_annotations); + PyObject *Eq_annotations = PyDict_New(); + if (!Eq_annotations) return 0; + cond = PyObject_SetAttrString(state->Eq_type, "_field_types", + Eq_annotations) == 0; + if (!cond) { + Py_DECREF(Eq_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Eq_type, "__annotations__", + Eq_annotations) == 0; + if (!cond) { + Py_DECREF(Eq_annotations); + return 0; + } + Py_DECREF(Eq_annotations); + PyObject *NotEq_annotations = PyDict_New(); + if (!NotEq_annotations) return 0; + cond = PyObject_SetAttrString(state->NotEq_type, "_field_types", + NotEq_annotations) == 0; + if (!cond) { + Py_DECREF(NotEq_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->NotEq_type, "__annotations__", + NotEq_annotations) == 0; + if (!cond) { + Py_DECREF(NotEq_annotations); + return 0; + } + Py_DECREF(NotEq_annotations); + PyObject *Lt_annotations = PyDict_New(); + if (!Lt_annotations) return 0; + cond = PyObject_SetAttrString(state->Lt_type, "_field_types", + Lt_annotations) == 0; + if (!cond) { + Py_DECREF(Lt_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Lt_type, "__annotations__", + Lt_annotations) == 0; + if (!cond) { + Py_DECREF(Lt_annotations); + return 0; + } + Py_DECREF(Lt_annotations); + PyObject *LtE_annotations = PyDict_New(); + if (!LtE_annotations) return 0; + cond = PyObject_SetAttrString(state->LtE_type, "_field_types", + LtE_annotations) == 0; + if (!cond) { + Py_DECREF(LtE_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->LtE_type, "__annotations__", + LtE_annotations) == 0; + if (!cond) { + Py_DECREF(LtE_annotations); + return 0; + } + Py_DECREF(LtE_annotations); + PyObject *Gt_annotations = PyDict_New(); + if (!Gt_annotations) return 0; + cond = PyObject_SetAttrString(state->Gt_type, "_field_types", + Gt_annotations) == 0; + if (!cond) { + Py_DECREF(Gt_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Gt_type, "__annotations__", + Gt_annotations) == 0; + if (!cond) { + Py_DECREF(Gt_annotations); + return 0; + } + Py_DECREF(Gt_annotations); + PyObject *GtE_annotations = PyDict_New(); + if (!GtE_annotations) return 0; + cond = PyObject_SetAttrString(state->GtE_type, "_field_types", + GtE_annotations) == 0; + if (!cond) { + Py_DECREF(GtE_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->GtE_type, "__annotations__", + GtE_annotations) == 0; + if (!cond) { + Py_DECREF(GtE_annotations); + return 0; + } + Py_DECREF(GtE_annotations); + PyObject *Is_annotations = PyDict_New(); + if (!Is_annotations) return 0; + cond = PyObject_SetAttrString(state->Is_type, "_field_types", + Is_annotations) == 0; + if (!cond) { + Py_DECREF(Is_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Is_type, "__annotations__", + Is_annotations) == 0; + if (!cond) { + Py_DECREF(Is_annotations); + return 0; + } + Py_DECREF(Is_annotations); + PyObject *IsNot_annotations = PyDict_New(); + if (!IsNot_annotations) return 0; + cond = PyObject_SetAttrString(state->IsNot_type, "_field_types", + IsNot_annotations) == 0; + if (!cond) { + Py_DECREF(IsNot_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->IsNot_type, "__annotations__", + IsNot_annotations) == 0; + if (!cond) { + Py_DECREF(IsNot_annotations); + return 0; + } + Py_DECREF(IsNot_annotations); + PyObject *In_annotations = PyDict_New(); + if (!In_annotations) return 0; + cond = PyObject_SetAttrString(state->In_type, "_field_types", + In_annotations) == 0; + if (!cond) { + Py_DECREF(In_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->In_type, "__annotations__", + In_annotations) == 0; + if (!cond) { + Py_DECREF(In_annotations); + return 0; + } + Py_DECREF(In_annotations); + PyObject *NotIn_annotations = PyDict_New(); + if (!NotIn_annotations) return 0; + cond = PyObject_SetAttrString(state->NotIn_type, "_field_types", + NotIn_annotations) == 0; + if (!cond) { + Py_DECREF(NotIn_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->NotIn_type, "__annotations__", + NotIn_annotations) == 0; + if (!cond) { + Py_DECREF(NotIn_annotations); + return 0; + } + Py_DECREF(NotIn_annotations); + PyObject *comprehension_annotations = PyDict_New(); + if (!comprehension_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(comprehension_annotations, "target", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(comprehension_annotations, "iter", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + cond = PyDict_SetItemString(comprehension_annotations, "ifs", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyLong_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(comprehension_annotations, "is_async", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->comprehension_type, "_field_types", + comprehension_annotations) == 0; + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->comprehension_type, "__annotations__", + comprehension_annotations) == 0; + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + Py_DECREF(comprehension_annotations); + PyObject *ExceptHandler_annotations = PyDict_New(); + if (!ExceptHandler_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + cond = PyDict_SetItemString(ExceptHandler_annotations, "type", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + cond = PyDict_SetItemString(ExceptHandler_annotations, "name", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + cond = PyDict_SetItemString(ExceptHandler_annotations, "body", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ExceptHandler_type, "_field_types", + ExceptHandler_annotations) == 0; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ExceptHandler_type, "__annotations__", + ExceptHandler_annotations) == 0; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + Py_DECREF(ExceptHandler_annotations); + PyObject *arguments_annotations = PyDict_New(); + if (!arguments_annotations) return 0; + { + PyObject *type = state->arg_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "posonlyargs", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->arg_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "args", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->arg_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "vararg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->arg_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "kwonlyargs", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "kw_defaults", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->arg_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "kwarg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "defaults", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->arguments_type, "_field_types", + arguments_annotations) == 0; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->arguments_type, "__annotations__", + arguments_annotations) == 0; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + Py_DECREF(arguments_annotations); + PyObject *arg_annotations = PyDict_New(); + if (!arg_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(arg_annotations, "arg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + cond = PyDict_SetItemString(arg_annotations, "annotation", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + cond = PyDict_SetItemString(arg_annotations, "type_comment", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->arg_type, "_field_types", + arg_annotations) == 0; + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->arg_type, "__annotations__", + arg_annotations) == 0; + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + Py_DECREF(arg_annotations); + PyObject *keyword_annotations = PyDict_New(); + if (!keyword_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + cond = PyDict_SetItemString(keyword_annotations, "arg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(keyword_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->keyword_type, "_field_types", + keyword_annotations) == 0; + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->keyword_type, "__annotations__", + keyword_annotations) == 0; + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + Py_DECREF(keyword_annotations); + PyObject *alias_annotations = PyDict_New(); + if (!alias_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(alias_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + cond = PyDict_SetItemString(alias_annotations, "asname", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->alias_type, "_field_types", + alias_annotations) == 0; + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->alias_type, "__annotations__", + alias_annotations) == 0; + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + Py_DECREF(alias_annotations); + PyObject *withitem_annotations = PyDict_New(); + if (!withitem_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(withitem_annotations, "context_expr", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + cond = PyDict_SetItemString(withitem_annotations, "optional_vars", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->withitem_type, "_field_types", + withitem_annotations) == 0; + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->withitem_type, "__annotations__", + withitem_annotations) == 0; + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + Py_DECREF(withitem_annotations); + PyObject *match_case_annotations = PyDict_New(); + if (!match_case_annotations) return 0; + { + PyObject *type = state->pattern_type; + Py_INCREF(type); + cond = PyDict_SetItemString(match_case_annotations, "pattern", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + cond = PyDict_SetItemString(match_case_annotations, "guard", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + cond = PyDict_SetItemString(match_case_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->match_case_type, "_field_types", + match_case_annotations) == 0; + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->match_case_type, "__annotations__", + match_case_annotations) == 0; + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + Py_DECREF(match_case_annotations); + PyObject *MatchValue_annotations = PyDict_New(); + if (!MatchValue_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(MatchValue_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchValue_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchValue_type, "_field_types", + MatchValue_annotations) == 0; + if (!cond) { + Py_DECREF(MatchValue_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchValue_type, "__annotations__", + MatchValue_annotations) == 0; + if (!cond) { + Py_DECREF(MatchValue_annotations); + return 0; + } + Py_DECREF(MatchValue_annotations); + PyObject *MatchSingleton_annotations = PyDict_New(); + if (!MatchSingleton_annotations) return 0; + { + PyObject *type = (PyObject *)&PyBaseObject_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(MatchSingleton_annotations, "value", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchSingleton_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchSingleton_type, "_field_types", + MatchSingleton_annotations) == 0; + if (!cond) { + Py_DECREF(MatchSingleton_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchSingleton_type, + "__annotations__", + MatchSingleton_annotations) == 0; + if (!cond) { + Py_DECREF(MatchSingleton_annotations); + return 0; + } + Py_DECREF(MatchSingleton_annotations); + PyObject *MatchSequence_annotations = PyDict_New(); + if (!MatchSequence_annotations) return 0; + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchSequence_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchSequence_annotations, "patterns", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchSequence_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchSequence_type, "_field_types", + MatchSequence_annotations) == 0; + if (!cond) { + Py_DECREF(MatchSequence_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchSequence_type, "__annotations__", + MatchSequence_annotations) == 0; + if (!cond) { + Py_DECREF(MatchSequence_annotations); + return 0; + } + Py_DECREF(MatchSequence_annotations); + PyObject *MatchMapping_annotations = PyDict_New(); + if (!MatchMapping_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchMapping_annotations, "keys", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + } + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchMapping_annotations, "patterns", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchMapping_annotations, "rest", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchMapping_type, "_field_types", + MatchMapping_annotations) == 0; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchMapping_type, "__annotations__", + MatchMapping_annotations) == 0; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + Py_DECREF(MatchMapping_annotations); + PyObject *MatchClass_annotations = PyDict_New(); + if (!MatchClass_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(MatchClass_annotations, "cls", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + } + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchClass_annotations, "patterns", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchClass_annotations, "kwd_attrs", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + } + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchClass_annotations, "kwd_patterns", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchClass_type, "_field_types", + MatchClass_annotations) == 0; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchClass_type, "__annotations__", + MatchClass_annotations) == 0; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + Py_DECREF(MatchClass_annotations); + PyObject *MatchStar_annotations = PyDict_New(); + if (!MatchStar_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchStar_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchStar_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchStar_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchStar_type, "_field_types", + MatchStar_annotations) == 0; + if (!cond) { + Py_DECREF(MatchStar_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchStar_type, "__annotations__", + MatchStar_annotations) == 0; + if (!cond) { + Py_DECREF(MatchStar_annotations); + return 0; + } + Py_DECREF(MatchStar_annotations); + PyObject *MatchAs_annotations = PyDict_New(); + if (!MatchAs_annotations) return 0; + { + PyObject *type = state->pattern_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchAs_annotations, "pattern", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchAs_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchAs_type, "_field_types", + MatchAs_annotations) == 0; + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchAs_type, "__annotations__", + MatchAs_annotations) == 0; + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + Py_DECREF(MatchAs_annotations); + PyObject *MatchOr_annotations = PyDict_New(); + if (!MatchOr_annotations) return 0; + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchOr_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchOr_annotations, "patterns", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchOr_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchOr_type, "_field_types", + MatchOr_annotations) == 0; + if (!cond) { + Py_DECREF(MatchOr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchOr_type, "__annotations__", + MatchOr_annotations) == 0; + if (!cond) { + Py_DECREF(MatchOr_annotations); + return 0; + } + Py_DECREF(MatchOr_annotations); + PyObject *TypeIgnore_annotations = PyDict_New(); + if (!TypeIgnore_annotations) return 0; + { + PyObject *type = (PyObject *)&PyLong_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeIgnore_annotations, "lineno", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeIgnore_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeIgnore_annotations, "tag", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeIgnore_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TypeIgnore_type, "_field_types", + TypeIgnore_annotations) == 0; + if (!cond) { + Py_DECREF(TypeIgnore_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TypeIgnore_type, "__annotations__", + TypeIgnore_annotations) == 0; + if (!cond) { + Py_DECREF(TypeIgnore_annotations); + return 0; + } + Py_DECREF(TypeIgnore_annotations); + PyObject *TypeVar_annotations = PyDict_New(); + if (!TypeVar_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeVar_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + cond = PyDict_SetItemString(TypeVar_annotations, "bound", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TypeVar_type, "_field_types", + TypeVar_annotations) == 0; + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TypeVar_type, "__annotations__", + TypeVar_annotations) == 0; + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + Py_DECREF(TypeVar_annotations); + PyObject *ParamSpec_annotations = PyDict_New(); + if (!ParamSpec_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(ParamSpec_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ParamSpec_type, "_field_types", + ParamSpec_annotations) == 0; + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ParamSpec_type, "__annotations__", + ParamSpec_annotations) == 0; + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + Py_DECREF(ParamSpec_annotations); + PyObject *TypeVarTuple_annotations = PyDict_New(); + if (!TypeVarTuple_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeVarTuple_annotations, "name", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TypeVarTuple_type, "_field_types", + TypeVarTuple_annotations) == 0; + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TypeVarTuple_type, "__annotations__", + TypeVarTuple_annotations) == 0; + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + Py_DECREF(TypeVarTuple_annotations); + + return 1; +} + + typedef struct { PyObject_HEAD @@ -860,7 +5026,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) Py_ssize_t i, numfields = 0; int res = -1; - PyObject *key, *value, *fields; + PyObject *key, *value, *fields, *remaining_fields = NULL; if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { goto cleanup; } @@ -869,6 +5035,13 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) if (numfields == -1) { goto cleanup; } + remaining_fields = PySet_New(fields); + } + else { + remaining_fields = PySet_New(NULL); + } + if (remaining_fields == NULL) { + goto cleanup; } res = 0; /* if no error occurs, this stays 0 to the end */ @@ -888,6 +5061,11 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) goto cleanup; } res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i)); + if (PySet_Discard(remaining_fields, name) < 0) { + res = -1; + Py_DECREF(name); + goto cleanup; + } Py_DECREF(name); if (res < 0) { goto cleanup; @@ -900,13 +5078,14 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) if (contains == -1) { res = -1; goto cleanup; - } else if (contains == 1) { - Py_ssize_t p = PySequence_Index(fields, key); + } + else if (contains == 1) { + int p = PySet_Discard(remaining_fields, key); if (p == -1) { res = -1; goto cleanup; } - if (p < PyTuple_GET_SIZE(args)) { + if (p == 0) { PyErr_Format(PyExc_TypeError, "%.400s got multiple values for argument '%U'", Py_TYPE(self)->tp_name, key); @@ -914,15 +5093,91 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) goto cleanup; } } + else if ( + PyUnicode_CompareWithASCIIString(key, "lineno") != 0 && + PyUnicode_CompareWithASCIIString(key, "col_offset") != 0 && + PyUnicode_CompareWithASCIIString(key, "end_lineno") != 0 && + PyUnicode_CompareWithASCIIString(key, "end_col_offset") != 0 + ) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ got an unexpected keyword argument '%U'. " + "Support for arbitrary keyword arguments is deprecated " + "and will be removed in Python 3.15.", + Py_TYPE(self)->tp_name, key + ) < 0) { + res = -1; + goto cleanup; + } + } res = PyObject_SetAttr(self, key, value); if (res < 0) { goto cleanup; } } } + Py_ssize_t size = PySet_Size(remaining_fields); + PyObject *field_types = NULL, *remaining_list = NULL; + if (size > 0) { + if (!PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), &_Py_ID(_field_types), + &field_types)) { + res = -1; + goto cleanup; + } + remaining_list = PySequence_List(remaining_fields); + if (!remaining_list) { + goto set_remaining_cleanup; + } + for (Py_ssize_t i = 0; i < size; i++) { + PyObject *name = PyList_GET_ITEM(remaining_list, i); + PyObject *type = PyDict_GetItemWithError(field_types, name); + if (!type) { + if (!PyErr_Occurred()) { + PyErr_SetObject(PyExc_KeyError, name); + } + goto set_remaining_cleanup; + } + if (_PyUnion_Check(type)) { + // optional field + // do nothing, we'll have set a None default on the class + } + else if (Py_IS_TYPE(type, &Py_GenericAliasType)) { + // list field + PyObject *empty = PyList_New(0); + if (!empty) { + goto set_remaining_cleanup; + } + res = PyObject_SetAttr(self, name, empty); + Py_DECREF(empty); + if (res < 0) { + goto set_remaining_cleanup; + } + } + else { + // simple field (e.g., identifier) + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ missing 1 required positional argument: '%U'. " + "This will become an error in Python 3.15.", + Py_TYPE(self)->tp_name, name + ) < 0) { + res = -1; + goto cleanup; + } + } + } + Py_DECREF(remaining_list); + Py_DECREF(field_types); + } cleanup: Py_XDECREF(fields); + Py_XDECREF(remaining_fields); return res; + set_remaining_cleanup: + Py_XDECREF(remaining_list); + Py_XDECREF(field_types); + res = -1; + goto cleanup; } /* Pickling support */ @@ -934,14 +5189,75 @@ ast_type_reduce(PyObject *self, PyObject *unused) return NULL; } - PyObject *dict; + PyObject *dict = NULL, *fields = NULL, *remaining_fields = NULL, + *remaining_dict = NULL, *positional_args = NULL; if (PyObject_GetOptionalAttr(self, state->__dict__, &dict) < 0) { return NULL; } + PyObject *result = NULL; if (dict) { - return Py_BuildValue("O()N", Py_TYPE(self), dict); + // Serialize the fields as positional args if possible, because if we + // serialize them as a dict, during unpickling they are set only *after* + // the object is constructed, which will now trigger a DeprecationWarning + // if the AST type has required fields. + if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { + goto cleanup; + } + if (fields) { + Py_ssize_t numfields = PySequence_Size(fields); + if (numfields == -1) { + Py_DECREF(dict); + goto cleanup; + } + remaining_dict = PyDict_Copy(dict); + Py_DECREF(dict); + if (!remaining_dict) { + goto cleanup; + } + positional_args = PyList_New(0); + if (!positional_args) { + goto cleanup; + } + for (Py_ssize_t i = 0; i < numfields; i++) { + PyObject *name = PySequence_GetItem(fields, i); + if (!name) { + goto cleanup; + } + PyObject *value = PyDict_GetItemWithError(remaining_dict, name); + if (!value) { + if (PyErr_Occurred()) { + goto cleanup; + } + break; + } + if (PyList_Append(positional_args, value) < 0) { + goto cleanup; + } + if (PyDict_DelItem(remaining_dict, name) < 0) { + goto cleanup; + } + Py_DECREF(name); + } + PyObject *args_tuple = PyList_AsTuple(positional_args); + if (!args_tuple) { + goto cleanup; + } + result = Py_BuildValue("ONO", Py_TYPE(self), args_tuple, + remaining_dict); + } + else { + result = Py_BuildValue("O()N", Py_TYPE(self), dict); + } + } + else { + result = Py_BuildValue("O()", Py_TYPE(self)); } - return Py_BuildValue("O()", Py_TYPE(self)); +cleanup: + Py_XDECREF(fields); + Py_XDECREF(remaining_fields); + Py_XDECREF(remaining_dict); + Py_XDECREF(positional_args); + return result; } static PyMemberDef ast_type_members[] = { @@ -1939,6 +6255,9 @@ init_types(struct ast_state *state) "TypeVarTuple(identifier name)"); if (!state->TypeVarTuple_type) return -1; + if (!add_ast_annotations(state)) { + return -1; + } return 0; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index e54d5cbacdc96f9..b0074962b737992 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -703,17 +703,34 @@ builtin_format_impl(PyObject *module, PyObject *value, PyObject *format_spec) /*[clinic input] chr as builtin_chr - i: int + i: object / Return a Unicode string of one character with ordinal i; 0 <= i <= 0x10ffff. [clinic start generated code]*/ static PyObject * -builtin_chr_impl(PyObject *module, int i) -/*[clinic end generated code: output=c733afcd200afcb7 input=3f604ef45a70750d]*/ +builtin_chr(PyObject *module, PyObject *i) +/*[clinic end generated code: output=d34f25b8035a9b10 input=f919867f0ba2f496]*/ { - return PyUnicode_FromOrdinal(i); + int overflow; + long v = PyLong_AsLongAndOverflow(i, &overflow); + if (v == -1 && PyErr_Occurred()) { + return NULL; + } + if (overflow) { + v = overflow < 0 ? INT_MIN : INT_MAX; + /* Allow PyUnicode_FromOrdinal() to raise an exception */ + } +#if SIZEOF_INT < SIZEOF_LONG + else if (v < INT_MIN) { + v = INT_MIN; + } + else if (v > INT_MAX) { + v = INT_MAX; + } +#endif + return PyUnicode_FromOrdinal(v); } @@ -2382,11 +2399,6 @@ builtin_round_impl(PyObject *module, PyObject *number, PyObject *ndigits) { PyObject *round, *result; - if (!_PyType_IsReady(Py_TYPE(number))) { - if (PyType_Ready(Py_TYPE(number)) < 0) - return NULL; - } - round = _PyObject_LookupSpecial(number, &_Py_ID(__round__)); if (round == NULL) { if (!PyErr_Occurred()) diff --git a/Python/brc.c b/Python/brc.c new file mode 100644 index 000000000000000..b73c721e71aef6b --- /dev/null +++ b/Python/brc.c @@ -0,0 +1,198 @@ +// Implementation of biased reference counting inter-thread queue. +// +// Biased reference counting maintains two refcount fields in each object: +// ob_ref_local and ob_ref_shared. The true refcount is the sum of these two +// fields. In some cases, when refcounting operations are split across threads, +// the ob_ref_shared field can be negative (although the total refcount must +// be at least zero). In this case, the thread that decremented the refcount +// requests that the owning thread give up ownership and merge the refcount +// fields. This file implements the mechanism for doing so. +// +// Each thread state maintains a queue of objects whose refcounts it should +// merge. The thread states are stored in a per-interpreter hash table by +// thread id. The hash table has a fixed size and uses a linked list to store +// thread states within each bucket. +// +// The queueing thread uses the eval breaker mechanism to notify the owning +// thread that it has objects to merge. Additionaly, all queued objects are +// merged during GC. +#include "Python.h" +#include "pycore_object.h" // _Py_ExplicitMergeRefcount +#include "pycore_brc.h" // struct _brc_thread_state +#include "pycore_ceval.h" // _Py_set_eval_breaker_bit +#include "pycore_llist.h" // struct llist_node +#include "pycore_pystate.h" // _PyThreadStateImpl + +#ifdef Py_GIL_DISABLED + +// Get the hashtable bucket for a given thread id. +static struct _brc_bucket * +get_bucket(PyInterpreterState *interp, uintptr_t tid) +{ + return &interp->brc.table[tid % _Py_BRC_NUM_BUCKETS]; +} + +// Find the thread state in a hash table bucket by thread id. +static _PyThreadStateImpl * +find_thread_state(struct _brc_bucket *bucket, uintptr_t thread_id) +{ + struct llist_node *node; + llist_for_each(node, &bucket->root) { + // Get the containing _PyThreadStateImpl from the linked-list node. + _PyThreadStateImpl *ts = llist_data(node, _PyThreadStateImpl, + brc.bucket_node); + if (ts->brc.tid == thread_id) { + return ts; + } + } + return NULL; +} + +// Enqueue an object to be merged by the owning thread. This steals a +// reference to the object. +void +_Py_brc_queue_object(PyObject *ob) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + + uintptr_t ob_tid = _Py_atomic_load_uintptr(&ob->ob_tid); + if (ob_tid == 0) { + // The owning thread may have concurrently decided to merge the + // refcount fields. + Py_DECREF(ob); + return; + } + + struct _brc_bucket *bucket = get_bucket(interp, ob_tid); + PyMutex_Lock(&bucket->mutex); + _PyThreadStateImpl *tstate = find_thread_state(bucket, ob_tid); + if (tstate == NULL) { + // If we didn't find the owning thread then it must have already exited. + // It's safe (and necessary) to merge the refcount. Subtract one when + // merging because we've stolen a reference. + Py_ssize_t refcount = _Py_ExplicitMergeRefcount(ob, -1); + PyMutex_Unlock(&bucket->mutex); + if (refcount == 0) { + _Py_Dealloc(ob); + } + return; + } + + if (_PyObjectStack_Push(&tstate->brc.objects_to_merge, ob) < 0) { + PyMutex_Unlock(&bucket->mutex); + + // Fall back to stopping all threads and manually merging the refcount + // if we can't enqueue the object to be merged. + _PyEval_StopTheWorld(interp); + Py_ssize_t refcount = _Py_ExplicitMergeRefcount(ob, -1); + _PyEval_StartTheWorld(interp); + + if (refcount == 0) { + _Py_Dealloc(ob); + } + return; + } + + // Notify owning thread + _Py_set_eval_breaker_bit(&tstate->base, _PY_EVAL_EXPLICIT_MERGE_BIT); + + PyMutex_Unlock(&bucket->mutex); +} + +static void +merge_queued_objects(_PyObjectStack *to_merge) +{ + PyObject *ob; + while ((ob = _PyObjectStack_Pop(to_merge)) != NULL) { + // Subtract one when merging because the queue had a reference. + Py_ssize_t refcount = _Py_ExplicitMergeRefcount(ob, -1); + if (refcount == 0) { + _Py_Dealloc(ob); + } + } +} + +// Process this thread's queue of objects to merge. +void +_Py_brc_merge_refcounts(PyThreadState *tstate) +{ + struct _brc_thread_state *brc = &((_PyThreadStateImpl *)tstate)->brc; + struct _brc_bucket *bucket = get_bucket(tstate->interp, brc->tid); + + // Append all objects into a local stack. We don't want to hold the lock + // while calling destructors. + PyMutex_Lock(&bucket->mutex); + _PyObjectStack_Merge(&brc->local_objects_to_merge, &brc->objects_to_merge); + PyMutex_Unlock(&bucket->mutex); + + // Process the local stack until it's empty + merge_queued_objects(&brc->local_objects_to_merge); +} + +void +_Py_brc_init_state(PyInterpreterState *interp) +{ + struct _brc_state *brc = &interp->brc; + for (Py_ssize_t i = 0; i < _Py_BRC_NUM_BUCKETS; i++) { + llist_init(&brc->table[i].root); + } +} + +void +_Py_brc_init_thread(PyThreadState *tstate) +{ + struct _brc_thread_state *brc = &((_PyThreadStateImpl *)tstate)->brc; + brc->tid = _Py_ThreadId(); + + // Add ourself to the hashtable + struct _brc_bucket *bucket = get_bucket(tstate->interp, brc->tid); + PyMutex_Lock(&bucket->mutex); + llist_insert_tail(&bucket->root, &brc->bucket_node); + PyMutex_Unlock(&bucket->mutex); +} + +void +_Py_brc_remove_thread(PyThreadState *tstate) +{ + struct _brc_thread_state *brc = &((_PyThreadStateImpl *)tstate)->brc; + struct _brc_bucket *bucket = get_bucket(tstate->interp, brc->tid); + + // We need to fully process any objects to merge before removing ourself + // from the hashtable. It is not safe to perform any refcount operations + // after we are removed. After that point, other threads treat our objects + // as abandoned and may merge the objects' refcounts directly. + bool empty = false; + while (!empty) { + // Process the local stack until it's empty + merge_queued_objects(&brc->local_objects_to_merge); + + PyMutex_Lock(&bucket->mutex); + empty = (brc->objects_to_merge.head == NULL); + if (empty) { + llist_remove(&brc->bucket_node); + } + else { + _PyObjectStack_Merge(&brc->local_objects_to_merge, + &brc->objects_to_merge); + } + PyMutex_Unlock(&bucket->mutex); + } + + assert(brc->local_objects_to_merge.head == NULL); + assert(brc->objects_to_merge.head == NULL); +} + +void +_Py_brc_after_fork(PyInterpreterState *interp) +{ + // Unlock all bucket mutexes. Some of the buckets may be locked because + // locks can be handed off to a parked thread (see lock.c). We don't have + // to worry about consistency here, becuase no thread can be actively + // modifying a bucket, but it might be paused (not yet woken up) on a + // PyMutex_Lock while holding that lock. + for (Py_ssize_t i = 0; i < _Py_BRC_NUM_BUCKETS; i++) { + _PyMutex_at_fork_reinit(&interp->brc.table[i].mutex); + } +} + +#endif /* Py_GIL_DISABLED */ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ebd5b06abb2d4e4..565379afc4b5a70 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -8,7 +8,6 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() -#include "pycore_ceval.h" // _PyEval_SignalAsyncExc() #include "pycore_code.h" #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "pycore_function.h" @@ -54,6 +53,8 @@ #define guard #define override #define specializing +#define split +#define replicate(TIMES) // Dummy variables for stack effects. static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub; @@ -133,32 +134,32 @@ dummy_func( switch (opcode) { // BEGIN BYTECODES // - inst(NOP, (--)) { + pure inst(NOP, (--)) { } family(RESUME, 0) = { RESUME_CHECK, }; - inst(RESUME, (--)) { - TIER_ONE_ONLY + tier1 inst(RESUME, (--)) { assert(frame == tstate->current_frame); - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - ERROR_IF(err, error); - next_instr = this_instr; - } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); + if (tstate->tracing == 0) { + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + ERROR_IF(err, error); + next_instr = this_instr; + DISPATCH(); } - this_instr->op.code = RESUME_CHECK; } + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } + this_instr->op.code = RESUME_CHECK; } inst(RESUME_CHECK, (--)) { @@ -166,16 +167,16 @@ dummy_func( DEOPT_IF(_Py_emscripten_signal_clock == 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version); + DEOPT_IF(eval_breaker != version && tstate->tracing == 0); } inst(INSTRUMENTED_RESUME, (--)) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - if (code_version != global_version) { + if (code_version != global_version && tstate->tracing == 0) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { GOTO_ERROR(error); } @@ -208,7 +209,7 @@ dummy_func( Py_INCREF(value); } - pure inst(LOAD_FAST, (-- value)) { + replicate(8) pure inst(LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); @@ -234,7 +235,7 @@ dummy_func( Py_INCREF(value); } - inst(STORE_FAST, (value --)) { + replicate(8) inst(STORE_FAST, (value --)) { SETLOCAL(oparg, value); } @@ -267,8 +268,7 @@ dummy_func( macro(END_FOR) = POP_TOP; - inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) { - TIER_ONE_ONLY + tier1 inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) { /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { @@ -285,8 +285,7 @@ dummy_func( Py_DECREF(receiver); } - inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { - TIER_ONE_ONLY + tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { @@ -318,7 +317,6 @@ dummy_func( }; specializing op(_SPECIALIZE_TO_BOOL, (counter/1, value -- value)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -340,12 +338,12 @@ dummy_func( macro(TO_BOOL) = _SPECIALIZE_TO_BOOL + unused/2 + _TO_BOOL; inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) { - DEOPT_IF(!PyBool_Check(value)); + EXIT_IF(!PyBool_Check(value)); STAT_INC(TO_BOOL, hit); } inst(TO_BOOL_INT, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyLong_CheckExact(value)); + EXIT_IF(!PyLong_CheckExact(value)); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -358,7 +356,7 @@ dummy_func( } inst(TO_BOOL_LIST, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyList_CheckExact(value)); + EXIT_IF(!PyList_CheckExact(value)); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; DECREF_INPUTS(); @@ -366,13 +364,13 @@ dummy_func( inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) { // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value)); + EXIT_IF(!Py_IsNone(value)); STAT_INC(TO_BOOL, hit); res = Py_False; } inst(TO_BOOL_STR, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyUnicode_CheckExact(value)); + EXIT_IF(!PyUnicode_CheckExact(value)); STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -388,7 +386,7 @@ dummy_func( inst(TO_BOOL_ALWAYS_TRUE, (unused/1, version/2, value -- res)) { // This one is a bit weird, because we expect *some* failures: assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version); + EXIT_IF(Py_TYPE(value)->tp_version_tag != version); STAT_INC(TO_BOOL, hit); DECREF_INPUTS(); res = Py_True; @@ -411,12 +409,12 @@ dummy_func( // BINARY_OP_INPLACE_ADD_UNICODE, // See comments at that opcode. }; - op(_GUARD_BOTH_INT, (left, right -- left: &PYLONG_TYPE, right: &PYLONG_TYPE)) { - DEOPT_IF(!PyLong_CheckExact(left)); - DEOPT_IF(!PyLong_CheckExact(right)); + op(_GUARD_BOTH_INT, (left, right -- left, right)) { + EXIT_IF(!PyLong_CheckExact(left)); + EXIT_IF(!PyLong_CheckExact(right)); } - pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res: &PYLONG_TYPE)) { + pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { STAT_INC(BINARY_OP, hit); res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -424,7 +422,7 @@ dummy_func( ERROR_IF(res == NULL, error); } - pure op(_BINARY_OP_ADD_INT, (left, right -- res: &PYLONG_TYPE)) { + pure op(_BINARY_OP_ADD_INT, (left, right -- res)) { STAT_INC(BINARY_OP, hit); res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -432,7 +430,7 @@ dummy_func( ERROR_IF(res == NULL, error); } - pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res: &PYLONG_TYPE)) { + pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { STAT_INC(BINARY_OP, hit); res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -447,12 +445,12 @@ dummy_func( macro(BINARY_OP_SUBTRACT_INT) = _GUARD_BOTH_INT + unused/1 + _BINARY_OP_SUBTRACT_INT; - op(_GUARD_BOTH_FLOAT, (left, right -- left: &PYFLOAT_TYPE, right: &PYFLOAT_TYPE)) { - DEOPT_IF(!PyFloat_CheckExact(left)); - DEOPT_IF(!PyFloat_CheckExact(right)); + op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { + EXIT_IF(!PyFloat_CheckExact(left)); + EXIT_IF(!PyFloat_CheckExact(right)); } - pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res: &PYFLOAT_TYPE)) { + pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval * @@ -460,7 +458,7 @@ dummy_func( DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - pure op(_BINARY_OP_ADD_FLOAT, (left, right -- res: &PYFLOAT_TYPE)) { + pure op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval + @@ -468,7 +466,7 @@ dummy_func( DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - pure op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res: &PYFLOAT_TYPE)) { + pure op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval - @@ -483,12 +481,12 @@ dummy_func( macro(BINARY_OP_SUBTRACT_FLOAT) = _GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_SUBTRACT_FLOAT; - op(_GUARD_BOTH_UNICODE, (left, right -- left: &PYUNICODE_TYPE, right: &PYUNICODE_TYPE)) { - DEOPT_IF(!PyUnicode_CheckExact(left)); - DEOPT_IF(!PyUnicode_CheckExact(right)); + op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { + EXIT_IF(!PyUnicode_CheckExact(left)); + EXIT_IF(!PyUnicode_CheckExact(right)); } - pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res: &PYUNICODE_TYPE)) { + pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { STAT_INC(BINARY_OP, hit); res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); @@ -505,8 +503,7 @@ dummy_func( // So the inputs are the same as for all BINARY_OP // specializations, but there is no output. // At the end we just skip over the STORE_FAST. - op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { - TIER_ONE_ONLY + tier1 op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { assert(next_instr->op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(next_instr->op.arg); DEOPT_IF(*target_local != left); @@ -544,7 +541,6 @@ dummy_func( }; specializing op(_SPECIALIZE_BINARY_SUBSCR, (counter/1, container, sub -- container, sub)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -692,7 +688,6 @@ dummy_func( }; specializing op(_SPECIALIZE_STORE_SUBSCR, (counter/1, container, sub -- container, sub)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -761,8 +756,7 @@ dummy_func( ERROR_IF(res == NULL, error); } - inst(RAISE_VARARGS, (args[oparg] -- )) { - TIER_ONE_ONLY + tier1 inst(RAISE_VARARGS, (args[oparg] -- )) { PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -786,8 +780,7 @@ dummy_func( ERROR_IF(true, error); } - inst(INTERPRETER_EXIT, (retval --)) { - TIER_ONE_ONLY + tier1 inst(INTERPRETER_EXIT, (retval --)) { assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ @@ -979,7 +972,6 @@ dummy_func( }; specializing op(_SPECIALIZE_SEND, (counter/1, receiver, unused -- receiver, unused)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -1074,8 +1066,7 @@ dummy_func( goto resume_frame; } - inst(YIELD_VALUE, (retval -- unused)) { - TIER_ONE_ONLY + tier1 inst(YIELD_VALUE, (retval -- unused)) { // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -1104,8 +1095,7 @@ dummy_func( Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); } - inst(RERAISE, (values[oparg], exc -- values[oparg])) { - TIER_ONE_ONLY + tier1 inst(RERAISE, (values[oparg], exc -- values[oparg])) { assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1126,8 +1116,7 @@ dummy_func( goto exception_unwind; } - inst(END_ASYNC_FOR, (awaitable, exc -- )) { - TIER_ONE_ONLY + tier1 inst(END_ASYNC_FOR, (awaitable, exc -- )) { assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { DECREF_INPUTS(); @@ -1140,8 +1129,7 @@ dummy_func( } } - inst(CLEANUP_THROW, (sub_iter, last_sent_val, exc_value -- none, value)) { - TIER_ONE_ONLY + tier1 inst(CLEANUP_THROW, (sub_iter, last_sent_val, exc_value -- none, value)) { assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { @@ -1213,7 +1201,6 @@ dummy_func( }; specializing op(_SPECIALIZE_UNPACK_SEQUENCE, (counter/1, seq -- seq)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -1283,7 +1270,6 @@ dummy_func( }; specializing op(_SPECIALIZE_STORE_ATTR, (counter/1, owner -- owner)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -1402,7 +1388,6 @@ dummy_func( }; specializing op(_SPECIALIZE_LOAD_GLOBAL, (counter/1 -- )) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); @@ -1730,7 +1715,6 @@ dummy_func( }; specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super, class, unused -- global_super, class, unused)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION int load_method = oparg & 1; if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -1743,8 +1727,7 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } - op(_LOAD_SUPER_ATTR, (global_super, class, self -- attr, null if (oparg & 1))) { - TIER_ONE_ONLY + tier1 op(_LOAD_SUPER_ATTR, (global_super, class, self -- attr, null if (oparg & 1))) { if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( @@ -1846,7 +1829,6 @@ dummy_func( }; specializing op(_SPECIALIZE_LOAD_ATTR, (counter/1, owner -- owner)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); @@ -1877,7 +1859,7 @@ dummy_func( something was returned by a descriptor protocol). Set the second element of the stack to NULL, to signal CALL that it's not a method call. - NULL | meth | arg1 | ... | argN + meth | NULL | arg1 | ... | argN */ DECREF_INPUTS(); ERROR_IF(attr == NULL, error); @@ -1901,10 +1883,10 @@ dummy_func( LOAD_ATTR, }; - op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner: &(GUARD_TYPE_VERSION_TYPE + type_version))) { + op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version); + EXIT_IF(tp->tp_version_tag != type_version); } op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { @@ -1914,7 +1896,7 @@ dummy_func( DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); } - op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { + split op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; DEOPT_IF(attr == NULL); @@ -1931,11 +1913,11 @@ dummy_func( _LOAD_ATTR_INSTANCE_VALUE + unused/5; // Skip over rest of cache - op(_CHECK_ATTR_MODULE, (type_version/2, owner -- owner)) { + op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) { DEOPT_IF(!PyModule_CheckExact(owner)); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version); + DEOPT_IF(dict->ma_keys->dk_version != dict_version); } op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { @@ -1995,7 +1977,7 @@ dummy_func( _LOAD_ATTR_WITH_HINT + unused/5; - op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { + split op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { char *addr = (char *)owner + index; attr = *(PyObject **)addr; DEOPT_IF(attr == NULL); @@ -2018,7 +2000,7 @@ dummy_func( } - op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { + split op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = Py_NewRef(descr); @@ -2082,7 +2064,7 @@ dummy_func( DISPATCH_INLINED(new_frame); } - op(_GUARD_DORV_VALUES, (owner -- owner: &GUARD_DORV_VALUES_TYPE)) { + op(_GUARD_DORV_VALUES, (owner -- owner)) { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv)); @@ -2171,7 +2153,6 @@ dummy_func( }; specializing op(_SPECIALIZE_COMPARE_OP, (counter/1, left, right -- left, right)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -2198,9 +2179,16 @@ dummy_func( macro(COMPARE_OP) = _SPECIALIZE_COMPARE_OP + _COMPARE_OP; - inst(COMPARE_OP_FLOAT, (unused/1, left, right -- res)) { - DEOPT_IF(!PyFloat_CheckExact(left)); - DEOPT_IF(!PyFloat_CheckExact(right)); + macro(COMPARE_OP_FLOAT) = + _GUARD_BOTH_FLOAT + unused/1 + _COMPARE_OP_FLOAT; + + macro(COMPARE_OP_INT) = + _GUARD_BOTH_INT + unused/1 + _COMPARE_OP_INT; + + macro(COMPARE_OP_STR) = + _GUARD_BOTH_UNICODE + unused/1 + _COMPARE_OP_STR; + + op(_COMPARE_OP_FLOAT, (left, right -- res)) { STAT_INC(COMPARE_OP, hit); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); @@ -2213,9 +2201,7 @@ dummy_func( } // Similar to COMPARE_OP_FLOAT - inst(COMPARE_OP_INT, (unused/1, left, right -- res)) { - DEOPT_IF(!PyLong_CheckExact(left)); - DEOPT_IF(!PyLong_CheckExact(right)); + op(_COMPARE_OP_INT, (left, right -- res)) { DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left)); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right)); STAT_INC(COMPARE_OP, hit); @@ -2232,9 +2218,7 @@ dummy_func( } // Similar to COMPARE_OP_FLOAT, but for ==, != only - inst(COMPARE_OP_STR, (unused/1, left, right -- res)) { - DEOPT_IF(!PyUnicode_CheckExact(left)); - DEOPT_IF(!PyUnicode_CheckExact(right)); + op(_COMPARE_OP_STR, (left, right -- res)) { STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left, right); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); @@ -2293,38 +2277,38 @@ dummy_func( b = res ? Py_True : Py_False; } - inst(IMPORT_NAME, (level, fromlist -- res)) { - TIER_ONE_ONLY + tier1 inst(IMPORT_NAME, (level, fromlist -- res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_name(tstate, frame, name, fromlist, level); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } - inst(IMPORT_FROM, (from -- from, res)) { - TIER_ONE_ONLY + tier1 inst(IMPORT_FROM, (from -- from, res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_from(tstate, from, name); ERROR_IF(res == NULL, error); } - inst(JUMP_FORWARD, (--)) { - TIER_ONE_ONLY + tier1 inst(JUMP_FORWARD, (--)) { JUMPBY(oparg); } - inst(JUMP_BACKWARD, (unused/1 --)) { + tier1 inst(JUMP_BACKWARD, (unused/1 --)) { CHECK_EVAL_BREAKER(); assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); #if ENABLE_SPECIALIZATION - this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); + uint16_t counter = this_instr[1].cache; + this_instr[1].cache = counter + (1 << OPTIMIZER_BITS_IN_COUNTER); /* We are using unsigned values, but we really want signed values, so - * do the 2s complement comparison manually */ - uint16_t ucounter = this_instr[1].cache + (1 << 15); - uint16_t threshold = tstate->interp->optimizer_backedge_threshold + (1 << 15); + * do the 2s complement adjustment manually */ + uint32_t offset_counter = counter ^ (1 << 15); + uint32_t threshold = tstate->interp->optimizer_backedge_threshold; + assert((threshold & OPTIMIZER_BITS_MASK) == 0); + // Use '>=' not '>' so that the optimizer/backoff bits do not effect the result. // Double-check that the opcode isn't instrumented or something: - if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { + if (offset_counter >= threshold && this_instr->op.code == JUMP_BACKWARD) { OPT_STAT_INC(attempts); _Py_CODEUNIT *start = this_instr; /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ @@ -2332,24 +2316,24 @@ dummy_func( oparg >>= 8; start--; } - int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer); + _PyExecutorObject *executor; + int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor); ERROR_IF(optimized < 0, error); if (optimized) { - // Rewind and enter the executor: - assert(start->op.code == ENTER_EXECUTOR); - next_instr = start; - this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + GOTO_TIER_TWO(executor); } else { - int backoff = this_instr[1].cache & ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); - if (backoff < MINIMUM_TIER2_BACKOFF) { - backoff = MINIMUM_TIER2_BACKOFF; + int backoff = this_instr[1].cache & OPTIMIZER_BITS_MASK; + backoff++; + if (backoff < MIN_TIER2_BACKOFF) { + backoff = MIN_TIER2_BACKOFF; } - else if (backoff < 15 - OPTIMIZER_BITS_IN_COUNTER) { - backoff++; + else if (backoff > MAX_TIER2_BACKOFF) { + backoff = MAX_TIER2_BACKOFF; } - assert(backoff <= 15 - OPTIMIZER_BITS_IN_COUNTER); - this_instr[1].cache = ((1 << 16) - ((1 << OPTIMIZER_BITS_IN_COUNTER) << backoff)) | backoff; + this_instr[1].cache = ((UINT16_MAX << OPTIMIZER_BITS_IN_COUNTER) << backoff) | backoff; } } #endif /* ENABLE_SPECIALIZATION */ @@ -2365,28 +2349,17 @@ dummy_func( JUMP_BACKWARD_NO_INTERRUPT, }; - inst(ENTER_EXECUTOR, (--)) { - TIER_ONE_ONLY + tier1 inst(ENTER_EXECUTOR, (--)) { CHECK_EVAL_BREAKER(); - PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; - if (executor->vm_data.valid) { - Py_INCREF(executor); - current_executor = executor; - GOTO_TIER_TWO(); - } - else { - /* ENTER_EXECUTOR will be the first code unit of the instruction */ - assert(oparg < 256); - code->co_executors->executors[oparg] = NULL; - opcode = this_instr->op.code = executor->vm_data.opcode; - this_instr->op.arg = executor->vm_data.oparg; - oparg = executor->vm_data.oparg; - Py_DECREF(executor); - next_instr = this_instr; - DISPATCH_GOTO(); - } + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + Py_INCREF(executor); + GOTO_TIER_TWO(executor); } replaced op(_POP_JUMP_IF_FALSE, (cond -- )) { @@ -2425,8 +2398,7 @@ dummy_func( macro(POP_JUMP_IF_NOT_NONE) = unused/1 + _IS_NONE + _POP_JUMP_IF_FALSE; - inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { - TIER_ONE_ONLY + tier1 inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -2522,7 +2494,6 @@ dummy_func( }; specializing op(_SPECIALIZE_FOR_ITER, (counter/1, iter -- iter)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -2614,11 +2585,14 @@ dummy_func( assert(Py_TYPE(iter) == &PyListIter_Type); STAT_INC(FOR_ITER, hit); PyListObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { + if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + it->it_index = -1; + #ifndef Py_GIL_DISABLED if (seq != NULL) { it->it_seq = NULL; Py_DECREF(seq); } + #endif Py_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ @@ -2633,7 +2607,7 @@ dummy_func( assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; DEOPT_IF(seq == NULL); - DEOPT_IF(it->it_index >= PyList_GET_SIZE(seq)); + DEOPT_IF((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)); } op(_ITER_NEXT_LIST, (iter -- iter, next)) { @@ -2722,7 +2696,7 @@ dummy_func( DEOPT_IF(r->len <= 0); } - op(_ITER_NEXT_RANGE, (iter -- iter, next: &PYLONG_TYPE)) { + op(_ITER_NEXT_RANGE, (iter -- iter, next)) { _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); @@ -2880,19 +2854,19 @@ dummy_func( exc_info->exc_value = Py_NewRef(new_exc); } - op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner: &GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_TYPE)) { + op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); } - op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner: &(GUARD_KEYS_VERSION_TYPE + keys_version))) { + op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) { PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version); } - op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { + split op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); @@ -3017,7 +2991,6 @@ dummy_func( }; specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -3101,7 +3074,7 @@ dummy_func( macro(CALL) = _SPECIALIZE_CALL + unused/2 + _CALL; - op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable: &PYMETHOD_TYPE, null: &NULL_TYPE, unused[oparg])) { + op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { DEOPT_IF(null != NULL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type); } @@ -3119,7 +3092,7 @@ dummy_func( DEOPT_IF(tstate->interp->eval_frame); } - op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable: &(PYFUNCTION_TYPE_VERSION_TYPE + func_version), self_or_null, unused[oparg])) { + op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { DEOPT_IF(!PyFunction_Check(callable)); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version); @@ -3134,7 +3107,7 @@ dummy_func( DEOPT_IF(tstate->py_recursion_remaining <= 1); } - pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { int argcount = oparg; if (self_or_null != NULL) { args--; @@ -3382,7 +3355,7 @@ dummy_func( STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ - res = ((_PyCFunctionFast)(void(*)(void))cfunc)( + res = ((PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable), args, total_args); @@ -3413,8 +3386,8 @@ dummy_func( DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable); res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3479,8 +3452,7 @@ dummy_func( } // This is secretly a super-instruction - inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, args[oparg] -- unused)) { - TIER_ONE_ONLY + tier1 inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, args[oparg] -- unused)) { assert(oparg == 1); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.list_append); @@ -3545,8 +3517,8 @@ dummy_func( DEOPT_IF(!Py_IS_TYPE(self, d_type)); STAT_INC(CALL, hit); int nargs = total_args - 1; - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; res = cfunc(self, args + 1, nargs, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3603,8 +3575,8 @@ dummy_func( PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); - _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; res = cfunc(self, args + 1, nargs); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3818,8 +3790,7 @@ dummy_func( } } - inst(RETURN_GENERATOR, (--)) { - TIER_ONE_ONLY + tier1 inst(RETURN_GENERATOR, (--)) { assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -3885,7 +3856,6 @@ dummy_func( } specializing op(_SPECIALIZE_BINARY_OP, (counter/1, lhs, rhs -- lhs, rhs)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -3991,8 +3961,7 @@ dummy_func( INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } - inst(EXTENDED_ARG, ( -- )) { - TIER_ONE_ONLY + tier1 inst(EXTENDED_ARG, ( -- )) { assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; @@ -4000,29 +3969,27 @@ dummy_func( DISPATCH_GOTO(); } - inst(CACHE, (--)) { - TIER_ONE_ONLY + tier1 inst(CACHE, (--)) { assert(0 && "Executing a cache."); - Py_UNREACHABLE(); + Py_FatalError("Executing a cache."); } - inst(RESERVED, (--)) { - TIER_ONE_ONLY + tier1 inst(RESERVED, (--)) { assert(0 && "Executing RESERVED instruction."); - Py_UNREACHABLE(); + Py_FatalError("Executing RESERVED instruction."); } ///////// Tier-2 only opcodes ///////// op (_GUARD_IS_TRUE_POP, (flag -- )) { SYNC_SP(); - DEOPT_IF(!Py_IsTrue(flag)); + EXIT_IF(!Py_IsTrue(flag)); assert(Py_IsTrue(flag)); } op (_GUARD_IS_FALSE_POP, (flag -- )) { SYNC_SP(); - DEOPT_IF(!Py_IsFalse(flag)); + EXIT_IF(!Py_IsFalse(flag)); assert(Py_IsFalse(flag)); } @@ -4030,25 +3997,25 @@ dummy_func( SYNC_SP(); if (!Py_IsNone(val)) { Py_DECREF(val); - DEOPT_IF(1); + EXIT_IF(1); } } op (_GUARD_IS_NOT_NONE_POP, (val -- )) { SYNC_SP(); - DEOPT_IF(Py_IsNone(val)); + EXIT_IF(Py_IsNone(val)); Py_DECREF(val); } op(_JUMP_TO_TOP, (--)) { - next_uop = current_executor->trace; +#ifndef _Py_JIT + next_uop = ¤t_executor->trace[1]; +#endif CHECK_EVAL_BREAKER(); } - op(_SET_IP, (--)) { - TIER_TWO_ONLY - // TODO: Put the code pointer in `operand` to avoid indirection via `frame` - frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + oparg; + tier2 op(_SET_IP, (instr_ptr/4 --)) { + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; } op(_SAVE_RETURN_OFFSET, (--)) { @@ -4060,22 +4027,38 @@ dummy_func( #endif } - op(_EXIT_TRACE, (--)) { - TIER_TWO_ONLY - DEOPT_IF(1); + tier2 op(_EXIT_TRACE, (--)) { + EXIT_IF(1); } - op(_CHECK_VALIDITY, (--)) { - TIER_TWO_ONLY + tier2 op(_CHECK_VALIDITY, (--)) { DEOPT_IF(!current_executor->vm_data.valid); } - op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + tier2 pure op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + value = Py_NewRef(ptr); + } + + tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + value = ptr; + } + + tier2 pure op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { value = Py_NewRef(ptr); + null = NULL; } - op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + tier2 pure op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { value = ptr; + null = NULL; + } + + tier2 op(_CHECK_GLOBALS, (dict/4 -- )) { + DEOPT_IF(GLOBALS() != dict); + } + + tier2 op(_CHECK_BUILTINS, (dict/4 -- )) { + DEOPT_IF(BUILTINS() != dict); } /* Internal -- for testing executors */ @@ -4084,6 +4067,59 @@ dummy_func( exe->count++; } + /* Only used for handling cold side exits, should never appear in + * a normal trace or as part of an instruction. + */ + tier2 op(_COLD_EXIT, (--)) { + _PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor; + _PyExitData *exit = &previous->exits[oparg]; + exit->temperature++; + PyCodeObject *code = _PyFrame_GetCode(frame); + _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; + if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) { + GOTO_TIER_ONE(target); + } + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } else { + int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); + if (optimized <= 0) { + int32_t new_temp = -1 * tstate->interp->optimizer_side_threshold; + exit->temperature = (new_temp < INT16_MIN) ? INT16_MIN : new_temp; + if (optimized < 0) { + Py_DECREF(previous); + tstate->previous_executor = Py_None; + ERROR_IF(1, error); + } + GOTO_TIER_ONE(target); + } + } + /* We need two references. One to store in exit->executor and + * one to keep the executor alive when executing. */ + Py_INCREF(executor); + exit->executor = executor; + GOTO_TIER_TWO(executor); + } + + tier2 op(_START_EXECUTOR, (executor/4 --)) { + Py_DECREF(tstate->previous_executor); + tstate->previous_executor = NULL; +#ifndef _Py_JIT + current_executor = (_PyExecutorObject*)executor; +#endif + } + + tier2 op(_FATAL_ERROR, (--)) { + assert(0); + Py_FatalError("Fatal error uop executed."); + } + + tier2 op(_CHECK_VALIDITY_AND_SET_IP, (instr_ptr/4 --)) { + DEOPT_IF(!current_executor->vm_data.valid); + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + } // END BYTECODES // diff --git a/Python/ceval.c b/Python/ceval.c index 49388cd20377c0c..41e9310938d826e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -5,16 +5,18 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_SignalAsyncExc() +#include "pycore_ceval.h" #include "pycore_code.h" #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "pycore_function.h" #include "pycore_instruments.h" #include "pycore_intrinsics.h" +#include "pycore_jit.h" #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_opcode_metadata.h" // EXTRA_CASES +#include "pycore_optimizer.h" // _PyUOpExecutor_Type #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -647,7 +649,10 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { extern const struct _PyCode_DEF(8) _Py_InitCleanup; -extern const char *_PyUOpName(int index); +#ifdef Py_DEBUG +extern void _PyUOpPrint(const _PyUOpInstruction *uop); +#endif + /* Disable unused label warnings. They are handy for debugging, even if computed gotos aren't used. */ @@ -737,15 +742,16 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int goto resume_with_error; } - /* State shared between Tier 1 and Tier 2 interpreter */ - _PyExecutorObject *current_executor = NULL; - /* Local "register" variables. * These are cached values from the frame and code object. */ - _Py_CODEUNIT *next_instr; PyObject **stack_pointer; +#ifndef _Py_JIT + /* Tier 2 interpreter state */ + _PyExecutorObject *current_executor = NULL; + const _PyUOpInstruction *next_uop = NULL; +#endif start_frame: if (_Py_EnterRecursivePy(tstate)) { @@ -794,17 +800,23 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int { _Py_CODEUNIT *prev = frame->instr_ptr; _Py_CODEUNIT *here = frame->instr_ptr = next_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - int original_opcode = _Py_call_instrumentation_line( - tstate, frame, here, prev); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = here+1; - goto error; - } - next_instr = frame->instr_ptr; - if (next_instr != here) { - DISPATCH(); + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + original_opcode = code->_co_monitoring->lines[(int)(here - _PyCode_CODE(code))].original_opcode; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, here, prev); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = here+1; + goto error; + } + next_instr = frame->instr_ptr; + if (next_instr != here) { + DISPATCH(); + } } if (_PyOpcode_Caches[original_opcode]) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); @@ -955,9 +967,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int -// The Tier 2 interpreter is also here! +// Tier 2 is also here! enter_tier_two: +#ifdef _Py_JIT + assert(0); +#else + #undef LOAD_IP #define LOAD_IP(UNUSED) (void)0 @@ -991,22 +1007,22 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif OPT_STAT_INC(traces_executed); - _PyUOpInstruction *next_uop = current_executor->trace; uint16_t uopcode; #ifdef Py_STATS uint64_t trace_uop_execution_counter = 0; #endif + assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); for (;;) { uopcode = next_uop->opcode; - DPRINTF(3, - "%4d: uop %s, oparg %d, operand %" PRIu64 ", target %d, stack_level %d\n", - (int)(next_uop - current_executor->trace), - _PyUOpName(uopcode), - next_uop->oparg, - next_uop->operand, - next_uop->target, +#ifdef Py_DEBUG + if (lltrace >= 3) { + printf("%4d uop: ", (int)(next_uop - (current_executor == NULL ? next_uop : current_executor->trace))); + _PyUOpPrint(next_uop); + printf(" stack_level=%d\n", (int)(stack_pointer - _PyFrame_Stackbase(frame))); + } +#endif next_uop++; OPT_STAT_INC(uops_executed); UOP_STAT_INC(uopcode, execution_count); @@ -1021,9 +1037,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int default: #ifdef Py_DEBUG { - fprintf(stderr, "Unknown uop %d, oparg %d, operand %" PRIu64 " @ %d\n", - opcode, next_uop[-1].oparg, next_uop[-1].operand, - (int)(next_uop - current_executor->trace - 1)); + printf("Unknown uop: "); + _PyUOpPrint(&next_uop[-1]); + printf(" @ %d\n", (int)(next_uop - current_executor->trace - 1)); Py_FatalError("Unknown uop"); } #else @@ -1051,29 +1067,63 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int pop_1_error_tier_two: STACK_SHRINK(1); error_tier_two: - DPRINTF(2, "Error: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d -> %s]\n", - uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, - (int)(next_uop - current_executor->trace - 1), - _PyOpcode_OpName[frame->instr_ptr->op.code]); +#ifdef Py_DEBUG + if (lltrace >= 2) { + printf("Error: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(" @ %d -> %s]\n", + (int)(next_uop - current_executor->trace - 1), + _PyOpcode_OpName[frame->instr_ptr->op.code]); + } +#endif OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); frame->return_offset = 0; // Don't leave this random _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(current_executor); + tstate->previous_executor = NULL; goto resume_with_error; // Jump here from DEOPT_IF() deoptimize: next_instr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); - DPRINTF(2, "DEOPT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d -> %s]\n", - uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, - (int)(next_uop - current_executor->trace - 1), - _PyOpcode_OpName[frame->instr_ptr->op.code]); +#ifdef Py_DEBUG + if (lltrace >= 2) { + printf("DEOPT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(" -> %s]\n", + _PyOpcode_OpName[frame->instr_ptr->op.code]); + } +#endif OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); UOP_STAT_INC(uopcode, miss); Py_DECREF(current_executor); + tstate->previous_executor = NULL; DISPATCH(); +// Jump here from EXIT_IF() +side_exit: + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + UOP_STAT_INC(uopcode, miss); + uint32_t exit_index = next_uop[-1].exit_index; + assert(exit_index < current_executor->exit_count); + _PyExitData *exit = ¤t_executor->exits[exit_index]; +#ifdef Py_DEBUG + if (lltrace >= 2) { + printf("SIDE EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %u, temp %d, target %d -> %s]\n", + exit_index, exit->temperature, exit->target, + _PyOpcode_OpName[frame->instr_ptr->op.code]); + } +#endif + Py_INCREF(exit->executor); + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_TWO(exit->executor); + +#endif // _Py_JIT + } + #if defined(__GNUC__) # pragma GCC diagnostic pop #elif defined(_MSC_VER) /* MS_WINDOWS */ diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index f3b169241535f37..f5c44307a513f83 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -56,60 +56,52 @@ #define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) _Py_atomic_load_relaxed(ATOMIC_VAL) #endif -/* bpo-40010: eval_breaker should be recomputed if there - is a pending signal: signal received by another thread which cannot - handle signals. - Similarly, we set CALLS_TO_DO and ASYNC_EXCEPTION to match the thread. -*/ +// Atomically copy the bits indicated by mask between two values. static inline void -update_eval_breaker_from_thread(PyInterpreterState *interp, PyThreadState *tstate) +copy_eval_breaker_bits(uintptr_t *from, uintptr_t *to, uintptr_t mask) { - if (tstate == NULL) { + uintptr_t from_bits = _Py_atomic_load_uintptr_relaxed(from) & mask; + uintptr_t old_value = _Py_atomic_load_uintptr_relaxed(to); + uintptr_t to_bits = old_value & mask; + if (from_bits == to_bits) { return; } - if (_Py_IsMainThread()) { - int32_t calls_to_do = _Py_atomic_load_int32_relaxed( - &_PyRuntime.ceval.pending_mainthread.calls_to_do); - if (calls_to_do) { - _Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 1); - } - if (_Py_ThreadCanHandleSignals(interp)) { - if (_Py_atomic_load_int(&_PyRuntime.signals.is_tripped)) { - _Py_set_eval_breaker_bit(interp, _PY_SIGNALS_PENDING_BIT, 1); - } - } - } - if (tstate->async_exc != NULL) { - _Py_set_eval_breaker_bit(interp, _PY_ASYNC_EXCEPTION_BIT, 1); - } + uintptr_t new_value; + do { + new_value = (old_value & ~mask) | from_bits; + } while (!_Py_atomic_compare_exchange_uintptr(to, &old_value, new_value)); } +// When attaching a thread, set the global instrumentation version and +// _PY_CALLS_TO_DO_BIT from the current state of the interpreter. static inline void -SET_GIL_DROP_REQUEST(PyInterpreterState *interp) +update_eval_breaker_for_thread(PyInterpreterState *interp, PyThreadState *tstate) { - _Py_set_eval_breaker_bit(interp, _PY_GIL_DROP_REQUEST_BIT, 1); -} - - -static inline void -RESET_GIL_DROP_REQUEST(PyInterpreterState *interp) -{ - _Py_set_eval_breaker_bit(interp, _PY_GIL_DROP_REQUEST_BIT, 0); -} - - -static inline void -SIGNAL_PENDING_CALLS(PyInterpreterState *interp) -{ - _Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 1); -} +#ifdef Py_GIL_DISABLED + // Free-threaded builds eagerly update the eval_breaker on *all* threads as + // needed, so this function doesn't apply. + return; +#endif + int32_t calls_to_do = _Py_atomic_load_int32_relaxed( + &interp->ceval.pending.calls_to_do); + if (calls_to_do) { + _Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); + } + else if (_Py_IsMainThread()) { + calls_to_do = _Py_atomic_load_int32_relaxed( + &_PyRuntime.ceval.pending_mainthread.calls_to_do); + if (calls_to_do) { + _Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); + } + } -static inline void -UNSIGNAL_PENDING_CALLS(PyInterpreterState *interp) -{ - _Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 0); + // _PY_CALLS_TO_DO_BIT was derived from other state above, so the only bits + // we copy from our interpreter's state are the instrumentation version. + copy_eval_breaker_bits(&interp->ceval.instrumentation_version, + &tstate->eval_breaker, + ~_PY_EVAL_EVENTS_MASK); } /* @@ -254,13 +246,14 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) the GIL, and that's the only time we might delete the interpreter, so checking tstate first prevents the crash. See https://github.com/python/cpython/issues/104341. */ - if (tstate != NULL && _Py_eval_breaker_bit_is_set(interp, _PY_GIL_DROP_REQUEST_BIT)) { + if (tstate != NULL && + _Py_eval_breaker_bit_is_set(tstate, _PY_GIL_DROP_REQUEST_BIT)) { MUTEX_LOCK(gil->switch_mutex); /* Not switched yet => wait */ if (((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) == tstate) { assert(_PyThreadState_CheckConsistency(tstate)); - RESET_GIL_DROP_REQUEST(tstate->interp); + _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); /* NOTE: if COND_WAIT does not atomically start waiting when releasing the mutex, another thread can run through, take the GIL and drop it again, and reset the condition @@ -321,6 +314,8 @@ take_gil(PyThreadState *tstate) _Py_atomic_load_int_relaxed(&gil->locked) && gil->switch_number == saved_switchnum) { + PyThreadState *holder_tstate = + (PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder); if (_PyThreadState_MustExit(tstate)) { MUTEX_UNLOCK(gil->mutex); // gh-96387: If the loop requested a drop request in a previous @@ -330,13 +325,13 @@ take_gil(PyThreadState *tstate) // may have to request again a drop request (iterate one more // time). if (drop_requested) { - RESET_GIL_DROP_REQUEST(interp); + _Py_unset_eval_breaker_bit(holder_tstate, _PY_GIL_DROP_REQUEST_BIT); } PyThread_exit_thread(); } assert(_PyThreadState_CheckConsistency(tstate)); - SET_GIL_DROP_REQUEST(interp); + _Py_set_eval_breaker_bit(holder_tstate, _PY_GIL_DROP_REQUEST_BIT); drop_requested = 1; } } @@ -369,13 +364,15 @@ take_gil(PyThreadState *tstate) in take_gil() while the main thread called wait_for_thread_shutdown() from Py_Finalize(). */ MUTEX_UNLOCK(gil->mutex); - drop_gil(interp, tstate); + /* Passing NULL to drop_gil() indicates that this thread is about to + terminate and will never hold the GIL again. */ + drop_gil(interp, NULL); PyThread_exit_thread(); } assert(_PyThreadState_CheckConsistency(tstate)); - RESET_GIL_DROP_REQUEST(interp); - update_eval_breaker_from_thread(interp, tstate); + _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); + update_eval_breaker_for_thread(interp, tstate); MUTEX_UNLOCK(gil->mutex); @@ -590,15 +587,6 @@ _PyEval_ReInitThreads(PyThreadState *tstate) } #endif -/* This function is used to signal that async exceptions are waiting to be - raised. */ - -void -_PyEval_SignalAsyncExc(PyInterpreterState *interp) -{ - _Py_set_eval_breaker_bit(interp, _PY_ASYNC_EXCEPTION_BIT, 1); -} - PyThreadState * PyEval_SaveThread(void) { @@ -610,8 +598,16 @@ PyEval_SaveThread(void) void PyEval_RestoreThread(PyThreadState *tstate) { +#ifdef MS_WINDOWS + int err = GetLastError(); +#endif + _Py_EnsureTstateNotNULL(tstate); _PyThreadState_Attach(tstate); + +#ifdef MS_WINDOWS + SetLastError(err); +#endif } @@ -638,11 +634,9 @@ PyEval_RestoreThread(PyThreadState *tstate) */ void -_PyEval_SignalReceived(PyInterpreterState *interp) +_PyEval_SignalReceived(void) { - if (_Py_ThreadCanHandleSignals(interp)) { - _Py_set_eval_breaker_bit(interp, _PY_SIGNALS_PENDING_BIT, 1); - } + _Py_set_eval_breaker_bit(_PyRuntime.main_tstate, _PY_SIGNALS_PENDING_BIT); } /* Push one item onto the queue while holding the lock. */ @@ -694,6 +688,26 @@ _pop_pending_call(struct _pending_calls *pending, } } +#ifndef Py_GIL_DISABLED +static void +signal_active_thread(PyInterpreterState *interp, uintptr_t bit) +{ + struct _gil_runtime_state *gil = interp->ceval.gil; + + // If a thread from the targeted interpreter is holding the GIL, signal + // that thread. Otherwise, the next thread to run from the targeted + // interpreter will have its bit set as part of taking the GIL. + MUTEX_LOCK(gil->mutex); + if (_Py_atomic_load_int_relaxed(&gil->locked)) { + PyThreadState *holder = (PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder); + if (holder->interp == interp) { + _Py_set_eval_breaker_bit(holder, bit); + } + } + MUTEX_UNLOCK(gil->mutex); +} +#endif + /* This implementation is thread-safe. It allows scheduling to be made from any thread, and even from an executing callback. @@ -703,10 +717,9 @@ int _PyEval_AddPendingCall(PyInterpreterState *interp, _Py_pending_call_func func, void *arg, int flags) { - assert(!(flags & _Py_PENDING_MAINTHREADONLY) - || _Py_IsMainInterpreter(interp)); struct _pending_calls *pending = &interp->ceval.pending; - if (flags & _Py_PENDING_MAINTHREADONLY) { + int main_only = (flags & _Py_PENDING_MAINTHREADONLY) != 0; + if (main_only) { /* The main thread only exists in the main interpreter. */ assert(_Py_IsMainInterpreter(interp)); pending = &_PyRuntime.ceval.pending_mainthread; @@ -716,8 +729,17 @@ _PyEval_AddPendingCall(PyInterpreterState *interp, int result = _push_pending_call(pending, func, arg, flags); PyMutex_Unlock(&pending->mutex); - /* signal main loop */ - SIGNAL_PENDING_CALLS(interp); + if (main_only) { + _Py_set_eval_breaker_bit(_PyRuntime.main_tstate, _PY_CALLS_TO_DO_BIT); + } + else { +#ifdef Py_GIL_DISABLED + _Py_set_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT); +#else + signal_active_thread(interp, _PY_CALLS_TO_DO_BIT); +#endif + } + return result; } @@ -734,13 +756,13 @@ static int handle_signals(PyThreadState *tstate) { assert(_PyThreadState_CheckConsistency(tstate)); - _Py_set_eval_breaker_bit(tstate->interp, _PY_SIGNALS_PENDING_BIT, 0); + _Py_unset_eval_breaker_bit(tstate, _PY_SIGNALS_PENDING_BIT); if (!_Py_ThreadCanHandleSignals(tstate->interp)) { return 0; } if (_PyErr_CheckSignalsTstate(tstate) < 0) { /* On failure, re-schedule a call to handle_signals(). */ - _Py_set_eval_breaker_bit(tstate->interp, _PY_SIGNALS_PENDING_BIT, 1); + _Py_set_eval_breaker_bit(tstate, _PY_SIGNALS_PENDING_BIT); return -1; } return 0; @@ -775,9 +797,30 @@ _make_pending_calls(struct _pending_calls *pending) return 0; } +static void +signal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + _Py_set_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT); +#else + _Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); +#endif +} + +static void +unsignal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + _Py_unset_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT); +#else + _Py_unset_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); +#endif +} + static int -make_pending_calls(PyInterpreterState *interp) +make_pending_calls(PyThreadState *tstate) { + PyInterpreterState *interp = tstate->interp; struct _pending_calls *pending = &interp->ceval.pending; struct _pending_calls *pending_main = &_PyRuntime.ceval.pending_mainthread; @@ -803,12 +846,12 @@ make_pending_calls(PyInterpreterState *interp) /* unsignal before starting to call callbacks, so that any callback added in-between re-signals */ - UNSIGNAL_PENDING_CALLS(interp); + unsignal_pending_calls(tstate, interp); if (_make_pending_calls(pending) != 0) { pending->busy = 0; /* There might not be more calls to make, but we play it safe. */ - SIGNAL_PENDING_CALLS(interp); + signal_pending_calls(tstate, interp); return -1; } @@ -816,7 +859,7 @@ make_pending_calls(PyInterpreterState *interp) if (_make_pending_calls(pending_main) != 0) { pending->busy = 0; /* There might not be more calls to make, but we play it safe. */ - SIGNAL_PENDING_CALLS(interp); + signal_pending_calls(tstate, interp); return -1; } } @@ -825,13 +868,37 @@ make_pending_calls(PyInterpreterState *interp) return 0; } +void +_Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit) +{ + _PyRuntimeState *runtime = &_PyRuntime; + + HEAD_LOCK(runtime); + for (PyThreadState *tstate = interp->threads.head; tstate != NULL; tstate = tstate->next) { + _Py_set_eval_breaker_bit(tstate, bit); + } + HEAD_UNLOCK(runtime); +} + +void +_Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit) +{ + _PyRuntimeState *runtime = &_PyRuntime; + + HEAD_LOCK(runtime); + for (PyThreadState *tstate = interp->threads.head; tstate != NULL; tstate = tstate->next) { + _Py_unset_eval_breaker_bit(tstate, bit); + } + HEAD_UNLOCK(runtime); +} + void _Py_FinishPendingCalls(PyThreadState *tstate) { assert(PyGILState_Check()); assert(_PyThreadState_CheckConsistency(tstate)); - if (make_pending_calls(tstate->interp) < 0) { + if (make_pending_calls(tstate) < 0) { PyObject *exc = _PyErr_GetRaisedException(tstate); PyErr_BadInternalCall(); _PyErr_ChainExceptions1(exc); @@ -854,7 +921,7 @@ _PyEval_MakePendingCalls(PyThreadState *tstate) } } - res = make_pending_calls(tstate->interp); + res = make_pending_calls(tstate); if (res != 0) { return res; } @@ -947,11 +1014,11 @@ _PyEval_InitState(PyInterpreterState *interp) int _Py_HandlePending(PyThreadState *tstate) { - PyInterpreterState *interp = tstate->interp; + uintptr_t breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); /* Stop-the-world */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_EVAL_PLEASE_STOP_BIT)) { - _Py_set_eval_breaker_bit(interp, _PY_EVAL_PLEASE_STOP_BIT, 0); + if ((breaker & _PY_EVAL_PLEASE_STOP_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_PLEASE_STOP_BIT); _PyThreadState_Suspend(tstate); /* The attach blocks until the stop-the-world event is complete. */ @@ -959,27 +1026,35 @@ _Py_HandlePending(PyThreadState *tstate) } /* Pending signals */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_SIGNALS_PENDING_BIT)) { + if ((breaker & _PY_SIGNALS_PENDING_BIT) != 0) { if (handle_signals(tstate) != 0) { return -1; } } /* Pending calls */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_CALLS_TO_DO_BIT)) { - if (make_pending_calls(interp) != 0) { + if ((breaker & _PY_CALLS_TO_DO_BIT) != 0) { + if (make_pending_calls(tstate) != 0) { return -1; } } +#ifdef Py_GIL_DISABLED + /* Objects with refcounts to merge */ + if ((breaker & _PY_EVAL_EXPLICIT_MERGE_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_EXPLICIT_MERGE_BIT); + _Py_brc_merge_refcounts(tstate); + } +#endif + /* GC scheduled to run */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_GC_SCHEDULED_BIT)) { - _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 0); + if ((breaker & _PY_GC_SCHEDULED_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT); _Py_RunGC(tstate); } /* GIL drop request */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_GIL_DROP_REQUEST_BIT)) { + if ((breaker & _PY_GIL_DROP_REQUEST_BIT) != 0) { /* Give another thread a chance */ _PyThreadState_Detach(tstate); @@ -989,11 +1064,10 @@ _Py_HandlePending(PyThreadState *tstate) } /* Check for asynchronous exception. */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_ASYNC_EXCEPTION_BIT)) { - _Py_set_eval_breaker_bit(interp, _PY_ASYNC_EXCEPTION_BIT, 0); - if (tstate->async_exc != NULL) { - PyObject *exc = tstate->async_exc; - tstate->async_exc = NULL; + if ((breaker & _PY_ASYNC_EXCEPTION_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_ASYNC_EXCEPTION_BIT); + PyObject *exc = _Py_atomic_exchange_ptr(&tstate->async_exc, NULL); + if (exc != NULL) { _PyErr_SetNone(tstate, exc); Py_DECREF(exc); return -1; @@ -1001,4 +1075,3 @@ _Py_HandlePending(PyThreadState *tstate) } return 0; } - diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index c2550f53ad6eaaa..6ad41950ea3b7e9 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -86,6 +86,12 @@ #define PRE_DISPATCH_GOTO() ((void)0) #endif +#ifdef Py_GIL_DISABLED +#define QSBR_QUIESCENT_STATE(tstate) _Py_qsbr_quiescent_state(((_PyThreadStateImpl *)tstate)->qsbr) +#else +#define QSBR_QUIESCENT_STATE(tstate) +#endif + /* Do interpreter dispatch accounting for tracing and instrumentation */ #define DISPATCH() \ @@ -117,7 +123,8 @@ #define CHECK_EVAL_BREAKER() \ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \ - if (_Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & _PY_EVAL_EVENTS_MASK) { \ + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { \ if (_Py_HandlePending(tstate) != 0) { \ GOTO_ERROR(error); \ } \ @@ -332,12 +339,16 @@ do { \ // for an exception handler, displaying the traceback, and so on #define INSTRUMENTED_JUMP(src, dest, event) \ do { \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (next_instr == NULL) { \ - next_instr = (dest)+1; \ - goto error; \ + if (tstate->tracing) {\ + next_instr = dest; \ + } else { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + if (next_instr == NULL) { \ + next_instr = (dest)+1; \ + goto error; \ + } \ } \ } while (0); @@ -365,12 +376,6 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { tstate->py_recursion_remaining++; } -/* Marker to specify tier 1 only instructions */ -#define TIER_ONE_ONLY - -/* Marker to specify tier 2 only instructions */ -#define TIER_TWO_ONLY - /* Implementation of "macros" that modify the instruction pointer, * stack pointer, or frame pointer. * These need to treated differently by tier 1 and 2. @@ -387,7 +392,36 @@ stack_pointer = _PyFrame_GetStackPointer(frame); /* Tier-switching macros. */ -#define GOTO_TIER_TWO() goto enter_tier_two; +#ifdef _Py_JIT +#define GOTO_TIER_TWO(EXECUTOR) \ +do { \ + jit_func jitted = (EXECUTOR)->jit_code; \ + next_instr = jitted(frame, stack_pointer, tstate); \ + Py_DECREF(tstate->previous_executor); \ + tstate->previous_executor = NULL; \ + frame = tstate->current_frame; \ + if (next_instr == NULL) { \ + goto resume_with_error; \ + } \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + DISPATCH(); \ +} while (0) +#else +#define GOTO_TIER_TWO(EXECUTOR) \ +do { \ + next_uop = (EXECUTOR)->trace; \ + assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); \ + goto enter_tier_two; \ +} while (0) +#endif + +#define GOTO_TIER_ONE(TARGET) \ +do { \ + Py_DECREF(tstate->previous_executor); \ + tstate->previous_executor = NULL; \ + next_instr = target; \ + DISPATCH(); \ +} while (0) #define CURRENT_OPARG() (next_uop[-1].oparg) diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 8d40e659b54a57e..3898f987cd61ea6 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -233,25 +233,6 @@ PyDoc_STRVAR(builtin_chr__doc__, #define BUILTIN_CHR_METHODDEF \ {"chr", (PyCFunction)builtin_chr, METH_O, builtin_chr__doc__}, -static PyObject * -builtin_chr_impl(PyObject *module, int i); - -static PyObject * -builtin_chr(PyObject *module, PyObject *arg) -{ - PyObject *return_value = NULL; - int i; - - i = PyLong_AsInt(arg); - if (i == -1 && PyErr_Occurred()) { - goto exit; - } - return_value = builtin_chr_impl(module, i); - -exit: - return return_value; -} - PyDoc_STRVAR(builtin_compile__doc__, "compile($module, /, source, filename, mode, flags=0,\n" " dont_inherit=False, optimize=-1, *, _feature_version=-1)\n" @@ -1212,4 +1193,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=31bded5d08647a57 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=643a8d5f900e0c36 input=a9049054013a1b77]*/ diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 93b8385a5b4097c..13f4ea81eb89842 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1131,6 +1131,24 @@ sys__clear_type_cache(PyObject *module, PyObject *Py_UNUSED(ignored)) return sys__clear_type_cache_impl(module); } +PyDoc_STRVAR(sys__clear_internal_caches__doc__, +"_clear_internal_caches($module, /)\n" +"--\n" +"\n" +"Clear all internal performance-related caches."); + +#define SYS__CLEAR_INTERNAL_CACHES_METHODDEF \ + {"_clear_internal_caches", (PyCFunction)sys__clear_internal_caches, METH_NOARGS, sys__clear_internal_caches__doc__}, + +static PyObject * +sys__clear_internal_caches_impl(PyObject *module); + +static PyObject * +sys__clear_internal_caches(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return sys__clear_internal_caches_impl(module); +} + PyDoc_STRVAR(sys_is_finalizing__doc__, "is_finalizing($module, /)\n" "--\n" @@ -1486,4 +1504,4 @@ sys__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=3dc3b2cb0ce38ebb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b8b1c53e04c3b20c input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index 7cf05dd06831195..6b17f3bcaf22644 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -160,7 +160,7 @@ _PyCompile_EnsureArrayLargeEnough(int idx, void **array, int *alloc, if (idx >= new_alloc) { new_alloc = idx + default_alloc; } - arr = PyObject_Calloc(new_alloc, item_size); + arr = PyMem_Calloc(new_alloc, item_size); if (arr == NULL) { PyErr_NoMemory(); return ERROR; @@ -181,7 +181,7 @@ _PyCompile_EnsureArrayLargeEnough(int idx, void **array, int *alloc, } assert(newsize > 0); - void *tmp = PyObject_Realloc(arr, newsize); + void *tmp = PyMem_Realloc(arr, newsize); if (tmp == NULL) { PyErr_NoMemory(); return ERROR; @@ -282,10 +282,10 @@ instr_sequence_insert_instruction(instr_sequence *seq, int pos, static void instr_sequence_fini(instr_sequence *seq) { - PyObject_Free(seq->s_labelmap); + PyMem_Free(seq->s_labelmap); seq->s_labelmap = NULL; - PyObject_Free(seq->s_instrs); + PyMem_Free(seq->s_instrs); seq->s_instrs = NULL; } @@ -690,7 +690,7 @@ compiler_unit_free(struct compiler_unit *u) Py_CLEAR(u->u_metadata.u_cellvars); Py_CLEAR(u->u_metadata.u_fasthidden); Py_CLEAR(u->u_private); - PyObject_Free(u); + PyMem_Free(u); } static int @@ -921,11 +921,10 @@ dict_add_o(PyObject *dict, PyObject *o) PyObject *v; Py_ssize_t arg; - v = PyDict_GetItemWithError(dict, o); + if (PyDict_GetItemRef(dict, o, &v) < 0) { + return ERROR; + } if (!v) { - if (PyErr_Occurred()) { - return ERROR; - } arg = PyDict_GET_SIZE(dict); v = PyLong_FromSsize_t(arg); if (!v) { @@ -935,10 +934,10 @@ dict_add_o(PyObject *dict, PyObject *o) Py_DECREF(v); return ERROR; } - Py_DECREF(v); } else arg = PyLong_AsLong(v); + Py_DECREF(v); return arg; } @@ -958,14 +957,15 @@ merge_consts_recursive(PyObject *const_cache, PyObject *o) return NULL; } - // t is borrowed reference - PyObject *t = PyDict_SetDefault(const_cache, key, key); - if (t != key) { - // o is registered in const_cache. Just use it. - Py_XINCREF(t); + PyObject *t; + int res = PyDict_SetDefaultRef(const_cache, key, key, &t); + if (res != 0) { + // o was not inserted into const_cache. t is either the existing value + // or NULL (on error). Py_DECREF(key); return t; } + Py_DECREF(t); // We registered o in const_cache. // When o is a tuple or frozenset, we want to merge its @@ -1262,8 +1262,7 @@ compiler_enter_scope(struct compiler *c, identifier name, struct compiler_unit *u; - u = (struct compiler_unit *)PyObject_Calloc(1, sizeof( - struct compiler_unit)); + u = (struct compiler_unit *)PyMem_Calloc(1, sizeof(struct compiler_unit)); if (!u) { PyErr_NoMemory(); return ERROR; @@ -1692,16 +1691,13 @@ compiler_unwind_fblock_stack(struct compiler *c, location *ploc, static int compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) { - int i = 0; - stmt_ty st; - PyObject *docstring; /* Set current line number to the line number of first statement. This way line number for SETUP_ANNOTATIONS will always coincide with the line number of first "real" statement in module. If body is empty, then lineno will be set later in optimize_and_assemble. */ if (c->u->u_scope_type == COMPILER_SCOPE_MODULE && asdl_seq_LEN(stmts)) { - st = (stmt_ty)asdl_seq_GET(stmts, 0); + stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); loc = LOC(st); } /* Every annotated class and module should have __annotations__. */ @@ -1711,16 +1707,17 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) if (!asdl_seq_LEN(stmts)) { return SUCCESS; } - /* if not -OO mode, set docstring */ - if (c->c_optimize < 2) { - docstring = _PyAST_GetDocString(stmts); - if (docstring) { + Py_ssize_t first_instr = 0; + PyObject *docstring = _PyAST_GetDocString(stmts); + if (docstring) { + first_instr = 1; + /* if not -OO mode, set docstring */ + if (c->c_optimize < 2) { PyObject *cleandoc = _PyCompile_CleanDoc(docstring); if (cleandoc == NULL) { return ERROR; } - i = 1; - st = (stmt_ty)asdl_seq_GET(stmts, 0); + stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); assert(st->kind == Expr_kind); location loc = LOC(st->v.Expr.value); ADDOP_LOAD_CONST(c, loc, cleandoc); @@ -1728,7 +1725,7 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); } } - for (; i < asdl_seq_LEN(stmts); i++) { + for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(stmts); i++) { VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i)); } return SUCCESS; @@ -1737,16 +1734,10 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) static int compiler_codegen(struct compiler *c, mod_ty mod) { - _Py_DECLARE_STR(anon_module, "<module>"); - RETURN_IF_ERROR( - compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, - mod, 1)); - location loc = LOCATION(1, 1, 0, 0); switch (mod->kind) { case Module_kind: if (compiler_body(c, loc, mod->v.Module.body) < 0) { - compiler_exit_scope(c); return ERROR; } break; @@ -1755,10 +1746,10 @@ compiler_codegen(struct compiler *c, mod_ty mod) ADDOP(c, loc, SETUP_ANNOTATIONS); } c->c_interactive = 1; - VISIT_SEQ_IN_SCOPE(c, stmt, mod->v.Interactive.body); + VISIT_SEQ(c, stmt, mod->v.Interactive.body); break; case Expression_kind: - VISIT_IN_SCOPE(c, expr, mod->v.Expression.body); + VISIT(c, expr, mod->v.Expression.body); break; default: PyErr_Format(PyExc_SystemError, @@ -1769,14 +1760,29 @@ compiler_codegen(struct compiler *c, mod_ty mod) return SUCCESS; } +static int +compiler_enter_anonymous_scope(struct compiler* c, mod_ty mod) +{ + _Py_DECLARE_STR(anon_module, "<module>"); + RETURN_IF_ERROR( + compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, + mod, 1)); + return SUCCESS; +} + static PyCodeObject * compiler_mod(struct compiler *c, mod_ty mod) { + PyCodeObject *co = NULL; int addNone = mod->kind != Expression_kind; - if (compiler_codegen(c, mod) < 0) { + if (compiler_enter_anonymous_scope(c, mod) < 0) { return NULL; } - PyCodeObject *co = optimize_and_assemble(c, addNone); + if (compiler_codegen(c, mod) < 0) { + goto finally; + } + co = optimize_and_assemble(c, addNone); +finally: compiler_exit_scope(c); return co; } @@ -2239,7 +2245,6 @@ static int compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t funcflags, int firstlineno) { - PyObject *docstring = NULL; arguments_ty args; identifier name; asdl_stmt_seq *body; @@ -2266,28 +2271,33 @@ compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t f RETURN_IF_ERROR( compiler_enter_scope(c, name, scope_type, (void *)s, firstlineno)); - /* if not -OO mode, add docstring */ - if (c->c_optimize < 2) { - docstring = _PyAST_GetDocString(body); - if (docstring) { + Py_ssize_t first_instr = 0; + PyObject *docstring = _PyAST_GetDocString(body); + if (docstring) { + first_instr = 1; + /* if not -OO mode, add docstring */ + if (c->c_optimize < 2) { docstring = _PyCompile_CleanDoc(docstring); if (docstring == NULL) { compiler_exit_scope(c); return ERROR; } } + else { + docstring = NULL; + } } if (compiler_add_const(c->c_const_cache, c->u, docstring ? docstring : Py_None) < 0) { Py_XDECREF(docstring); compiler_exit_scope(c); return ERROR; } - Py_XDECREF(docstring); + Py_CLEAR(docstring); c->u->u_metadata.u_argcount = asdl_seq_LEN(args->args); c->u->u_metadata.u_posonlyargcount = asdl_seq_LEN(args->posonlyargs); c->u->u_metadata.u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs); - for (Py_ssize_t i = docstring ? 1 : 0; i < asdl_seq_LEN(body); i++) { + for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(body); i++) { VISIT_IN_SCOPE(c, stmt, (stmt_ty)asdl_seq_GET(body, i)); } if (c->u->u_ste->ste_coroutine || c->u->u_ste->ste_generator) { @@ -6657,7 +6667,7 @@ ensure_fail_pop(struct compiler *c, pattern_context *pc, Py_ssize_t n) return SUCCESS; } Py_ssize_t needed = sizeof(jump_target_label) * size; - jump_target_label *resized = PyObject_Realloc(pc->fail_pop, needed); + jump_target_label *resized = PyMem_Realloc(pc->fail_pop, needed); if (resized == NULL) { PyErr_NoMemory(); return ERROR; @@ -6696,13 +6706,13 @@ emit_and_reset_fail_pop(struct compiler *c, location loc, USE_LABEL(c, pc->fail_pop[pc->fail_pop_size]); if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, loc) < 0) { pc->fail_pop_size = 0; - PyObject_Free(pc->fail_pop); + PyMem_Free(pc->fail_pop); pc->fail_pop = NULL; return ERROR; } } USE_LABEL(c, pc->fail_pop[0]); - PyObject_Free(pc->fail_pop); + PyMem_Free(pc->fail_pop); pc->fail_pop = NULL; return SUCCESS; } @@ -7206,7 +7216,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) Py_DECREF(pc->stores); *pc = old_pc; Py_INCREF(pc->stores); - // Need to NULL this for the PyObject_Free call in the error block. + // Need to NULL this for the PyMem_Free call in the error block. old_pc.fail_pop = NULL; // No match. Pop the remaining copy of the subject and fail: if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, LOC(p)) < 0 || @@ -7252,7 +7262,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) diff: compiler_error(c, LOC(p), "alternative patterns bind different names"); error: - PyObject_Free(old_pc.fail_pop); + PyMem_Free(old_pc.fail_pop); Py_DECREF(old_pc.stores); Py_XDECREF(control); return ERROR; @@ -7453,7 +7463,7 @@ compiler_match(struct compiler *c, stmt_ty s) pattern_context pc; pc.fail_pop = NULL; int result = compiler_match_inner(c, s, &pc); - PyObject_Free(pc.fail_pop); + PyMem_Free(pc.fail_pop); return result; } @@ -7528,22 +7538,26 @@ _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj) return ERROR; } - // t is borrowed reference - PyObject *t = PyDict_SetDefault(const_cache, key, key); + PyObject *t; + int res = PyDict_SetDefaultRef(const_cache, key, key, &t); Py_DECREF(key); - if (t == NULL) { + if (res < 0) { return ERROR; } - if (t == key) { // obj is new constant. + if (res == 0) { // inserted: obj is new constant. + Py_DECREF(t); return SUCCESS; } if (PyTuple_CheckExact(t)) { - // t is still borrowed reference - t = PyTuple_GET_ITEM(t, 1); + PyObject *item = PyTuple_GET_ITEM(t, 1); + Py_SETREF(*obj, Py_NewRef(item)); + Py_DECREF(t); + } + else { + Py_SETREF(*obj, t); } - Py_SETREF(*obj, Py_NewRef(t)); return SUCCESS; } @@ -7914,15 +7928,20 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return NULL; } + metadata = PyDict_New(); + if (metadata == NULL) { + return NULL; + } + + if (compiler_enter_anonymous_scope(c, mod) < 0) { + return NULL; + } if (compiler_codegen(c, mod) < 0) { goto finally; } _PyCompile_CodeUnitMetadata *umd = &c->u->u_metadata; - metadata = PyDict_New(); - if (metadata == NULL) { - goto finally; - } + #define SET_MATADATA_ITEM(key, value) \ if (value != NULL) { \ if (PyDict_SetItemString(metadata, key, value) < 0) goto finally; \ diff --git a/Python/condvar.h b/Python/condvar.h index d54db94f2c871d4..dcabed6d55928ce 100644 --- a/Python/condvar.h +++ b/Python/condvar.h @@ -260,13 +260,13 @@ PyMUTEX_UNLOCK(PyMUTEX_T *cs) return 0; } - Py_LOCAL_INLINE(int) PyCOND_INIT(PyCOND_T *cv) { InitializeConditionVariable(cv); return 0; } + Py_LOCAL_INLINE(int) PyCOND_FINI(PyCOND_T *cv) { @@ -279,27 +279,32 @@ PyCOND_WAIT(PyCOND_T *cv, PyMUTEX_T *cs) return SleepConditionVariableSRW(cv, cs, INFINITE, 0) ? 0 : -1; } -/* This implementation makes no distinction about timeouts. Signal - * 2 to indicate that we don't know. - */ +/* return 0 for success, 1 on timeout, -1 on error */ Py_LOCAL_INLINE(int) PyCOND_TIMEDWAIT(PyCOND_T *cv, PyMUTEX_T *cs, long long us) { - return SleepConditionVariableSRW(cv, cs, (DWORD)(us/1000), 0) ? 2 : -1; + BOOL success = SleepConditionVariableSRW(cv, cs, (DWORD)(us/1000), 0); + if (!success) { + if (GetLastError() == ERROR_TIMEOUT) { + return 1; + } + return -1; + } + return 0; } Py_LOCAL_INLINE(int) PyCOND_SIGNAL(PyCOND_T *cv) { - WakeConditionVariable(cv); - return 0; + WakeConditionVariable(cv); + return 0; } Py_LOCAL_INLINE(int) PyCOND_BROADCAST(PyCOND_T *cv) { - WakeAllConditionVariable(cv); - return 0; + WakeAllConditionVariable(cv); + return 0; } diff --git a/Python/context.c b/Python/context.c index 294485e5b407dfe..3937819b3c386cb 100644 --- a/Python/context.c +++ b/Python/context.c @@ -65,11 +65,11 @@ contextvar_del(PyContextVar *var); #ifdef WITH_FREELISTS -static struct _Py_context_state * -get_context_state(void) +static struct _Py_context_freelist * +get_context_freelist(void) { - _PyFreeListState *state = _PyFreeListState_GET(); - return &state->contexts; + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + return &freelists->contexts; } #endif @@ -341,11 +341,11 @@ _context_alloc(void) { PyContext *ctx; #ifdef WITH_FREELISTS - struct _Py_context_state *state = get_context_state(); - if (state->numfree > 0) { - state->numfree--; - ctx = state->freelist; - state->freelist = (PyContext *)ctx->ctx_weakreflist; + struct _Py_context_freelist *context_freelist = get_context_freelist(); + if (context_freelist->numfree > 0) { + context_freelist->numfree--; + ctx = context_freelist->items; + context_freelist->items = (PyContext *)ctx->ctx_weakreflist; OBJECT_STAT_INC(from_freelist); ctx->ctx_weakreflist = NULL; _Py_NewReference((PyObject *)ctx); @@ -468,11 +468,11 @@ context_tp_dealloc(PyContext *self) (void)context_tp_clear(self); #ifdef WITH_FREELISTS - struct _Py_context_state *state = get_context_state(); - if (state->numfree >= 0 && state->numfree < PyContext_MAXFREELIST) { - state->numfree++; - self->ctx_weakreflist = (PyObject *)state->freelist; - state->freelist = self; + struct _Py_context_freelist *context_freelist = get_context_freelist(); + if (context_freelist->numfree >= 0 && context_freelist->numfree < PyContext_MAXFREELIST) { + context_freelist->numfree++; + self->ctx_weakreflist = (PyObject *)context_freelist->items; + context_freelist->items = self; OBJECT_STAT_INC(to_freelist); } else @@ -1267,13 +1267,13 @@ get_token_missing(void) void -_PyContext_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) +_PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) { #ifdef WITH_FREELISTS - struct _Py_context_state *state = &freelist_state->contexts; + struct _Py_context_freelist *state = &freelists->contexts; for (; state->numfree > 0; state->numfree--) { - PyContext *ctx = state->freelist; - state->freelist = (PyContext *)ctx->ctx_weakreflist; + PyContext *ctx = state->items; + state->items = (PyContext *)ctx->ctx_weakreflist; ctx->ctx_weakreflist = NULL; PyObject_GC_Del(ctx); } @@ -1284,13 +1284,6 @@ _PyContext_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) } -void -_PyContext_Fini(_PyFreeListState *state) -{ - _PyContext_ClearFreeList(state, 1); -} - - PyStatus _PyContext_Init(PyInterpreterState *interp) { diff --git a/Python/crossinterp.c b/Python/crossinterp.c index c6ed7daeb1074a8..143b261f9a53960 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -7,7 +7,6 @@ #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_namespace.h" //_PyNamespace_New() #include "pycore_pyerrors.h" // _PyErr_Clear() -#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_typeobject.h" // _PyType_GetModuleName() #include "pycore_weakref.h" // _PyWeakref_GET_REF() @@ -16,47 +15,12 @@ /* exceptions */ /**************/ -/* InterpreterError extends Exception */ - -static PyTypeObject _PyExc_InterpreterError = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "InterpreterError", - .tp_doc = PyDoc_STR("An interpreter was not found."), - //.tp_base = (PyTypeObject *)PyExc_BaseException, -}; -PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError; - -/* InterpreterNotFoundError extends InterpreterError */ - -static PyTypeObject _PyExc_InterpreterNotFoundError = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "InterpreterNotFoundError", - .tp_doc = PyDoc_STR("An interpreter was not found."), - .tp_base = &_PyExc_InterpreterError, -}; -PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFoundError; - -/* lifecycle */ - -static int -init_exceptions(PyInterpreterState *interp) -{ - _PyExc_InterpreterError.tp_base = (PyTypeObject *)PyExc_BaseException; - if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterError) < 0) { - return -1; - } - if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterNotFoundError) < 0) { - return -1; - } - return 0; -} - -static void -fini_exceptions(PyInterpreterState *interp) -{ - _PyStaticType_Dealloc(interp, &_PyExc_InterpreterNotFoundError); - _PyStaticType_Dealloc(interp, &_PyExc_InterpreterError); -} +static int init_exceptions(PyInterpreterState *); +static void fini_exceptions(PyInterpreterState *); +static int _init_not_shareable_error_type(PyInterpreterState *); +static void _fini_not_shareable_error_type(PyInterpreterState *); +static PyObject * _get_not_shareable_error_type(PyInterpreterState *); +#include "crossinterp_exceptions.h" /***************************/ @@ -67,7 +31,7 @@ int _Py_CallInInterpreter(PyInterpreterState *interp, _Py_simple_func func, void *arg) { - if (interp == _PyThreadState_GetCurrent()->interp) { + if (interp == PyInterpreterState_Get()) { return func(arg); } // XXX Emit a warning if this fails? @@ -79,7 +43,7 @@ int _Py_CallInInterpreterAndRawFree(PyInterpreterState *interp, _Py_simple_func func, void *arg) { - if (interp == _PyThreadState_GetCurrent()->interp) { + if (interp == PyInterpreterState_Get()) { int res = func(arg); PyMem_RawFree(arg); return res; @@ -94,6 +58,20 @@ _Py_CallInInterpreterAndRawFree(PyInterpreterState *interp, /* cross-interpreter data */ /**************************/ +/* registry of {type -> crossinterpdatafunc} */ + +/* For now we use a global registry of shareable classes. An + alternative would be to add a tp_* slot for a class's + crossinterpdatafunc. It would be simpler and more efficient. */ + +static void xid_lookup_init(PyInterpreterState *); +static void xid_lookup_fini(PyInterpreterState *); +static crossinterpdatafunc lookup_getdata(PyInterpreterState *, PyObject *); +#include "crossinterp_data_lookup.h" + + +/* lifecycle */ + _PyCrossInterpreterData * _PyCrossInterpreterData_New(void) { @@ -114,38 +92,6 @@ _PyCrossInterpreterData_Free(_PyCrossInterpreterData *xid) } -/* exceptions */ - -static PyStatus -_init_not_shareable_error_type(PyInterpreterState *interp) -{ - const char *name = "_interpreters.NotShareableError"; - PyObject *base = PyExc_ValueError; - PyObject *ns = NULL; - PyObject *exctype = PyErr_NewException(name, base, ns); - if (exctype == NULL) { - PyErr_Clear(); - return _PyStatus_ERR("could not initialize NotShareableError"); - } - - interp->xi.PyExc_NotShareableError = exctype; - return _PyStatus_OK(); -} - -static void -_fini_not_shareable_error_type(PyInterpreterState *interp) -{ - Py_CLEAR(interp->xi.PyExc_NotShareableError); -} - -static PyObject * -_get_not_shareable_error_type(PyInterpreterState *interp) -{ - assert(interp->xi.PyExc_NotShareableError != NULL); - return interp->xi.PyExc_NotShareableError; -} - - /* defining cross-interpreter data */ static inline void @@ -156,7 +102,7 @@ _xidata_init(_PyCrossInterpreterData *data) assert(data->data == NULL); assert(data->obj == NULL); *data = (_PyCrossInterpreterData){0}; - data->interpid = -1; + _PyCrossInterpreterData_INTERPID(data) = -1; } static inline void @@ -193,7 +139,9 @@ _PyCrossInterpreterData_Init(_PyCrossInterpreterData *data, // Ideally every object would know its owning interpreter. // Until then, we have to rely on the caller to identify it // (but we don't need it in all cases). - data->interpid = (interp != NULL) ? interp->id : -1; + _PyCrossInterpreterData_INTERPID(data) = (interp != NULL) + ? PyInterpreterState_GetID(interp) + : -1; data->new_object = new_object; } @@ -223,8 +171,8 @@ _PyCrossInterpreterData_Clear(PyInterpreterState *interp, assert(data != NULL); // This must be called in the owning interpreter. assert(interp == NULL - || data->interpid == -1 - || data->interpid == interp->id); + || _PyCrossInterpreterData_INTERPID(data) == -1 + || _PyCrossInterpreterData_INTERPID(data) == PyInterpreterState_GetID(interp)); _xidata_clear(data); } @@ -238,13 +186,13 @@ _check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) // data->obj may be NULL, so we don't check it. - if (data->interpid < 0) { - _PyErr_SetString(tstate, PyExc_SystemError, "missing interp"); + if (_PyCrossInterpreterData_INTERPID(data) < 0) { + PyErr_SetString(PyExc_SystemError, "missing interp"); return -1; } if (data->new_object == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, "missing new_object func"); + PyErr_SetString(PyExc_SystemError, "missing new_object func"); return -1; } @@ -253,25 +201,6 @@ _check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) return 0; } -static crossinterpdatafunc _lookup_getdata_from_registry( - PyInterpreterState *, PyObject *); - -static crossinterpdatafunc -_lookup_getdata(PyInterpreterState *interp, PyObject *obj) -{ - /* Cross-interpreter objects are looked up by exact match on the class. - We can reassess this policy when we move from a global registry to a - tp_* slot. */ - return _lookup_getdata_from_registry(interp, obj); -} - -crossinterpdatafunc -_PyCrossInterpreterData_Lookup(PyObject *obj) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - return _lookup_getdata(interp, obj); -} - static inline void _set_xid_lookup_failure(PyInterpreterState *interp, PyObject *obj, const char *msg) @@ -295,8 +224,8 @@ _set_xid_lookup_failure(PyInterpreterState *interp, int _PyObject_CheckCrossInterpreterData(PyObject *obj) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - crossinterpdatafunc getdata = _lookup_getdata(interp, obj); + PyInterpreterState *interp = PyInterpreterState_Get(); + crossinterpdatafunc getdata = lookup_getdata(interp, obj); if (getdata == NULL) { if (!PyErr_Occurred()) { _set_xid_lookup_failure(interp, obj, NULL); @@ -309,20 +238,16 @@ _PyObject_CheckCrossInterpreterData(PyObject *obj) int _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) { - PyThreadState *tstate = _PyThreadState_GetCurrent(); -#ifdef Py_DEBUG - // The caller must hold the GIL - _Py_EnsureTstateNotNULL(tstate); -#endif + PyThreadState *tstate = PyThreadState_Get(); PyInterpreterState *interp = tstate->interp; // Reset data before re-populating. *data = (_PyCrossInterpreterData){0}; - data->interpid = -1; + _PyCrossInterpreterData_INTERPID(data) = -1; // Call the "getdata" func for the object. Py_INCREF(obj); - crossinterpdatafunc getdata = _lookup_getdata(interp, obj); + crossinterpdatafunc getdata = lookup_getdata(interp, obj); if (getdata == NULL) { Py_DECREF(obj); if (!PyErr_Occurred()) { @@ -337,7 +262,7 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) } // Fill in the blanks and validate the result. - data->interpid = interp->id; + _PyCrossInterpreterData_INTERPID(data) = PyInterpreterState_GetID(interp); if (_check_xidata(tstate, data) != 0) { (void)_PyCrossInterpreterData_Release(data); return -1; @@ -374,7 +299,8 @@ _xidata_release(_PyCrossInterpreterData *data, int rawfree) } // Switch to the original interpreter. - PyInterpreterState *interp = _PyInterpreterState_LookUpID(data->interpid); + PyInterpreterState *interp = _PyInterpreterState_LookUpID( + _PyCrossInterpreterData_INTERPID(data)); if (interp == NULL) { // The interpreter was already destroyed. // This function shouldn't have been called. @@ -408,538 +334,6 @@ _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *data) } -/* registry of {type -> crossinterpdatafunc} */ - -/* For now we use a global registry of shareable classes. An - alternative would be to add a tp_* slot for a class's - crossinterpdatafunc. It would be simpler and more efficient. */ - -static inline struct _xidregistry * -_get_global_xidregistry(_PyRuntimeState *runtime) -{ - return &runtime->xi.registry; -} - -static inline struct _xidregistry * -_get_xidregistry(PyInterpreterState *interp) -{ - return &interp->xi.registry; -} - -static inline struct _xidregistry * -_get_xidregistry_for_type(PyInterpreterState *interp, PyTypeObject *cls) -{ - struct _xidregistry *registry = _get_global_xidregistry(interp->runtime); - if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { - registry = _get_xidregistry(interp); - } - return registry; -} - -static int -_xidregistry_add_type(struct _xidregistry *xidregistry, - PyTypeObject *cls, crossinterpdatafunc getdata) -{ - struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem)); - if (newhead == NULL) { - return -1; - } - *newhead = (struct _xidregitem){ - // We do not keep a reference, to avoid keeping the class alive. - .cls = cls, - .refcount = 1, - .getdata = getdata, - }; - if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { - // XXX Assign a callback to clear the entry from the registry? - newhead->weakref = PyWeakref_NewRef((PyObject *)cls, NULL); - if (newhead->weakref == NULL) { - PyMem_RawFree(newhead); - return -1; - } - } - newhead->next = xidregistry->head; - if (newhead->next != NULL) { - newhead->next->prev = newhead; - } - xidregistry->head = newhead; - return 0; -} - -static struct _xidregitem * -_xidregistry_remove_entry(struct _xidregistry *xidregistry, - struct _xidregitem *entry) -{ - struct _xidregitem *next = entry->next; - if (entry->prev != NULL) { - assert(entry->prev->next == entry); - entry->prev->next = next; - } - else { - assert(xidregistry->head == entry); - xidregistry->head = next; - } - if (next != NULL) { - next->prev = entry->prev; - } - Py_XDECREF(entry->weakref); - PyMem_RawFree(entry); - return next; -} - -static void -_xidregistry_clear(struct _xidregistry *xidregistry) -{ - struct _xidregitem *cur = xidregistry->head; - xidregistry->head = NULL; - while (cur != NULL) { - struct _xidregitem *next = cur->next; - Py_XDECREF(cur->weakref); - PyMem_RawFree(cur); - cur = next; - } -} - -static void -_xidregistry_lock(struct _xidregistry *registry) -{ - if (registry->global) { - PyMutex_Lock(®istry->mutex); - } - // else: Within an interpreter we rely on the GIL instead of a separate lock. -} - -static void -_xidregistry_unlock(struct _xidregistry *registry) -{ - if (registry->global) { - PyMutex_Unlock(®istry->mutex); - } -} - -static struct _xidregitem * -_xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls) -{ - struct _xidregitem *cur = xidregistry->head; - while (cur != NULL) { - if (cur->weakref != NULL) { - // cur is/was a heap type. - PyObject *registered = _PyWeakref_GET_REF(cur->weakref); - if (registered == NULL) { - // The weakly ref'ed object was freed. - cur = _xidregistry_remove_entry(xidregistry, cur); - continue; - } - assert(PyType_Check(registered)); - assert(cur->cls == (PyTypeObject *)registered); - assert(cur->cls->tp_flags & Py_TPFLAGS_HEAPTYPE); - Py_DECREF(registered); - } - if (cur->cls == cls) { - return cur; - } - cur = cur->next; - } - return NULL; -} - -int -_PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, - crossinterpdatafunc getdata) -{ - if (!PyType_Check(cls)) { - PyErr_Format(PyExc_ValueError, "only classes may be registered"); - return -1; - } - if (getdata == NULL) { - PyErr_Format(PyExc_ValueError, "missing 'getdata' func"); - return -1; - } - - int res = 0; - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls); - _xidregistry_lock(xidregistry); - - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); - if (matched != NULL) { - assert(matched->getdata == getdata); - matched->refcount += 1; - goto finally; - } - - res = _xidregistry_add_type(xidregistry, cls, getdata); - -finally: - _xidregistry_unlock(xidregistry); - return res; -} - -int -_PyCrossInterpreterData_UnregisterClass(PyTypeObject *cls) -{ - int res = 0; - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls); - _xidregistry_lock(xidregistry); - - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); - if (matched != NULL) { - assert(matched->refcount > 0); - matched->refcount -= 1; - if (matched->refcount == 0) { - (void)_xidregistry_remove_entry(xidregistry, matched); - } - res = 1; - } - - _xidregistry_unlock(xidregistry); - return res; -} - -static crossinterpdatafunc -_lookup_getdata_from_registry(PyInterpreterState *interp, PyObject *obj) -{ - PyTypeObject *cls = Py_TYPE(obj); - - struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls); - _xidregistry_lock(xidregistry); - - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); - crossinterpdatafunc func = matched != NULL ? matched->getdata : NULL; - - _xidregistry_unlock(xidregistry); - return func; -} - -/* cross-interpreter data for builtin types */ - -// bytes - -struct _shared_bytes_data { - char *bytes; - Py_ssize_t len; -}; - -static PyObject * -_new_bytes_object(_PyCrossInterpreterData *data) -{ - struct _shared_bytes_data *shared = (struct _shared_bytes_data *)(data->data); - return PyBytes_FromStringAndSize(shared->bytes, shared->len); -} - -static int -_bytes_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - if (_PyCrossInterpreterData_InitWithSize( - data, tstate->interp, sizeof(struct _shared_bytes_data), obj, - _new_bytes_object - ) < 0) - { - return -1; - } - struct _shared_bytes_data *shared = (struct _shared_bytes_data *)data->data; - if (PyBytes_AsStringAndSize(obj, &shared->bytes, &shared->len) < 0) { - _PyCrossInterpreterData_Clear(tstate->interp, data); - return -1; - } - return 0; -} - -// str - -struct _shared_str_data { - int kind; - const void *buffer; - Py_ssize_t len; -}; - -static PyObject * -_new_str_object(_PyCrossInterpreterData *data) -{ - struct _shared_str_data *shared = (struct _shared_str_data *)(data->data); - return PyUnicode_FromKindAndData(shared->kind, shared->buffer, shared->len); -} - -static int -_str_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - if (_PyCrossInterpreterData_InitWithSize( - data, tstate->interp, sizeof(struct _shared_str_data), obj, - _new_str_object - ) < 0) - { - return -1; - } - struct _shared_str_data *shared = (struct _shared_str_data *)data->data; - shared->kind = PyUnicode_KIND(obj); - shared->buffer = PyUnicode_DATA(obj); - shared->len = PyUnicode_GET_LENGTH(obj); - return 0; -} - -// int - -static PyObject * -_new_long_object(_PyCrossInterpreterData *data) -{ - return PyLong_FromSsize_t((Py_ssize_t)(data->data)); -} - -static int -_long_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - /* Note that this means the size of shareable ints is bounded by - * sys.maxsize. Hence on 32-bit architectures that is half the - * size of maximum shareable ints on 64-bit. - */ - Py_ssize_t value = PyLong_AsSsize_t(obj); - if (value == -1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) { - PyErr_SetString(PyExc_OverflowError, "try sending as bytes"); - } - return -1; - } - _PyCrossInterpreterData_Init(data, tstate->interp, (void *)value, NULL, - _new_long_object); - // data->obj and data->free remain NULL - return 0; -} - -// float - -static PyObject * -_new_float_object(_PyCrossInterpreterData *data) -{ - double * value_ptr = data->data; - return PyFloat_FromDouble(*value_ptr); -} - -static int -_float_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - if (_PyCrossInterpreterData_InitWithSize( - data, tstate->interp, sizeof(double), NULL, - _new_float_object - ) < 0) - { - return -1; - } - double *shared = (double *)data->data; - *shared = PyFloat_AsDouble(obj); - return 0; -} - -// None - -static PyObject * -_new_none_object(_PyCrossInterpreterData *data) -{ - // XXX Singleton refcounts are problematic across interpreters... - return Py_NewRef(Py_None); -} - -static int -_none_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - _PyCrossInterpreterData_Init(data, tstate->interp, NULL, NULL, - _new_none_object); - // data->data, data->obj and data->free remain NULL - return 0; -} - -// bool - -static PyObject * -_new_bool_object(_PyCrossInterpreterData *data) -{ - if (data->data){ - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; -} - -static int -_bool_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - _PyCrossInterpreterData_Init(data, tstate->interp, - (void *) (Py_IsTrue(obj) ? (uintptr_t) 1 : (uintptr_t) 0), NULL, - _new_bool_object); - // data->obj and data->free remain NULL - return 0; -} - -// tuple - -struct _shared_tuple_data { - Py_ssize_t len; - _PyCrossInterpreterData **data; -}; - -static PyObject * -_new_tuple_object(_PyCrossInterpreterData *data) -{ - struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data->data); - PyObject *tuple = PyTuple_New(shared->len); - if (tuple == NULL) { - return NULL; - } - - for (Py_ssize_t i = 0; i < shared->len; i++) { - PyObject *item = _PyCrossInterpreterData_NewObject(shared->data[i]); - if (item == NULL){ - Py_DECREF(tuple); - return NULL; - } - PyTuple_SET_ITEM(tuple, i, item); - } - return tuple; -} - -static void -_tuple_shared_free(void* data) -{ - struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data); -#ifndef NDEBUG - int64_t interpid = PyInterpreterState_GetID(_PyInterpreterState_GET()); -#endif - for (Py_ssize_t i = 0; i < shared->len; i++) { - if (shared->data[i] != NULL) { - assert(shared->data[i]->interpid == interpid); - _PyCrossInterpreterData_Release(shared->data[i]); - PyMem_RawFree(shared->data[i]); - shared->data[i] = NULL; - } - } - PyMem_Free(shared->data); - PyMem_RawFree(shared); -} - -static int -_tuple_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - Py_ssize_t len = PyTuple_GET_SIZE(obj); - if (len < 0) { - return -1; - } - struct _shared_tuple_data *shared = PyMem_RawMalloc(sizeof(struct _shared_tuple_data)); - if (shared == NULL){ - PyErr_NoMemory(); - return -1; - } - - shared->len = len; - shared->data = (_PyCrossInterpreterData **) PyMem_Calloc(shared->len, sizeof(_PyCrossInterpreterData *)); - if (shared->data == NULL) { - PyErr_NoMemory(); - return -1; - } - - for (Py_ssize_t i = 0; i < shared->len; i++) { - _PyCrossInterpreterData *data = _PyCrossInterpreterData_New(); - if (data == NULL) { - goto error; // PyErr_NoMemory already set - } - PyObject *item = PyTuple_GET_ITEM(obj, i); - - int res = -1; - if (!_Py_EnterRecursiveCallTstate(tstate, " while sharing a tuple")) { - res = _PyObject_GetCrossInterpreterData(item, data); - _Py_LeaveRecursiveCallTstate(tstate); - } - if (res < 0) { - PyMem_RawFree(data); - goto error; - } - shared->data[i] = data; - } - _PyCrossInterpreterData_Init( - data, tstate->interp, shared, obj, _new_tuple_object); - data->free = _tuple_shared_free; - return 0; - -error: - _tuple_shared_free(shared); - return -1; -} - -// registration - -static void -_register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) -{ - // None - if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { - Py_FatalError("could not register None for cross-interpreter sharing"); - } - - // int - if (_xidregistry_add_type(xidregistry, &PyLong_Type, _long_shared) != 0) { - Py_FatalError("could not register int for cross-interpreter sharing"); - } - - // bytes - if (_xidregistry_add_type(xidregistry, &PyBytes_Type, _bytes_shared) != 0) { - Py_FatalError("could not register bytes for cross-interpreter sharing"); - } - - // str - if (_xidregistry_add_type(xidregistry, &PyUnicode_Type, _str_shared) != 0) { - Py_FatalError("could not register str for cross-interpreter sharing"); - } - - // bool - if (_xidregistry_add_type(xidregistry, &PyBool_Type, _bool_shared) != 0) { - Py_FatalError("could not register bool for cross-interpreter sharing"); - } - - // float - if (_xidregistry_add_type(xidregistry, &PyFloat_Type, _float_shared) != 0) { - Py_FatalError("could not register float for cross-interpreter sharing"); - } - - // tuple - if (_xidregistry_add_type(xidregistry, &PyTuple_Type, _tuple_shared) != 0) { - Py_FatalError("could not register tuple for cross-interpreter sharing"); - } -} - -/* registry lifecycle */ - -static void -_xidregistry_init(struct _xidregistry *registry) -{ - if (registry->initialized) { - return; - } - registry->initialized = 1; - - if (registry->global) { - // Registering the builtins is cheap so we don't bother doing it lazily. - assert(registry->head == NULL); - _register_builtins_for_crossinterpreter_data(registry); - } -} - -static void -_xidregistry_fini(struct _xidregistry *registry) -{ - if (!registry->initialized) { - return; - } - registry->initialized = 0; - - _xidregistry_clear(registry); -} - - /*************************/ /* convenience utilities */ /*************************/ @@ -1023,6 +417,10 @@ _convert_exc_to_TracebackException(PyObject *exc, PyObject **p_tbexc) return -1; } +// We accommodate backports here. +#ifndef _Py_EMPTY_STR +# define _Py_EMPTY_STR &_Py_STR(empty) +#endif static const char * _format_TracebackException(PyObject *tbexc) @@ -1031,7 +429,8 @@ _format_TracebackException(PyObject *tbexc) if (lines == NULL) { return NULL; } - PyObject *formatted_obj = PyUnicode_Join(&_Py_STR(empty), lines); + assert(_Py_EMPTY_STR != NULL); + PyObject *formatted_obj = PyUnicode_Join(_Py_EMPTY_STR, lines); Py_DECREF(lines); if (formatted_obj == NULL) { return NULL; @@ -1600,7 +999,7 @@ _sharednsitem_has_value(_PyXI_namespace_item *item, int64_t *p_interpid) return 0; } if (p_interpid != NULL) { - *p_interpid = item->data->interpid; + *p_interpid = _PyCrossInterpreterData_INTERPID(item->data); } return 1; } @@ -1874,7 +1273,7 @@ _PyXI_FreeNamespace(_PyXI_namespace *ns) return; } - if (interpid == PyInterpreterState_GetID(_PyInterpreterState_GET())) { + if (interpid == PyInterpreterState_GetID(PyInterpreterState_Get())) { _sharedns_free(ns); } else { @@ -2014,7 +1413,7 @@ _enter_session(_PyXI_session *session, PyInterpreterState *interp) PyThreadState *prev = tstate; if (interp != tstate->interp) { tstate = PyThreadState_New(interp); - tstate->_whence = _PyThreadState_WHENCE_EXEC; + _PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_EXEC); // XXX Possible GILState issues? session->prev_tstate = PyThreadState_Swap(tstate); assert(session->prev_tstate == prev); @@ -2073,7 +1472,7 @@ _propagate_not_shareable_error(_PyXI_session *session) if (session == NULL) { return; } - PyInterpreterState *interp = _PyInterpreterState_GET(); + PyInterpreterState *interp = PyInterpreterState_Get(); if (PyErr_ExceptionMatches(_get_not_shareable_error_type(interp))) { // We want to propagate the exception directly. session->_error_override = _PyXI_ERR_NOT_SHAREABLE; @@ -2245,18 +1644,12 @@ _PyXI_Exit(_PyXI_session *session) PyStatus _PyXI_Init(PyInterpreterState *interp) { - PyStatus status; - - // Initialize the XID registry. - if (_Py_IsMainInterpreter(interp)) { - _xidregistry_init(_get_global_xidregistry(interp->runtime)); - } - _xidregistry_init(_get_xidregistry(interp)); + // Initialize the XID lookup state (e.g. registry). + xid_lookup_init(interp); // Initialize exceptions (heap types). - status = _init_not_shareable_error_type(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; + if (_init_not_shareable_error_type(interp) < 0) { + return _PyStatus_ERR("failed to initialize NotShareableError"); } return _PyStatus_OK(); @@ -2271,11 +1664,8 @@ _PyXI_Fini(PyInterpreterState *interp) // Finalize exceptions (heap types). _fini_not_shareable_error_type(interp); - // Finalize the XID registry. - _xidregistry_fini(_get_xidregistry(interp)); - if (_Py_IsMainInterpreter(interp)) { - _xidregistry_fini(_get_global_xidregistry(interp->runtime)); - } + // Finalize the XID lookup state (e.g. registry). + xid_lookup_fini(interp); } PyStatus diff --git a/Python/crossinterp_data_lookup.h b/Python/crossinterp_data_lookup.h new file mode 100644 index 000000000000000..863919ad42fb97a --- /dev/null +++ b/Python/crossinterp_data_lookup.h @@ -0,0 +1,594 @@ + +static crossinterpdatafunc _lookup_getdata_from_registry( + PyInterpreterState *, PyObject *); + +static crossinterpdatafunc +lookup_getdata(PyInterpreterState *interp, PyObject *obj) +{ + /* Cross-interpreter objects are looked up by exact match on the class. + We can reassess this policy when we move from a global registry to a + tp_* slot. */ + return _lookup_getdata_from_registry(interp, obj); +} + +crossinterpdatafunc +_PyCrossInterpreterData_Lookup(PyObject *obj) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + return lookup_getdata(interp, obj); +} + + +/***********************************************/ +/* a registry of {type -> crossinterpdatafunc} */ +/***********************************************/ + +/* For now we use a global registry of shareable classes. An + alternative would be to add a tp_* slot for a class's + crossinterpdatafunc. It would be simpler and more efficient. */ + + +/* registry lifecycle */ + +static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *); + +static void +_xidregistry_init(struct _xidregistry *registry) +{ + if (registry->initialized) { + return; + } + registry->initialized = 1; + + if (registry->global) { + // Registering the builtins is cheap so we don't bother doing it lazily. + assert(registry->head == NULL); + _register_builtins_for_crossinterpreter_data(registry); + } +} + +static void _xidregistry_clear(struct _xidregistry *); + +static void +_xidregistry_fini(struct _xidregistry *registry) +{ + if (!registry->initialized) { + return; + } + registry->initialized = 0; + + _xidregistry_clear(registry); +} + +static inline struct _xidregistry * _get_global_xidregistry(_PyRuntimeState *); +static inline struct _xidregistry * _get_xidregistry(PyInterpreterState *); + +static void +xid_lookup_init(PyInterpreterState *interp) +{ + if (_Py_IsMainInterpreter(interp)) { + _xidregistry_init(_get_global_xidregistry(interp->runtime)); + } + _xidregistry_init(_get_xidregistry(interp)); +} + +static void +xid_lookup_fini(PyInterpreterState *interp) +{ + _xidregistry_fini(_get_xidregistry(interp)); + if (_Py_IsMainInterpreter(interp)) { + _xidregistry_fini(_get_global_xidregistry(interp->runtime)); + } +} + + +/* registry thread safety */ + +static void +_xidregistry_lock(struct _xidregistry *registry) +{ + if (registry->global) { + PyMutex_Lock(®istry->mutex); + } + // else: Within an interpreter we rely on the GIL instead of a separate lock. +} + +static void +_xidregistry_unlock(struct _xidregistry *registry) +{ + if (registry->global) { + PyMutex_Unlock(®istry->mutex); + } +} + + +/* accessing the registry */ + +static inline struct _xidregistry * +_get_global_xidregistry(_PyRuntimeState *runtime) +{ + return &runtime->xi.registry; +} + +static inline struct _xidregistry * +_get_xidregistry(PyInterpreterState *interp) +{ + return &interp->xi.registry; +} + +static inline struct _xidregistry * +_get_xidregistry_for_type(PyInterpreterState *interp, PyTypeObject *cls) +{ + struct _xidregistry *registry = _get_global_xidregistry(interp->runtime); + if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { + registry = _get_xidregistry(interp); + } + return registry; +} + +static struct _xidregitem * _xidregistry_remove_entry( + struct _xidregistry *, struct _xidregitem *); + +static struct _xidregitem * +_xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls) +{ + struct _xidregitem *cur = xidregistry->head; + while (cur != NULL) { + if (cur->weakref != NULL) { + // cur is/was a heap type. + PyObject *registered = _PyWeakref_GET_REF(cur->weakref); + if (registered == NULL) { + // The weakly ref'ed object was freed. + cur = _xidregistry_remove_entry(xidregistry, cur); + continue; + } + assert(PyType_Check(registered)); + assert(cur->cls == (PyTypeObject *)registered); + assert(cur->cls->tp_flags & Py_TPFLAGS_HEAPTYPE); + Py_DECREF(registered); + } + if (cur->cls == cls) { + return cur; + } + cur = cur->next; + } + return NULL; +} + +static crossinterpdatafunc +_lookup_getdata_from_registry(PyInterpreterState *interp, PyObject *obj) +{ + PyTypeObject *cls = Py_TYPE(obj); + + struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls); + _xidregistry_lock(xidregistry); + + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + crossinterpdatafunc func = matched != NULL ? matched->getdata : NULL; + + _xidregistry_unlock(xidregistry); + return func; +} + + +/* updating the registry */ + +static int +_xidregistry_add_type(struct _xidregistry *xidregistry, + PyTypeObject *cls, crossinterpdatafunc getdata) +{ + struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem)); + if (newhead == NULL) { + return -1; + } + *newhead = (struct _xidregitem){ + // We do not keep a reference, to avoid keeping the class alive. + .cls = cls, + .refcount = 1, + .getdata = getdata, + }; + if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { + // XXX Assign a callback to clear the entry from the registry? + newhead->weakref = PyWeakref_NewRef((PyObject *)cls, NULL); + if (newhead->weakref == NULL) { + PyMem_RawFree(newhead); + return -1; + } + } + newhead->next = xidregistry->head; + if (newhead->next != NULL) { + newhead->next->prev = newhead; + } + xidregistry->head = newhead; + return 0; +} + +static struct _xidregitem * +_xidregistry_remove_entry(struct _xidregistry *xidregistry, + struct _xidregitem *entry) +{ + struct _xidregitem *next = entry->next; + if (entry->prev != NULL) { + assert(entry->prev->next == entry); + entry->prev->next = next; + } + else { + assert(xidregistry->head == entry); + xidregistry->head = next; + } + if (next != NULL) { + next->prev = entry->prev; + } + Py_XDECREF(entry->weakref); + PyMem_RawFree(entry); + return next; +} + +static void +_xidregistry_clear(struct _xidregistry *xidregistry) +{ + struct _xidregitem *cur = xidregistry->head; + xidregistry->head = NULL; + while (cur != NULL) { + struct _xidregitem *next = cur->next; + Py_XDECREF(cur->weakref); + PyMem_RawFree(cur); + cur = next; + } +} + +int +_PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, + crossinterpdatafunc getdata) +{ + if (!PyType_Check(cls)) { + PyErr_Format(PyExc_ValueError, "only classes may be registered"); + return -1; + } + if (getdata == NULL) { + PyErr_Format(PyExc_ValueError, "missing 'getdata' func"); + return -1; + } + + int res = 0; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls); + _xidregistry_lock(xidregistry); + + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + if (matched != NULL) { + assert(matched->getdata == getdata); + matched->refcount += 1; + goto finally; + } + + res = _xidregistry_add_type(xidregistry, cls, getdata); + +finally: + _xidregistry_unlock(xidregistry); + return res; +} + +int +_PyCrossInterpreterData_UnregisterClass(PyTypeObject *cls) +{ + int res = 0; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls); + _xidregistry_lock(xidregistry); + + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + if (matched != NULL) { + assert(matched->refcount > 0); + matched->refcount -= 1; + if (matched->refcount == 0) { + (void)_xidregistry_remove_entry(xidregistry, matched); + } + res = 1; + } + + _xidregistry_unlock(xidregistry); + return res; +} + + +/********************************************/ +/* cross-interpreter data for builtin types */ +/********************************************/ + +// bytes + +struct _shared_bytes_data { + char *bytes; + Py_ssize_t len; +}; + +static PyObject * +_new_bytes_object(_PyCrossInterpreterData *data) +{ + struct _shared_bytes_data *shared = (struct _shared_bytes_data *)(data->data); + return PyBytes_FromStringAndSize(shared->bytes, shared->len); +} + +static int +_bytes_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(struct _shared_bytes_data), obj, + _new_bytes_object + ) < 0) + { + return -1; + } + struct _shared_bytes_data *shared = (struct _shared_bytes_data *)data->data; + if (PyBytes_AsStringAndSize(obj, &shared->bytes, &shared->len) < 0) { + _PyCrossInterpreterData_Clear(tstate->interp, data); + return -1; + } + return 0; +} + +// str + +struct _shared_str_data { + int kind; + const void *buffer; + Py_ssize_t len; +}; + +static PyObject * +_new_str_object(_PyCrossInterpreterData *data) +{ + struct _shared_str_data *shared = (struct _shared_str_data *)(data->data); + return PyUnicode_FromKindAndData(shared->kind, shared->buffer, shared->len); +} + +static int +_str_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(struct _shared_str_data), obj, + _new_str_object + ) < 0) + { + return -1; + } + struct _shared_str_data *shared = (struct _shared_str_data *)data->data; + shared->kind = PyUnicode_KIND(obj); + shared->buffer = PyUnicode_DATA(obj); + shared->len = PyUnicode_GET_LENGTH(obj); + return 0; +} + +// int + +static PyObject * +_new_long_object(_PyCrossInterpreterData *data) +{ + return PyLong_FromSsize_t((Py_ssize_t)(data->data)); +} + +static int +_long_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + /* Note that this means the size of shareable ints is bounded by + * sys.maxsize. Hence on 32-bit architectures that is half the + * size of maximum shareable ints on 64-bit. + */ + Py_ssize_t value = PyLong_AsSsize_t(obj); + if (value == -1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_SetString(PyExc_OverflowError, "try sending as bytes"); + } + return -1; + } + _PyCrossInterpreterData_Init(data, tstate->interp, (void *)value, NULL, + _new_long_object); + // data->obj and data->free remain NULL + return 0; +} + +// float + +static PyObject * +_new_float_object(_PyCrossInterpreterData *data) +{ + double * value_ptr = data->data; + return PyFloat_FromDouble(*value_ptr); +} + +static int +_float_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(double), NULL, + _new_float_object + ) < 0) + { + return -1; + } + double *shared = (double *)data->data; + *shared = PyFloat_AsDouble(obj); + return 0; +} + +// None + +static PyObject * +_new_none_object(_PyCrossInterpreterData *data) +{ + // XXX Singleton refcounts are problematic across interpreters... + return Py_NewRef(Py_None); +} + +static int +_none_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + _PyCrossInterpreterData_Init(data, tstate->interp, NULL, NULL, + _new_none_object); + // data->data, data->obj and data->free remain NULL + return 0; +} + +// bool + +static PyObject * +_new_bool_object(_PyCrossInterpreterData *data) +{ + if (data->data){ + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +static int +_bool_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + _PyCrossInterpreterData_Init(data, tstate->interp, + (void *) (Py_IsTrue(obj) ? (uintptr_t) 1 : (uintptr_t) 0), NULL, + _new_bool_object); + // data->obj and data->free remain NULL + return 0; +} + +// tuple + +struct _shared_tuple_data { + Py_ssize_t len; + _PyCrossInterpreterData **data; +}; + +static PyObject * +_new_tuple_object(_PyCrossInterpreterData *data) +{ + struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data->data); + PyObject *tuple = PyTuple_New(shared->len); + if (tuple == NULL) { + return NULL; + } + + for (Py_ssize_t i = 0; i < shared->len; i++) { + PyObject *item = _PyCrossInterpreterData_NewObject(shared->data[i]); + if (item == NULL){ + Py_DECREF(tuple); + return NULL; + } + PyTuple_SET_ITEM(tuple, i, item); + } + return tuple; +} + +static void +_tuple_shared_free(void* data) +{ + struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data); +#ifndef NDEBUG + int64_t interpid = PyInterpreterState_GetID(_PyInterpreterState_GET()); +#endif + for (Py_ssize_t i = 0; i < shared->len; i++) { + if (shared->data[i] != NULL) { + assert(_PyCrossInterpreterData_INTERPID(shared->data[i]) == interpid); + _PyCrossInterpreterData_Release(shared->data[i]); + PyMem_RawFree(shared->data[i]); + shared->data[i] = NULL; + } + } + PyMem_Free(shared->data); + PyMem_RawFree(shared); +} + +static int +_tuple_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + Py_ssize_t len = PyTuple_GET_SIZE(obj); + if (len < 0) { + return -1; + } + struct _shared_tuple_data *shared = PyMem_RawMalloc(sizeof(struct _shared_tuple_data)); + if (shared == NULL){ + PyErr_NoMemory(); + return -1; + } + + shared->len = len; + shared->data = (_PyCrossInterpreterData **) PyMem_Calloc(shared->len, sizeof(_PyCrossInterpreterData *)); + if (shared->data == NULL) { + PyErr_NoMemory(); + return -1; + } + + for (Py_ssize_t i = 0; i < shared->len; i++) { + _PyCrossInterpreterData *data = _PyCrossInterpreterData_New(); + if (data == NULL) { + goto error; // PyErr_NoMemory already set + } + PyObject *item = PyTuple_GET_ITEM(obj, i); + + int res = -1; + if (!_Py_EnterRecursiveCallTstate(tstate, " while sharing a tuple")) { + res = _PyObject_GetCrossInterpreterData(item, data); + _Py_LeaveRecursiveCallTstate(tstate); + } + if (res < 0) { + PyMem_RawFree(data); + goto error; + } + shared->data[i] = data; + } + _PyCrossInterpreterData_Init( + data, tstate->interp, shared, obj, _new_tuple_object); + data->free = _tuple_shared_free; + return 0; + +error: + _tuple_shared_free(shared); + return -1; +} + +// registration + +static void +_register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) +{ + // None + if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { + Py_FatalError("could not register None for cross-interpreter sharing"); + } + + // int + if (_xidregistry_add_type(xidregistry, &PyLong_Type, _long_shared) != 0) { + Py_FatalError("could not register int for cross-interpreter sharing"); + } + + // bytes + if (_xidregistry_add_type(xidregistry, &PyBytes_Type, _bytes_shared) != 0) { + Py_FatalError("could not register bytes for cross-interpreter sharing"); + } + + // str + if (_xidregistry_add_type(xidregistry, &PyUnicode_Type, _str_shared) != 0) { + Py_FatalError("could not register str for cross-interpreter sharing"); + } + + // bool + if (_xidregistry_add_type(xidregistry, &PyBool_Type, _bool_shared) != 0) { + Py_FatalError("could not register bool for cross-interpreter sharing"); + } + + // float + if (_xidregistry_add_type(xidregistry, &PyFloat_Type, _float_shared) != 0) { + Py_FatalError("could not register float for cross-interpreter sharing"); + } + + // tuple + if (_xidregistry_add_type(xidregistry, &PyTuple_Type, _tuple_shared) != 0) { + Py_FatalError("could not register tuple for cross-interpreter sharing"); + } +} diff --git a/Python/crossinterp_exceptions.h b/Python/crossinterp_exceptions.h new file mode 100644 index 000000000000000..e418cf91d4a7af6 --- /dev/null +++ b/Python/crossinterp_exceptions.h @@ -0,0 +1,80 @@ + +/* InterpreterError extends Exception */ + +static PyTypeObject _PyExc_InterpreterError = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "interpreters.InterpreterError", + .tp_doc = PyDoc_STR("A cross-interpreter operation failed"), + //.tp_base = (PyTypeObject *)PyExc_BaseException, +}; +PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError; + +/* InterpreterNotFoundError extends InterpreterError */ + +static PyTypeObject _PyExc_InterpreterNotFoundError = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "interpreters.InterpreterNotFoundError", + .tp_doc = PyDoc_STR("An interpreter was not found"), + .tp_base = &_PyExc_InterpreterError, +}; +PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFoundError; + +/* NotShareableError extends ValueError */ + +static int +_init_not_shareable_error_type(PyInterpreterState *interp) +{ + const char *name = "interpreters.NotShareableError"; + PyObject *base = PyExc_ValueError; + PyObject *ns = NULL; + PyObject *exctype = PyErr_NewException(name, base, ns); + if (exctype == NULL) { + return -1; + } + + _PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError = exctype; + return 0; +} + +static void +_fini_not_shareable_error_type(PyInterpreterState *interp) +{ + Py_CLEAR(_PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError); +} + +static PyObject * +_get_not_shareable_error_type(PyInterpreterState *interp) +{ + assert(_PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError != NULL); + return _PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError; +} + + +/* lifecycle */ + +static int +init_exceptions(PyInterpreterState *interp) +{ + // builtin static types + _PyExc_InterpreterError.tp_base = (PyTypeObject *)PyExc_BaseException; + if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterError) < 0) { + return -1; + } + if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterNotFoundError) < 0) { + return -1; + } + + // heap types + // We would call _init_not_shareable_error_type() here too, + // but that leads to ref leaks + + return 0; +} + +static void +fini_exceptions(PyInterpreterState *interp) +{ + // Likewise with _fini_not_shareable_error_type(). + _PyStaticType_Dealloc(interp, &_PyExc_InterpreterNotFoundError); + _PyStaticType_Dealloc(interp, &_PyExc_InterpreterError); +} diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 241b9056207715d..20fab8f4c61eb5c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -17,10 +17,10 @@ if (_Py_emscripten_signal_clock == 0) goto deoptimize; _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) goto deoptimize; + if (eval_breaker != version && tstate->tracing == 0) goto deoptimize; break; } @@ -37,6 +37,102 @@ break; } + case _LOAD_FAST_0: { + PyObject *value; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_1: { + PyObject *value; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_2: { + PyObject *value; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_3: { + PyObject *value; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_4: { + PyObject *value; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_5: { + PyObject *value; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_6: { + PyObject *value; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_7: { + PyObject *value; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + case _LOAD_FAST: { PyObject *value; oparg = CURRENT_OPARG(); @@ -69,6 +165,86 @@ break; } + case _STORE_FAST_0: { + PyObject *value; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_1: { + PyObject *value; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_2: { + PyObject *value; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_3: { + PyObject *value; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_4: { + PyObject *value; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_5: { + PyObject *value; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_6: { + PyObject *value; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_7: { + PyObject *value; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + case _STORE_FAST: { PyObject *value; oparg = CURRENT_OPARG(); @@ -141,7 +317,7 @@ case _TO_BOOL_BOOL: { PyObject *value; value = stack_pointer[-1]; - if (!PyBool_Check(value)) goto deoptimize; + if (!PyBool_Check(value)) goto side_exit; STAT_INC(TO_BOOL, hit); break; } @@ -150,7 +326,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyLong_CheckExact(value)) goto deoptimize; + if (!PyLong_CheckExact(value)) goto side_exit; STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -168,7 +344,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyList_CheckExact(value)) goto deoptimize; + if (!PyList_CheckExact(value)) goto side_exit; STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; Py_DECREF(value); @@ -181,7 +357,7 @@ PyObject *res; value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - if (!Py_IsNone(value)) goto deoptimize; + if (!Py_IsNone(value)) goto side_exit; STAT_INC(TO_BOOL, hit); res = Py_False; stack_pointer[-1] = res; @@ -192,7 +368,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyUnicode_CheckExact(value)) goto deoptimize; + if (!PyUnicode_CheckExact(value)) goto side_exit; STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -214,7 +390,7 @@ uint32_t version = (uint32_t)CURRENT_OPERAND(); // This one is a bit weird, because we expect *some* failures: assert(version); - if (Py_TYPE(value)->tp_version_tag != version) goto deoptimize; + if (Py_TYPE(value)->tp_version_tag != version) goto side_exit; STAT_INC(TO_BOOL, hit); Py_DECREF(value); res = Py_True; @@ -238,8 +414,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyLong_CheckExact(left)) goto deoptimize; - if (!PyLong_CheckExact(right)) goto deoptimize; + if (!PyLong_CheckExact(left)) goto side_exit; + if (!PyLong_CheckExact(right)) goto side_exit; break; } @@ -296,8 +472,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyFloat_CheckExact(left)) goto deoptimize; - if (!PyFloat_CheckExact(right)) goto deoptimize; + if (!PyFloat_CheckExact(left)) goto side_exit; + if (!PyFloat_CheckExact(right)) goto side_exit; break; } @@ -354,8 +530,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyUnicode_CheckExact(left)) goto deoptimize; - if (!PyUnicode_CheckExact(right)) goto deoptimize; + if (!PyUnicode_CheckExact(left)) goto side_exit; + if (!PyUnicode_CheckExact(right)) goto side_exit; break; } @@ -1534,7 +1710,7 @@ Py_DECREF(self); if (attr == NULL) goto pop_3_error_tier_two; stack_pointer[-3] = attr; - stack_pointer += -2 + ((0) ? 1 : 0); + stack_pointer += -2; break; } @@ -1598,7 +1774,7 @@ something was returned by a descriptor protocol). Set the second element of the stack to NULL, to signal CALL that it's not a method call. - NULL | meth | arg1 | ... | argN + meth | NULL | arg1 | ... | argN */ Py_DECREF(owner); if (attr == NULL) goto pop_1_error_tier_two; @@ -1623,7 +1799,7 @@ uint32_t type_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - if (tp->tp_version_tag != type_version) goto deoptimize; + if (tp->tp_version_tag != type_version) goto side_exit; break; } @@ -1637,11 +1813,11 @@ break; } - case _LOAD_ATTR_INSTANCE_VALUE: { + case _LOAD_ATTR_INSTANCE_VALUE_0: { PyObject *owner; PyObject *attr; PyObject *null = NULL; - oparg = CURRENT_OPARG(); + (void)null; owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); @@ -1652,19 +1828,39 @@ null = NULL; Py_DECREF(owner); stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); break; } + case _LOAD_ATTR_INSTANCE_VALUE_1: { + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + (void)null; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)CURRENT_OPERAND(); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + attr = _PyDictOrValues_GetValues(dorv)->values[index]; + if (attr == NULL) goto deoptimize; + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + stack_pointer[-1] = attr; + stack_pointer[0] = null; + stack_pointer += 1; + break; + } + + /* _LOAD_ATTR_INSTANCE_VALUE is split on (oparg & 1) */ + case _CHECK_ATTR_MODULE: { PyObject *owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND(); + uint32_t dict_version = (uint32_t)CURRENT_OPERAND(); if (!PyModule_CheckExact(owner)) goto deoptimize; PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - if (dict->ma_keys->dk_version != type_version) goto deoptimize; + if (dict->ma_keys->dk_version != dict_version) goto deoptimize; break; } @@ -1735,11 +1931,11 @@ break; } - case _LOAD_ATTR_SLOT: { + case _LOAD_ATTR_SLOT_0: { PyObject *owner; PyObject *attr; PyObject *null = NULL; - oparg = CURRENT_OPARG(); + (void)null; owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); char *addr = (char *)owner + index; @@ -1750,11 +1946,31 @@ null = NULL; Py_DECREF(owner); stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); break; } + case _LOAD_ATTR_SLOT_1: { + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + (void)null; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)CURRENT_OPERAND(); + char *addr = (char *)owner + index; + attr = *(PyObject **)addr; + if (attr == NULL) goto deoptimize; + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + stack_pointer[-1] = attr; + stack_pointer[0] = null; + stack_pointer += 1; + break; + } + + /* _LOAD_ATTR_SLOT is split on (oparg & 1) */ + case _CHECK_ATTR_CLASS: { PyObject *owner; owner = stack_pointer[-1]; @@ -1765,11 +1981,11 @@ break; } - case _LOAD_ATTR_CLASS: { + case _LOAD_ATTR_CLASS_0: { PyObject *owner; PyObject *attr; PyObject *null = NULL; - oparg = CURRENT_OPARG(); + (void)null; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND(); STAT_INC(LOAD_ATTR, hit); @@ -1778,11 +1994,29 @@ null = NULL; Py_DECREF(owner); stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); break; } + case _LOAD_ATTR_CLASS_1: { + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + (void)null; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)CURRENT_OPERAND(); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + null = NULL; + Py_DECREF(owner); + stack_pointer[-1] = attr; + stack_pointer[0] = null; + stack_pointer += 1; + break; + } + + /* _LOAD_ATTR_CLASS is split on (oparg & 1) */ + /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 */ /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ @@ -1866,8 +2100,6 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyFloat_CheckExact(left)) goto deoptimize; - if (!PyFloat_CheckExact(right)) goto deoptimize; STAT_INC(COMPARE_OP, hit); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); @@ -1889,8 +2121,6 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyLong_CheckExact(left)) goto deoptimize; - if (!PyLong_CheckExact(right)) goto deoptimize; if (!_PyLong_IsCompact((PyLongObject *)left)) goto deoptimize; if (!_PyLong_IsCompact((PyLongObject *)right)) goto deoptimize; STAT_INC(COMPARE_OP, hit); @@ -1916,8 +2146,6 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyUnicode_CheckExact(left)) goto deoptimize; - if (!PyUnicode_CheckExact(right)) goto deoptimize; STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left, right); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); @@ -2013,8 +2241,6 @@ break; } - /* _JUMP_BACKWARD is not a viable micro-op for tier 2 */ - /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ @@ -2202,7 +2428,7 @@ assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; if (seq == NULL) goto deoptimize; - if (it->it_index >= PyList_GET_SIZE(seq)) goto deoptimize; + if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) goto deoptimize; break; } @@ -2467,8 +2693,8 @@ assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); self = owner; stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; break; } @@ -2487,8 +2713,8 @@ attr = Py_NewRef(descr); self = owner; stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; break; } @@ -2504,7 +2730,6 @@ Py_DECREF(owner); attr = Py_NewRef(descr); stack_pointer[-1] = attr; - stack_pointer += ((0) ? 1 : 0); break; } @@ -2521,7 +2746,6 @@ Py_DECREF(owner); attr = Py_NewRef(descr); stack_pointer[-1] = attr; - stack_pointer += ((0) ? 1 : 0); break; } @@ -2550,8 +2774,8 @@ attr = Py_NewRef(descr); self = owner; stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; break; } @@ -2618,6 +2842,136 @@ break; } + case _INIT_CALL_PY_EXACT_ARGS_0: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer += -1 - oparg; + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_1: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer += -1 - oparg; + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_2: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer += -1 - oparg; + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_3: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer += -1 - oparg; + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_4: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer += -1 - oparg; + break; + } + case _INIT_CALL_PY_EXACT_ARGS: { PyObject **args; PyObject *self_or_null; @@ -2663,7 +3017,6 @@ goto exit_unwind; } #endif - stack_pointer += ((0) ? 1 : 0); break; } @@ -2844,7 +3197,7 @@ STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ - res = ((_PyCFunctionFast)(void(*)(void))cfunc)( + res = ((PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable), args, total_args); @@ -2885,8 +3238,8 @@ if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) goto deoptimize; STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable); res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3037,8 +3390,8 @@ if (!Py_IS_TYPE(self, d_type)) goto deoptimize; STAT_INC(CALL, hit); int nargs = total_args - 1; - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; res = cfunc(self, args + 1, nargs, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ @@ -3116,8 +3469,8 @@ PyObject *self = args[0]; if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; STAT_INC(CALL, hit); - _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; res = cfunc(self, args + 1, nargs); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3319,7 +3672,7 @@ PyObject *flag; flag = stack_pointer[-1]; stack_pointer += -1; - if (!Py_IsTrue(flag)) goto deoptimize; + if (!Py_IsTrue(flag)) goto side_exit; assert(Py_IsTrue(flag)); break; } @@ -3328,7 +3681,7 @@ PyObject *flag; flag = stack_pointer[-1]; stack_pointer += -1; - if (!Py_IsFalse(flag)) goto deoptimize; + if (!Py_IsFalse(flag)) goto side_exit; assert(Py_IsFalse(flag)); break; } @@ -3339,7 +3692,7 @@ stack_pointer += -1; if (!Py_IsNone(val)) { Py_DECREF(val); - if (1) goto deoptimize; + if (1) goto side_exit; } break; } @@ -3348,22 +3701,22 @@ PyObject *val; val = stack_pointer[-1]; stack_pointer += -1; - if (Py_IsNone(val)) goto deoptimize; + if (Py_IsNone(val)) goto side_exit; Py_DECREF(val); break; } case _JUMP_TO_TOP: { - next_uop = current_executor->trace; + #ifndef _Py_JIT + next_uop = ¤t_executor->trace[1]; + #endif CHECK_EVAL_BREAKER(); break; } case _SET_IP: { - oparg = CURRENT_OPARG(); - TIER_TWO_ONLY - // TODO: Put the code pointer in `operand` to avoid indirection via `frame` - frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + oparg; + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; break; } @@ -3379,13 +3732,11 @@ } case _EXIT_TRACE: { - TIER_TWO_ONLY - if (1) goto deoptimize; + if (1) goto side_exit; break; } case _CHECK_VALIDITY: { - TIER_TWO_ONLY if (!current_executor->vm_data.valid) goto deoptimize; break; } @@ -3408,6 +3759,42 @@ break; } + case _LOAD_CONST_INLINE_WITH_NULL: { + PyObject *value; + PyObject *null; + PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + value = Py_NewRef(ptr); + null = NULL; + stack_pointer[0] = value; + stack_pointer[1] = null; + stack_pointer += 2; + break; + } + + case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { + PyObject *value; + PyObject *null; + PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + value = ptr; + null = NULL; + stack_pointer[0] = value; + stack_pointer[1] = null; + stack_pointer += 2; + break; + } + + case _CHECK_GLOBALS: { + PyObject *dict = (PyObject *)CURRENT_OPERAND(); + if (GLOBALS() != dict) goto deoptimize; + break; + } + + case _CHECK_BUILTINS: { + PyObject *dict = (PyObject *)CURRENT_OPERAND(); + if (BUILTINS() != dict) goto deoptimize; + break; + } + case _INTERNAL_INCREMENT_OPT_COUNTER: { PyObject *opt; opt = stack_pointer[-1]; @@ -3417,4 +3804,62 @@ break; } + case _COLD_EXIT: { + oparg = CURRENT_OPARG(); + _PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor; + _PyExitData *exit = &previous->exits[oparg]; + exit->temperature++; + PyCodeObject *code = _PyFrame_GetCode(frame); + _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; + if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) { + GOTO_TIER_ONE(target); + } + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } else { + int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); + if (optimized <= 0) { + int32_t new_temp = -1 * tstate->interp->optimizer_side_threshold; + exit->temperature = (new_temp < INT16_MIN) ? INT16_MIN : new_temp; + if (optimized < 0) { + Py_DECREF(previous); + tstate->previous_executor = Py_None; + if (1) goto error_tier_two; + } + GOTO_TIER_ONE(target); + } + } + /* We need two references. One to store in exit->executor and + * one to keep the executor alive when executing. */ + Py_INCREF(executor); + exit->executor = executor; + GOTO_TIER_TWO(executor); + break; + } + + case _START_EXECUTOR: { + PyObject *executor = (PyObject *)CURRENT_OPERAND(); + Py_DECREF(tstate->previous_executor); + tstate->previous_executor = NULL; + #ifndef _Py_JIT + current_executor = (_PyExecutorObject*)executor; + #endif + break; + } + + case _FATAL_ERROR: { + assert(0); + Py_FatalError("Fatal error uop executed."); + break; + } + + case _CHECK_VALIDITY_AND_SET_IP: { + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); + if (!current_executor->vm_data.valid) goto deoptimize; + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + break; + } + #undef TIER_TWO diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 96610b3cb11a43d..2f47e47bf9d29d0 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -162,7 +162,7 @@ basicblock_last_instr(const basicblock *b) { static basicblock * cfg_builder_new_block(cfg_builder *g) { - basicblock *b = (basicblock *)PyObject_Calloc(1, sizeof(basicblock)); + basicblock *b = (basicblock *)PyMem_Calloc(1, sizeof(basicblock)); if (b == NULL) { PyErr_NoMemory(); return NULL; @@ -212,14 +212,14 @@ basicblock_add_jump(basicblock *b, int opcode, basicblock *target, location loc) } static inline int -basicblock_append_instructions(basicblock *target, basicblock *source) +basicblock_append_instructions(basicblock *to, basicblock *from) { - for (int i = 0; i < source->b_iused; i++) { - int n = basicblock_next_instr(target); + for (int i = 0; i < from->b_iused; i++) { + int n = basicblock_next_instr(to); if (n < 0) { return ERROR; } - target->b_instr[n] = source->b_instr[i]; + to->b_instr[n] = from->b_instr[i]; } return SUCCESS; } @@ -292,9 +292,9 @@ static void dump_basicblock(const basicblock *b) { const char *b_return = basicblock_returns(b) ? "return " : ""; - fprintf(stderr, "%d: [EH=%d CLD=%d WRM=%d NO_FT=%d %p] used: %d, depth: %d, %s\n", + fprintf(stderr, "%d: [EH=%d CLD=%d WRM=%d NO_FT=%d %p] used: %d, depth: %d, preds: %d %s\n", b->b_label.id, b->b_except_handler, b->b_cold, b->b_warm, BB_NO_FALLTHROUGH(b), b, b->b_iused, - b->b_startdepth, b_return); + b->b_startdepth, b->b_predecessors, b_return); if (b->b_instr) { int i; for (i = 0; i < b->b_iused; i++) { @@ -437,10 +437,10 @@ _PyCfgBuilder_Free(cfg_builder *g) basicblock *b = g->g_block_list; while (b != NULL) { if (b->b_instr) { - PyObject_Free((void *)b->b_instr); + PyMem_Free((void *)b->b_instr); } basicblock *next = b->b_list; - PyObject_Free((void *)b); + PyMem_Free((void *)b); b = next; } PyMem_Free(g); @@ -665,12 +665,6 @@ translate_jump_labels_to_targets(basicblock *entryblock) return SUCCESS; } -int -_PyCfg_JumpLabelsToTargets(cfg_builder *g) -{ - return translate_jump_labels_to_targets(g->g_entryblock); -} - static int mark_except_handlers(basicblock *entryblock) { #ifndef NDEBUG @@ -1165,15 +1159,26 @@ remove_redundant_jumps(cfg_builder *g) { return changes; } +static inline bool +basicblock_has_no_lineno(basicblock *b) { + for (int i = 0; i < b->b_iused; i++) { + if (b->b_instr[i].i_loc.lineno >= 0) { + return false; + } + } + return true; +} + /* Maximum size of basic block that should be copied in optimizer */ #define MAX_COPY_SIZE 4 -/* If this block ends with an unconditional jump to a small exit block, then +/* If this block ends with an unconditional jump to a small exit block or + * a block that has no line numbers (and no fallthrough), then * remove the jump and extend this block with the target. * Returns 1 if extended, 0 if no change, and -1 on error. */ static int -inline_small_exit_blocks(basicblock *bb) { +basicblock_inline_small_or_no_lineno_blocks(basicblock *bb) { cfg_instr *last = basicblock_last_instr(bb); if (last == NULL) { return 0; @@ -1182,14 +1187,46 @@ inline_small_exit_blocks(basicblock *bb) { return 0; } basicblock *target = last->i_target; - if (basicblock_exits_scope(target) && target->b_iused <= MAX_COPY_SIZE) { + bool small_exit_block = (basicblock_exits_scope(target) && + target->b_iused <= MAX_COPY_SIZE); + bool no_lineno_no_fallthrough = (basicblock_has_no_lineno(target) && + !BB_HAS_FALLTHROUGH(target)); + if (small_exit_block || no_lineno_no_fallthrough) { + assert(is_jump(last)); + int removed_jump_opcode = last->i_opcode; INSTR_SET_OP0(last, NOP); RETURN_IF_ERROR(basicblock_append_instructions(bb, target)); + if (no_lineno_no_fallthrough) { + last = basicblock_last_instr(bb); + if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode) && + removed_jump_opcode == JUMP) + { + /* Make sure we don't lose eval breaker checks */ + last->i_opcode = JUMP; + } + } + target->b_predecessors--; return 1; } return 0; } +static int +inline_small_or_no_lineno_blocks(basicblock *entryblock) { + bool changes; + do { + changes = false; + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + int res = basicblock_inline_small_or_no_lineno_blocks(b); + RETURN_IF_ERROR(res); + if (res) { + changes = true; + } + } + } while(changes); /* every change removes a jump, ensuring convergence */ + return changes; +} + // Attempt to eliminate jumps to jumps by updating inst to jump to // target->i_target using the provided opcode. Return whether or not the // optimization was successful. @@ -1804,9 +1841,7 @@ optimize_cfg(cfg_builder *g, PyObject *consts, PyObject *const_cache, int firstl { assert(PyDict_CheckExact(const_cache)); RETURN_IF_ERROR(check_cfg(g)); - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - RETURN_IF_ERROR(inline_small_exit_blocks(b)); - } + RETURN_IF_ERROR(inline_small_or_no_lineno_blocks(g->g_entryblock)); RETURN_IF_ERROR(remove_unreachable(g->g_entryblock)); RETURN_IF_ERROR(resolve_line_numbers(g, firstlineno)); RETURN_IF_ERROR(optimize_load_const(const_cache, g, consts)); @@ -1814,9 +1849,6 @@ optimize_cfg(cfg_builder *g, PyObject *consts, PyObject *const_cache, int firstl RETURN_IF_ERROR(optimize_basic_block(const_cache, b, consts)); } RETURN_IF_ERROR(remove_redundant_nops_and_pairs(g->g_entryblock)); - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - RETURN_IF_ERROR(inline_small_exit_blocks(b)); - } RETURN_IF_ERROR(remove_unreachable(g->g_entryblock)); int removed_nops, removed_jumps; @@ -2333,12 +2365,7 @@ convert_pseudo_ops(cfg_builder *g) static inline bool is_exit_or_eval_check_without_lineno(basicblock *b) { if (basicblock_exits_scope(b) || basicblock_has_eval_break(b)) { - for (int i = 0; i < b->b_iused; i++) { - if (b->b_instr[i].i_loc.lineno >= 0) { - return false; - } - } - return true; + return basicblock_has_no_lineno(b); } else { return false; @@ -2696,7 +2723,7 @@ _PyCfg_ToInstructionSequence(cfg_builder *g, _PyCompile_InstructionSequence *seq RETURN_IF_ERROR(_PyCompile_InstructionSequence_UseLabel(seq, b->b_label.id)); for (int i = 0; i < b->b_iused; i++) { cfg_instr *instr = &b->b_instr[i]; - if (OPCODE_HAS_JUMP(instr->i_opcode)) { + if (OPCODE_HAS_JUMP(instr->i_opcode) || is_block_push(instr)) { instr->i_oparg = instr->i_target->b_label.id; } RETURN_IF_ERROR( @@ -2757,3 +2784,14 @@ _PyCfg_OptimizedCfgToInstructionSequence(cfg_builder *g, return SUCCESS; } + +/* This is used by _PyCompile_Assemble to fill in the jump and exception + * targets in a synthetic CFG (which is not the ouptut of the builtin compiler). + */ +int +_PyCfg_JumpLabelsToTargets(cfg_builder *g) +{ + RETURN_IF_ERROR(translate_jump_labels_to_targets(g->g_entryblock)); + RETURN_IF_ERROR(label_exception_targets(g->g_entryblock)); + return SUCCESS; +} diff --git a/Python/frame.c b/Python/frame.c index 2865b2eab603c27..ddf6ef6ba5465cf 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -139,7 +139,6 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) for (int i = 0; i < frame->stacktop; i++) { Py_XDECREF(frame->localsplus[i]); } - Py_XDECREF(frame->frame_obj); Py_XDECREF(frame->f_locals); Py_DECREF(frame->f_funcobj); } diff --git a/Python/gc.c b/Python/gc.c index 466467602915264..ea3b596d1713dfc 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -12,6 +12,7 @@ #include "pycore_object_alloc.h" // _PyObject_MallocWithType() #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_time.h" // _PyTime_PerfCounterUnchecked() #include "pycore_weakref.h" // _PyWeakref_ClearRef() #include "pydtrace.h" @@ -394,16 +395,17 @@ update_refs(PyGC_Head *containers) while (gc != containers) { next = GC_NEXT(gc); + PyObject *op = FROM_GC(gc); /* Move any object that might have become immortal to the * permanent generation as the reference count is not accurately * reflecting the actual number of live references to this object */ - if (_Py_IsImmortal(FROM_GC(gc))) { + if (_Py_IsImmortal(op)) { gc_list_move(gc, &get_gc_state()->permanent_generation.head); gc = next; continue; } - gc_reset_refs(gc, Py_REFCNT(FROM_GC(gc))); + gc_reset_refs(gc, Py_REFCNT(op)); /* Python's cyclic gc should never see an incoming refcount * of 0: if something decref'ed to 0, it should have been * deallocated immediately at that time. @@ -422,7 +424,7 @@ update_refs(PyGC_Head *containers) * so serious that maybe this should be a release-build * check instead of an assert? */ - _PyObject_ASSERT(FROM_GC(gc), gc_get_refs(gc) != 0); + _PyObject_ASSERT(op, gc_get_refs(gc) != 0); gc = next; } } @@ -488,7 +490,7 @@ visit_reachable(PyObject *op, void *arg) } // It would be a logic error elsewhere if the collecting flag were set on // an untracked object. - assert(gc->_gc_next != 0); + _PyObject_ASSERT(op, gc->_gc_next != 0); if (gc->_gc_next & NEXT_MASK_UNREACHABLE) { /* This had gc_refs = 0 when move_unreachable got @@ -660,7 +662,9 @@ static void move_legacy_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers) { PyGC_Head *gc, *next; - assert((unreachable->_gc_next & NEXT_MASK_UNREACHABLE) == 0); + _PyObject_ASSERT( + FROM_GC(unreachable), + (unreachable->_gc_next & NEXT_MASK_UNREACHABLE) == 0); /* March over unreachable. Move objects with finalizers into * `finalizers`. @@ -683,10 +687,14 @@ static inline void clear_unreachable_mask(PyGC_Head *unreachable) { /* Check that the list head does not have the unreachable bit set */ - assert(((uintptr_t)unreachable & NEXT_MASK_UNREACHABLE) == 0); + _PyObject_ASSERT( + FROM_GC(unreachable), + ((uintptr_t)unreachable & NEXT_MASK_UNREACHABLE) == 0); + _PyObject_ASSERT( + FROM_GC(unreachable), + (unreachable->_gc_next & NEXT_MASK_UNREACHABLE) == 0); PyGC_Head *gc, *next; - assert((unreachable->_gc_next & NEXT_MASK_UNREACHABLE) == 0); for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { _PyObject_ASSERT((PyObject*)FROM_GC(gc), gc->_gc_next & NEXT_MASK_UNREACHABLE); gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; @@ -840,7 +848,7 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) */ if (gc_is_collecting(AS_GC((PyObject *)wr))) { /* it should already have been cleared above */ - assert(wr->wr_object == Py_None); + _PyObject_ASSERT((PyObject*)wr, wr->wr_object == Py_None); continue; } @@ -851,9 +859,8 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) /* Move wr to wrcb_to_call, for the next pass. */ wrasgc = AS_GC((PyObject *)wr); - assert(wrasgc != next); /* wrasgc is reachable, but - next isn't, so they can't - be the same */ + // wrasgc is reachable, but next isn't, so they can't be the same + _PyObject_ASSERT((PyObject *)wr, wrasgc != next); gc_list_move(wrasgc, &wrcb_to_call); } } @@ -1279,7 +1286,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) PyGC_Head unreachable; /* non-problematic unreachable trash */ PyGC_Head finalizers; /* objects with, & reachable from, __del__ */ PyGC_Head *gc; - _PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ + PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ GCState *gcstate = &tstate->interp->gc; // gc_collect_main() must not be called before _PyGC_Init @@ -1320,7 +1327,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) if (gcstate->debug & _PyGC_DEBUG_STATS) { PySys_WriteStderr("gc: collecting generation %d...\n", generation); show_stats_each_generations(gcstate); - t1 = _PyTime_GetPerfCounter(); + t1 = _PyTime_PerfCounterUnchecked(); } if (PyDTrace_GC_START_ENABLED()) { @@ -1421,7 +1428,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) debug_cycle("uncollectable", FROM_GC(gc)); } if (gcstate->debug & _PyGC_DEBUG_STATS) { - double d = _PyTime_AsSecondsDouble(_PyTime_GetPerfCounter() - t1); + double d = PyTime_AsSecondsDouble(_PyTime_PerfCounterUnchecked() - t1); PySys_WriteStderr( "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", n+m, n, d); @@ -1765,21 +1772,25 @@ PyObject_IS_GC(PyObject *obj) } void -_Py_ScheduleGC(PyInterpreterState *interp) +_Py_ScheduleGC(PyThreadState *tstate) { - _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 1); + if (!_Py_eval_breaker_bit_is_set(tstate, _PY_GC_SCHEDULED_BIT)) + { + _Py_set_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT); + } } void _PyObject_GC_Link(PyObject *op) { - PyGC_Head *g = AS_GC(op); - assert(((uintptr_t)g & (sizeof(uintptr_t)-1)) == 0); // g must be correctly aligned + PyGC_Head *gc = AS_GC(op); + // gc must be correctly aligned + _PyObject_ASSERT(op, ((uintptr_t)gc & (sizeof(uintptr_t)-1)) == 0); PyThreadState *tstate = _PyThreadState_GET(); GCState *gcstate = &tstate->interp->gc; - g->_gc_next = 0; - g->_gc_prev = 0; + gc->_gc_next = 0; + gc->_gc_prev = 0; gcstate->generations[0].count++; /* number of allocated GC objects */ if (gcstate->generations[0].count > gcstate->generations[0].threshold && gcstate->enabled && @@ -1787,7 +1798,7 @@ _PyObject_GC_Link(PyObject *op) !_Py_atomic_load_int_relaxed(&gcstate->collecting) && !_PyErr_Occurred(tstate)) { - _Py_ScheduleGC(tstate->interp); + _Py_ScheduleGC(tstate); } } diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index f2cd84981461a4f..d4fb50106093ee6 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1,5 +1,6 @@ // Cyclic garbage collector implementation for free-threaded build. #include "Python.h" +#include "pycore_brc.h" // struct _brc_thread_state #include "pycore_ceval.h" // _Py_set_eval_breaker_bit() #include "pycore_context.h" #include "pycore_dict.h" // _PyDict_MaybeUntrack() @@ -10,6 +11,7 @@ #include "pycore_object_stack.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_time.h" // _PyTime_GetPerfCounter() #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_weakref.h" // _PyWeakref_ClearRef() #include "pydtrace.h" @@ -22,10 +24,18 @@ typedef struct _gc_runtime_state GCState; # define GC_DEBUG #endif +// Each thread buffers the count of allocated objects in a thread-local +// variable up to +/- this amount to reduce the overhead of updating +// the global count. +#define LOCAL_ALLOC_COUNT_THRESHOLD 512 + // Automatically choose the generation that needs collecting. #define GENERATION_AUTO (-1) -// A linked-list of objects using the `ob_tid` field as the next pointer. +// A linked list of objects using the `ob_tid` field as the next pointer. +// The linked list pointers are distinct from any real thread ids, because the +// thread ids returned by _Py_ThreadId() are also pointers to distinct objects. +// No thread will confuse its own id with a linked list pointer. struct worklist { uintptr_t head; }; @@ -46,6 +56,7 @@ struct collection_state { GCState *gcstate; Py_ssize_t collected; Py_ssize_t uncollectable; + Py_ssize_t long_lived_total; struct worklist unreachable; struct worklist legacy_finalizers; struct worklist wrcb_to_call; @@ -148,8 +159,7 @@ gc_decref(PyObject *op) op->ob_tid -= 1; } -// Merge refcounts while the world is stopped. -static void +static Py_ssize_t merge_refcount(PyObject *op, Py_ssize_t extra) { assert(_PyInterpreterState_GET()->stoptheworld.world_stopped); @@ -165,6 +175,7 @@ merge_refcount(PyObject *op, Py_ssize_t extra) op->ob_tid = 0; op->ob_ref_local = 0; op->ob_ref_shared = _Py_REF_SHARED(refcount, _Py_REF_MERGED); + return refcount; } static void @@ -220,7 +231,7 @@ gc_visit_heaps_lock_held(PyInterpreterState *interp, mi_block_visit_fun *visitor struct visitor_args *arg) { // Offset of PyObject header from start of memory block. - Py_ssize_t offset_base = sizeof(PyGC_Head); + Py_ssize_t offset_base = 0; if (_PyMem_DebugEnabled()) { // The debug allocator adds two words at the beginning of each block. offset_base += 2 * sizeof(size_t); @@ -278,6 +289,41 @@ gc_visit_heaps(PyInterpreterState *interp, mi_block_visit_fun *visitor, return err; } +static void +merge_queued_objects(_PyThreadStateImpl *tstate, struct collection_state *state) +{ + struct _brc_thread_state *brc = &tstate->brc; + _PyObjectStack_Merge(&brc->local_objects_to_merge, &brc->objects_to_merge); + + PyObject *op; + while ((op = _PyObjectStack_Pop(&brc->local_objects_to_merge)) != NULL) { + // Subtract one when merging because the queue had a reference. + Py_ssize_t refcount = merge_refcount(op, -1); + + if (!_PyObject_GC_IS_TRACKED(op) && refcount == 0) { + // GC objects with zero refcount are handled subsequently by the + // GC as if they were cyclic trash, but we have to handle dead + // non-GC objects here. Add one to the refcount so that we can + // decref and deallocate the object once we start the world again. + op->ob_ref_shared += (1 << _Py_REF_SHARED_SHIFT); +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyInterpreterState_GET()); +#endif + worklist_push(&state->objs_to_decref, op); + } + } +} + +static void +merge_all_queued_objects(PyInterpreterState *interp, struct collection_state *state) +{ + HEAD_LOCK(&_PyRuntime); + for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + merge_queued_objects((_PyThreadStateImpl *)p, state); + } + HEAD_UNLOCK(&_PyRuntime); +} + // Subtract an incoming reference from the computed "gc_refs" refcount. static int visit_decref(PyObject *op, void *arg) @@ -330,8 +376,14 @@ update_refs(const mi_heap_t *heap, const mi_heap_area_t *area, Py_ssize_t refcount = Py_REFCNT(op); _PyObject_ASSERT(op, refcount >= 0); - // Add the actual refcount to ob_tid. + // We repurpose ob_tid to compute "gc_refs", the number of external + // references to the object (i.e., from outside the GC heaps). This means + // that ob_tid is no longer a valid thread id until it is restored by + // scan_heap_visitor(). Until then, we cannot use the standard reference + // counting functions or allow other threads to run Python code. gc_maybe_init_refs(op); + + // Add the actual refcount to ob_tid. gc_add_refs(op, refcount); // Subtract internal references from ob_tid. Objects with ob_tid > 0 @@ -443,7 +495,7 @@ scan_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, else { // object is reachable, restore `ob_tid`; we're done with these objects gc_restore_tid(op); - state->gcstate->long_lived_total++; + state->long_lived_total++; } return true; @@ -605,6 +657,8 @@ get_gc_state(void) void _PyGC_InitState(GCState *gcstate) { + // TODO: move to pycore_runtime_init.h once the incremental GC lands. + gcstate->generations[0].threshold = 2000; } @@ -885,62 +939,6 @@ invoke_gc_callback(PyThreadState *tstate, const char *phase, assert(!_PyErr_Occurred(tstate)); } - -/* Find the oldest generation (highest numbered) where the count - * exceeds the threshold. Objects in the that generation and - * generations younger than it will be collected. */ -static int -gc_select_generation(GCState *gcstate) -{ - for (int i = NUM_GENERATIONS-1; i >= 0; i--) { - if (gcstate->generations[i].count > gcstate->generations[i].threshold) { - /* Avoid quadratic performance degradation in number - of tracked objects (see also issue #4074): - - To limit the cost of garbage collection, there are two strategies; - - make each collection faster, e.g. by scanning fewer objects - - do less collections - This heuristic is about the latter strategy. - - In addition to the various configurable thresholds, we only trigger a - full collection if the ratio - - long_lived_pending / long_lived_total - - is above a given value (hardwired to 25%). - - The reason is that, while "non-full" collections (i.e., collections of - the young and middle generations) will always examine roughly the same - number of objects -- determined by the aforementioned thresholds --, - the cost of a full collection is proportional to the total number of - long-lived objects, which is virtually unbounded. - - Indeed, it has been remarked that doing a full collection every - <constant number> of object creations entails a dramatic performance - degradation in workloads which consist in creating and storing lots of - long-lived objects (e.g. building a large list of GC-tracked objects would - show quadratic performance, instead of linear as expected: see issue #4074). - - Using the above ratio, instead, yields amortized linear performance in - the total number of objects (the effect of which can be summarized - thusly: "each full garbage collection is more and more costly as the - number of objects grows, but we do fewer and fewer of them"). - - This heuristic was suggested by Martin von Löwis on python-dev in - June 2008. His original analysis and proposal can be found at: - http://mail.python.org/pipermail/python-dev/2008-June/080579.html - */ - if (i == NUM_GENERATIONS - 1 - && gcstate->long_lived_pending < gcstate->long_lived_total / 4) - { - continue; - } - return i; - } - } - return -1; -} - static void cleanup_worklist(struct worklist *worklist) { @@ -952,10 +950,63 @@ cleanup_worklist(struct worklist *worklist) } } +static bool +gc_should_collect(GCState *gcstate) +{ + int count = _Py_atomic_load_int_relaxed(&gcstate->generations[0].count); + int threshold = gcstate->generations[0].threshold; + if (count <= threshold || threshold == 0 || !gcstate->enabled) { + return false; + } + // Avoid quadratic behavior by scaling threshold to the number of live + // objects. A few tests rely on immediate scheduling of the GC so we ignore + // the scaled threshold if generations[1].threshold is set to zero. + return (count > gcstate->long_lived_total / 4 || + gcstate->generations[1].threshold == 0); +} + +static void +record_allocation(PyThreadState *tstate) +{ + struct _gc_thread_state *gc = &((_PyThreadStateImpl *)tstate)->gc; + + // We buffer the allocation count to avoid the overhead of atomic + // operations for every allocation. + gc->alloc_count++; + if (gc->alloc_count >= LOCAL_ALLOC_COUNT_THRESHOLD) { + // TODO: Use Py_ssize_t for the generation count. + GCState *gcstate = &tstate->interp->gc; + _Py_atomic_add_int(&gcstate->generations[0].count, (int)gc->alloc_count); + gc->alloc_count = 0; + + if (gc_should_collect(gcstate) && + !_Py_atomic_load_int_relaxed(&gcstate->collecting)) + { + _Py_ScheduleGC(tstate); + } + } +} + +static void +record_deallocation(PyThreadState *tstate) +{ + struct _gc_thread_state *gc = &((_PyThreadStateImpl *)tstate)->gc; + + gc->alloc_count--; + if (gc->alloc_count <= -LOCAL_ALLOC_COUNT_THRESHOLD) { + GCState *gcstate = &tstate->interp->gc; + _Py_atomic_add_int(&gcstate->generations[0].count, (int)gc->alloc_count); + gc->alloc_count = 0; + } +} + static void gc_collect_internal(PyInterpreterState *interp, struct collection_state *state) { _PyEval_StopTheWorld(interp); + // merge refcounts for all queued objects + merge_all_queued_objects(interp, state); + // Find unreachable objects int err = deduce_unreachable_heap(interp, state); if (err < 0) { @@ -971,10 +1022,16 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state) } } + // Record the number of live GC objects + interp->gc.long_lived_total = state->long_lived_total; + // Clear weakrefs and enqueue callbacks (but do not call them). clear_weakrefs(state); _PyEval_StartTheWorld(interp); + // Deallocate any object from the refcount merge step + cleanup_worklist(&state->objs_to_decref); + // Call weakref callbacks and finalizers after unpausing other threads to // avoid potential deadlocks. call_weakref_callbacks(state); @@ -1015,7 +1072,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) int i; Py_ssize_t m = 0; /* # objects collected */ Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ - _PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ + PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ GCState *gcstate = &tstate->interp->gc; // gc_collect_main() must not be called before _PyGC_Init @@ -1029,15 +1086,10 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) return 0; } - if (generation == GENERATION_AUTO) { - // Select the oldest generation that needs collecting. We will collect - // objects from that generation and all generations younger than it. - generation = gc_select_generation(gcstate); - if (generation < 0) { - // No generation needs to be collected. - _Py_atomic_store_int(&gcstate->collecting, 0); - return 0; - } + if (reason == _Py_GC_REASON_HEAP && !gc_should_collect(gcstate)) { + // Don't collect if the threshold is not exceeded. + _Py_atomic_store_int(&gcstate->collecting, 0); + return 0; } assert(generation >= 0 && generation < NUM_GENERATIONS); @@ -1056,7 +1108,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) if (gcstate->debug & _PyGC_DEBUG_STATS) { PySys_WriteStderr("gc: collecting generation %d...\n", generation); show_stats_each_generations(gcstate); - t1 = _PyTime_GetPerfCounter(); + t1 = _PyTime_PerfCounterUnchecked(); } if (PyDTrace_GC_START_ENABLED()) { @@ -1084,7 +1136,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) n = state.uncollectable; if (gcstate->debug & _PyGC_DEBUG_STATS) { - double d = _PyTime_AsSecondsDouble(_PyTime_GetPerfCounter() - t1); + double d = PyTime_AsSecondsDouble(_PyTime_PerfCounterUnchecked() - t1); PySys_WriteStderr( "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", n+m, n, d); @@ -1513,31 +1565,24 @@ PyObject_IS_GC(PyObject *obj) } void -_Py_ScheduleGC(PyInterpreterState *interp) +_Py_ScheduleGC(PyThreadState *tstate) { - _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 1); + if (!_Py_eval_breaker_bit_is_set(tstate, _PY_GC_SCHEDULED_BIT)) + { + _Py_set_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT); + } } void _PyObject_GC_Link(PyObject *op) { - PyThreadState *tstate = _PyThreadState_GET(); - GCState *gcstate = &tstate->interp->gc; - gcstate->generations[0].count++; /* number of allocated GC objects */ - if (gcstate->generations[0].count > gcstate->generations[0].threshold && - gcstate->enabled && - gcstate->generations[0].threshold && - !_Py_atomic_load_int_relaxed(&gcstate->collecting) && - !_PyErr_Occurred(tstate)) - { - _Py_ScheduleGC(tstate->interp); - } + record_allocation(_PyThreadState_GET()); } void _Py_RunGC(PyThreadState *tstate) { - gc_collect_main(tstate, GENERATION_AUTO, _Py_GC_REASON_HEAP); + gc_collect_main(tstate, 0, _Py_GC_REASON_HEAP); } static PyObject * @@ -1552,10 +1597,12 @@ gc_alloc(PyTypeObject *tp, size_t basicsize, size_t presize) if (mem == NULL) { return _PyErr_NoMemory(tstate); } - ((PyObject **)mem)[0] = NULL; - ((PyObject **)mem)[1] = NULL; + if (presize) { + ((PyObject **)mem)[0] = NULL; + ((PyObject **)mem)[1] = NULL; + } PyObject *op = (PyObject *)(mem + presize); - _PyObject_GC_Link(op); + record_allocation(tstate); return op; } @@ -1637,29 +1684,22 @@ PyObject_GC_Del(void *op) PyErr_SetRaisedException(exc); #endif } - GCState *gcstate = get_gc_state(); - if (gcstate->generations[0].count > 0) { - gcstate->generations[0].count--; - } + + record_deallocation(_PyThreadState_GET()); + PyObject_Free(((char *)op)-presize); } int PyObject_GC_IsTracked(PyObject* obj) { - if (_PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)) { - return 1; - } - return 0; + return _PyObject_GC_IS_TRACKED(obj); } int PyObject_GC_IsFinalized(PyObject *obj) { - if (_PyObject_IS_GC(obj) && _PyGC_FINALIZED(obj)) { - return 1; - } - return 0; + return _PyGC_FINALIZED(obj); } struct custom_visitor_args { @@ -1709,12 +1749,10 @@ PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) void _PyGC_ClearAllFreeLists(PyInterpreterState *interp) { - _PyDict_ClearFreeList(interp); - HEAD_LOCK(&_PyRuntime); _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)interp->threads.head; while (tstate != NULL) { - _Py_ClearFreeLists(&tstate->freelist_state, 0); + _PyObject_ClearFreeLists(&tstate->freelists, 0); tstate = (_PyThreadStateImpl *)tstate->base.next; } HEAD_UNLOCK(&_PyRuntime); diff --git a/Python/gc_gil.c b/Python/gc_gil.c index 04c1c184250c609..48646c7af86b7fd 100644 --- a/Python/gc_gil.c +++ b/Python/gc_gil.c @@ -11,9 +11,7 @@ void _PyGC_ClearAllFreeLists(PyInterpreterState *interp) { - _PyDict_ClearFreeList(interp); - - _Py_ClearFreeLists(&interp->freelist_state, 0); + _PyObject_ClearFreeLists(&interp->object_state.freelists, 0); } #endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 16f1db30620d722..bb26dac0e2be203 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -112,7 +112,6 @@ lhs = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -242,7 +241,6 @@ /* Skip 1 cache entry */ // _BINARY_OP_INPLACE_ADD_UNICODE { - TIER_ONE_ONLY assert(next_instr->op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(next_instr->op.arg); DEOPT_IF(*target_local != left, BINARY_OP); @@ -429,7 +427,6 @@ container = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -739,9 +736,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); - TIER_ONE_ONLY assert(0 && "Executing a cache."); - Py_UNREACHABLE(); + Py_FatalError("Executing a cache."); + DISPATCH(); } TARGET(CALL) { @@ -760,7 +757,6 @@ callable = stack_pointer[-2 - oparg]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -1004,7 +1000,6 @@ } #endif } - stack_pointer += ((0) ? 1 : 0); DISPATCH(); } @@ -1069,7 +1064,7 @@ STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ - res = ((_PyCFunctionFast)(void(*)(void))cfunc)( + res = ((PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable), args, total_args); @@ -1115,8 +1110,8 @@ DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable); res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -1477,7 +1472,6 @@ args = &stack_pointer[-oparg]; self = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - TIER_ONE_ONLY assert(oparg == 1); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.list_append, CALL); @@ -1523,8 +1517,8 @@ PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); - _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; res = cfunc(self, args + 1, nargs); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -1568,8 +1562,8 @@ DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; res = cfunc(self, args + 1, nargs, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ @@ -1754,7 +1748,6 @@ } #endif } - stack_pointer += ((0) ? 1 : 0); DISPATCH(); } @@ -1954,7 +1947,6 @@ exc_value = stack_pointer[-1]; last_sent_val = stack_pointer[-2]; sub_iter = stack_pointer[-3]; - TIER_ONE_ONLY assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { @@ -1989,7 +1981,6 @@ left = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -2027,20 +2018,26 @@ PyObject *right; PyObject *left; PyObject *res; - /* Skip 1 cache entry */ + // _GUARD_BOTH_FLOAT right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. + { + DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_FLOAT + { + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left); + double dright = PyFloat_AS_DOUBLE(right); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + } stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); @@ -2054,24 +2051,30 @@ PyObject *right; PyObject *left; PyObject *res; - /* Skip 1 cache entry */ + // _GUARD_BOTH_INT right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && + { + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_INT + { + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + } stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); @@ -2085,21 +2088,27 @@ PyObject *right; PyObject *left; PyObject *res; - /* Skip 1 cache entry */ + // _GUARD_BOTH_UNICODE right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. + { + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_STR + { + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left, right); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + } stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); @@ -2321,7 +2330,6 @@ PyObject *awaitable; exc = stack_pointer[-1]; awaitable = stack_pointer[-2]; - TIER_ONE_ONLY assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { Py_DECREF(awaitable); @@ -2363,29 +2371,19 @@ } TARGET(ENTER_EXECUTOR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); - TIER_ONE_ONLY CHECK_EVAL_BREAKER(); PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; - if (executor->vm_data.valid) { - Py_INCREF(executor); - current_executor = executor; - GOTO_TIER_TWO(); - } - else { - /* ENTER_EXECUTOR will be the first code unit of the instruction */ - assert(oparg < 256); - code->co_executors->executors[oparg] = NULL; - opcode = this_instr->op.code = executor->vm_data.opcode; - this_instr->op.arg = executor->vm_data.oparg; - oparg = executor->vm_data.oparg; - Py_DECREF(executor); - next_instr = this_instr; - DISPATCH_GOTO(); - } + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + Py_INCREF(executor); + GOTO_TIER_TWO(executor); DISPATCH(); } @@ -2410,7 +2408,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); - TIER_ONE_ONLY assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; @@ -2469,7 +2466,6 @@ iter = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -2552,11 +2548,14 @@ assert(Py_TYPE(iter) == &PyListIter_Type); STAT_INC(FOR_ITER, hit); PyListObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { + if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + it->it_index = -1; + #ifndef Py_GIL_DISABLED if (seq != NULL) { it->it_seq = NULL; Py_DECREF(seq); } + #endif Py_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ @@ -2856,7 +2855,6 @@ PyObject *from; PyObject *res; from = stack_pointer[-1]; - TIER_ONE_ONLY PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; @@ -2874,7 +2872,6 @@ PyObject *res; fromlist = stack_pointer[-1]; level = stack_pointer[-2]; - TIER_ONE_ONLY PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_name(tstate, frame, name, fromlist, level); Py_DECREF(level); @@ -2934,7 +2931,6 @@ PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - TIER_ONE_ONLY /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { @@ -2957,7 +2953,6 @@ PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - TIER_ONE_ONLY if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { @@ -3128,9 +3123,9 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RESUME); - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - if (code_version != global_version) { + if (code_version != global_version && tstate->tracing == 0) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { GOTO_ERROR(error); } @@ -3237,7 +3232,6 @@ INSTRUCTION_STATS(INTERPRETER_EXIT); PyObject *retval; retval = stack_pointer[-1]; - TIER_ONE_ONLY assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ @@ -3274,13 +3268,16 @@ assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); #if ENABLE_SPECIALIZATION - this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); + uint16_t counter = this_instr[1].cache; + this_instr[1].cache = counter + (1 << OPTIMIZER_BITS_IN_COUNTER); /* We are using unsigned values, but we really want signed values, so - * do the 2s complement comparison manually */ - uint16_t ucounter = this_instr[1].cache + (1 << 15); - uint16_t threshold = tstate->interp->optimizer_backedge_threshold + (1 << 15); + * do the 2s complement adjustment manually */ + uint32_t offset_counter = counter ^ (1 << 15); + uint32_t threshold = tstate->interp->optimizer_backedge_threshold; + assert((threshold & OPTIMIZER_BITS_MASK) == 0); + // Use '>=' not '>' so that the optimizer/backoff bits do not effect the result. // Double-check that the opcode isn't instrumented or something: - if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { + if (offset_counter >= threshold && this_instr->op.code == JUMP_BACKWARD) { OPT_STAT_INC(attempts); _Py_CODEUNIT *start = this_instr; /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ @@ -3288,24 +3285,24 @@ oparg >>= 8; start--; } - int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer); + _PyExecutorObject *executor; + int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor); if (optimized < 0) goto error; if (optimized) { - // Rewind and enter the executor: - assert(start->op.code == ENTER_EXECUTOR); - next_instr = start; - this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + GOTO_TIER_TWO(executor); } else { - int backoff = this_instr[1].cache & ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); - if (backoff < MINIMUM_TIER2_BACKOFF) { - backoff = MINIMUM_TIER2_BACKOFF; + int backoff = this_instr[1].cache & OPTIMIZER_BITS_MASK; + backoff++; + if (backoff < MIN_TIER2_BACKOFF) { + backoff = MIN_TIER2_BACKOFF; } - else if (backoff < 15 - OPTIMIZER_BITS_IN_COUNTER) { - backoff++; + else if (backoff > MAX_TIER2_BACKOFF) { + backoff = MAX_TIER2_BACKOFF; } - assert(backoff <= 15 - OPTIMIZER_BITS_IN_COUNTER); - this_instr[1].cache = ((1 << 16) - ((1 << OPTIMIZER_BITS_IN_COUNTER) << backoff)) | backoff; + this_instr[1].cache = ((UINT16_MAX << OPTIMIZER_BITS_IN_COUNTER) << backoff) | backoff; } } #endif /* ENABLE_SPECIALIZATION */ @@ -3316,7 +3313,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - TIER_ONE_ONLY /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -3330,7 +3326,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_FORWARD); - TIER_ONE_ONLY JUMPBY(oparg); DISPATCH(); } @@ -3399,7 +3394,6 @@ owner = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); @@ -3431,7 +3425,7 @@ something was returned by a descriptor protocol). Set the second element of the stack to NULL, to signal CALL that it's not a method call. - NULL | meth | arg1 | ... | argN + meth | NULL | arg1 | ... | argN */ Py_DECREF(owner); if (attr == NULL) goto pop_1_error; @@ -3598,8 +3592,8 @@ self = owner; } stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; DISPATCH(); } @@ -3633,8 +3627,8 @@ self = owner; } stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; DISPATCH(); } @@ -3680,8 +3674,8 @@ self = owner; } stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; DISPATCH(); } @@ -3697,11 +3691,11 @@ // _CHECK_ATTR_MODULE owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t dict_version = read_u32(&this_instr[2].cache); DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); + DEOPT_IF(dict->ma_keys->dk_version != dict_version, LOAD_ATTR); } // _LOAD_ATTR_MODULE { @@ -3752,7 +3746,6 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; - stack_pointer += ((0) ? 1 : 0); DISPATCH(); } @@ -3795,7 +3788,6 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; - stack_pointer += ((0) ? 1 : 0); DISPATCH(); } @@ -4096,7 +4088,6 @@ // _SPECIALIZE_LOAD_GLOBAL { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); @@ -4298,7 +4289,6 @@ global_super = stack_pointer[-3]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION int load_method = oparg & 1; if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -4313,7 +4303,6 @@ // _LOAD_SUPER_ATTR self = stack_pointer[-1]; { - TIER_ONE_ONLY if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( @@ -4381,7 +4370,7 @@ Py_DECREF(self); if (attr == NULL) goto pop_3_error; stack_pointer[-3] = attr; - stack_pointer += -2 + ((0) ? 1 : 0); + stack_pointer += -2; DISPATCH(); } @@ -4724,7 +4713,6 @@ INSTRUCTION_STATS(RAISE_VARARGS); PyObject **args; args = &stack_pointer[-oparg]; - TIER_ONE_ONLY PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -4756,7 +4744,6 @@ PyObject **values; exc = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; - TIER_ONE_ONLY assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -4781,9 +4768,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); - TIER_ONE_ONLY assert(0 && "Executing RESERVED instruction."); - Py_UNREACHABLE(); + Py_FatalError("Executing RESERVED instruction."); + DISPATCH(); } TARGET(RESUME) { @@ -4792,24 +4779,24 @@ INSTRUCTION_STATS(RESUME); PREDICTED(RESUME); _Py_CODEUNIT *this_instr = next_instr - 1; - TIER_ONE_ONLY assert(frame == tstate->current_frame); - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - if (err) goto error; - next_instr = this_instr; - } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); + if (tstate->tracing == 0) { + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + if (err) goto error; + next_instr = this_instr; + DISPATCH(); } - this_instr->op.code = RESUME_CHECK; } + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } + this_instr->op.code = RESUME_CHECK; DISPATCH(); } @@ -4822,10 +4809,10 @@ DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); + DEOPT_IF(eval_breaker != version && tstate->tracing == 0, RESUME); DISPATCH(); } @@ -4870,7 +4857,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); - TIER_ONE_ONLY assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4937,7 +4923,6 @@ receiver = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -5124,7 +5109,6 @@ owner = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -5416,7 +5400,6 @@ container = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -5518,7 +5501,6 @@ value = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -5727,7 +5709,6 @@ seq = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -5862,7 +5843,6 @@ INSTRUCTION_STATS(YIELD_VALUE); PyObject *retval; retval = stack_pointer[-1]; - TIER_ONE_ONLY // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. diff --git a/Python/getargs.c b/Python/getargs.c index 0c4ce282f487648..08e97ee3e627b5a 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -8,6 +8,7 @@ #include "pycore_modsupport.h" // export _PyArg_NoKeywords() #include "pycore_pylifecycle.h" // _PyArg_Fini #include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_pyerrors.h" // _Py_CalculateSuggestions() /* Export Stable ABIs (abi only) */ PyAPI_FUNC(int) _PyArg_Parse_SizeT(PyObject *, const char *, ...); @@ -1424,12 +1425,31 @@ error_unexpected_keyword_arg(PyObject *kwargs, PyObject *kwnames, PyObject *kwtu int match = PySequence_Contains(kwtuple, keyword); if (match <= 0) { if (!match) { - PyErr_Format(PyExc_TypeError, - "'%S' is an invalid keyword " - "argument for %.200s%s", - keyword, - (fname == NULL) ? "this function" : fname, - (fname == NULL) ? "" : "()"); + PyObject *kwlist = PySequence_List(kwtuple); + if (!kwlist) { + return; + } + PyObject *suggestion_keyword = _Py_CalculateSuggestions(kwlist, keyword); + Py_DECREF(kwlist); + + if (suggestion_keyword) { + PyErr_Format(PyExc_TypeError, + "%.200s%s got an unexpected keyword argument '%S'." + " Did you mean '%S'?", + (fname == NULL) ? "this function" : fname, + (fname == NULL) ? "" : "()", + keyword, + suggestion_keyword); + Py_DECREF(suggestion_keyword); + } + else { + PyErr_Format(PyExc_TypeError, + "%.200s%s got an unexpected keyword argument '%S'", + (fname == NULL) ? "this function" : fname, + (fname == NULL) ? "" : "()", + keyword); + } + } return; } @@ -1457,6 +1477,9 @@ PyArg_ValidateKeywordArguments(PyObject *kwargs) return 1; } +static PyObject * +new_kwtuple(const char * const *keywords, int total, int pos); + #define IS_END_OF_FORMAT(c) (c == '\0' || c == ';' || c == ':') static int @@ -1722,12 +1745,35 @@ vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, } } if (!match) { - PyErr_Format(PyExc_TypeError, - "'%U' is an invalid keyword " - "argument for %.200s%s", - key, - (fname == NULL) ? "this function" : fname, - (fname == NULL) ? "" : "()"); + PyObject *_pykwtuple = new_kwtuple(kwlist, len, pos); + if (!_pykwtuple) { + return cleanreturn(0, &freelist); + } + PyObject *pykwlist = PySequence_List(_pykwtuple); + Py_DECREF(_pykwtuple); + if (!pykwlist) { + return cleanreturn(0, &freelist); + } + PyObject *suggestion_keyword = _Py_CalculateSuggestions(pykwlist, key); + Py_DECREF(pykwlist); + + if (suggestion_keyword) { + PyErr_Format(PyExc_TypeError, + "%.200s%s got an unexpected keyword argument '%S'." + " Did you mean '%S'?", + (fname == NULL) ? "this function" : fname, + (fname == NULL) ? "" : "()", + key, + suggestion_keyword); + Py_DECREF(suggestion_keyword); + } + else { + PyErr_Format(PyExc_TypeError, + "%.200s%s got an unexpected keyword argument '%S'", + (fname == NULL) ? "this function" : fname, + (fname == NULL) ? "" : "()", + key); + } return cleanreturn(0, &freelist); } } diff --git a/Python/import.c b/Python/import.c index 2dd95d8364a0be0..dc92708c8b6ea0b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -13,6 +13,7 @@ #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_sysmodule.h" // _PySys_Audit() +#include "pycore_time.h" // _PyTime_PerfCounterUnchecked() #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include "marshal.h" // PyMarshal_ReadObjectFromString() @@ -2719,7 +2720,7 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) #define import_level FIND_AND_LOAD(interp).import_level #define accumulated FIND_AND_LOAD(interp).accumulated - _PyTime_t t1 = 0, accumulated_copy = accumulated; + PyTime_t t1 = 0, accumulated_copy = accumulated; PyObject *sys_path = PySys_GetObject("path"); PyObject *sys_meta_path = PySys_GetObject("meta_path"); @@ -2747,7 +2748,7 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) #undef header import_level++; - t1 = _PyTime_GetPerfCounter(); + t1 = _PyTime_PerfCounterUnchecked(); accumulated = 0; } @@ -2762,7 +2763,7 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) mod != NULL); if (import_time) { - _PyTime_t cum = _PyTime_GetPerfCounter() - t1; + PyTime_t cum = _PyTime_PerfCounterUnchecked() - t1; import_level--; fprintf(stderr, "import time: %9ld | %10ld | %*s%s\n", @@ -3544,7 +3545,7 @@ _imp_get_frozen_object_impl(PyObject *module, PyObject *name, struct frozen_info info = {0}; Py_buffer buf = {0}; if (PyObject_CheckBuffer(dataobj)) { - if (PyObject_GetBuffer(dataobj, &buf, PyBUF_READ) != 0) { + if (PyObject_GetBuffer(dataobj, &buf, PyBUF_SIMPLE) != 0) { return NULL; } info.data = (const char *)buf.buf; diff --git a/Python/initconfig.c b/Python/initconfig.c index a6d8c1761566179..74f28f3b39175b1 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -153,7 +153,8 @@ Options (and corresponding environment variables):\n\ .pyc extension; also PYTHONOPTIMIZE=x\n\ -OO : do -O changes and also discard docstrings; add .opt-2 before\n\ .pyc extension\n\ --P : don't prepend a potentially unsafe path to sys.path; also PYTHONSAFEPATH\n\ +-P : don't prepend a potentially unsafe path to sys.path; also\n\ + PYTHONSAFEPATH\n\ -q : don't print version and copyright messages on interactive startup\n\ -s : don't add user site directory to sys.path; also PYTHONNOUSERSITE\n\ -S : don't imply 'import site' on initialization\n\ @@ -169,9 +170,10 @@ Options (and corresponding environment variables):\n\ -X opt : set implementation-specific option\n\ --check-hash-based-pycs always|default|never:\n\ control how Python invalidates hash-based .pyc files\n\ ---help-env : print help about Python environment variables and exit\n\ ---help-xoptions : print help about implementation-specific -X options and exit\n\ ---help-all : print complete help information and exit\n\ +--help-env: print help about Python environment variables and exit\n\ +--help-xoptions: print help about implementation-specific -X options and exit\n\ +--help-all: print complete help information and exit\n\ +\n\ Arguments:\n\ file : program read from script file\n\ - : program read from stdin (default; interactive mode if a tty)\n\ @@ -180,73 +182,61 @@ arg ...: arguments passed to program in sys.argv[1:]\n\ static const char usage_xoptions[] = "\ The following implementation-specific options are available:\n\ -\n\ -X faulthandler: enable faulthandler\n\ -\n\ -X showrefcount: output the total reference count and number of used\n\ - memory blocks when the program finishes or after each statement in the\n\ - interactive interpreter. This only works on debug builds\n\ -\n\ + memory blocks when the program finishes or after each statement in\n\ + the interactive interpreter. This only works on debug builds\n\ -X tracemalloc: start tracing Python memory allocations using the\n\ - tracemalloc module. By default, only the most recent frame is stored in a\n\ - traceback of a trace. Use -X tracemalloc=NFRAME to start tracing with a\n\ - traceback limit of NFRAME frames\n\ -\n\ --X importtime: show how long each import takes. It shows module name,\n\ - cumulative time (including nested imports) and self time (excluding\n\ - nested imports). Note that its output may be broken in multi-threaded\n\ - application. Typical usage is python3 -X importtime -c 'import asyncio'\n\ -\n\ --X dev: enable CPython's \"development mode\", introducing additional runtime\n\ - checks which are too expensive to be enabled by default. Effect of the\n\ - developer mode:\n\ - * Add default warning filter, as -W default\n\ - * Install debug hooks on memory allocators: see the PyMem_SetupDebugHooks()\n\ - C function\n\ - * Enable the faulthandler module to dump the Python traceback on a crash\n\ - * Enable asyncio debug mode\n\ - * Set the dev_mode attribute of sys.flags to True\n\ - * io.IOBase destructor logs close() exceptions\n\ -\n\ --X utf8: enable UTF-8 mode for operating system interfaces, overriding the default\n\ - locale-aware mode. -X utf8=0 explicitly disables UTF-8 mode (even when it would\n\ - otherwise activate automatically)\n\ -\n\ --X pycache_prefix=PATH: enable writing .pyc files to a parallel tree rooted at the\n\ - given directory instead of to the code tree\n\ -\n\ + tracemalloc module. By default, only the most recent frame is stored\n\ + in a traceback of a trace. Use -X tracemalloc=NFRAME to start\n\ + tracing with a traceback limit of NFRAME frames\n\ +-X importtime: show how long each import takes. It shows module name,\n\ + cumulative time (including nested imports) and self time (excluding\n\ + nested imports). Note that its output may be broken in\n\ + multi-threaded application.\n\ + Typical usage is python3 -X importtime -c 'import asyncio'\n\ +-X dev : enable CPython's \"development mode\", introducing additional runtime\n\ + checks which are too expensive to be enabled by default. Effect of\n\ + the developer mode:\n\ + * Add default warning filter, as -W default\n\ + * Install debug hooks on memory allocators: see the\n\ + PyMem_SetupDebugHooks() C function\n\ + * Enable the faulthandler module to dump the Python traceback on\n\ + a crash\n\ + * Enable asyncio debug mode\n\ + * Set the dev_mode attribute of sys.flags to True\n\ + * io.IOBase destructor logs close() exceptions\n\ +-X utf8: enable UTF-8 mode for operating system interfaces, overriding the\n\ + default locale-aware mode. -X utf8=0 explicitly disables UTF-8 mode\n\ + (even when it would otherwise activate automatically)\n\ +-X pycache_prefix=PATH: enable writing .pyc files to a parallel tree rooted\n\ + at the given directory instead of to the code tree\n\ -X warn_default_encoding: enable opt-in EncodingWarning for 'encoding=None'\n\ -\n\ --X no_debug_ranges: disable the inclusion of the tables mapping extra location \n\ - information (end line, start column offset and end column offset) to every \n\ - instruction in code objects. This is useful when smaller code objects and pyc \n\ - files are desired as well as suppressing the extra visual location indicators \n\ - when the interpreter displays tracebacks.\n\ -\n\ --X perf: activate support for the Linux \"perf\" profiler by activating the \"perf\"\n\ - trampoline. When this option is activated, the Linux \"perf\" profiler will be \n\ - able to report Python calls. This option is only available on some platforms and will \n\ - do nothing if is not supported on the current system. The default value is \"off\".\n\ -\n\ +-X no_debug_ranges: disable the inclusion of the tables mapping extra location\n\ + information (end line, start column offset and end column offset) to\n\ + every instruction in code objects. This is useful when smaller code\n\ + objects and pyc files are desired as well as suppressing the extra\n\ + visual location indicators when the interpreter displays tracebacks.\n\ +-X perf: activate support for the Linux \"perf\" profiler by activating the\n\ + \"perf\" trampoline. When this option is activated, the Linux \"perf\"\n\ + profiler will be able to report Python calls. This option is only\n\ + available on some platforms and will do nothing if is not supported\n\ + on the current system. The default value is \"off\".\n\ -X frozen_modules=[on|off]: whether or not frozen modules should be used.\n\ - The default is \"on\" (or \"off\" if you are running a local build).\n\ -\n\ + The default is \"on\" (or \"off\" if you are running a local build).\n\ -X int_max_str_digits=number: limit the size of int<->str conversions.\n\ - This helps avoid denial of service attacks when parsing untrusted data.\n\ - The default is sys.int_info.default_max_str_digits. 0 disables.\n\ -\n\ + This helps avoid denial of service attacks when parsing untrusted\n\ + data. The default is sys.int_info.default_max_str_digits.\n\ + 0 disables.\n\ -X cpu_count=[n|default]: Override the return value of os.cpu_count(),\n\ - os.process_cpu_count(), and multiprocessing.cpu_count(). This can help users who need\n\ - to limit resources in a container." - + os.process_cpu_count(), and multiprocessing.cpu_count(). This can\n\ + help users who need to limit resources in a container." #ifdef Py_STATS "\n\ -\n\ -X pystats: Enable pystats collection at startup." #endif #ifdef Py_DEBUG "\n\ -\n\ -X presite=package.module: import this module before site.py is run." #endif ; @@ -254,65 +244,73 @@ The following implementation-specific options are available:\n\ /* Envvars that don't have equivalent command-line options are listed first */ static const char usage_envvars[] = "Environment variables that change behavior:\n" -"PYTHONSTARTUP: file executed on interactive startup (no default)\n" -"PYTHONPATH : '%lc'-separated list of directories prefixed to the\n" -" default module search path. The result is sys.path.\n" -"PYTHONHOME : alternate <prefix> directory (or <prefix>%lc<exec_prefix>).\n" -" The default module search path uses %s.\n" -"PYTHONPLATLIBDIR : override sys.platlibdir.\n" -"PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" -"PYTHONUTF8: if set to 1, enable the UTF-8 mode.\n" +"PYTHONSTARTUP : file executed on interactive startup (no default)\n" +"PYTHONPATH : '%lc'-separated list of directories prefixed to the\n" +" default module search path. The result is sys.path.\n" +"PYTHONHOME : alternate <prefix> directory (or <prefix>%lc<exec_prefix>).\n" +" The default module search path uses %s.\n" +"PYTHONPLATLIBDIR: override sys.platlibdir.\n" +"PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" +"PYTHONUTF8 : if set to 1, enable the UTF-8 mode.\n" "PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" "PYTHONFAULTHANDLER: dump the Python traceback on fatal errors.\n" -"PYTHONHASHSEED: if this variable is set to 'random', a random value is used\n" -" to seed the hashes of str and bytes objects. It can also be set to an\n" -" integer in the range [0,4294967295] to get hash values with a\n" -" predictable seed.\n" +"PYTHONHASHSEED : if this variable is set to 'random', a random value is used\n" +" to seed the hashes of str and bytes objects. It can also be\n" +" set to an integer in the range [0,4294967295] to get hash\n" +" values with a predictable seed.\n" "PYTHONINTMAXSTRDIGITS: limits the maximum digit characters in an int value\n" -" when converting from a string and when converting an int back to a str.\n" -" A value of 0 disables the limit. Conversions to or from bases 2, 4, 8,\n" -" 16, and 32 are never limited.\n" -"PYTHONMALLOC: set the Python memory allocators and/or install debug hooks\n" -" on Python memory allocators. Use PYTHONMALLOC=debug to install debug\n" -" hooks.\n" +" when converting from a string and when converting an int\n" +" back to a str. A value of 0 disables the limit.\n" +" Conversions to or from bases 2, 4, 8, 16, and 32 are never\n" +" limited.\n" +"PYTHONMALLOC : set the Python memory allocators and/or install debug hooks\n" +" on Python memory allocators. Use PYTHONMALLOC=debug to\n" +" install debug hooks.\n" "PYTHONCOERCECLOCALE: if this variable is set to 0, it disables the locale\n" -" coercion behavior. Use PYTHONCOERCECLOCALE=warn to request display of\n" -" locale coercion and locale compatibility warnings on stderr.\n" +" coercion behavior. Use PYTHONCOERCECLOCALE=warn to request\n" +" display of locale coercion and locale compatibility warnings\n" +" on stderr.\n" "PYTHONBREAKPOINT: if this variable is set to 0, it disables the default\n" -" debugger. It can be set to the callable of your debugger of choice.\n" +" debugger. It can be set to the callable of your debugger of\n" +" choice.\n" "PYTHON_CPU_COUNT: Overrides the return value of os.process_cpu_count(),\n" -" os.cpu_count(), and multiprocessing.cpu_count() if set to a positive integer.\n" -"PYTHONDEVMODE: enable the development mode.\n" +" os.cpu_count(), and multiprocessing.cpu_count() if set to\n" +" a positive integer.\n" +"PYTHONDEVMODE : enable the development mode.\n" "PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files.\n" "PYTHONWARNDEFAULTENCODING: enable opt-in EncodingWarning for 'encoding=None'.\n" -"PYTHONNODEBUGRANGES: if this variable is set, it disables the inclusion of the \n" -" tables mapping extra location information (end line, start column offset \n" -" and end column offset) to every instruction in code objects. This is useful \n" -" when smaller code objects and pyc files are desired as well as suppressing the \n" -" extra visual location indicators when the interpreter displays tracebacks.\n" -"PYTHON_FROZEN_MODULES : if this variable is set, it determines whether or not \n" -" frozen modules should be used. The default is \"on\" (or \"off\" if you are \n" -" running a local build).\n" -"PYTHON_COLORS : If this variable is set to 1, the interpreter will" -" colorize various kinds of output. Setting it to 0 deactivates this behavior.\n" -"PYTHON_HISTORY : the location of a .python_history file.\n" -"These variables have equivalent command-line parameters (see --help for details):\n" -"PYTHONDEBUG : enable parser debug mode (-d)\n" -"PYTHONDONTWRITEBYTECODE : don't write .pyc files (-B)\n" -"PYTHONINSPECT : inspect interactively after running script (-i)\n" -"PYTHONINTMAXSTRDIGITS : limit max digit characters in an int value\n" -" (-X int_max_str_digits=number)\n" -"PYTHONNOUSERSITE : disable user site directory (-s)\n" -"PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" -"PYTHONSAFEPATH : don't prepend a potentially unsafe path to sys.path (-P)\n" -"PYTHONUNBUFFERED : disable stdout/stderr buffering (-u)\n" -"PYTHONVERBOSE : trace import statements (-v)\n" -"PYTHONWARNINGS=arg : warning control (-W arg)\n" +"PYTHONNODEBUGRANGES: if this variable is set, it disables the inclusion of\n" +" the tables mapping extra location information (end line,\n" +" start column offset and end column offset) to every\n" +" instruction in code objects. This is useful when smaller\n" +" code objects and pyc files are desired as well as\n" +" suppressing the extra visual location indicators when the\n" +" interpreter displays tracebacks.\n" +"PYTHON_FROZEN_MODULES: if this variable is set, it determines whether or not\n" +" frozen modules should be used. The default is \"on\" (or\n" +" \"off\" if you are running a local build).\n" +"PYTHON_COLORS : if this variable is set to 1, the interpreter will colorize\n" +" various kinds of output. Setting it to 0 deactivates\n" +" this behavior.\n" +"PYTHON_HISTORY : the location of a .python_history file.\n" +"\n" +"These variables have equivalent command-line options (see --help for details):\n" +"PYTHONDEBUG : enable parser debug mode (-d)\n" +"PYTHONDONTWRITEBYTECODE: don't write .pyc files (-B)\n" +"PYTHONINSPECT : inspect interactively after running script (-i)\n" +"PYTHONINTMAXSTRDIGITS: limit max digit characters in an int value\n" +" (-X int_max_str_digits=number)\n" +"PYTHONNOUSERSITE: disable user site directory (-s)\n" +"PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" +"PYTHONSAFEPATH : don't prepend a potentially unsafe path to sys.path.\n" +"PYTHONUNBUFFERED: disable stdout/stderr buffering (-u)\n" +"PYTHONVERBOSE : trace import statements (-v)\n" +"PYTHONWARNINGS=arg: warning control (-W arg)\n" #ifdef Py_STATS -"PYTHONSTATS : turns on statistics gathering\n" +"PYTHONSTATS : turns on statistics gathering\n" #endif #ifdef Py_DEBUG -"PYTHON_PRESITE=pkg.mod : import this module before site.py is run\n" +"PYTHON_PRESITE=pkg.mod: import this module before site.py is run\n" #endif ; @@ -2387,9 +2385,9 @@ static void config_complete_usage(const wchar_t* program) { config_usage(0, program); - puts("\n"); + putchar('\n'); config_envvars_usage(); - puts("\n"); + putchar('\n'); config_xoptions_usage(); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 533aece210202bb..4b7d8b5a5875044 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -891,18 +891,43 @@ static inline int most_significant_bit(uint8_t bits) { static uint32_t global_version(PyInterpreterState *interp) { - return interp->ceval.eval_breaker & ~_PY_EVAL_EVENTS_MASK; + return (uint32_t)_Py_atomic_load_uintptr_relaxed( + &interp->ceval.instrumentation_version); } +/* Atomically set the given version in the given location, without touching + anything in _PY_EVAL_EVENTS_MASK. */ static void -set_global_version(PyInterpreterState *interp, uint32_t version) +set_version_raw(uintptr_t *ptr, uint32_t version) { - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - uintptr_t old = _Py_atomic_load_uintptr(&interp->ceval.eval_breaker); - intptr_t new; + uintptr_t old = _Py_atomic_load_uintptr_relaxed(ptr); + uintptr_t new; do { new = (old & _PY_EVAL_EVENTS_MASK) | version; - } while (!_Py_atomic_compare_exchange_uintptr(&interp->ceval.eval_breaker, &old, new)); + } while (!_Py_atomic_compare_exchange_uintptr(ptr, &old, new)); +} + +static void +set_global_version(PyThreadState *tstate, uint32_t version) +{ + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + PyInterpreterState *interp = tstate->interp; + set_version_raw(&interp->ceval.instrumentation_version, version); + +#ifdef Py_GIL_DISABLED + // Set the version on all threads in free-threaded builds. + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); + for (tstate = interp->threads.head; tstate; + tstate = PyThreadState_Next(tstate)) { + set_version_raw(&tstate->eval_breaker, version); + }; + HEAD_UNLOCK(runtime); +#else + // Normal builds take the current version from instrumentation_version when + // attaching a thread, so we only have to set the current thread's version. + set_version_raw(&tstate->eval_breaker, version); +#endif } static bool @@ -1131,15 +1156,13 @@ int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev) { PyCodeObject *code = _PyFrame_GetCode(frame); + assert(tstate->tracing == 0); assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); int i = (int)(instr - _PyCode_CODE(code)); _PyCoMonitoringData *monitoring = code->_co_monitoring; _PyCoLineInstrumentationData *line_data = &monitoring->lines[i]; - if (tstate->tracing) { - goto done; - } PyInterpreterState *interp = tstate->interp; int8_t line_delta = line_data->line_delta; int line = compute_line(code, i, line_delta); @@ -1566,7 +1589,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) { if (is_version_up_to_date(code, interp)) { assert( - (interp->ceval.eval_breaker & ~_PY_EVAL_EVENTS_MASK) == 0 || + interp->ceval.instrumentation_version == 0 || instrumentation_cross_checks(interp, code) ); return 0; @@ -1574,7 +1597,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (code->co_executors != NULL) { _PyCode_Clear_Executors(code); } - _Py_Executors_InvalidateDependency(interp, code); + _Py_Executors_InvalidateDependency(interp, code, 1); int code_len = (int)Py_SIZE(code); /* Exit early to avoid creating instrumentation * data for potential statically allocated code @@ -1778,7 +1801,8 @@ int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); - PyInterpreterState *interp = _PyInterpreterState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); + PyInterpreterState *interp = tstate->interp; assert(events < (1 << _PY_MONITORING_UNGROUPED_EVENTS)); if (check_tool(interp, tool_id)) { return -1; @@ -1793,8 +1817,8 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) PyErr_Format(PyExc_OverflowError, "events set too many times"); return -1; } - set_global_version(interp, new_version); - _Py_Executors_InvalidateAll(interp); + set_global_version(tstate, new_version); + _Py_Executors_InvalidateAll(interp, 1); return instrument_all_executing_code_objects(interp); } @@ -1824,7 +1848,7 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent /* Force instrumentation update */ code->_co_instrumentation_version -= MONITORING_VERSION_INCREMENT; } - _Py_Executors_InvalidateDependency(interp, code); + _Py_Executors_InvalidateDependency(interp, code, 1); if (_Py_Instrument(code, interp)) { return -1; } @@ -2122,7 +2146,8 @@ monitoring_restart_events_impl(PyObject *module) * last restart version > instrumented version for all code objects * last restart version < current version */ - PyInterpreterState *interp = _PyInterpreterState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); + PyInterpreterState *interp = tstate->interp; uint32_t restart_version = global_version(interp) + MONITORING_VERSION_INCREMENT; uint32_t new_version = restart_version + MONITORING_VERSION_INCREMENT; if (new_version <= MONITORING_VERSION_INCREMENT) { @@ -2130,7 +2155,7 @@ monitoring_restart_events_impl(PyObject *module) return NULL; } interp->last_restart_version = restart_version; - set_global_version(interp, new_version); + set_global_version(tstate, new_version); if (instrument_all_executing_code_objects(interp)) { return NULL; } diff --git a/Python/jit.c b/Python/jit.c new file mode 100644 index 000000000000000..ac2c60ed925a264 --- /dev/null +++ b/Python/jit.c @@ -0,0 +1,461 @@ +#ifdef _Py_JIT + +#include "Python.h" + +#include "pycore_abstract.h" +#include "pycore_call.h" +#include "pycore_ceval.h" +#include "pycore_dict.h" +#include "pycore_intrinsics.h" +#include "pycore_long.h" +#include "pycore_opcode_metadata.h" +#include "pycore_opcode_utils.h" +#include "pycore_optimizer.h" +#include "pycore_pyerrors.h" +#include "pycore_setobject.h" +#include "pycore_sliceobject.h" +#include "pycore_jit.h" + +#include "jit_stencils.h" + +// Memory management stuff: //////////////////////////////////////////////////// + +#ifndef MS_WINDOWS + #include <sys/mman.h> +#endif + +static size_t +get_page_size(void) +{ +#ifdef MS_WINDOWS + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; +#else + return sysconf(_SC_PAGESIZE); +#endif +} + +static void +jit_error(const char *message) +{ +#ifdef MS_WINDOWS + int hint = GetLastError(); +#else + int hint = errno; +#endif + PyErr_Format(PyExc_RuntimeWarning, "JIT %s (%d)", message, hint); +} + +static unsigned char * +jit_alloc(size_t size) +{ + assert(size); + assert(size % get_page_size() == 0); +#ifdef MS_WINDOWS + int flags = MEM_COMMIT | MEM_RESERVE; + unsigned char *memory = VirtualAlloc(NULL, size, flags, PAGE_READWRITE); + int failed = memory == NULL; +#else + int flags = MAP_ANONYMOUS | MAP_PRIVATE; + unsigned char *memory = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0); + int failed = memory == MAP_FAILED; +#endif + if (failed) { + jit_error("unable to allocate memory"); + return NULL; + } + return memory; +} + +static int +jit_free(unsigned char *memory, size_t size) +{ + assert(size); + assert(size % get_page_size() == 0); +#ifdef MS_WINDOWS + int failed = !VirtualFree(memory, 0, MEM_RELEASE); +#else + int failed = munmap(memory, size); +#endif + if (failed) { + jit_error("unable to free memory"); + return -1; + } + return 0; +} + +static int +mark_executable(unsigned char *memory, size_t size) +{ + if (size == 0) { + return 0; + } + assert(size % get_page_size() == 0); + // Do NOT ever leave the memory writable! Also, don't forget to flush the + // i-cache (I cannot begin to tell you how horrible that is to debug): +#ifdef MS_WINDOWS + if (!FlushInstructionCache(GetCurrentProcess(), memory, size)) { + jit_error("unable to flush instruction cache"); + return -1; + } + int old; + int failed = !VirtualProtect(memory, size, PAGE_EXECUTE_READ, &old); +#else + __builtin___clear_cache((char *)memory, (char *)memory + size); + int failed = mprotect(memory, size, PROT_EXEC | PROT_READ); +#endif + if (failed) { + jit_error("unable to protect executable memory"); + return -1; + } + return 0; +} + +static int +mark_readable(unsigned char *memory, size_t size) +{ + if (size == 0) { + return 0; + } + assert(size % get_page_size() == 0); +#ifdef MS_WINDOWS + DWORD old; + int failed = !VirtualProtect(memory, size, PAGE_READONLY, &old); +#else + int failed = mprotect(memory, size, PROT_READ); +#endif + if (failed) { + jit_error("unable to protect readable memory"); + return -1; + } + return 0; +} + +// JIT compiler stuff: ///////////////////////////////////////////////////////// + +// Warning! AArch64 requires you to get your hands dirty. These are your gloves: + +// value[value_start : value_start + len] +static uint32_t +get_bits(uint64_t value, uint8_t value_start, uint8_t width) +{ + assert(width <= 32); + return (value >> value_start) & ((1ULL << width) - 1); +} + +// *loc[loc_start : loc_start + width] = value[value_start : value_start + width] +static void +set_bits(uint32_t *loc, uint8_t loc_start, uint64_t value, uint8_t value_start, + uint8_t width) +{ + assert(loc_start + width <= 32); + // Clear the bits we're about to patch: + *loc &= ~(((1ULL << width) - 1) << loc_start); + assert(get_bits(*loc, loc_start, width) == 0); + // Patch the bits: + *loc |= get_bits(value, value_start, width) << loc_start; + assert(get_bits(*loc, loc_start, width) == get_bits(value, value_start, width)); +} + +// See https://developer.arm.com/documentation/ddi0602/2023-09/Base-Instructions +// for instruction encodings: +#define IS_AARCH64_ADD_OR_SUB(I) (((I) & 0x11C00000) == 0x11000000) +#define IS_AARCH64_ADRP(I) (((I) & 0x9F000000) == 0x90000000) +#define IS_AARCH64_BRANCH(I) (((I) & 0x7C000000) == 0x14000000) +#define IS_AARCH64_LDR_OR_STR(I) (((I) & 0x3B000000) == 0x39000000) +#define IS_AARCH64_MOV(I) (((I) & 0x9F800000) == 0x92800000) + +// Fill all of stencil's holes in the memory pointed to by base, using the +// values in patches. +static void +patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) +{ + for (uint64_t i = 0; i < stencil->holes_size; i++) { + const Hole *hole = &stencil->holes[i]; + unsigned char *location = base + hole->offset; + uint64_t value = patches[hole->value] + (uint64_t)hole->symbol + hole->addend; + uint8_t *loc8 = (uint8_t *)location; + uint32_t *loc32 = (uint32_t *)location; + uint64_t *loc64 = (uint64_t *)location; + // LLD is a great reference for performing relocations... just keep in + // mind that Tools/jit/build.py does filtering and preprocessing for us! + // Here's a good place to start for each platform: + // - aarch64-apple-darwin: + // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64.cpp + // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.cpp + // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.h + // - aarch64-unknown-linux-gnu: + // - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/AArch64.cpp + // - i686-pc-windows-msvc: + // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp + // - x86_64-apple-darwin: + // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/X86_64.cpp + // - x86_64-pc-windows-msvc: + // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp + // - x86_64-unknown-linux-gnu: + // - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/X86_64.cpp + switch (hole->kind) { + case HoleKind_IMAGE_REL_I386_DIR32: + // 32-bit absolute address. + // Check that we're not out of range of 32 unsigned bits: + assert(value < (1ULL << 32)); + *loc32 = (uint32_t)value; + continue; + case HoleKind_ARM64_RELOC_UNSIGNED: + case HoleKind_IMAGE_REL_AMD64_ADDR64: + case HoleKind_R_AARCH64_ABS64: + case HoleKind_X86_64_RELOC_UNSIGNED: + case HoleKind_R_X86_64_64: + // 64-bit absolute address. + *loc64 = value; + continue; + case HoleKind_R_X86_64_GOTPCRELX: + case HoleKind_R_X86_64_REX_GOTPCRELX: + case HoleKind_X86_64_RELOC_GOT: + case HoleKind_X86_64_RELOC_GOT_LOAD: { + // 32-bit relative address. + // Try to relax the GOT load into an immediate value: + uint64_t relaxed = *(uint64_t *)(value + 4) - 4; + if ((int64_t)relaxed - (int64_t)location >= -(1LL << 31) && + (int64_t)relaxed - (int64_t)location + 1 < (1LL << 31)) + { + if (loc8[-2] == 0x8B) { + // mov reg, dword ptr [rip + AAA] -> lea reg, [rip + XXX] + loc8[-2] = 0x8D; + value = relaxed; + } + else if (loc8[-2] == 0xFF && loc8[-1] == 0x15) { + // call qword ptr [rip + AAA] -> nop; call XXX + loc8[-2] = 0x90; + loc8[-1] = 0xE8; + value = relaxed; + } + else if (loc8[-2] == 0xFF && loc8[-1] == 0x25) { + // jmp qword ptr [rip + AAA] -> nop; jmp XXX + loc8[-2] = 0x90; + loc8[-1] = 0xE9; + value = relaxed; + } + } + } + // Fall through... + case HoleKind_R_X86_64_GOTPCREL: + case HoleKind_R_X86_64_PC32: + case HoleKind_X86_64_RELOC_SIGNED: + case HoleKind_X86_64_RELOC_BRANCH: + // 32-bit relative address. + value -= (uint64_t)location; + // Check that we're not out of range of 32 signed bits: + assert((int64_t)value >= -(1LL << 31)); + assert((int64_t)value < (1LL << 31)); + loc32[0] = (uint32_t)value; + continue; + case HoleKind_R_AARCH64_CALL26: + case HoleKind_R_AARCH64_JUMP26: + // 28-bit relative branch. + assert(IS_AARCH64_BRANCH(*loc32)); + value -= (uint64_t)location; + // Check that we're not out of range of 28 signed bits: + assert((int64_t)value >= -(1 << 27)); + assert((int64_t)value < (1 << 27)); + // Since instructions are 4-byte aligned, only use 26 bits: + assert(get_bits(value, 0, 2) == 0); + set_bits(loc32, 0, value, 2, 26); + continue; + case HoleKind_R_AARCH64_MOVW_UABS_G0_NC: + // 16-bit low part of an absolute address. + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 0 of 3"): + assert(get_bits(*loc32, 21, 2) == 0); + set_bits(loc32, 5, value, 0, 16); + continue; + case HoleKind_R_AARCH64_MOVW_UABS_G1_NC: + // 16-bit middle-low part of an absolute address. + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 1 of 3"): + assert(get_bits(*loc32, 21, 2) == 1); + set_bits(loc32, 5, value, 16, 16); + continue; + case HoleKind_R_AARCH64_MOVW_UABS_G2_NC: + // 16-bit middle-high part of an absolute address. + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 2 of 3"): + assert(get_bits(*loc32, 21, 2) == 2); + set_bits(loc32, 5, value, 32, 16); + continue; + case HoleKind_R_AARCH64_MOVW_UABS_G3: + // 16-bit high part of an absolute address. + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 3 of 3"): + assert(get_bits(*loc32, 21, 2) == 3); + set_bits(loc32, 5, value, 48, 16); + continue; + case HoleKind_ARM64_RELOC_GOT_LOAD_PAGE21: + case HoleKind_R_AARCH64_ADR_GOT_PAGE: + // 21-bit count of pages between this page and an absolute address's + // page... I know, I know, it's weird. Pairs nicely with + // ARM64_RELOC_GOT_LOAD_PAGEOFF12 (below). + assert(IS_AARCH64_ADRP(*loc32)); + // Try to relax the pair of GOT loads into an immediate value: + const Hole *next_hole = &stencil->holes[i + 1]; + if (i + 1 < stencil->holes_size && + (next_hole->kind == HoleKind_ARM64_RELOC_GOT_LOAD_PAGEOFF12 || + next_hole->kind == HoleKind_R_AARCH64_LD64_GOT_LO12_NC) && + next_hole->offset == hole->offset + 4 && + next_hole->symbol == hole->symbol && + next_hole->addend == hole->addend && + next_hole->value == hole->value) + { + unsigned char rd = get_bits(loc32[0], 0, 5); + assert(IS_AARCH64_LDR_OR_STR(loc32[1])); + unsigned char rt = get_bits(loc32[1], 0, 5); + unsigned char rn = get_bits(loc32[1], 5, 5); + assert(rd == rn && rn == rt); + uint64_t relaxed = *(uint64_t *)value; + if (relaxed < (1UL << 16)) { + // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; nop + loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | rd; + loc32[1] = 0xD503201F; + i++; + continue; + } + if (relaxed < (1ULL << 32)) { + // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; movk reg, YYY + loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | rd; + loc32[1] = 0xF2A00000 | (get_bits(relaxed, 16, 16) << 5) | rd; + i++; + continue; + } + relaxed = (uint64_t)value - (uint64_t)location; + if ((relaxed & 0x3) == 0 && + (int64_t)relaxed >= -(1L << 19) && + (int64_t)relaxed < (1L << 19)) + { + // adrp reg, AAA; ldr reg, [reg + BBB] -> ldr x0, XXX; nop + loc32[0] = 0x58000000 | (get_bits(relaxed, 2, 19) << 5) | rd; + loc32[1] = 0xD503201F; + i++; + continue; + } + } + // Number of pages between this page and the value's page: + value = (value >> 12) - ((uint64_t)location >> 12); + // Check that we're not out of range of 21 signed bits: + assert((int64_t)value >= -(1 << 20)); + assert((int64_t)value < (1 << 20)); + // value[0:2] goes in loc[29:31]: + set_bits(loc32, 29, value, 0, 2); + // value[2:21] goes in loc[5:26]: + set_bits(loc32, 5, value, 2, 19); + continue; + case HoleKind_ARM64_RELOC_GOT_LOAD_PAGEOFF12: + case HoleKind_R_AARCH64_LD64_GOT_LO12_NC: + // 12-bit low part of an absolute address. Pairs nicely with + // ARM64_RELOC_GOT_LOAD_PAGE21 (above). + assert(IS_AARCH64_LDR_OR_STR(*loc32) || IS_AARCH64_ADD_OR_SUB(*loc32)); + // There might be an implicit shift encoded in the instruction: + uint8_t shift = 0; + if (IS_AARCH64_LDR_OR_STR(*loc32)) { + shift = (uint8_t)get_bits(*loc32, 30, 2); + // If both of these are set, the shift is supposed to be 4. + // That's pretty weird, and it's never actually been observed... + assert(get_bits(*loc32, 23, 1) == 0 || get_bits(*loc32, 26, 1) == 0); + } + value = get_bits(value, 0, 12); + assert(get_bits(value, 0, shift) == 0); + set_bits(loc32, 10, value, shift, 12); + continue; + } + Py_UNREACHABLE(); + } +} + +static void +copy_and_patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) +{ + memcpy(base, stencil->body, stencil->body_size); + patch(base, stencil, patches); +} + +static void +emit(const StencilGroup *group, uint64_t patches[]) +{ + copy_and_patch((unsigned char *)patches[HoleValue_DATA], &group->data, patches); + copy_and_patch((unsigned char *)patches[HoleValue_CODE], &group->code, patches); +} + +// Compiles executor in-place. Don't forget to call _PyJIT_Free later! +int +_PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length) +{ + // Loop once to find the total compiled size: + size_t code_size = 0; + size_t data_size = 0; + for (size_t i = 0; i < length; i++) { + _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; + const StencilGroup *group = &stencil_groups[instruction->opcode]; + code_size += group->code.body_size; + data_size += group->data.body_size; + } + // Round up to the nearest page (code and data need separate pages): + size_t page_size = get_page_size(); + assert((page_size & (page_size - 1)) == 0); + code_size += page_size - (code_size & (page_size - 1)); + data_size += page_size - (data_size & (page_size - 1)); + unsigned char *memory = jit_alloc(code_size + data_size); + if (memory == NULL) { + return -1; + } + // Loop again to emit the code: + unsigned char *code = memory; + unsigned char *data = memory + code_size; + unsigned char *top = code; + if (trace[0].opcode == _START_EXECUTOR) { + // Don't want to execute this more than once: + top += stencil_groups[_START_EXECUTOR].code.body_size; + } + for (size_t i = 0; i < length; i++) { + _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; + const StencilGroup *group = &stencil_groups[instruction->opcode]; + // Think of patches as a dictionary mapping HoleValue to uint64_t: + uint64_t patches[] = GET_PATCHES(); + patches[HoleValue_CODE] = (uint64_t)code; + patches[HoleValue_CONTINUE] = (uint64_t)code + group->code.body_size; + patches[HoleValue_DATA] = (uint64_t)data; + patches[HoleValue_EXECUTOR] = (uint64_t)executor; + patches[HoleValue_OPARG] = instruction->oparg; + patches[HoleValue_OPERAND] = instruction->operand; + patches[HoleValue_TARGET] = instruction->target; + patches[HoleValue_TOP] = (uint64_t)top; + patches[HoleValue_ZERO] = 0; + emit(group, patches); + code += group->code.body_size; + data += group->data.body_size; + } + if (mark_executable(memory, code_size) || + mark_readable(memory + code_size, data_size)) + { + jit_free(memory, code_size + data_size); + return -1; + } + executor->jit_code = memory; + executor->jit_size = code_size + data_size; + return 0; +} + +void +_PyJIT_Free(_PyExecutorObject *executor) +{ + unsigned char *memory = (unsigned char *)executor->jit_code; + size_t size = executor->jit_size; + if (memory) { + executor->jit_code = NULL; + executor->jit_size = 0; + if (jit_free(memory, size)) { + PyErr_WriteUnraisable(NULL); + } + } +} + +#endif // _Py_JIT diff --git a/Python/lock.c b/Python/lock.c index f0ff1176941da8f..5fa8bf78da23808 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -5,18 +5,19 @@ #include "pycore_lock.h" #include "pycore_parking_lot.h" #include "pycore_semaphore.h" +#include "pycore_time.h" // _PyTime_MonotonicUnchecked() #ifdef MS_WINDOWS -#define WIN32_LEAN_AND_MEAN -#include <windows.h> // SwitchToThread() +# define WIN32_LEAN_AND_MEAN +# include <windows.h> // SwitchToThread() #elif defined(HAVE_SCHED_H) -#include <sched.h> // sched_yield() +# include <sched.h> // sched_yield() #endif // If a thread waits on a lock for longer than TIME_TO_BE_FAIR_NS (1 ms), then // the unlocking thread directly hands off ownership of the lock. This avoids // starvation. -static const _PyTime_t TIME_TO_BE_FAIR_NS = 1000*1000; +static const PyTime_t TIME_TO_BE_FAIR_NS = 1000*1000; // Spin for a bit before parking the thread. This is only enabled for // `--disable-gil` builds because it is unlikely to be helpful if the GIL is @@ -30,7 +31,7 @@ static const int MAX_SPIN_COUNT = 0; struct mutex_entry { // The time after which the unlocking thread should hand off lock ownership // directly to the waiting thread. Written by the waiting thread. - _PyTime_t time_to_be_fair; + PyTime_t time_to_be_fair; // Set to 1 if the lock was handed off. Written by the unlocking thread. int handed_off; @@ -53,7 +54,7 @@ _PyMutex_LockSlow(PyMutex *m) } PyLockStatus -_PyMutex_LockTimed(PyMutex *m, _PyTime_t timeout, _PyLockFlags flags) +_PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) { uint8_t v = _Py_atomic_load_uint8_relaxed(&m->v); if ((v & _Py_LOCKED) == 0) { @@ -65,8 +66,8 @@ _PyMutex_LockTimed(PyMutex *m, _PyTime_t timeout, _PyLockFlags flags) return PY_LOCK_FAILURE; } - _PyTime_t now = _PyTime_GetMonotonicClock(); - _PyTime_t endtime = 0; + PyTime_t now = _PyTime_MonotonicUnchecked(); + PyTime_t endtime = 0; if (timeout > 0) { endtime = _PyTime_Add(now, timeout); } @@ -142,7 +143,7 @@ mutex_unpark(PyMutex *m, struct mutex_entry *entry, int has_more_waiters) { uint8_t v = 0; if (entry) { - _PyTime_t now = _PyTime_GetMonotonicClock(); + PyTime_t now = _PyTime_MonotonicUnchecked(); int should_be_fair = now > entry->time_to_be_fair; entry->handed_off = should_be_fair; @@ -274,7 +275,7 @@ PyEvent_Wait(PyEvent *evt) } int -PyEvent_WaitTimed(PyEvent *evt, _PyTime_t timeout_ns) +PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns) { for (;;) { uint8_t v = _Py_atomic_load_uint8(&evt->v); @@ -459,3 +460,74 @@ _PyRWMutex_Unlock(_PyRWMutex *rwmutex) _PyParkingLot_UnparkAll(&rwmutex->bits); } } + +#define SEQLOCK_IS_UPDATING(sequence) (sequence & 0x01) + +void _PySeqLock_LockWrite(_PySeqLock *seqlock) +{ + // lock the entry by setting by moving to an odd sequence number + uint32_t prev = _Py_atomic_load_uint32_relaxed(&seqlock->sequence); + while (1) { + if (SEQLOCK_IS_UPDATING(prev)) { + // Someone else is currently updating the cache + _Py_yield(); + prev = _Py_atomic_load_uint32_relaxed(&seqlock->sequence); + } + else if (_Py_atomic_compare_exchange_uint32(&seqlock->sequence, &prev, prev + 1)) { + // We've locked the cache + break; + } + else { + _Py_yield(); + } + } +} + +void _PySeqLock_AbandonWrite(_PySeqLock *seqlock) +{ + uint32_t new_seq = seqlock->sequence - 1; + assert(!SEQLOCK_IS_UPDATING(new_seq)); + _Py_atomic_store_uint32(&seqlock->sequence, new_seq); +} + +void _PySeqLock_UnlockWrite(_PySeqLock *seqlock) +{ + uint32_t new_seq = seqlock->sequence + 1; + assert(!SEQLOCK_IS_UPDATING(new_seq)); + _Py_atomic_store_uint32(&seqlock->sequence, new_seq); +} + +uint32_t _PySeqLock_BeginRead(_PySeqLock *seqlock) +{ + uint32_t sequence = _Py_atomic_load_uint32_acquire(&seqlock->sequence); + while (SEQLOCK_IS_UPDATING(sequence)) { + _Py_yield(); + sequence = _Py_atomic_load_uint32_acquire(&seqlock->sequence); + } + + return sequence; +} + +uint32_t _PySeqLock_EndRead(_PySeqLock *seqlock, uint32_t previous) +{ + // Synchronize again and validate that the entry hasn't been updated + // while we were readying the values. + if (_Py_atomic_load_uint32_acquire(&seqlock->sequence) == previous) { + return 1; + } + + _Py_yield(); + return 0; +} + +uint32_t _PySeqLock_AfterFork(_PySeqLock *seqlock) +{ + // Synchronize again and validate that the entry hasn't been updated + // while we were readying the values. + if (SEQLOCK_IS_UPDATING(seqlock->sequence)) { + seqlock->sequence = 0; + return 1; + } + + return 0; +} diff --git a/Python/object_stack.c b/Python/object_stack.c index 8544892eb71dcb6..bd9696822c8a9d4 100644 --- a/Python/object_stack.c +++ b/Python/object_stack.c @@ -8,22 +8,22 @@ extern _PyObjectStackChunk *_PyObjectStackChunk_New(void); extern void _PyObjectStackChunk_Free(_PyObjectStackChunk *); -static struct _Py_object_stack_state * -get_state(void) +static struct _Py_object_stack_freelist * +get_object_stack_freelist(void) { - _PyFreeListState *state = _PyFreeListState_GET(); - return &state->object_stacks; + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + return &freelists->object_stacks; } _PyObjectStackChunk * _PyObjectStackChunk_New(void) { _PyObjectStackChunk *buf; - struct _Py_object_stack_state *state = get_state(); - if (state->numfree > 0) { - buf = state->free_list; - state->free_list = buf->prev; - state->numfree--; + struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist(); + if (obj_stack_freelist->numfree > 0) { + buf = obj_stack_freelist->items; + obj_stack_freelist->items = buf->prev; + obj_stack_freelist->numfree--; } else { // NOTE: we use PyMem_RawMalloc() here because this is used by the GC @@ -43,13 +43,13 @@ void _PyObjectStackChunk_Free(_PyObjectStackChunk *buf) { assert(buf->n == 0); - struct _Py_object_stack_state *state = get_state(); - if (state->numfree >= 0 && - state->numfree < _PyObjectStackChunk_MAXFREELIST) + struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist(); + if (obj_stack_freelist->numfree >= 0 && + obj_stack_freelist->numfree < _PyObjectStackChunk_MAXFREELIST) { - buf->prev = state->free_list; - state->free_list = buf; - state->numfree++; + buf->prev = obj_stack_freelist->items; + obj_stack_freelist->items = buf; + obj_stack_freelist->numfree++; } else { PyMem_RawFree(buf); @@ -68,7 +68,28 @@ _PyObjectStack_Clear(_PyObjectStack *queue) } void -_PyObjectStackChunk_ClearFreeList(_PyFreeListState *free_lists, int is_finalization) +_PyObjectStack_Merge(_PyObjectStack *dst, _PyObjectStack *src) +{ + if (src->head == NULL) { + return; + } + + if (dst->head != NULL) { + // First, append dst to the bottom of src + _PyObjectStackChunk *last = src->head; + while (last->prev != NULL) { + last = last->prev; + } + last->prev = dst->head; + } + + // Now that src has all the chunks, set dst to src + dst->head = src->head; + src->head = NULL; +} + +void +_PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) { if (!is_finalization) { // Ignore requests to clear the free list during GC. We use object @@ -76,12 +97,12 @@ _PyObjectStackChunk_ClearFreeList(_PyFreeListState *free_lists, int is_finalizat return; } - struct _Py_object_stack_state *state = &free_lists->object_stacks; - while (state->numfree > 0) { - _PyObjectStackChunk *buf = state->free_list; - state->free_list = buf->prev; - state->numfree--; + struct _Py_object_stack_freelist *freelist = &freelists->object_stacks; + while (freelist->numfree > 0) { + _PyObjectStackChunk *buf = freelist->items; + freelist->items = buf->prev; + freelist->numfree--; PyMem_RawFree(buf); } - state->numfree = -1; + freelist->numfree = -1; } diff --git a/Python/optimizer.c b/Python/optimizer.c index db615068ff517f6..acd6d52c4a885f2 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -2,11 +2,13 @@ #include "opcode.h" #include "pycore_interp.h" #include "pycore_bitutils.h" // _Py_popcount32() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_opcode_metadata.h" // _PyOpcode_OpName[] #include "pycore_opcode_utils.h" // MAX_REAL_OPCODE #include "pycore_optimizer.h" // _Py_uop_analyze_and_optimize() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_uop_ids.h" +#include "pycore_jit.h" #include "cpython/optimizer.h" #include <stdbool.h> #include <stdint.h> @@ -16,8 +18,6 @@ #include "pycore_uop_metadata.h" // Uop tables #undef NEED_OPCODE_METADATA -#define UOP_MAX_TRACE_LENGTH 512 - #define MAX_EXECUTORS_SIZE 256 @@ -72,25 +72,21 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO Py_INCREF(executor); if (instr->op.code == ENTER_EXECUTOR) { assert(index == instr->op.arg); - _PyExecutorObject *old = code->co_executors->executors[index]; - executor->vm_data.opcode = old->vm_data.opcode; - executor->vm_data.oparg = old->vm_data.oparg; - old->vm_data.opcode = 0; - code->co_executors->executors[index] = executor; - Py_DECREF(old); + _Py_ExecutorClear(code->co_executors->executors[index]); } else { assert(code->co_executors->size == index); assert(code->co_executors->capacity > index); - executor->vm_data.opcode = instr->op.code; - executor->vm_data.oparg = instr->op.arg; - code->co_executors->executors[index] = executor; - assert(index < MAX_EXECUTORS_SIZE); - instr->op.code = ENTER_EXECUTOR; - instr->op.arg = index; code->co_executors->size++; } - return; + executor->vm_data.opcode = instr->op.code; + executor->vm_data.oparg = instr->op.arg; + executor->vm_data.code = code; + executor->vm_data.index = (int)(instr - _PyCode_CODE(code)); + code->co_executors->executors[index] = executor; + assert(index < MAX_EXECUTORS_SIZE); + instr->op.code = ENTER_EXECUTOR; + instr->op.arg = index; } int @@ -107,16 +103,17 @@ PyUnstable_Replace_Executor(PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutor } static int -error_optimize( +never_optimize( _PyOptimizerObject* self, - PyCodeObject *code, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _PyExecutorObject **exec, int Py_UNUSED(stack_entries)) { - assert(0); - PyErr_Format(PyExc_SystemError, "Should never call error_optimize"); - return -1; + /* Although it should be benign for this to be called, + * it shouldn't happen, so fail in debug builds. */ + assert(0 && "never optimize should never be called"); + return 0; } PyTypeObject _PyDefaultOptimizer_Type = { @@ -127,43 +124,92 @@ PyTypeObject _PyDefaultOptimizer_Type = { .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, }; -_PyOptimizerObject _PyOptimizer_Default = { +static _PyOptimizerObject _PyOptimizer_Default = { PyObject_HEAD_INIT(&_PyDefaultOptimizer_Type) - .optimize = error_optimize, - .resume_threshold = INT16_MAX, - .backedge_threshold = INT16_MAX, + .optimize = never_optimize, + .resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD, + .backedge_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD, + .side_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD, }; +static uint32_t +shift_and_offset_threshold(uint32_t threshold) +{ + return (threshold << OPTIMIZER_BITS_IN_COUNTER) + (1 << 15); +} + _PyOptimizerObject * PyUnstable_GetOptimizer(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp->optimizer_backedge_threshold == + shift_and_offset_threshold(interp->optimizer->backedge_threshold)); + assert(interp->optimizer_resume_threshold == + shift_and_offset_threshold(interp->optimizer->resume_threshold)); if (interp->optimizer == &_PyOptimizer_Default) { return NULL; } - assert(interp->optimizer_backedge_threshold == interp->optimizer->backedge_threshold); - assert(interp->optimizer_resume_threshold == interp->optimizer->resume_threshold); Py_INCREF(interp->optimizer); return interp->optimizer; } -void -PyUnstable_SetOptimizer(_PyOptimizerObject *optimizer) +static _PyExecutorObject * +make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies); + +static int +init_cold_exit_executor(_PyExecutorObject *executor, int oparg); + +static int cold_exits_initialized = 0; +static _PyExecutorObject COLD_EXITS[UOP_MAX_TRACE_LENGTH] = { 0 }; + +static const _PyBloomFilter EMPTY_FILTER = { 0 }; + +_PyOptimizerObject * +_Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer) { - PyInterpreterState *interp = _PyInterpreterState_GET(); if (optimizer == NULL) { optimizer = &_PyOptimizer_Default; } + else if (cold_exits_initialized == 0) { + cold_exits_initialized = 1; + for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { + if (init_cold_exit_executor(&COLD_EXITS[i], i)) { + return NULL; + } + } + } _PyOptimizerObject *old = interp->optimizer; + if (old == NULL) { + old = &_PyOptimizer_Default; + } Py_INCREF(optimizer); interp->optimizer = optimizer; - interp->optimizer_backedge_threshold = optimizer->backedge_threshold; - interp->optimizer_resume_threshold = optimizer->resume_threshold; - Py_DECREF(old); + interp->optimizer_backedge_threshold = shift_and_offset_threshold(optimizer->backedge_threshold); + interp->optimizer_resume_threshold = shift_and_offset_threshold(optimizer->resume_threshold); + interp->optimizer_side_threshold = optimizer->side_threshold; + if (optimizer == &_PyOptimizer_Default) { + assert(interp->optimizer_backedge_threshold > (1 << 16)); + assert(interp->optimizer_resume_threshold > (1 << 16)); + } + return old; } int -_PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer) +PyUnstable_SetOptimizer(_PyOptimizerObject *optimizer) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyOptimizerObject *old = _Py_SetOptimizer(interp, optimizer); + Py_XDECREF(old); + return old == NULL ? -1 : 0; +} + +/* Returns 1 if optimized, 0 if not optimized, and -1 for an error. + * If optimized, *executor_ptr contains a new reference to the executor + */ +int +_PyOptimizer_Optimize( + _PyInterpreterFrame *frame, _Py_CODEUNIT *start, + PyObject **stack_pointer, _PyExecutorObject **executor_ptr) { PyCodeObject *code = (PyCodeObject *)frame->f_executable; assert(PyCode_Check(code)); @@ -172,12 +218,11 @@ _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject return 0; } _PyOptimizerObject *opt = interp->optimizer; - _PyExecutorObject *executor = NULL; - int err = opt->optimize(opt, code, start, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); + int err = opt->optimize(opt, frame, start, executor_ptr, (int)(stack_pointer - _PyFrame_Stackbase(frame))); if (err <= 0) { - assert(executor == NULL); return err; } + assert(*executor_ptr != NULL); int index = get_index_for_executor(code, start); if (index < 0) { /* Out of memory. Don't raise and assume that the @@ -186,11 +231,11 @@ _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject * If an optimizer has already produced an executor, * it might get confused by the executor disappearing, * but there is not much we can do about that here. */ - Py_DECREF(executor); + Py_DECREF(*executor_ptr); return 0; } - insert_executor(code, start, index, executor); - Py_DECREF(executor); + insert_executor(code, start, index, *executor_ptr); + assert((*executor_ptr)->vm_data.valid); return 1; } @@ -217,8 +262,22 @@ is_valid(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyBool_FromLong(((_PyExecutorObject *)self)->vm_data.valid); } +static PyObject * +get_opcode(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyLong_FromUnsignedLong(((_PyExecutorObject *)self)->vm_data.opcode); +} + +static PyObject * +get_oparg(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyLong_FromUnsignedLong(((_PyExecutorObject *)self)->vm_data.oparg); +} + static PyMethodDef executor_methods[] = { { "is_valid", is_valid, METH_NOARGS, NULL }, + { "get_opcode", get_opcode, METH_NOARGS, NULL }, + { "get_oparg", get_oparg, METH_NOARGS, NULL }, { NULL, NULL }, }; @@ -226,20 +285,45 @@ static PyMethodDef executor_methods[] = { static void uop_dealloc(_PyExecutorObject *self) { + _PyObject_GC_UNTRACK(self); _Py_ExecutorClear(self); - PyObject_Free(self); +#ifdef _Py_JIT + _PyJIT_Free(self); +#endif + PyObject_GC_Del(self); } const char * _PyUOpName(int index) { + if (index < 0 || index > MAX_UOP_ID) { + return NULL; + } return _PyOpcode_uop_name[index]; } +#ifdef Py_DEBUG +void +_PyUOpPrint(const _PyUOpInstruction *uop) +{ + const char *name = _PyUOpName(uop->opcode); + if (name == NULL) { + printf("<uop %d>", uop->opcode); + } + else { + printf("%s", name); + } + printf(" (%d, target=%d, operand=%" PRIx64 ")", + uop->oparg, + uop->target, + (uint64_t)uop->operand); +} +#endif + static Py_ssize_t uop_len(_PyExecutorObject *self) { - return Py_SIZE(self); + return self->code_size; } static PyObject * @@ -263,14 +347,21 @@ uop_item(_PyExecutorObject *self, Py_ssize_t index) Py_DECREF(oname); return NULL; } + PyObject *target = PyLong_FromUnsignedLong(self->trace[index].target); + if (oparg == NULL) { + Py_DECREF(oparg); + Py_DECREF(oname); + return NULL; + } PyObject *operand = PyLong_FromUnsignedLongLong(self->trace[index].operand); if (operand == NULL) { + Py_DECREF(target); Py_DECREF(oparg); Py_DECREF(oname); return NULL; } - PyObject *args[3] = { oname, oparg, operand }; - return _PyTuple_FromArraySteal(args, 3); + PyObject *args[4] = { oname, oparg, target, operand }; + return _PyTuple_FromArraySteal(args, 4); } PySequenceMethods uop_as_sequence = { @@ -278,15 +369,34 @@ PySequenceMethods uop_as_sequence = { .sq_item = (ssizeargfunc)uop_item, }; +static int +executor_clear(PyObject *o) +{ + _Py_ExecutorClear((_PyExecutorObject *)o); + return 0; +} + +static int +executor_traverse(PyObject *o, visitproc visit, void *arg) +{ + _PyExecutorObject *executor = (_PyExecutorObject *)o; + for (uint32_t i = 0; i < executor->exit_count; i++) { + Py_VISIT(executor->exits[i].executor); + } + return 0; +} + PyTypeObject _PyUOpExecutor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "uop_executor", - .tp_basicsize = offsetof(_PyExecutorObject, trace), - .tp_itemsize = sizeof(_PyUOpInstruction), - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_basicsize = offsetof(_PyExecutorObject, exits), + .tp_itemsize = 1, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_HAVE_GC, .tp_dealloc = (destructor)uop_dealloc, .tp_as_sequence = &uop_as_sequence, .tp_methods = executor_methods, + .tp_traverse = executor_traverse, + .tp_clear = executor_clear, }; /* TO DO -- Generate these tables */ @@ -310,7 +420,6 @@ BRANCH_TO_GUARD[4][2] = { [POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NOT_NONE_POP, }; -#define TRACE_STACK_SIZE 5 #define CONFIDENCE_RANGE 1000 #define CONFIDENCE_CUTOFF 333 @@ -323,18 +432,29 @@ BRANCH_TO_GUARD[4][2] = { #endif +// Beware: Macro arg order differs from struct member order +#ifdef Py_DEBUG #define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ - DPRINTF(2, \ - " ADD_TO_TRACE(%s, %d, %" PRIu64 ")\n", \ - _PyUOpName(OPCODE), \ - (OPARG), \ - (uint64_t)(OPERAND)); \ assert(trace_length < max_length); \ trace[trace_length].opcode = (OPCODE); \ trace[trace_length].oparg = (OPARG); \ + trace[trace_length].target = (TARGET); \ trace[trace_length].operand = (OPERAND); \ + if (lltrace >= 2) { \ + printf("%4d ADD_TO_TRACE: ", trace_length); \ + _PyUOpPrint(&trace[trace_length]); \ + printf("\n"); \ + } \ + trace_length++; +#else +#define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ + assert(trace_length < max_length); \ + trace[trace_length].opcode = (OPCODE); \ + trace[trace_length].oparg = (OPARG); \ trace[trace_length].target = (TARGET); \ + trace[trace_length].operand = (OPERAND); \ trace_length++; +#endif #define INSTR_IP(INSTR, CODE) \ ((uint32_t)((INSTR) - ((_Py_CODEUNIT *)(CODE)->co_code_adaptive))) @@ -359,7 +479,8 @@ BRANCH_TO_GUARD[4][2] = { ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); \ goto done; \ } \ - trace_stack[trace_stack_depth].code = code; \ + assert(func->func_code == (PyObject *)code); \ + trace_stack[trace_stack_depth].func = func; \ trace_stack[trace_stack_depth].instr = instr; \ trace_stack_depth++; #define TRACE_STACK_POP() \ @@ -367,7 +488,8 @@ BRANCH_TO_GUARD[4][2] = { Py_FatalError("Trace stack underflow\n"); \ } \ trace_stack_depth--; \ - code = trace_stack[trace_stack_depth].code; \ + func = trace_stack[trace_stack_depth].func; \ + code = (PyCodeObject *)trace_stack[trace_stack_depth].func->func_code; \ instr = trace_stack[trace_stack_depth].instr; /* Returns 1 on success, @@ -376,20 +498,23 @@ BRANCH_TO_GUARD[4][2] = { */ static int translate_bytecode_to_trace( - PyCodeObject *code, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _PyUOpInstruction *trace, int buffer_size, _PyBloomFilter *dependencies) { bool progress_needed = true; + PyCodeObject *code = (PyCodeObject *)frame->f_executable; + PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + assert(PyFunction_Check(func)); PyCodeObject *initial_code = code; _Py_BloomFilter_Add(dependencies, initial_code); _Py_CODEUNIT *initial_instr = instr; int trace_length = 0; int max_length = buffer_size; struct { - PyCodeObject *code; + PyFunctionObject *func; _Py_CODEUNIT *instr; } trace_stack[TRACE_STACK_SIZE]; int trace_stack_depth = 0; @@ -414,14 +539,15 @@ translate_bytecode_to_trace( top: // Jump here after _PUSH_FRAME or likely branches for (;;) { target = INSTR_IP(instr, code); - RESERVE_RAW(3, "epilogue"); // Always need space for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE - ADD_TO_TRACE(_SET_IP, target, 0, target); - ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); + RESERVE_RAW(2, "epilogue"); // Always need space for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE + ADD_TO_TRACE(_CHECK_VALIDITY_AND_SET_IP, 0, (uintptr_t)instr, target); uint32_t opcode = instr->op.code; uint32_t oparg = instr->op.arg; uint32_t extended = 0; + DPRINTF(3, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); + if (opcode == ENTER_EXECUTOR) { assert(oparg < 256); _PyExecutorObject *executor = code->co_executors->executors[oparg]; @@ -469,21 +595,25 @@ translate_bytecode_to_trace( int counter = instr[1].cache; int bitcount = _Py_popcount32(counter); int jump_likely = bitcount > 8; + /* If bitcount is 8 (half the jumps were taken), adjust confidence by 50%. + If it's 16 or 0 (all or none were taken), adjust by 10% + (since the future is still somewhat uncertain). + For values in between, adjust proportionally. */ if (jump_likely) { - confidence = confidence * bitcount / 16; + confidence = confidence * (bitcount + 2) / 20; } else { - confidence = confidence * (16 - bitcount) / 16; + confidence = confidence * (18 - bitcount) / 20; } + uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; + DPRINTF(2, "%d: %s(%d): counter=%x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", + target, _PyOpcode_OpName[opcode], oparg, + counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode)); if (confidence < CONFIDENCE_CUTOFF) { - DPRINTF(2, "Confidence too low (%d)\n", confidence); + DPRINTF(2, "Confidence too low (%d < %d)\n", confidence, CONFIDENCE_CUTOFF); OPT_STAT_INC(low_confidence); goto done; } - uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; - DPRINTF(2, "%s(%d): counter=%x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", - _PyOpcode_OpName[opcode], oparg, - counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode)); _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; _Py_CODEUNIT *target_instr = next_instr + oparg; if (jump_likely) { @@ -589,9 +719,9 @@ translate_bytecode_to_trace( ADD_TO_TRACE(uop, oparg, operand, target); if (uop == _POP_FRAME) { TRACE_STACK_POP(); - /* Set the operand to the code object returned to, + /* Set the operand to the function object returned to, * to assist optimization passes */ - trace[trace_length-1].operand = (uintptr_t)code; + trace[trace_length-1].operand = (uintptr_t)func; DPRINTF(2, "Returning to %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), @@ -607,10 +737,10 @@ translate_bytecode_to_trace( // Add one to account for the actual opcode/oparg pair: + 1; uint32_t func_version = read_u32(&instr[func_version_offset].cache); - PyFunctionObject *func = _PyFunction_LookupByVersion(func_version); + PyFunctionObject *new_func = _PyFunction_LookupByVersion(func_version); DPRINTF(3, "Function object: %p\n", func); - if (func != NULL) { - PyCodeObject *new_code = (PyCodeObject *)PyFunction_GET_CODE(func); + if (new_func != NULL) { + PyCodeObject *new_code = (PyCodeObject *)PyFunction_GET_CODE(new_func); if (new_code == code) { // Recursive call, bail (we could be here forever). DPRINTF(2, "Bailing on recursive call to %s (%s:%d)\n", @@ -635,8 +765,9 @@ translate_bytecode_to_trace( _Py_BloomFilter_Add(dependencies, new_code); /* Set the operand to the callee's code object, * to assist optimization passes */ - trace[trace_length-1].operand = (uintptr_t)new_code; + trace[trace_length-1].operand = (uintptr_t)new_func; code = new_code; + func = new_func; instr = _PyCode_CODE(code); DPRINTF(2, "Continuing in %s (%s:%d) at byte offset %d\n", @@ -708,9 +839,10 @@ translate_bytecode_to_trace( * NOPs are excluded from the count. */ static int -compute_used(_PyUOpInstruction *buffer, uint32_t *used) +compute_used(_PyUOpInstruction *buffer, uint32_t *used, int *exit_count_ptr) { int count = 0; + int exit_count = 0; SET_BIT(used, 0); for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { if (!BIT_IS_SET(used, i)) { @@ -718,6 +850,9 @@ compute_used(_PyUOpInstruction *buffer, uint32_t *used) } count++; int opcode = buffer[i].opcode; + if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { + exit_count++; + } if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) { continue; } @@ -733,44 +868,76 @@ compute_used(_PyUOpInstruction *buffer, uint32_t *used) UNSET_BIT(used, i); } } + *exit_count_ptr = exit_count; return count; } +/* Executor side exits */ + +static _PyExecutorObject * +allocate_executor(int exit_count, int length) +{ + int size = exit_count*sizeof(_PyExitData) + length*sizeof(_PyUOpInstruction); + _PyExecutorObject *res = PyObject_GC_NewVar(_PyExecutorObject, &_PyUOpExecutor_Type, size); + if (res == NULL) { + return NULL; + } + res->trace = (_PyUOpInstruction *)(res->exits + exit_count); + res->code_size = length; + res->exit_count = exit_count; + return res; +} + /* Makes an executor from a buffer of uops. * Account for the buffer having gaps and NOPs by computing a "used" * bit vector and only copying the used uops. Here "used" means reachable * and not a NOP. */ static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) +make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies) { uint32_t used[(UOP_MAX_TRACE_LENGTH + 31)/32] = { 0 }; - int length = compute_used(buffer, used); - _PyExecutorObject *executor = PyObject_NewVar(_PyExecutorObject, &_PyUOpExecutor_Type, length); + int exit_count; + int length = compute_used(buffer, used, &exit_count); + _PyExecutorObject *executor = allocate_executor(exit_count, length+1); if (executor == NULL) { return NULL; } - int dest = length - 1; + /* Initialize exits */ + for (int i = 0; i < exit_count; i++) { + executor->exits[i].executor = &COLD_EXITS[i]; + executor->exits[i].temperature = 0; + } + int next_exit = exit_count-1; + _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; /* Scan backwards, so that we see the destinations of jumps before the jumps themselves. */ for (int i = UOP_MAX_TRACE_LENGTH-1; i >= 0; i--) { if (!BIT_IS_SET(used, i)) { continue; } - executor->trace[dest] = buffer[i]; + *dest = buffer[i]; int opcode = buffer[i].opcode; if (opcode == _POP_JUMP_IF_FALSE || opcode == _POP_JUMP_IF_TRUE) { /* The oparg of the target will already have been set to its new offset */ - int oparg = executor->trace[dest].oparg; - executor->trace[dest].oparg = buffer[oparg].oparg; + int oparg = dest->oparg; + dest->oparg = buffer[oparg].oparg; + } + if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { + executor->exits[next_exit].target = buffer[i].target; + dest->exit_index = next_exit; + next_exit--; } /* Set the oparg to be the destination offset, * so that we can set the oparg of earlier jumps correctly. */ - buffer[i].oparg = dest; + buffer[i].oparg = (uint16_t)(dest - executor->trace); dest--; } - assert(dest == -1); + assert(next_exit == -1); + assert(dest == executor->trace); + dest->opcode = _START_EXECUTOR; + dest->operand = (uintptr_t)executor; _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -781,22 +948,53 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) if (lltrace >= 2) { printf("Optimized executor (length %d):\n", length); for (int i = 0; i < length; i++) { - printf("%4d %s(%d, %d, %" PRIu64 ")\n", - i, - _PyUOpName(executor->trace[i].opcode), - executor->trace[i].oparg, - executor->trace[i].target, - executor->trace[i].operand); + printf("%4d OPTIMIZED: ", i); + _PyUOpPrint(&executor->trace[i]); + printf("\n"); } } #endif +#ifdef _Py_JIT + executor->jit_code = NULL; + executor->jit_size = 0; + if (_PyJIT_Compile(executor, executor->trace, length+1)) { + Py_DECREF(executor); + return NULL; + } +#endif + _PyObject_GC_TRACK(executor); return executor; } +static int +init_cold_exit_executor(_PyExecutorObject *executor, int oparg) +{ + _Py_SetImmortal(executor); + Py_SET_TYPE(executor, &_PyUOpExecutor_Type); + executor->trace = (_PyUOpInstruction *)executor->exits; + executor->code_size = 1; + executor->exit_count = 0; + _PyUOpInstruction *inst = (_PyUOpInstruction *)&executor->trace[0]; + inst->opcode = _COLD_EXIT; + inst->oparg = oparg; + executor->vm_data.valid = true; + for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { + assert(executor->vm_data.bloom.bits[i] == 0); + } +#ifdef _Py_JIT + executor->jit_code = NULL; + executor->jit_size = 0; + if (_PyJIT_Compile(executor, executor->trace, 1)) { + return -1; + } +#endif + return 0; +} + static int uop_optimize( _PyOptimizerObject *self, - PyCodeObject *code, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, int curr_stackentries) @@ -804,18 +1002,36 @@ uop_optimize( _PyBloomFilter dependencies; _Py_BloomFilter_Init(&dependencies); _PyUOpInstruction buffer[UOP_MAX_TRACE_LENGTH]; - int err = translate_bytecode_to_trace(code, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies); + int err = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies); if (err <= 0) { // Error or nothing translated return err; } OPT_STAT_INC(traces_created); - char *uop_optimize = Py_GETENV("PYTHONUOPSOPTIMIZE"); - if (uop_optimize == NULL || *uop_optimize > '0') { - err = _Py_uop_analyze_and_optimize(code, buffer, UOP_MAX_TRACE_LENGTH, curr_stackentries); - if (err < 0) { - return -1; + char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); + if (env_var == NULL || *env_var == '\0' || *env_var > '0') { + err = _Py_uop_analyze_and_optimize(frame, buffer, + UOP_MAX_TRACE_LENGTH, + curr_stackentries, &dependencies); + if (err <= 0) { + return err; + } + } + assert(err == 1); + /* Fix up */ + for (int pc = 0; pc < UOP_MAX_TRACE_LENGTH; pc++) { + int opcode = buffer[pc].opcode; + int oparg = buffer[pc].oparg; + if (_PyUop_Flags[opcode] & HAS_OPARG_AND_1_FLAG) { + buffer[pc].opcode = opcode + 1 + (oparg & 1); + } + else if (oparg < _PyUop_Replication[opcode]) { + buffer[pc].opcode = opcode + oparg + 1; } + else if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) { + break; + } + assert(_PyOpcode_uop_name[buffer[pc].opcode]); } _PyExecutorObject *executor = make_executor_from_uops(buffer, &dependencies); if (executor == NULL) { @@ -848,16 +1064,18 @@ PyUnstable_Optimizer_NewUOpOptimizer(void) return NULL; } opt->optimize = uop_optimize; - opt->resume_threshold = INT16_MAX; - // Need at least 3 iterations to settle specializations. - // A few lower bits of the counter are reserved for other flags. - opt->backedge_threshold = 16 << OPTIMIZER_BITS_IN_COUNTER; + opt->resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD; + // Need a few iterations to settle specializations, + // and to ammortize the cost of optimization. + opt->side_threshold = 16; + opt->backedge_threshold = 16; return (PyObject *)opt; } static void counter_dealloc(_PyExecutorObject *self) { - PyObject *opt = (PyObject *)self->trace[0].operand; + /* The optimizer is the operand of the second uop. */ + PyObject *opt = (PyObject *)self->trace[1].operand; Py_DECREF(opt); uop_dealloc(self); } @@ -865,22 +1083,25 @@ counter_dealloc(_PyExecutorObject *self) { PyTypeObject _PyCounterExecutor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "counting_executor", - .tp_basicsize = offsetof(_PyExecutorObject, trace), - .tp_itemsize = sizeof(_PyUOpInstruction), - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_basicsize = offsetof(_PyExecutorObject, exits), + .tp_itemsize = 1, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_HAVE_GC, .tp_dealloc = (destructor)counter_dealloc, .tp_methods = executor_methods, + .tp_traverse = executor_traverse, + .tp_clear = executor_clear, }; static int counter_optimize( _PyOptimizerObject* self, - PyCodeObject *code, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, int Py_UNUSED(curr_stackentries) ) { + PyCodeObject *code = (PyCodeObject *)frame->f_executable; int oparg = instr->op.arg; while (instr->op.code == EXTENDED_ARG) { instr++; @@ -896,9 +1117,7 @@ counter_optimize( { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)) } }; - _PyBloomFilter empty; - _Py_BloomFilter_Init(&empty); - _PyExecutorObject *executor = make_executor_from_uops(buffer, &empty); + _PyExecutorObject *executor = make_executor_from_uops(buffer, &EMPTY_FILTER); if (executor == NULL) { return -1; } @@ -937,7 +1156,8 @@ PyUnstable_Optimizer_NewCounter(void) return NULL; } opt->base.optimize = counter_optimize; - opt->base.resume_threshold = INT16_MAX; + opt->base.resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD; + opt->base.side_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD; opt->base.backedge_threshold = 0; opt->count = 0; return (PyObject *)opt; @@ -1053,7 +1273,7 @@ link_executor(_PyExecutorObject *executor) } head->vm_data.links.next = executor; } - executor->vm_data.linked = true; + executor->vm_data.valid = true; /* executor_list_head must be first in list */ assert(interp->executor_list_head->vm_data.links.previous == NULL); } @@ -1061,9 +1281,6 @@ link_executor(_PyExecutorObject *executor) static void unlink_executor(_PyExecutorObject *executor) { - if (!executor->vm_data.linked) { - return; - } _PyExecutorLinkListNode *links = &executor->vm_data.links; _PyExecutorObject *next = links->next; _PyExecutorObject *prev = links->previous; @@ -1079,12 +1296,12 @@ unlink_executor(_PyExecutorObject *executor) assert(interp->executor_list_head == executor); interp->executor_list_head = next; } - executor->vm_data.linked = false; + executor->vm_data.valid = false; } /* This must be called by optimizers before using the executor */ void -_Py_ExecutorInit(_PyExecutorObject *executor, _PyBloomFilter *dependency_set) +_Py_ExecutorInit(_PyExecutorObject *executor, const _PyBloomFilter *dependency_set) { executor->vm_data.valid = true; for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { @@ -1097,13 +1314,33 @@ _Py_ExecutorInit(_PyExecutorObject *executor, _PyBloomFilter *dependency_set) void _Py_ExecutorClear(_PyExecutorObject *executor) { + if (!executor->vm_data.valid) { + return; + } unlink_executor(executor); + PyCodeObject *code = executor->vm_data.code; + if (code == NULL) { + return; + } + for (uint32_t i = 0; i < executor->exit_count; i++) { + Py_DECREF(executor->exits[i].executor); + executor->exits[i].executor = &COLD_EXITS[i]; + executor->exits[i].temperature = INT16_MIN; + } + _Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index]; + assert(instruction->op.code == ENTER_EXECUTOR); + int index = instruction->op.arg; + assert(code->co_executors->executors[index] == executor); + instruction->op.code = executor->vm_data.opcode; + instruction->op.arg = executor->vm_data.oparg; + executor->vm_data.code = NULL; + Py_CLEAR(code->co_executors->executors[index]); } void _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj) { - assert(executor->vm_data.valid = true); + assert(executor->vm_data.valid); _Py_BloomFilter_Add(&executor->vm_data.bloom, obj); } @@ -1111,7 +1348,7 @@ _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj) * May cause other executors to be invalidated as well */ void -_Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj) +_Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation) { _PyBloomFilter obj_filter; _Py_BloomFilter_Init(&obj_filter); @@ -1122,8 +1359,10 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj) assert(exec->vm_data.valid); _PyExecutorObject *next = exec->vm_data.links.next; if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter)) { - exec->vm_data.valid = false; - unlink_executor(exec); + _Py_ExecutorClear(exec); + if (is_invalidation) { + OPT_STAT_INC(executors_invalidated); + } } exec = next; } @@ -1131,17 +1370,19 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj) /* Invalidate all executors */ void -_Py_Executors_InvalidateAll(PyInterpreterState *interp) +_Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) { - /* Walk the list of executors */ - for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { - assert(exec->vm_data.valid); - _PyExecutorObject *next = exec->vm_data.links.next; - exec->vm_data.links.next = NULL; - exec->vm_data.links.previous = NULL; - exec->vm_data.valid = false; - exec->vm_data.linked = false; - exec = next; + while (interp->executor_list_head) { + _PyExecutorObject *executor = interp->executor_list_head; + if (executor->vm_data.code) { + // Clear the entire code object so its co_executors array be freed: + _PyCode_Clear_Executors(executor->vm_data.code); + } + else { + _Py_ExecutorClear(executor); + } + if (is_invalidation) { + OPT_STAT_INC(executors_invalidated); + } } - interp->executor_list_head = NULL; } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index d1225997e10be2b..2a7ef4ec919eebe 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -1,98 +1,531 @@ +/* + * This file contains the support code for CPython's uops optimizer. + * It also performs some simple optimizations. + * It performs a traditional data-flow analysis[1] over the trace of uops. + * Using the information gained, it chooses to emit, or skip certain instructions + * if possible. + * + * [1] For information on data-flow analysis, please see + * https://clang.llvm.org/docs/DataFlowAnalysisIntro.html + * + * */ #include "Python.h" #include "opcode.h" +#include "pycore_dict.h" #include "pycore_interp.h" #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_uop_metadata.h" +#include "pycore_dict.h" #include "pycore_long.h" #include "cpython/optimizer.h" +#include "pycore_optimizer.h" +#include "pycore_object.h" +#include "pycore_dict.h" +#include "pycore_function.h" +#include "pycore_uop_metadata.h" +#include "pycore_uop_ids.h" +#include "pycore_range.h" + +#include <stdarg.h> #include <stdbool.h> #include <stdint.h> #include <stddef.h> -#include "pycore_optimizer.h" + +#ifdef Py_DEBUG + extern const char *_PyUOpName(int index); + static const char *const DEBUG_ENV = "PYTHON_OPT_DEBUG"; + static inline int get_lltrace(void) { + char *uop_debug = Py_GETENV(DEBUG_ENV); + int lltrace = 0; + if (uop_debug != NULL && *uop_debug >= '0') { + lltrace = *uop_debug - '0'; // TODO: Parse an int and all that + } + return lltrace; + } + #define DPRINTF(level, ...) \ + if (get_lltrace() >= (level)) { printf(__VA_ARGS__); } +#else + #define DPRINTF(level, ...) +#endif + + + +static inline bool +op_is_end(uint32_t opcode) +{ + return opcode == _EXIT_TRACE || opcode == _JUMP_TO_TOP; +} + +static int +get_mutations(PyObject* dict) { + assert(PyDict_CheckExact(dict)); + PyDictObject *d = (PyDictObject *)dict; + return (d->ma_version_tag >> DICT_MAX_WATCHERS) & ((1 << DICT_WATCHED_MUTATION_BITS)-1); +} static void -peephole_opt(PyCodeObject *co, _PyUOpInstruction *buffer, int buffer_size) +increment_mutations(PyObject* dict) { + assert(PyDict_CheckExact(dict)); + PyDictObject *d = (PyDictObject *)dict; + d->ma_version_tag += (1 << DICT_MAX_WATCHERS); +} + +/* The first two dict watcher IDs are reserved for CPython, + * so we don't need to check that they haven't been used */ +#define BUILTINS_WATCHER_ID 0 +#define GLOBALS_WATCHER_ID 1 + +static int +globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, + PyObject* key, PyObject* new_value) { + RARE_EVENT_STAT_INC(watched_globals_modification); + assert(get_mutations(dict) < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS); + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict, 1); + increment_mutations(dict); + PyDict_Unwatch(GLOBALS_WATCHER_ID, dict); + return 0; +} + +static PyObject * +convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj) +{ + assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS || inst->opcode == _LOAD_ATTR_MODULE); + assert(PyDict_CheckExact(obj)); + PyDictObject *dict = (PyDictObject *)obj; + assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + assert(inst->operand <= UINT16_MAX); + if ((int)inst->operand >= dict->ma_keys->dk_nentries) { + return NULL; + } + PyObject *res = entries[inst->operand].me_value; + if (res == NULL) { + return NULL; + } + if (_Py_IsImmortal(res)) { + inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_BORROW_WITH_NULL : _LOAD_CONST_INLINE_BORROW; + } + else { + inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_WITH_NULL : _LOAD_CONST_INLINE; + } + inst->operand = (uint64_t)res; + return res; +} + +static int +incorrect_keys(_PyUOpInstruction *inst, PyObject *obj) +{ + if (!PyDict_CheckExact(obj)) { + return 1; + } + PyDictObject *dict = (PyDictObject *)obj; + if (dict->ma_keys->dk_version != inst->operand) { + return 1; + } + return 0; +} + +/* Returns 1 if successfully optimized + * 0 if the trace is not suitable for optimization (yet) + * -1 if there was an error. */ +static int +remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, + int buffer_size, _PyBloomFilter *dependencies) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *builtins = frame->f_builtins; + if (builtins != interp->builtins) { + return 1; + } + PyObject *globals = frame->f_globals; + assert(PyFunction_Check(((PyFunctionObject *)frame->f_funcobj))); + assert(((PyFunctionObject *)frame->f_funcobj)->func_builtins == builtins); + assert(((PyFunctionObject *)frame->f_funcobj)->func_globals == globals); + /* In order to treat globals as constants, we need to + * know that the globals dict is the one we expected, and + * that it hasn't changed + * In order to treat builtins as constants, we need to + * know that the builtins dict is the one we expected, and + * that it hasn't changed and that the global dictionary's + * keys have not changed */ + + /* These values represent stacks of booleans (one bool per bit). + * Pushing a frame shifts left, popping a frame shifts right. */ + uint32_t builtins_checked = 0; + uint32_t builtins_watched = 0; + uint32_t globals_checked = 0; + uint32_t globals_watched = 0; + if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { + interp->dict_state.watchers[GLOBALS_WATCHER_ID] = globals_watcher_callback; + } for (int pc = 0; pc < buffer_size; pc++) { - int opcode = buffer[pc].opcode; + _PyUOpInstruction *inst = &buffer[pc]; + int opcode = inst->opcode; switch(opcode) { - case _LOAD_CONST: { - assert(co != NULL); - PyObject *val = PyTuple_GET_ITEM(co->co_consts, buffer[pc].oparg); - buffer[pc].opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; - buffer[pc].operand = (uintptr_t)val; + case _GUARD_BUILTINS_VERSION: + if (incorrect_keys(inst, builtins)) { + return 0; + } + if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { + continue; + } + if ((builtins_watched & 1) == 0) { + PyDict_Watch(BUILTINS_WATCHER_ID, builtins); + builtins_watched |= 1; + } + if (builtins_checked & 1) { + buffer[pc].opcode = NOP; + } + else { + buffer[pc].opcode = _CHECK_BUILTINS; + buffer[pc].operand = (uintptr_t)builtins; + builtins_checked |= 1; + } break; - } - case _CHECK_PEP_523: + case _GUARD_GLOBALS_VERSION: + if (incorrect_keys(inst, globals)) { + return 0; + } + uint64_t watched_mutations = get_mutations(globals); + if (watched_mutations >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + continue; + } + if ((globals_watched & 1) == 0) { + PyDict_Watch(GLOBALS_WATCHER_ID, globals); + _Py_BloomFilter_Add(dependencies, globals); + globals_watched |= 1; + } + if (globals_checked & 1) { + buffer[pc].opcode = NOP; + } + else { + buffer[pc].opcode = _CHECK_GLOBALS; + buffer[pc].operand = (uintptr_t)globals; + globals_checked |= 1; + } + break; + case _LOAD_GLOBAL_BUILTINS: + if (globals_checked & builtins_checked & globals_watched & builtins_watched & 1) { + convert_global_to_const(inst, builtins); + } + break; + case _LOAD_GLOBAL_MODULE: + if (globals_checked & globals_watched & 1) { + convert_global_to_const(inst, globals); + } + break; + case _PUSH_FRAME: { - /* Setting the eval frame function invalidates - * all executors, so no need to check dynamically */ - if (_PyInterpreterState_GET()->eval_frame == NULL) { - buffer[pc].opcode = _NOP; + globals_checked <<= 1; + globals_watched <<= 1; + builtins_checked <<= 1; + builtins_watched <<= 1; + PyFunctionObject *func = (PyFunctionObject *)buffer[pc].operand; + if (func == NULL) { + return 1; + } + assert(PyFunction_Check(func)); + globals = func->func_globals; + builtins = func->func_builtins; + if (builtins != interp->builtins) { + return 1; } break; } - case _PUSH_FRAME: case _POP_FRAME: - co = (PyCodeObject *)buffer[pc].operand; + { + globals_checked >>= 1; + globals_watched >>= 1; + builtins_checked >>= 1; + builtins_watched >>= 1; + PyFunctionObject *func = (PyFunctionObject *)buffer[pc].operand; + assert(PyFunction_Check(func)); + globals = func->func_globals; + builtins = func->func_builtins; break; - case _JUMP_TO_TOP: - case _EXIT_TRACE: - return; + } + default: + if (op_is_end(opcode)) { + return 1; + } + break; + } + } + return 0; +} + + + +#define STACK_LEVEL() ((int)(stack_pointer - ctx->frame->stack)) + +#define GETLOCAL(idx) ((ctx->frame->locals[idx])) + +#define REPLACE_OP(INST, OP, ARG, OPERAND) \ + INST->opcode = OP; \ + INST->oparg = ARG; \ + INST->operand = OPERAND; + +#define OUT_OF_SPACE_IF_NULL(EXPR) \ + do { \ + if ((EXPR) == NULL) { \ + goto out_of_space; \ + } \ + } while (0); + +#define _LOAD_ATTR_NOT_NULL \ + do { \ + OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); \ + OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); \ + } while (0); + + +/* Shortened forms for convenience, used in optimizer_bytecodes.c */ +#define sym_is_not_null _Py_uop_sym_is_not_null +#define sym_is_const _Py_uop_sym_is_const +#define sym_get_const _Py_uop_sym_get_const +#define sym_new_unknown _Py_uop_sym_new_unknown +#define sym_new_not_null _Py_uop_sym_new_not_null +#define sym_new_type _Py_uop_sym_new_type +#define sym_is_null _Py_uop_sym_is_null +#define sym_new_const _Py_uop_sym_new_const +#define sym_new_null _Py_uop_sym_new_null +#define sym_matches_type _Py_uop_sym_matches_type +#define sym_set_null _Py_uop_sym_set_null +#define sym_set_non_null _Py_uop_sym_set_non_null +#define sym_set_type _Py_uop_sym_set_type +#define sym_set_const _Py_uop_sym_set_const +#define sym_is_bottom _Py_uop_sym_is_bottom +#define frame_new _Py_uop_frame_new +#define frame_pop _Py_uop_frame_pop + + +/* 1 for success, 0 for not ready, cannot error at the moment. */ +static int +optimize_uops( + PyCodeObject *co, + _PyUOpInstruction *trace, + int trace_len, + int curr_stacklen, + _PyBloomFilter *dependencies +) +{ + + _Py_UOpsContext context; + _Py_UOpsContext *ctx = &context; + + if (_Py_uop_abstractcontext_init(ctx) < 0) { + goto out_of_space; + } + _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, ctx->n_consumed, 0, curr_stacklen); + if (frame == NULL) { + return -1; + } + ctx->curr_frame_depth++; + ctx->frame = frame; + + for (_PyUOpInstruction *this_instr = trace; + this_instr < trace + trace_len && !op_is_end(this_instr->opcode); + this_instr++) { + + int oparg = this_instr->oparg; + uint32_t opcode = this_instr->opcode; + + _Py_UopsSymbol **stack_pointer = ctx->frame->stack_pointer; + + DPRINTF(3, "Abstract interpreting %s:%d ", + _PyUOpName(opcode), + oparg); + switch (opcode) { +#include "optimizer_cases.c.h" + + default: + DPRINTF(1, "Unknown opcode in abstract interpreter\n"); + Py_UNREACHABLE(); } + assert(ctx->frame != NULL); + DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); + ctx->frame->stack_pointer = stack_pointer; + assert(STACK_LEVEL() >= 0); } + + _Py_uop_abstractcontext_fini(ctx); + return 1; + +out_of_space: + DPRINTF(1, "Out of space in abstract interpreter\n"); + _Py_uop_abstractcontext_fini(ctx); + return 0; + +error: + DPRINTF(1, "Encountered error in abstract interpreter\n"); + _Py_uop_abstractcontext_fini(ctx); + return 0; } + static void remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) { + /* Remove _SET_IP and _CHECK_VALIDITY where possible. + * _SET_IP is needed if the following instruction escapes or + * could error. _CHECK_VALIDITY is needed if the previous + * instruction could have escaped. */ int last_set_ip = -1; - bool maybe_invalid = false; + bool may_have_escaped = true; for (int pc = 0; pc < buffer_size; pc++) { int opcode = buffer[pc].opcode; - if (opcode == _SET_IP) { - buffer[pc].opcode = NOP; - last_set_ip = pc; - } - else if (opcode == _CHECK_VALIDITY) { - if (maybe_invalid) { - maybe_invalid = false; - } - else { + switch (opcode) { + case _SET_IP: buffer[pc].opcode = NOP; + last_set_ip = pc; + break; + case _CHECK_VALIDITY: + if (may_have_escaped) { + may_have_escaped = false; + } + else { + buffer[pc].opcode = NOP; + } + break; + case _CHECK_VALIDITY_AND_SET_IP: + if (may_have_escaped) { + may_have_escaped = false; + buffer[pc].opcode = _CHECK_VALIDITY; + } + else { + buffer[pc].opcode = NOP; + } + last_set_ip = pc; + break; + case _POP_TOP: + { + _PyUOpInstruction *last = &buffer[pc-1]; + while (last->opcode == _NOP) { + last--; + } + if (last->opcode == _LOAD_CONST_INLINE || + last->opcode == _LOAD_CONST_INLINE_BORROW || + last->opcode == _LOAD_FAST || + last->opcode == _COPY + ) { + last->opcode = _NOP; + buffer[pc].opcode = NOP; + } + break; + } + case _JUMP_TO_TOP: + case _EXIT_TRACE: + return; + default: + { + bool needs_ip = false; + if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { + needs_ip = true; + may_have_escaped = true; + } + if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { + needs_ip = true; + } + if (needs_ip && last_set_ip >= 0) { + if (buffer[last_set_ip].opcode == _CHECK_VALIDITY) { + buffer[last_set_ip].opcode = _CHECK_VALIDITY_AND_SET_IP; + } + else { + assert(buffer[last_set_ip].opcode == _NOP); + buffer[last_set_ip].opcode = _SET_IP; + } + last_set_ip = -1; + } } } - else if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) { - break; - } - else { - if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { - maybe_invalid = true; - if (last_set_ip >= 0) { - buffer[last_set_ip].opcode = _SET_IP; + } +} + +static void +peephole_opt(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_size) +{ + PyCodeObject *co = (PyCodeObject *)frame->f_executable; + for (int pc = 0; pc < buffer_size; pc++) { + int opcode = buffer[pc].opcode; + switch(opcode) { + case _LOAD_CONST: { + assert(co != NULL); + PyObject *val = PyTuple_GET_ITEM(co->co_consts, buffer[pc].oparg); + buffer[pc].opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; + buffer[pc].operand = (uintptr_t)val; + break; + } + case _CHECK_PEP_523: + { + /* Setting the eval frame function invalidates + * all executors, so no need to check dynamically */ + if (_PyInterpreterState_GET()->eval_frame == NULL) { + buffer[pc].opcode = _NOP; } + break; } - if ((_PyUop_Flags[opcode] & HAS_ERROR_FLAG) || opcode == _PUSH_FRAME) { - if (last_set_ip >= 0) { - buffer[last_set_ip].opcode = _SET_IP; + case _PUSH_FRAME: + case _POP_FRAME: + { + PyFunctionObject *func = (PyFunctionObject *)buffer[pc].operand; + if (func == NULL) { + co = NULL; + } + else { + assert(PyFunction_Check(func)); + co = (PyCodeObject *)func->func_code; } + break; } + case _JUMP_TO_TOP: + case _EXIT_TRACE: + return; } } } - +// 0 - failure, no error raised, just fall back to Tier 1 +// -1 - failure, and raise error +// 1 - optimizer success int _Py_uop_analyze_and_optimize( - PyCodeObject *co, + _PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_size, - int curr_stacklen + int curr_stacklen, + _PyBloomFilter *dependencies ) { - peephole_opt(co, buffer, buffer_size); + OPT_STAT_INC(optimizer_attempts); + + int err = remove_globals(frame, buffer, buffer_size, dependencies); + if (err == 0) { + goto not_ready; + } + if (err < 0) { + goto error; + } + + peephole_opt(frame, buffer, buffer_size); + + err = optimize_uops( + (PyCodeObject *)frame->f_executable, buffer, + buffer_size, curr_stacklen, dependencies); + + if (err == 0) { + goto not_ready; + } + assert(err == 1); + remove_unneeded_uops(buffer, buffer_size); + + OPT_STAT_INC(optimizer_successes); + return 1; +not_ready: return 0; +error: + return -1; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c new file mode 100644 index 000000000000000..928c22da16b9998 --- /dev/null +++ b/Python/optimizer_bytecodes.c @@ -0,0 +1,453 @@ +#include "Python.h" +#include "pycore_optimizer.h" +#include "pycore_uops.h" +#include "pycore_uop_ids.h" +#include "internal/pycore_moduleobject.h" + +#define op(name, ...) /* NAME is ignored */ + +typedef struct _Py_UopsSymbol _Py_UopsSymbol; +typedef struct _Py_UOpsContext _Py_UOpsContext; +typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; + +/* Shortened forms for convenience */ +#define sym_is_not_null _Py_uop_sym_is_not_null +#define sym_is_const _Py_uop_sym_is_const +#define sym_get_const _Py_uop_sym_get_const +#define sym_new_unknown _Py_uop_sym_new_unknown +#define sym_new_not_null _Py_uop_sym_new_not_null +#define sym_new_type _Py_uop_sym_new_type +#define sym_is_null _Py_uop_sym_is_null +#define sym_new_const _Py_uop_sym_new_const +#define sym_new_null _Py_uop_sym_new_null +#define sym_matches_type _Py_uop_sym_matches_type +#define sym_set_null _Py_uop_sym_set_null +#define sym_set_non_null _Py_uop_sym_set_non_null +#define sym_set_type _Py_uop_sym_set_type +#define sym_set_const _Py_uop_sym_set_const +#define sym_is_bottom _Py_uop_sym_is_bottom +#define frame_new _Py_uop_frame_new +#define frame_pop _Py_uop_frame_pop + +static int +dummy_func(void) { + + PyCodeObject *code; + int oparg; + _Py_UopsSymbol *flag; + _Py_UopsSymbol *left; + _Py_UopsSymbol *right; + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; + _Py_UopsSymbol *iter; + _Py_UopsSymbol *top; + _Py_UopsSymbol *bottom; + _Py_UOpsAbstractFrame *frame; + _Py_UOpsContext *ctx; + _PyUOpInstruction *this_instr; + _PyBloomFilter *dependencies; + int modified; + +// BEGIN BYTECODES // + + op(_LOAD_FAST_CHECK, (-- value)) { + value = GETLOCAL(oparg); + // We guarantee this will error - just bail and don't optimize it. + if (sym_is_null(value)) { + goto out_of_space; + } + } + + op(_LOAD_FAST, (-- value)) { + value = GETLOCAL(oparg); + } + + op(_LOAD_FAST_AND_CLEAR, (-- value)) { + value = GETLOCAL(oparg); + _Py_UopsSymbol *temp; + OUT_OF_SPACE_IF_NULL(temp = sym_new_null(ctx)); + GETLOCAL(oparg) = temp; + } + + op(_STORE_FAST, (value --)) { + GETLOCAL(oparg) = value; + } + + op(_PUSH_NULL, (-- res)) { + res = sym_new_null(ctx); + if (res == NULL) { + goto out_of_space; + }; + } + + op(_GUARD_BOTH_INT, (left, right -- left, right)) { + if (sym_matches_type(left, &PyLong_Type) && + sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(left, &PyLong_Type); + sym_set_type(right, &PyLong_Type); + } + + op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { + if (sym_matches_type(left, &PyFloat_Type) && + sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + sym_set_type(left, &PyFloat_Type); + sym_set_type(right, &PyFloat_Type); + } + + op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { + if (sym_matches_type(left, &PyUnicode_Type) && + sym_matches_type(right, &PyUnicode_Type)) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + sym_set_type(left, &PyUnicode_Type); + sym_set_type(right, &PyUnicode_Type); + } + + op(_BINARY_OP_ADD_INT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + } + } + + op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + } + } + + op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + } + } + + op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) + + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + } + } + + op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) - + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + } + } + + op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) * + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + } + } + + op(_LOAD_CONST, (-- value)) { + // There should be no LOAD_CONST. It should be all + // replaced by peephole_opt. + Py_UNREACHABLE(); + } + + op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + } + + op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + } + + op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + } + + op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + } + + + op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { + assert(oparg > 0); + top = bottom; + } + + op(_SWAP, (bottom, unused[oparg-2], top -- + top, unused[oparg-2], bottom)) { + } + + op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { + _LOAD_ATTR_NOT_NULL + (void)index; + (void)owner; + } + + op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) { + (void)dict_version; + if (sym_is_const(owner)) { + PyObject *cnst = sym_get_const(owner); + if (PyModule_CheckExact(cnst)) { + PyModuleObject *mod = (PyModuleObject *)cnst; + PyObject *dict = mod->md_dict; + uint64_t watched_mutations = get_mutations(dict); + if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + PyDict_Watch(GLOBALS_WATCHER_ID, dict); + _Py_BloomFilter_Add(dependencies, dict); + this_instr->opcode = _NOP; + } + } + } + } + + op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { + (void)index; + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + attr = NULL; + if (this_instr[-1].opcode == _NOP) { + // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. + assert(sym_is_const(owner)); + PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); + assert(PyModule_CheckExact(mod)); + PyObject *dict = mod->md_dict; + PyObject *res = convert_global_to_const(this_instr, dict); + if (res != NULL) { + this_instr[-1].opcode = _POP_TOP; + OUT_OF_SPACE_IF_NULL(attr = sym_new_const(ctx, res)); + } + } + if (attr == NULL) { + /* No conversion made. We don't know what `attr` is. */ + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + } + } + + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { + _LOAD_ATTR_NOT_NULL + (void)hint; + (void)owner; + } + + op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { + _LOAD_ATTR_NOT_NULL + (void)index; + (void)owner; + } + + op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { + _LOAD_ATTR_NOT_NULL + (void)descr; + (void)owner; + } + + op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { + (void)descr; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + self = owner; + } + + op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) { + (void)descr; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + self = owner; + } + + op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self if (1))) { + (void)descr; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + self = owner; + } + + op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { + (void)callable; + OUT_OF_SPACE_IF_NULL(func = sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(self = sym_new_not_null(ctx)); + } + + + op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + sym_set_type(callable, &PyFunction_Type); + (void)self_or_null; + (void)func_version; + } + + op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { + sym_set_null(null); + sym_set_type(callable, &PyMethod_Type); + } + + op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + int argcount = oparg; + + (void)callable; + + PyFunctionObject *func = (PyFunctionObject *)(this_instr + 2)->operand; + if (func == NULL) { + goto error; + } + PyCodeObject *co = (PyCodeObject *)func->func_code; + + assert(self_or_null != NULL); + assert(args != NULL); + if (sym_is_not_null(self_or_null)) { + // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM + args--; + argcount++; + } + + _Py_UopsSymbol **localsplus_start = ctx->n_consumed; + int n_locals_already_filled = 0; + // Can determine statically, so we interleave the new locals + // and make the current stack the new locals. + // This also sets up for true call inlining. + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + localsplus_start = args; + n_locals_already_filled = argcount; + } + OUT_OF_SPACE_IF_NULL(new_frame = + frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); + } + + op(_POP_FRAME, (retval -- res)) { + SYNC_SP(); + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = retval; + } + + op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- unused if (0))) { + SYNC_SP(); + ctx->frame->stack_pointer = stack_pointer; + ctx->frame = new_frame; + ctx->curr_frame_depth++; + stack_pointer = new_frame->stack_pointer; + } + + op(_UNPACK_SEQUENCE, (seq -- values[oparg])) { + /* This has to be done manually */ + (void)seq; + for (int i = 0; i < oparg; i++) { + OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + } + } + + op(_UNPACK_EX, (seq -- values[oparg & 0xFF], unused, unused[oparg >> 8])) { + /* This has to be done manually */ + (void)seq; + int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; + for (int i = 0; i < totalargs; i++) { + OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + } + } + + op(_ITER_NEXT_RANGE, (iter -- iter, next)) { + OUT_OF_SPACE_IF_NULL(next = sym_new_type(ctx, &PyLong_Type)); + (void)iter; + } + + + + +// END BYTECODES // + +} diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h new file mode 100644 index 000000000000000..b38c03bed4b4d07 --- /dev/null +++ b/Python/optimizer_cases.c.h @@ -0,0 +1,1826 @@ +// This file is generated by Tools/cases_generator/optimizer_generator.py +// from: +// Python/optimizer_bytecodes.c +// Do not edit! + + case _NOP: { + break; + } + + case _RESUME_CHECK: { + break; + } + + /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 */ + + case _LOAD_FAST_CHECK: { + _Py_UopsSymbol *value; + value = GETLOCAL(oparg); + // We guarantee this will error - just bail and don't optimize it. + if (sym_is_null(value)) { + goto out_of_space; + } + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST: { + _Py_UopsSymbol *value; + value = GETLOCAL(oparg); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_AND_CLEAR: { + _Py_UopsSymbol *value; + value = GETLOCAL(oparg); + _Py_UopsSymbol *temp; + OUT_OF_SPACE_IF_NULL(temp = sym_new_null(ctx)); + GETLOCAL(oparg) = temp; + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_CONST: { + _Py_UopsSymbol *value; + // There should be no LOAD_CONST. It should be all + // replaced by peephole_opt. + Py_UNREACHABLE(); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _STORE_FAST: { + _Py_UopsSymbol *value; + value = stack_pointer[-1]; + GETLOCAL(oparg) = value; + stack_pointer += -1; + break; + } + + case _POP_TOP: { + stack_pointer += -1; + break; + } + + case _PUSH_NULL: { + _Py_UopsSymbol *res; + res = sym_new_null(ctx); + if (res == NULL) { + goto out_of_space; + }; + stack_pointer[0] = res; + stack_pointer += 1; + break; + } + + case _END_SEND: { + _Py_UopsSymbol *value; + value = sym_new_unknown(ctx); + if (value == NULL) goto out_of_space; + stack_pointer[-2] = value; + stack_pointer += -1; + break; + } + + case _UNARY_NEGATIVE: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-1] = res; + break; + } + + case _UNARY_NOT: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_BOOL: { + break; + } + + case _TO_BOOL_INT: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_LIST: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_NONE: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_STR: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_ALWAYS_TRUE: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-1] = res; + break; + } + + case _UNARY_INVERT: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-1] = res; + break; + } + + case _GUARD_BOTH_INT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_matches_type(left, &PyLong_Type) && + sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(left, &PyLong_Type); + sym_set_type(right, &PyLong_Type); + break; + } + + case _BINARY_OP_MULTIPLY_INT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + } + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _BINARY_OP_ADD_INT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + } + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _BINARY_OP_SUBTRACT_INT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + } + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _GUARD_BOTH_FLOAT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_matches_type(left, &PyFloat_Type) && + sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + sym_set_type(left, &PyFloat_Type); + sym_set_type(right, &PyFloat_Type); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) * + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + } + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _BINARY_OP_ADD_FLOAT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) + + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + } + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) - + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + } + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _GUARD_BOTH_UNICODE: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_matches_type(left, &PyUnicode_Type) && + sym_matches_type(right, &PyUnicode_Type)) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + sym_set_type(left, &PyUnicode_Type); + sym_set_type(right, &PyUnicode_Type); + break; + } + + case _BINARY_OP_ADD_UNICODE: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _BINARY_SUBSCR: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _BINARY_SLICE: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-3] = res; + stack_pointer += -2; + break; + } + + case _STORE_SLICE: { + stack_pointer += -4; + break; + } + + case _BINARY_SUBSCR_LIST_INT: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _BINARY_SUBSCR_STR_INT: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _BINARY_SUBSCR_TUPLE_INT: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _BINARY_SUBSCR_DICT: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + /* _BINARY_SUBSCR_GETITEM is not a viable micro-op for tier 2 */ + + case _LIST_APPEND: { + stack_pointer += -1; + break; + } + + case _SET_ADD: { + stack_pointer += -1; + break; + } + + case _STORE_SUBSCR: { + stack_pointer += -3; + break; + } + + case _STORE_SUBSCR_LIST_INT: { + stack_pointer += -3; + break; + } + + case _STORE_SUBSCR_DICT: { + stack_pointer += -3; + break; + } + + case _DELETE_SUBSCR: { + stack_pointer += -2; + break; + } + + case _CALL_INTRINSIC_1: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-1] = res; + break; + } + + case _CALL_INTRINSIC_2: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _POP_FRAME: { + _Py_UopsSymbol *retval; + _Py_UopsSymbol *res; + retval = stack_pointer[-1]; + stack_pointer += -1; + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = retval; + stack_pointer[0] = res; + stack_pointer += 1; + break; + } + + /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 */ + + case _GET_AITER: { + _Py_UopsSymbol *iter; + iter = sym_new_unknown(ctx); + if (iter == NULL) goto out_of_space; + stack_pointer[-1] = iter; + break; + } + + case _GET_ANEXT: { + _Py_UopsSymbol *awaitable; + awaitable = sym_new_unknown(ctx); + if (awaitable == NULL) goto out_of_space; + stack_pointer[0] = awaitable; + stack_pointer += 1; + break; + } + + case _GET_AWAITABLE: { + _Py_UopsSymbol *iter; + iter = sym_new_unknown(ctx); + if (iter == NULL) goto out_of_space; + stack_pointer[-1] = iter; + break; + } + + /* _SEND is not a viable micro-op for tier 2 */ + + /* _SEND_GEN is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 */ + + case _POP_EXCEPT: { + stack_pointer += -1; + break; + } + + case _LOAD_ASSERTION_ERROR: { + _Py_UopsSymbol *value; + value = sym_new_unknown(ctx); + if (value == NULL) goto out_of_space; + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_BUILD_CLASS: { + _Py_UopsSymbol *bc; + bc = sym_new_unknown(ctx); + if (bc == NULL) goto out_of_space; + stack_pointer[0] = bc; + stack_pointer += 1; + break; + } + + case _STORE_NAME: { + stack_pointer += -1; + break; + } + + case _DELETE_NAME: { + break; + } + + case _UNPACK_SEQUENCE: { + _Py_UopsSymbol *seq; + _Py_UopsSymbol **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + /* This has to be done manually */ + (void)seq; + for (int i = 0; i < oparg; i++) { + OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + } + stack_pointer += -1 + oparg; + break; + } + + case _UNPACK_SEQUENCE_TWO_TUPLE: { + _Py_UopsSymbol **values; + values = &stack_pointer[-1]; + for (int _i = oparg; --_i >= 0;) { + values[_i] = sym_new_unknown(ctx); + if (values[_i] == NULL) goto out_of_space; + } + stack_pointer += -1 + oparg; + break; + } + + case _UNPACK_SEQUENCE_TUPLE: { + _Py_UopsSymbol **values; + values = &stack_pointer[-1]; + for (int _i = oparg; --_i >= 0;) { + values[_i] = sym_new_unknown(ctx); + if (values[_i] == NULL) goto out_of_space; + } + stack_pointer += -1 + oparg; + break; + } + + case _UNPACK_SEQUENCE_LIST: { + _Py_UopsSymbol **values; + values = &stack_pointer[-1]; + for (int _i = oparg; --_i >= 0;) { + values[_i] = sym_new_unknown(ctx); + if (values[_i] == NULL) goto out_of_space; + } + stack_pointer += -1 + oparg; + break; + } + + case _UNPACK_EX: { + _Py_UopsSymbol *seq; + _Py_UopsSymbol **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + /* This has to be done manually */ + (void)seq; + int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; + for (int i = 0; i < totalargs; i++) { + OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + } + stack_pointer += (oparg >> 8) + (oparg & 0xFF); + break; + } + + case _STORE_ATTR: { + stack_pointer += -2; + break; + } + + case _DELETE_ATTR: { + stack_pointer += -1; + break; + } + + case _STORE_GLOBAL: { + stack_pointer += -1; + break; + } + + case _DELETE_GLOBAL: { + break; + } + + case _LOAD_LOCALS: { + _Py_UopsSymbol *locals; + locals = sym_new_unknown(ctx); + if (locals == NULL) goto out_of_space; + stack_pointer[0] = locals; + stack_pointer += 1; + break; + } + + case _LOAD_FROM_DICT_OR_GLOBALS: { + _Py_UopsSymbol *v; + v = sym_new_unknown(ctx); + if (v == NULL) goto out_of_space; + stack_pointer[-1] = v; + break; + } + + case _LOAD_NAME: { + _Py_UopsSymbol *v; + v = sym_new_unknown(ctx); + if (v == NULL) goto out_of_space; + stack_pointer[0] = v; + stack_pointer += 1; + break; + } + + case _LOAD_GLOBAL: { + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + null = sym_new_null(ctx); + if (null == NULL) goto out_of_space; + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + break; + } + + case _GUARD_GLOBALS_VERSION: { + break; + } + + case _GUARD_BUILTINS_VERSION: { + break; + } + + case _LOAD_GLOBAL_MODULE: { + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + null = sym_new_null(ctx); + if (null == NULL) goto out_of_space; + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + break; + } + + case _LOAD_GLOBAL_BUILTINS: { + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + null = sym_new_null(ctx); + if (null == NULL) goto out_of_space; + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + break; + } + + case _DELETE_FAST: { + break; + } + + case _MAKE_CELL: { + break; + } + + case _DELETE_DEREF: { + break; + } + + case _LOAD_FROM_DICT_OR_DEREF: { + _Py_UopsSymbol *value; + value = sym_new_unknown(ctx); + if (value == NULL) goto out_of_space; + stack_pointer[-1] = value; + break; + } + + case _LOAD_DEREF: { + _Py_UopsSymbol *value; + value = sym_new_unknown(ctx); + if (value == NULL) goto out_of_space; + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _STORE_DEREF: { + stack_pointer += -1; + break; + } + + case _COPY_FREE_VARS: { + break; + } + + case _BUILD_STRING: { + _Py_UopsSymbol *str; + str = sym_new_unknown(ctx); + if (str == NULL) goto out_of_space; + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + break; + } + + case _BUILD_TUPLE: { + _Py_UopsSymbol *tup; + tup = sym_new_unknown(ctx); + if (tup == NULL) goto out_of_space; + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + break; + } + + case _BUILD_LIST: { + _Py_UopsSymbol *list; + list = sym_new_unknown(ctx); + if (list == NULL) goto out_of_space; + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + break; + } + + case _LIST_EXTEND: { + stack_pointer += -1; + break; + } + + case _SET_UPDATE: { + stack_pointer += -1; + break; + } + + case _BUILD_SET: { + _Py_UopsSymbol *set; + set = sym_new_unknown(ctx); + if (set == NULL) goto out_of_space; + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + break; + } + + case _BUILD_MAP: { + _Py_UopsSymbol *map; + map = sym_new_unknown(ctx); + if (map == NULL) goto out_of_space; + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + break; + } + + case _SETUP_ANNOTATIONS: { + break; + } + + case _BUILD_CONST_KEY_MAP: { + _Py_UopsSymbol *map; + map = sym_new_unknown(ctx); + if (map == NULL) goto out_of_space; + stack_pointer[-1 - oparg] = map; + stack_pointer += -oparg; + break; + } + + case _DICT_UPDATE: { + stack_pointer += -1; + break; + } + + case _DICT_MERGE: { + stack_pointer += -1; + break; + } + + case _MAP_ADD: { + stack_pointer += -2; + break; + } + + /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ + + case _LOAD_SUPER_ATTR_ATTR: { + _Py_UopsSymbol *attr; + attr = sym_new_unknown(ctx); + if (attr == NULL) goto out_of_space; + stack_pointer[-3] = attr; + stack_pointer += -2; + break; + } + + case _LOAD_SUPER_ATTR_METHOD: { + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self_or_null; + attr = sym_new_unknown(ctx); + if (attr == NULL) goto out_of_space; + self_or_null = sym_new_unknown(ctx); + if (self_or_null == NULL) goto out_of_space; + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; + break; + } + + case _LOAD_ATTR: { + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self_or_null = NULL; + attr = sym_new_unknown(ctx); + if (attr == NULL) goto out_of_space; + self_or_null = sym_new_unknown(ctx); + if (self_or_null == NULL) goto out_of_space; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); + break; + } + + case _GUARD_TYPE_VERSION: { + break; + } + + case _CHECK_MANAGED_OBJECT_HAS_VALUES: { + break; + } + + case _LOAD_ATTR_INSTANCE_VALUE: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)this_instr->operand; + _LOAD_ATTR_NOT_NULL + (void)index; + (void)owner; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + break; + } + + case _CHECK_ATTR_MODULE: { + _Py_UopsSymbol *owner; + owner = stack_pointer[-1]; + uint32_t dict_version = (uint32_t)this_instr->operand; + (void)dict_version; + if (sym_is_const(owner)) { + PyObject *cnst = sym_get_const(owner); + if (PyModule_CheckExact(cnst)) { + PyModuleObject *mod = (PyModuleObject *)cnst; + PyObject *dict = mod->md_dict; + uint64_t watched_mutations = get_mutations(dict); + if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + PyDict_Watch(GLOBALS_WATCHER_ID, dict); + _Py_BloomFilter_Add(dependencies, dict); + this_instr->opcode = _NOP; + } + } + } + break; + } + + case _LOAD_ATTR_MODULE: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)this_instr->operand; + (void)index; + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + attr = NULL; + if (this_instr[-1].opcode == _NOP) { + // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. + assert(sym_is_const(owner)); + PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); + assert(PyModule_CheckExact(mod)); + PyObject *dict = mod->md_dict; + PyObject *res = convert_global_to_const(this_instr, dict); + if (res != NULL) { + this_instr[-1].opcode = _POP_TOP; + OUT_OF_SPACE_IF_NULL(attr = sym_new_const(ctx, res)); + } + } + if (attr == NULL) { + /* No conversion made. We don't know what `attr` is. */ + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + break; + } + + case _CHECK_ATTR_WITH_HINT: { + break; + } + + case _LOAD_ATTR_WITH_HINT: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; + owner = stack_pointer[-1]; + uint16_t hint = (uint16_t)this_instr->operand; + _LOAD_ATTR_NOT_NULL + (void)hint; + (void)owner; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + break; + } + + case _LOAD_ATTR_SLOT: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)this_instr->operand; + _LOAD_ATTR_NOT_NULL + (void)index; + (void)owner; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + break; + } + + case _CHECK_ATTR_CLASS: { + break; + } + + case _LOAD_ATTR_CLASS: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + _LOAD_ATTR_NOT_NULL + (void)descr; + (void)owner; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + break; + } + + /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 */ + + /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ + + case _GUARD_DORV_VALUES: { + break; + } + + case _STORE_ATTR_INSTANCE_VALUE: { + stack_pointer += -2; + break; + } + + /* _STORE_ATTR_WITH_HINT is not a viable micro-op for tier 2 */ + + case _STORE_ATTR_SLOT: { + stack_pointer += -2; + break; + } + + case _COMPARE_OP: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _COMPARE_OP_FLOAT: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _COMPARE_OP_INT: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _COMPARE_OP_STR: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _IS_OP: { + _Py_UopsSymbol *b; + b = sym_new_unknown(ctx); + if (b == NULL) goto out_of_space; + stack_pointer[-2] = b; + stack_pointer += -1; + break; + } + + case _CONTAINS_OP: { + _Py_UopsSymbol *b; + b = sym_new_unknown(ctx); + if (b == NULL) goto out_of_space; + stack_pointer[-2] = b; + stack_pointer += -1; + break; + } + + case _CHECK_EG_MATCH: { + _Py_UopsSymbol *rest; + _Py_UopsSymbol *match; + rest = sym_new_unknown(ctx); + if (rest == NULL) goto out_of_space; + match = sym_new_unknown(ctx); + if (match == NULL) goto out_of_space; + stack_pointer[-2] = rest; + stack_pointer[-1] = match; + break; + } + + case _CHECK_EXC_MATCH: { + _Py_UopsSymbol *b; + b = sym_new_unknown(ctx); + if (b == NULL) goto out_of_space; + stack_pointer[-1] = b; + break; + } + + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + + /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + + case _IS_NONE: { + _Py_UopsSymbol *b; + b = sym_new_unknown(ctx); + if (b == NULL) goto out_of_space; + stack_pointer[-1] = b; + break; + } + + case _GET_LEN: { + _Py_UopsSymbol *len_o; + len_o = sym_new_unknown(ctx); + if (len_o == NULL) goto out_of_space; + stack_pointer[0] = len_o; + stack_pointer += 1; + break; + } + + case _MATCH_CLASS: { + _Py_UopsSymbol *attrs; + attrs = sym_new_unknown(ctx); + if (attrs == NULL) goto out_of_space; + stack_pointer[-3] = attrs; + stack_pointer += -2; + break; + } + + case _MATCH_MAPPING: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[0] = res; + stack_pointer += 1; + break; + } + + case _MATCH_SEQUENCE: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[0] = res; + stack_pointer += 1; + break; + } + + case _MATCH_KEYS: { + _Py_UopsSymbol *values_or_none; + values_or_none = sym_new_unknown(ctx); + if (values_or_none == NULL) goto out_of_space; + stack_pointer[0] = values_or_none; + stack_pointer += 1; + break; + } + + case _GET_ITER: { + _Py_UopsSymbol *iter; + iter = sym_new_unknown(ctx); + if (iter == NULL) goto out_of_space; + stack_pointer[-1] = iter; + break; + } + + case _GET_YIELD_FROM_ITER: { + _Py_UopsSymbol *iter; + iter = sym_new_unknown(ctx); + if (iter == NULL) goto out_of_space; + stack_pointer[-1] = iter; + break; + } + + /* _FOR_ITER is not a viable micro-op for tier 2 */ + + case _FOR_ITER_TIER_TWO: { + _Py_UopsSymbol *next; + next = sym_new_unknown(ctx); + if (next == NULL) goto out_of_space; + stack_pointer[0] = next; + stack_pointer += 1; + break; + } + + /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 */ + + case _ITER_CHECK_LIST: { + break; + } + + /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 */ + + case _GUARD_NOT_EXHAUSTED_LIST: { + break; + } + + case _ITER_NEXT_LIST: { + _Py_UopsSymbol *next; + next = sym_new_unknown(ctx); + if (next == NULL) goto out_of_space; + stack_pointer[0] = next; + stack_pointer += 1; + break; + } + + case _ITER_CHECK_TUPLE: { + break; + } + + /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 */ + + case _GUARD_NOT_EXHAUSTED_TUPLE: { + break; + } + + case _ITER_NEXT_TUPLE: { + _Py_UopsSymbol *next; + next = sym_new_unknown(ctx); + if (next == NULL) goto out_of_space; + stack_pointer[0] = next; + stack_pointer += 1; + break; + } + + case _ITER_CHECK_RANGE: { + break; + } + + /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 */ + + case _GUARD_NOT_EXHAUSTED_RANGE: { + break; + } + + case _ITER_NEXT_RANGE: { + _Py_UopsSymbol *iter; + _Py_UopsSymbol *next; + iter = stack_pointer[-1]; + OUT_OF_SPACE_IF_NULL(next = sym_new_type(ctx, &PyLong_Type)); + (void)iter; + stack_pointer[0] = next; + stack_pointer += 1; + break; + } + + /* _FOR_ITER_GEN is not a viable micro-op for tier 2 */ + + case _BEFORE_ASYNC_WITH: { + _Py_UopsSymbol *exit; + _Py_UopsSymbol *res; + exit = sym_new_unknown(ctx); + if (exit == NULL) goto out_of_space; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; + break; + } + + case _BEFORE_WITH: { + _Py_UopsSymbol *exit; + _Py_UopsSymbol *res; + exit = sym_new_unknown(ctx); + if (exit == NULL) goto out_of_space; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; + break; + } + + case _WITH_EXCEPT_START: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[0] = res; + stack_pointer += 1; + break; + } + + case _PUSH_EXC_INFO: { + _Py_UopsSymbol *prev_exc; + _Py_UopsSymbol *new_exc; + prev_exc = sym_new_unknown(ctx); + if (prev_exc == NULL) goto out_of_space; + new_exc = sym_new_unknown(ctx); + if (new_exc == NULL) goto out_of_space; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + break; + } + + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { + break; + } + + case _GUARD_KEYS_VERSION: { + break; + } + + case _LOAD_ATTR_METHOD_WITH_VALUES: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + (void)descr; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + self = owner; + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + break; + } + + case _LOAD_ATTR_METHOD_NO_DICT: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + (void)descr; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + self = owner; + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + break; + } + + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { + _Py_UopsSymbol *attr; + attr = sym_new_unknown(ctx); + if (attr == NULL) goto out_of_space; + stack_pointer[-1] = attr; + break; + } + + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { + _Py_UopsSymbol *attr; + attr = sym_new_unknown(ctx); + if (attr == NULL) goto out_of_space; + stack_pointer[-1] = attr; + break; + } + + case _CHECK_ATTR_METHOD_LAZY_DICT: { + break; + } + + case _LOAD_ATTR_METHOD_LAZY_DICT: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + (void)descr; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + self = owner; + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + break; + } + + /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 */ + + /* _CALL is not a viable micro-op for tier 2 */ + + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { + _Py_UopsSymbol *null; + _Py_UopsSymbol *callable; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + sym_set_null(null); + sym_set_type(callable, &PyMethod_Type); + break; + } + + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { + _Py_UopsSymbol *callable; + _Py_UopsSymbol *func; + _Py_UopsSymbol *self; + callable = stack_pointer[-2 - oparg]; + (void)callable; + OUT_OF_SPACE_IF_NULL(func = sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(self = sym_new_not_null(ctx)); + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = self; + break; + } + + case _CHECK_PEP_523: { + break; + } + + case _CHECK_FUNCTION_EXACT_ARGS: { + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)this_instr->operand; + sym_set_type(callable, &PyFunction_Type); + (void)self_or_null; + (void)func_version; + break; + } + + case _CHECK_STACK_SPACE: { + break; + } + + case _INIT_CALL_PY_EXACT_ARGS: { + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + _Py_UOpsAbstractFrame *new_frame; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + (void)callable; + PyFunctionObject *func = (PyFunctionObject *)(this_instr + 2)->operand; + if (func == NULL) { + goto error; + } + PyCodeObject *co = (PyCodeObject *)func->func_code; + assert(self_or_null != NULL); + assert(args != NULL); + if (sym_is_not_null(self_or_null)) { + // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM + args--; + argcount++; + } + _Py_UopsSymbol **localsplus_start = ctx->n_consumed; + int n_locals_already_filled = 0; + // Can determine statically, so we interleave the new locals + // and make the current stack the new locals. + // This also sets up for true call inlining. + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + localsplus_start = args; + n_locals_already_filled = argcount; + } + OUT_OF_SPACE_IF_NULL(new_frame = + frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); + stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer += -1 - oparg; + break; + } + + case _PUSH_FRAME: { + _Py_UOpsAbstractFrame *new_frame; + new_frame = (_Py_UOpsAbstractFrame *)stack_pointer[-1]; + stack_pointer += -1; + ctx->frame->stack_pointer = stack_pointer; + ctx->frame = new_frame; + ctx->curr_frame_depth++; + stack_pointer = new_frame->stack_pointer; + break; + } + + /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 */ + + case _CALL_TYPE_1: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + break; + } + + case _CALL_STR_1: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + break; + } + + case _CALL_TUPLE_1: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + break; + } + + /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 */ + + case _EXIT_INIT_CHECK: { + stack_pointer += -1; + break; + } + + case _CALL_BUILTIN_CLASS: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + break; + } + + case _CALL_BUILTIN_O: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + break; + } + + case _CALL_BUILTIN_FAST: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + break; + } + + case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + break; + } + + case _CALL_LEN: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + break; + } + + case _CALL_ISINSTANCE: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + break; + } + + case _CALL_METHOD_DESCRIPTOR_O: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + break; + } + + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + break; + } + + case _CALL_METHOD_DESCRIPTOR_NOARGS: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + break; + } + + case _CALL_METHOD_DESCRIPTOR_FAST: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + break; + } + + /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */ + + /* _CALL_KW is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + + /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + + case _MAKE_FUNCTION: { + _Py_UopsSymbol *func; + func = sym_new_unknown(ctx); + if (func == NULL) goto out_of_space; + stack_pointer[-1] = func; + break; + } + + case _SET_FUNCTION_ATTRIBUTE: { + _Py_UopsSymbol *func; + func = sym_new_unknown(ctx); + if (func == NULL) goto out_of_space; + stack_pointer[-2] = func; + stack_pointer += -1; + break; + } + + case _BUILD_SLICE: { + _Py_UopsSymbol *slice; + slice = sym_new_unknown(ctx); + if (slice == NULL) goto out_of_space; + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + break; + } + + case _CONVERT_VALUE: { + _Py_UopsSymbol *result; + result = sym_new_unknown(ctx); + if (result == NULL) goto out_of_space; + stack_pointer[-1] = result; + break; + } + + case _FORMAT_SIMPLE: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-1] = res; + break; + } + + case _FORMAT_WITH_SPEC: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _COPY: { + _Py_UopsSymbol *bottom; + _Py_UopsSymbol *top; + bottom = stack_pointer[-1 - (oparg-1)]; + assert(oparg > 0); + top = bottom; + stack_pointer[0] = top; + stack_pointer += 1; + break; + } + + case _BINARY_OP: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _SWAP: { + _Py_UopsSymbol *top; + _Py_UopsSymbol *bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; + stack_pointer[-2 - (oparg-2)] = top; + stack_pointer[-1] = bottom; + break; + } + + /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ + + case _GUARD_IS_TRUE_POP: { + stack_pointer += -1; + break; + } + + case _GUARD_IS_FALSE_POP: { + stack_pointer += -1; + break; + } + + case _GUARD_IS_NONE_POP: { + stack_pointer += -1; + break; + } + + case _GUARD_IS_NOT_NONE_POP: { + stack_pointer += -1; + break; + } + + case _JUMP_TO_TOP: { + break; + } + + case _SET_IP: { + break; + } + + case _SAVE_RETURN_OFFSET: { + break; + } + + case _EXIT_TRACE: { + break; + } + + case _CHECK_VALIDITY: { + break; + } + + case _LOAD_CONST_INLINE: { + _Py_UopsSymbol *value; + PyObject *ptr = (PyObject *)this_instr->operand; + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_CONST_INLINE_BORROW: { + _Py_UopsSymbol *value; + PyObject *ptr = (PyObject *)this_instr->operand; + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_CONST_INLINE_WITH_NULL: { + _Py_UopsSymbol *value; + _Py_UopsSymbol *null; + PyObject *ptr = (PyObject *)this_instr->operand; + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + stack_pointer[0] = value; + stack_pointer[1] = null; + stack_pointer += 2; + break; + } + + case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { + _Py_UopsSymbol *value; + _Py_UopsSymbol *null; + PyObject *ptr = (PyObject *)this_instr->operand; + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + stack_pointer[0] = value; + stack_pointer[1] = null; + stack_pointer += 2; + break; + } + + case _CHECK_GLOBALS: { + break; + } + + case _CHECK_BUILTINS: { + break; + } + + case _INTERNAL_INCREMENT_OPT_COUNTER: { + stack_pointer += -1; + break; + } + + case _COLD_EXIT: { + break; + } + + case _START_EXECUTOR: { + break; + } + + case _FATAL_ERROR: { + break; + } + + case _CHECK_VALIDITY_AND_SET_IP: { + break; + } + diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c new file mode 100644 index 000000000000000..a529cc2f5cf2150 --- /dev/null +++ b/Python/optimizer_symbols.c @@ -0,0 +1,441 @@ + +#include "Python.h" + +#include "cpython/optimizer.h" +#include "pycore_code.h" +#include "pycore_frame.h" +#include "pycore_optimizer.h" + +#include <stdbool.h> +#include <stdint.h> +#include <stddef.h> + +/* Symbols + ======= + + See the diagram at + https://github.com/faster-cpython/ideas/blob/main/3.13/redundancy_eliminator.md + + We represent the nodes in the diagram as follows + (the flag bits are only defined in optimizer_symbols.c): + - Top: no flag bits, typ and const_val are NULL. + - NULL: IS_NULL flag set, type and const_val NULL. + - Not NULL: NOT_NULL flag set, type and const_val NULL. + - None/not None: not used. (None could be represented as any other constant.) + - Known type: NOT_NULL flag set and typ set; const_val is NULL. + - Known constant: NOT_NULL flag set, type set, const_val set. + - Bottom: IS_NULL and NOT_NULL flags set, type and const_val NULL. + */ + +// Flags for below. +#define IS_NULL 1 << 0 +#define NOT_NULL 1 << 1 + +#ifdef Py_DEBUG +static inline int get_lltrace(void) { + char *uop_debug = Py_GETENV("PYTHON_OPT_DEBUG"); + int lltrace = 0; + if (uop_debug != NULL && *uop_debug >= '0') { + lltrace = *uop_debug - '0'; // TODO: Parse an int and all that + } + return lltrace; +} +#define DPRINTF(level, ...) \ + if (get_lltrace() >= (level)) { printf(__VA_ARGS__); } +#else +#define DPRINTF(level, ...) +#endif + +static _Py_UopsSymbol * +sym_new(_Py_UOpsContext *ctx) +{ + _Py_UopsSymbol *self = &ctx->t_arena.arena[ctx->t_arena.ty_curr_number]; + if (ctx->t_arena.ty_curr_number >= ctx->t_arena.ty_max_number) { + OPT_STAT_INC(optimizer_failure_reason_no_memory); + DPRINTF(1, "out of space for symbolic expression type\n"); + return NULL; + } + ctx->t_arena.ty_curr_number++; + self->flags = 0; + self->typ = NULL; + self->const_val = NULL; + + return self; +} + +static inline void +sym_set_flag(_Py_UopsSymbol *sym, int flag) +{ + sym->flags |= flag; +} + +static inline void +sym_set_bottom(_Py_UopsSymbol *sym) +{ + sym_set_flag(sym, IS_NULL | NOT_NULL); + sym->typ = NULL; + Py_CLEAR(sym->const_val); +} + +bool +_Py_uop_sym_is_bottom(_Py_UopsSymbol *sym) +{ + if ((sym->flags & IS_NULL) && (sym->flags & NOT_NULL)) { + assert(sym->flags == (IS_NULL | NOT_NULL)); + assert(sym->typ == NULL); + assert(sym->const_val == NULL); + return true; + } + return false; +} + +bool +_Py_uop_sym_is_not_null(_Py_UopsSymbol *sym) +{ + return sym->flags == NOT_NULL; +} + +bool +_Py_uop_sym_is_null(_Py_UopsSymbol *sym) +{ + return sym->flags == IS_NULL; +} + +bool +_Py_uop_sym_is_const(_Py_UopsSymbol *sym) +{ + return sym->const_val != NULL; +} + +PyObject * +_Py_uop_sym_get_const(_Py_UopsSymbol *sym) +{ + return sym->const_val; +} + +void +_Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *typ) +{ + assert(typ != NULL && PyType_Check(typ)); + if (sym->flags & IS_NULL) { + sym_set_bottom(sym); + return; + } + if (sym->typ != NULL) { + if (sym->typ != typ) { + sym_set_bottom(sym); + } + return; + } + sym_set_flag(sym, NOT_NULL); + sym->typ = typ; +} + +void +_Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val) +{ + assert(const_val != NULL); + if (sym->flags & IS_NULL) { + sym_set_bottom(sym); + return; + } + PyTypeObject *typ = Py_TYPE(const_val); + if (sym->typ != NULL && sym->typ != typ) { + sym_set_bottom(sym); + return; + } + if (sym->const_val != NULL) { + if (sym->const_val != const_val) { + // TODO: What if they're equal? + sym_set_bottom(sym); + } + return; + } + sym_set_flag(sym, NOT_NULL); + sym->typ = typ; + sym->const_val = Py_NewRef(const_val); +} + + +void +_Py_uop_sym_set_null(_Py_UopsSymbol *sym) +{ + sym_set_flag(sym, IS_NULL); +} + +void +_Py_uop_sym_set_non_null(_Py_UopsSymbol *sym) +{ + sym_set_flag(sym, NOT_NULL); +} + + +_Py_UopsSymbol * +_Py_uop_sym_new_unknown(_Py_UOpsContext *ctx) +{ + return sym_new(ctx); +} + +_Py_UopsSymbol * +_Py_uop_sym_new_not_null(_Py_UOpsContext *ctx) +{ + _Py_UopsSymbol *res = _Py_uop_sym_new_unknown(ctx); + if (res == NULL) { + return NULL; + } + sym_set_flag(res, NOT_NULL); + return res; +} + +_Py_UopsSymbol * +_Py_uop_sym_new_type(_Py_UOpsContext *ctx, PyTypeObject *typ) +{ + _Py_UopsSymbol *res = sym_new(ctx); + if (res == NULL) { + return NULL; + } + _Py_uop_sym_set_type(res, typ); + return res; +} + +// Adds a new reference to const_val, owned by the symbol. +_Py_UopsSymbol * +_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val) +{ + assert(const_val != NULL); + _Py_UopsSymbol *res = sym_new(ctx); + if (res == NULL) { + return NULL; + } + _Py_uop_sym_set_const(res, const_val); + return res; +} + +_Py_UopsSymbol * +_Py_uop_sym_new_null(_Py_UOpsContext *ctx) +{ + _Py_UopsSymbol *null_sym = _Py_uop_sym_new_unknown(ctx); + if (null_sym == NULL) { + return NULL; + } + _Py_uop_sym_set_null(null_sym); + return null_sym; +} + +bool +_Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ) +{ + assert(typ != NULL && PyType_Check(typ)); + if (_Py_uop_sym_is_bottom(sym)) { + return false; + } + return sym->typ == typ; +} + +// 0 on success, -1 on error. +_Py_UOpsAbstractFrame * +_Py_uop_frame_new( + _Py_UOpsContext *ctx, + PyCodeObject *co, + _Py_UopsSymbol **localsplus_start, + int n_locals_already_filled, + int curr_stackentries) +{ + assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); + _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; + + frame->stack_len = co->co_stacksize; + frame->locals_len = co->co_nlocalsplus; + + frame->locals = localsplus_start; + frame->stack = frame->locals + co->co_nlocalsplus; + frame->stack_pointer = frame->stack + curr_stackentries; + ctx->n_consumed = localsplus_start + (co->co_nlocalsplus + co->co_stacksize); + if (ctx->n_consumed >= ctx->limit) { + return NULL; + } + + + // Initialize with the initial state of all local variables + for (int i = n_locals_already_filled; i < co->co_nlocalsplus; i++) { + _Py_UopsSymbol *local = _Py_uop_sym_new_unknown(ctx); + if (local == NULL) { + return NULL; + } + frame->locals[i] = local; + } + + + // Initialize the stack as well + for (int i = 0; i < curr_stackentries; i++) { + _Py_UopsSymbol *stackvar = _Py_uop_sym_new_unknown(ctx); + if (stackvar == NULL) { + return NULL; + } + frame->stack[i] = stackvar; + } + + return frame; +} + +void +_Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx) +{ + if (ctx == NULL) { + return; + } + ctx->curr_frame_depth = 0; + int tys = ctx->t_arena.ty_curr_number; + for (int i = 0; i < tys; i++) { + Py_CLEAR(ctx->t_arena.arena[i].const_val); + } +} + +int +_Py_uop_abstractcontext_init(_Py_UOpsContext *ctx) +{ + ctx->limit = ctx->locals_and_stack + MAX_ABSTRACT_INTERP_SIZE; + ctx->n_consumed = ctx->locals_and_stack; +#ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. + for (int i = 0 ; i < MAX_ABSTRACT_INTERP_SIZE; i++) { + ctx->locals_and_stack[i] = NULL; + } +#endif + + // Setup the arena for sym expressions. + ctx->t_arena.ty_curr_number = 0; + ctx->t_arena.ty_max_number = TY_ARENA_SIZE; + + // Frame setup + ctx->curr_frame_depth = 0; + + return 0; +} + +int +_Py_uop_frame_pop(_Py_UOpsContext *ctx) +{ + _Py_UOpsAbstractFrame *frame = ctx->frame; + ctx->n_consumed = frame->locals; + ctx->curr_frame_depth--; + assert(ctx->curr_frame_depth >= 1); + ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; + + return 0; +} + +#define TEST_PREDICATE(PRED, MSG) \ +do { \ + if (!(PRED)) { \ + PyErr_SetString( \ + PyExc_AssertionError, \ + (MSG)); \ + goto fail; \ + } \ +} while (0) + +static _Py_UopsSymbol * +make_bottom(_Py_UOpsContext *ctx) +{ + _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); + _Py_uop_sym_set_null(sym); + _Py_uop_sym_set_non_null(sym); + return sym; +} + +PyObject * +_Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) +{ + _Py_UOpsContext context; + _Py_UOpsContext *ctx = &context; + _Py_uop_abstractcontext_init(ctx); + PyObject *val_42 = NULL; + PyObject *val_43 = NULL; + + // Use a single 'sym' variable so copy-pasting tests is easier. + _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); + if (sym == NULL) { + goto fail; + } + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "top is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "top as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_bottom(sym), "top is bottom"); + + sym = make_bottom(ctx); + if (sym == NULL) { + goto fail; + } + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "bottom is NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "bottom is not NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "bottom matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "bottom is a constant is not false"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "bottom as constant is not NULL"); + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "bottom isn't bottom"); + + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (sym == NULL) { + goto fail; + } + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "int is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "int isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "int isn't int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "int matches float"); + TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "int is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "int as constant is not NULL"); + + _Py_uop_sym_set_type(sym, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(int and int) isn't int"); + + _Py_uop_sym_set_type(sym, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(int and float) isn't bottom"); + + val_42 = PyLong_FromLong(42); + assert(val_42 != NULL); + assert(_Py_IsImmortal(val_42)); + + val_43 = PyLong_FromLong(43); + assert(val_43 != NULL); + assert(_Py_IsImmortal(val_43)); + + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (sym == NULL) { + goto fail; + } + _Py_uop_sym_set_const(sym, val_42); + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "42 isn't an int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "42 matches float"); + TEST_PREDICATE(_Py_uop_sym_is_const(sym), "42 is not a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) != NULL, "42 as constant is NULL"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "42 as constant isn't 42"); + + _Py_uop_sym_set_type(sym, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(42 and 42) isn't an int"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "(42 and 42) as constant isn't 42"); + + _Py_uop_sym_set_type(sym, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and float) isn't bottom"); + + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (sym == NULL) { + goto fail; + } + _Py_uop_sym_set_const(sym, val_42); + _Py_uop_sym_set_const(sym, val_43); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and 43) isn't bottom"); + + _Py_uop_abstractcontext_fini(ctx); + Py_DECREF(val_42); + Py_DECREF(val_43); + Py_RETURN_NONE; + +fail: + _Py_uop_abstractcontext_fini(ctx); + Py_XDECREF(val_42); + Py_XDECREF(val_43); + return NULL; +} diff --git a/Python/parking_lot.c b/Python/parking_lot.c index d44c1b4b93b4d2b..0a897f9952f648e 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -1,11 +1,12 @@ #include "Python.h" #include "pycore_llist.h" -#include "pycore_lock.h" // _PyRawMutex +#include "pycore_lock.h" // _PyRawMutex #include "pycore_parking_lot.h" -#include "pycore_pyerrors.h" // _Py_FatalErrorFormat -#include "pycore_pystate.h" // _PyThreadState_GET -#include "pycore_semaphore.h" // _PySemaphore +#include "pycore_pyerrors.h" // _Py_FatalErrorFormat +#include "pycore_pystate.h" // _PyThreadState_GET +#include "pycore_semaphore.h" // _PySemaphore +#include "pycore_time.h" //_PyTime_MonotonicUnchecked() #include <stdbool.h> @@ -91,7 +92,7 @@ _PySemaphore_Destroy(_PySemaphore *sema) } static int -_PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout) +_PySemaphore_PlatformWait(_PySemaphore *sema, PyTime_t timeout) { int res; #if defined(MS_WINDOWS) @@ -119,13 +120,13 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout) struct timespec ts; #if defined(CLOCK_MONOTONIC) && defined(HAVE_SEM_CLOCKWAIT) - _PyTime_t deadline = _PyTime_Add(_PyTime_GetMonotonicClock(), timeout); + PyTime_t deadline = _PyTime_Add(_PyTime_MonotonicUnchecked(), timeout); _PyTime_AsTimespec_clamp(deadline, &ts); err = sem_clockwait(&sema->platform_sem, CLOCK_MONOTONIC, &ts); #else - _PyTime_t deadline = _PyTime_Add(_PyTime_GetSystemClock(), timeout); + PyTime_t deadline = _PyTime_Add(_PyTime_TimeUnchecked(), timeout); _PyTime_AsTimespec_clamp(deadline, &ts); @@ -158,11 +159,15 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout) if (sema->counter == 0) { if (timeout >= 0) { struct timespec ts; - - _PyTime_t deadline = _PyTime_Add(_PyTime_GetSystemClock(), timeout); +#if defined(HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP) + _PyTime_AsTimespec_clamp(timeout, &ts); + err = pthread_cond_timedwait_relative_np(&sema->cond, &sema->mutex, &ts); +#else + PyTime_t deadline = _PyTime_Add(_PyTime_TimeUnchecked(), timeout); _PyTime_AsTimespec_clamp(deadline, &ts); err = pthread_cond_timedwait(&sema->cond, &sema->mutex, &ts); +#endif // HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP } else { err = pthread_cond_wait(&sema->cond, &sema->mutex); @@ -184,7 +189,7 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout) } int -_PySemaphore_Wait(_PySemaphore *sema, _PyTime_t timeout, int detach) +_PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout, int detach) { PyThreadState *tstate = NULL; if (detach) { @@ -240,6 +245,7 @@ dequeue(Bucket *bucket, const void *address) if (wait->addr == (uintptr_t)address) { llist_remove(node); --bucket->num_waiters; + wait->is_unparking = true; return wait; } } @@ -258,6 +264,7 @@ dequeue_all(Bucket *bucket, const void *address, struct llist_node *dst) llist_remove(node); llist_insert_tail(dst, node); --bucket->num_waiters; + wait->is_unparking = true; } } } @@ -277,7 +284,7 @@ atomic_memcmp(const void *addr, const void *expected, size_t addr_size) int _PyParkingLot_Park(const void *addr, const void *expected, size_t size, - _PyTime_t timeout_ns, void *park_arg, int detach) + PyTime_t timeout_ns, void *park_arg, int detach) { struct wait_entry wait = { .park_arg = park_arg, @@ -333,8 +340,6 @@ _PyParkingLot_Unpark(const void *addr, _Py_unpark_fn_t *fn, void *arg) _PyRawMutex_Lock(&bucket->mutex); struct wait_entry *waiter = dequeue(bucket, addr); if (waiter) { - waiter->is_unparking = true; - int has_more_waiters = (bucket->num_waiters > 0); fn(arg, waiter->park_arg, has_more_waiters); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index fff64dd63d6b219..3a2c0a450ac9d97 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -32,6 +32,7 @@ #include "pycore_typevarobject.h" // _Py_clear_generic_types() #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() #include "pycore_weakref.h" // _PyWeakref_GET_REF() +#include "cpython/optimizer.h" // _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS #include "pycore_obmalloc.h" // _PyMem_init_obmalloc() #include "opcode.h" @@ -609,7 +610,11 @@ init_interp_create_gil(PyThreadState *tstate, int gil) static int builtins_dict_watcher(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value) { - RARE_EVENT_INC(builtin_dict); + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->rare_events.builtin_dict < _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { + _Py_Executors_InvalidateAll(interp, 1); + } + RARE_EVENT_INTERP_INC(interp, builtin_dict); return 0; } @@ -658,6 +663,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime, if (tstate == NULL) { return _PyStatus_ERR("can't make first thread"); } + runtime->main_tstate = tstate; _PyThreadState_Bind(tstate); init_interp_create_gil(tstate, config.gil); @@ -1104,7 +1110,6 @@ run_presite(PyThreadState *tstate) ); if (presite_modname == NULL) { fprintf(stderr, "Could not convert pre-site module name to unicode\n"); - Py_DECREF(presite_modname); } else { PyObject *presite = PyImport_Import(presite_modname); @@ -1240,17 +1245,26 @@ init_interp_main(PyThreadState *tstate) // Turn on experimental tier 2 (uops-based) optimizer if (is_main_interp) { +#ifndef _Py_JIT + // No JIT, maybe use the tier two interpreter: char *envvar = Py_GETENV("PYTHON_UOPS"); int enabled = envvar != NULL && *envvar > '0'; if (_Py_get_xoption(&config->xoptions, L"uops") != NULL) { enabled = 1; } if (enabled) { +#else + // Always enable tier two for JIT builds (ignoring the environment + // variable and command-line option above): + if (true) { +#endif PyObject *opt = PyUnstable_Optimizer_NewUOpOptimizer(); if (opt == NULL) { return _PyStatus_ERR("can't initialize optimizer"); } - PyUnstable_SetOptimizer((_PyOptimizerObject *)opt); + if (PyUnstable_SetOptimizer((_PyOptimizerObject *)opt)) { + return _PyStatus_ERR("can't initialize optimizer"); + } Py_DECREF(opt); } } @@ -1280,11 +1294,9 @@ init_interp_main(PyThreadState *tstate) } } - if ((interp->rare_events.builtins_dict_watcher_id = PyDict_AddWatcher(&builtins_dict_watcher)) == -1) { - return _PyStatus_ERR("failed to add builtin dict watcher"); - } - if (PyDict_Watch(interp->rare_events.builtins_dict_watcher_id, interp->builtins) != 0) { + interp->dict_state.watchers[0] = &builtins_dict_watcher; + if (PyDict_Watch(0, interp->builtins) != 0) { return _PyStatus_ERR("failed to set builtin dict watcher"); } @@ -1615,8 +1627,13 @@ finalize_modules(PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; - // Stop collecting stats on __builtin__ modifications during teardown - PyDict_Unwatch(interp->rare_events.builtins_dict_watcher_id, interp->builtins); + // Invalidate all executors and turn off tier 2 optimizer + _Py_Executors_InvalidateAll(interp, 0); + _PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL); + Py_XDECREF(old); + + // Stop watching __builtin__ modifications + PyDict_Unwatch(0, interp->builtins); PyObject *modules = _PyImport_GetModules(interp); if (modules == NULL) { @@ -1775,16 +1792,14 @@ finalize_interp_types(PyInterpreterState *interp) // a dict internally. _PyUnicode_ClearInterned(interp); - _PyDict_Fini(interp); _PyUnicode_Fini(interp); - _PyFreeListState *state = _PyFreeListState_GET(); - _PyTuple_Fini(state); - _PyList_Fini(state); - _PyFloat_Fini(state); - _PySlice_Fini(state); - _PyContext_Fini(state); - _PyAsyncGen_Fini(state); +#ifndef Py_GIL_DISABLED + // With Py_GIL_DISABLED: + // the freelists for the current thread state have already been cleared. + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + _PyObject_ClearFreeLists(freelists, 1); +#endif #ifdef Py_DEBUG _PyStaticObjects_CheckRefcnt(interp); @@ -1822,6 +1837,9 @@ finalize_interp_clear(PyThreadState *tstate) finalize_interp_types(tstate->interp); + /* Free any delayed free requests immediately */ + _PyMem_FiniDelayed(tstate->interp); + /* finalize_interp_types may allocate Python objects so we may need to abandon mimalloc segments again */ _PyThreadState_ClearMimallocHeaps(tstate); diff --git a/Python/pystate.c b/Python/pystate.c index 8e097c848cf4a18..a370fff857af852 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -67,7 +67,7 @@ _Py_thread_local PyThreadState *_Py_tss_tstate = NULL; #endif static inline PyThreadState * -current_fast_get(_PyRuntimeState *Py_UNUSED(runtime)) +current_fast_get(void) { #ifdef HAVE_THREAD_LOCAL return _Py_tss_tstate; @@ -101,14 +101,14 @@ current_fast_clear(_PyRuntimeState *Py_UNUSED(runtime)) } #define tstate_verify_not_active(tstate) \ - if (tstate == current_fast_get((tstate)->interp->runtime)) { \ + if (tstate == current_fast_get()) { \ _Py_FatalErrorFormat(__func__, "tstate %p is still current", tstate); \ } PyThreadState * _PyThreadState_GetCurrent(void) { - return current_fast_get(&_PyRuntime); + return current_fast_get(); } @@ -360,10 +360,9 @@ holds_gil(PyThreadState *tstate) // XXX Fall back to tstate->interp->runtime->ceval.gil.last_holder // (and tstate->interp->runtime->ceval.gil.locked). assert(tstate != NULL); - _PyRuntimeState *runtime = tstate->interp->runtime; /* Must be the tstate for this thread */ - assert(tstate == gilstate_tss_get(runtime)); - return tstate == current_fast_get(runtime); + assert(tstate == gilstate_tss_get(tstate->interp->runtime)); + return tstate == current_fast_get(); } @@ -396,6 +395,7 @@ _Py_COMP_DIAG_POP &(runtime)->atexit.mutex, \ &(runtime)->audit_hooks.mutex, \ &(runtime)->allocators.mutex, \ + &(runtime)->_main_interpreter.types.mutex, \ } static void @@ -500,6 +500,8 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime) _PyMutex_at_fork_reinit(locks[i]); } + _PyTypes_AfterFork(); + /* bpo-42540: id_mutex is freed by _PyInterpreterState_Delete, which does * not force the default allocator. */ if (_PyThread_at_fork_reinit(&runtime->interpreters.main->id_mutex) < 0) { @@ -518,6 +520,8 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime) return _PyStatus_NO_MEMORY(); } + _PyThread_AfterFork(&runtime->threads); + return _PyStatus_OK(); } #endif @@ -610,6 +614,10 @@ init_interpreter(PyInterpreterState *interp, _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); +#ifdef Py_GIL_DISABLED + _Py_brc_init_state(interp); +#endif + llist_init(&interp->mem_free_queue.head); for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->monitors.tools[i] = 0; } @@ -621,9 +629,7 @@ init_interpreter(PyInterpreterState *interp, } interp->sys_profile_initialized = false; interp->sys_trace_initialized = false; - interp->optimizer = &_PyOptimizer_Default; - interp->optimizer_backedge_threshold = _PyOptimizer_Default.backedge_threshold; - interp->optimizer_resume_threshold = _PyOptimizer_Default.backedge_threshold; + (void)_Py_SetOptimizer(interp, NULL); interp->next_func_version = 1; interp->executor_list_head = NULL; if (interp != &runtime->_main_interpreter) { @@ -723,7 +729,7 @@ PyInterpreterState * PyInterpreterState_New(void) { // tstate can be NULL - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); PyInterpreterState *interp; PyStatus status = _PyInterpreterState_New(tstate, &interp); @@ -776,10 +782,9 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) tstate->_status.cleared = 0; } - Py_CLEAR(interp->optimizer); - interp->optimizer = &_PyOptimizer_Default; - interp->optimizer_backedge_threshold = _PyOptimizer_Default.backedge_threshold; - interp->optimizer_resume_threshold = _PyOptimizer_Default.backedge_threshold; + _PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL); + assert(old != NULL); + Py_DECREF(old); /* It is possible that any of the objects below have a finalizer that runs Python code or otherwise relies on a thread state @@ -790,9 +795,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->audit_hooks); - // At this time, all the threads should be cleared so we don't need - // atomic operations for eval_breaker - interp->ceval.eval_breaker = 0; + interp->ceval.instrumentation_version = 0; for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->monitors.tools[i] = 0; @@ -882,7 +885,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp) // Use the current Python thread state to call audit hooks and to collect // garbage. It can be different than the current Python thread state // of 'interp'. - PyThreadState *current_tstate = current_fast_get(interp->runtime); + PyThreadState *current_tstate = current_fast_get(); _PyImport_ClearCore(interp); interpreter_clear(interp, current_tstate); } @@ -908,7 +911,7 @@ PyInterpreterState_Delete(PyInterpreterState *interp) // XXX Clearing the "current" thread state should happen before // we start finalizing the interpreter (or the current thread state). - PyThreadState *tcur = current_fast_get(runtime); + PyThreadState *tcur = current_fast_get(); if (tcur != NULL && interp == tcur->interp) { /* Unset current thread. After this, many C API calls become crashy. */ _PyThreadState_Detach(tcur); @@ -950,6 +953,8 @@ PyInterpreterState_Delete(PyInterpreterState *interp) PyThread_free_lock(interp->id_mutex); } + _Py_qsbr_fini(interp); + _PyObject_FiniState(interp); free_interpreter(interp); @@ -1010,7 +1015,7 @@ _PyInterpreterState_SetRunningMain(PyInterpreterState *interp) if (_PyInterpreterState_FailIfRunningMain(interp) < 0) { return -1; } - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); if (tstate->interp != interp) { PyErr_SetString(PyExc_RuntimeError, @@ -1025,7 +1030,7 @@ void _PyInterpreterState_SetNotRunningMain(PyInterpreterState *interp) { PyThreadState *tstate = interp->threads.main; - assert(tstate == current_fast_get(&_PyRuntime)); + assert(tstate == current_fast_get()); if (tstate->on_delete != NULL) { // The threading module was imported for the first time in this @@ -1044,7 +1049,14 @@ _PyInterpreterState_SetNotRunningMain(PyInterpreterState *interp) int _PyInterpreterState_IsRunningMain(PyInterpreterState *interp) { - return (interp->threads.main != NULL); + if (interp->threads.main != NULL) { + return 1; + } + // For now, we assume the main interpreter is always running. + if (_Py_IsMainInterpreter(interp)) { + return 1; + } + return 0; } int @@ -1178,7 +1190,7 @@ PyInterpreterState_GetDict(PyInterpreterState *interp) PyInterpreterState* PyInterpreterState_Get(void) { - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); PyInterpreterState *interp = tstate->interp; if (interp == NULL) { @@ -1305,6 +1317,8 @@ init_threadstate(_PyThreadStateImpl *_tstate, assert(interp != NULL); tstate->interp = interp; + tstate->eval_breaker = + _Py_atomic_load_uintptr_relaxed(&interp->ceval.instrumentation_version); // next/prev are set in add_threadstate(). assert(tstate->next == NULL); @@ -1334,6 +1348,13 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->datastack_top = NULL; tstate->datastack_limit = NULL; tstate->what_event = -1; + tstate->previous_executor = NULL; + +#ifdef Py_GIL_DISABLED + // Initialize biased reference counting inter-thread queue + _Py_brc_init_thread(tstate); +#endif + llist_init(&_tstate->mem_free_queue); if (interp->stoptheworld.requested || _PyRuntime.stoptheworld.requested) { // Start in the suspended state if there is an ongoing stop-the-world. @@ -1371,6 +1392,14 @@ new_threadstate(PyInterpreterState *interp, int whence) if (new_tstate == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + Py_ssize_t qsbr_idx = _Py_qsbr_reserve(interp); + if (qsbr_idx < 0) { + PyMem_RawFree(new_tstate); + return NULL; + } +#endif + /* We serialize concurrent creation to protect global state. */ HEAD_LOCK(runtime); @@ -1405,6 +1434,12 @@ new_threadstate(PyInterpreterState *interp, int whence) // Must be called with lock unlocked to avoid re-entrancy deadlock. PyMem_RawFree(new_tstate); } + +#ifdef Py_GIL_DISABLED + // Must be called with lock unlocked to avoid lock ordering deadlocks. + _Py_qsbr_register(tstate, interp, qsbr_idx); +#endif + return (PyThreadState *)tstate; } @@ -1459,22 +1494,11 @@ clear_datastack(PyThreadState *tstate) } } -void -_Py_ClearFreeLists(_PyFreeListState *state, int is_finalization) -{ - _PyFloat_ClearFreeList(state, is_finalization); - _PyTuple_ClearFreeList(state, is_finalization); - _PyList_ClearFreeList(state, is_finalization); - _PyContext_ClearFreeList(state, is_finalization); - _PyAsyncGen_ClearFreeLists(state, is_finalization); - _PyObjectStackChunk_ClearFreeList(state, is_finalization); -} - void PyThreadState_Clear(PyThreadState *tstate) { assert(tstate->_status.initialized && !tstate->_status.cleared); - assert(current_fast_get(&_PyRuntime)->interp == tstate->interp); + assert(current_fast_get()->interp == tstate->interp); // XXX assert(!tstate->_status.bound || tstate->_status.unbound); tstate->_status.finalizing = 1; // just in case @@ -1552,13 +1576,19 @@ PyThreadState_Clear(PyThreadState *tstate) // don't call _PyInterpreterState_SetNotRunningMain() yet. tstate->on_delete(tstate->on_delete_data); } + #ifdef Py_GIL_DISABLED // Each thread should clear own freelists in free-threading builds. - _PyFreeListState *freelist_state = &((_PyThreadStateImpl*)tstate)->freelist_state; - _Py_ClearFreeLists(freelist_state, 1); - _PySlice_ClearCache(freelist_state); + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + _PyObject_ClearFreeLists(freelists, 1); + + // Remove ourself from the biased reference counting table of threads. + _Py_brc_remove_thread(tstate); #endif + // Merge our queue of pointers to be freed into the interpreter queue. + _PyMem_AbandonDelayed(tstate); + _PyThreadState_ClearMimallocHeaps(tstate); tstate->_status.cleared = 1; @@ -1605,6 +1635,10 @@ tstate_delete_common(PyThreadState *tstate) } HEAD_UNLOCK(runtime); +#ifdef Py_GIL_DISABLED + _Py_qsbr_unregister((_PyThreadStateImpl *)tstate); +#endif + // XXX Unbind in PyThreadState_Clear(), or earlier // (and assert not-equal here)? if (tstate->_status.bound_gilstate) { @@ -1646,6 +1680,9 @@ void _PyThreadState_DeleteCurrent(PyThreadState *tstate) { _Py_EnsureTstateNotNULL(tstate); +#ifdef Py_GIL_DISABLED + _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); +#endif tstate_set_detached(tstate); tstate_delete_common(tstate); current_fast_clear(tstate->interp->runtime); @@ -1656,7 +1693,7 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate) void PyThreadState_DeleteCurrent(void) { - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); _PyThreadState_DeleteCurrent(tstate); } @@ -1732,7 +1769,7 @@ _PyThreadState_GetDict(PyThreadState *tstate) PyObject * PyThreadState_GetDict(void) { - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); if (tstate == NULL) { return NULL; } @@ -1853,7 +1890,7 @@ _PyThreadState_Attach(PyThreadState *tstate) #endif _Py_EnsureTstateNotNULL(tstate); - if (current_fast_get(&_PyRuntime) != NULL) { + if (current_fast_get() != NULL) { Py_FatalError("non-NULL old thread state"); } @@ -1867,6 +1904,10 @@ _PyThreadState_Attach(PyThreadState *tstate) tstate_wait_attach(tstate); } +#ifdef Py_GIL_DISABLED + _Py_qsbr_attach(((_PyThreadStateImpl *)tstate)->qsbr); +#endif + // Resume previous critical section. This acquires the lock(s) from the // top-most critical section. if (tstate->critical_section != 0) { @@ -1883,10 +1924,13 @@ detach_thread(PyThreadState *tstate, int detached_state) { // XXX assert(tstate_is_alive(tstate) && tstate_is_bound(tstate)); assert(tstate->state == _Py_THREAD_ATTACHED); - assert(tstate == current_fast_get(&_PyRuntime)); + assert(tstate == current_fast_get()); if (tstate->critical_section != 0) { _PyCriticalSection_SuspendAll(tstate); } +#ifdef Py_GIL_DISABLED + _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); +#endif tstate_deactivate(tstate); tstate_set_detached(tstate); current_fast_clear(&_PyRuntime); @@ -1983,8 +2027,7 @@ park_detached_threads(struct _stoptheworld_state *stw) } } else if (state == _Py_THREAD_ATTACHED && t != stw->requester) { - // TODO: set this per-thread, rather than per-interpreter. - _Py_set_eval_breaker_bit(t->interp, _PY_EVAL_PLEASE_STOP_BIT, 1); + _Py_set_eval_breaker_bit(t, _PY_EVAL_PLEASE_STOP_BIT); } } stw->thread_countdown -= num_parked; @@ -2035,7 +2078,7 @@ stop_the_world(struct _stoptheworld_state *stw) break; } - _PyTime_t wait_ns = 1000*1000; // 1ms (arbitrary, may need tuning) + PyTime_t wait_ns = 1000*1000; // 1ms (arbitrary, may need tuning) if (PyEvent_WaitTimed(&stw->stop_event, wait_ns)) { assert(stw->thread_countdown == 0); break; @@ -2148,19 +2191,18 @@ PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) * deadlock, we need to release head_mutex before * the decref. */ - PyObject *old_exc = tstate->async_exc; - tstate->async_exc = Py_XNewRef(exc); + Py_XINCREF(exc); + PyObject *old_exc = _Py_atomic_exchange_ptr(&tstate->async_exc, exc); HEAD_UNLOCK(runtime); Py_XDECREF(old_exc); - _PyEval_SignalAsyncExc(tstate->interp); + _Py_set_eval_breaker_bit(tstate, _PY_ASYNC_EXCEPTION_BIT); return 1; } HEAD_UNLOCK(runtime); return 0; } - //--------------------------------- // API for the current thread state //--------------------------------- @@ -2168,14 +2210,14 @@ PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) PyThreadState * PyThreadState_GetUnchecked(void) { - return current_fast_get(&_PyRuntime); + return current_fast_get(); } PyThreadState * PyThreadState_Get(void) { - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); return tstate; } @@ -2183,7 +2225,7 @@ PyThreadState_Get(void) PyThreadState * _PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) { - PyThreadState *oldts = current_fast_get(runtime); + PyThreadState *oldts = current_fast_get(); if (oldts != NULL) { _PyThreadState_Detach(oldts); } @@ -2278,7 +2320,7 @@ PyObject * _PyThread_CurrentFrames(void) { _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = current_fast_get(runtime); + PyThreadState *tstate = current_fast_get(); if (_PySys_Audit(tstate, "sys._current_frames", NULL) < 0) { return NULL; } @@ -2339,7 +2381,7 @@ PyObject * _PyThread_CurrentExceptions(void) { _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = current_fast_get(runtime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); @@ -2481,12 +2523,13 @@ PyGILState_Check(void) return 1; } - PyThreadState *tstate = current_fast_get(runtime); + PyThreadState *tstate = current_fast_get(); if (tstate == NULL) { return 0; } - return (tstate == gilstate_tss_get(runtime)); + PyThreadState *tcur = gilstate_tss_get(runtime); + return (tstate == tcur); } PyGILState_STATE @@ -2579,7 +2622,7 @@ PyGILState_Release(PyGILState_STATE oldstate) * races; see bugs 225673 and 1061968 (that nasty bug has a * habit of coming back). */ - assert(current_fast_get(runtime) == tstate); + assert(current_fast_get() == tstate); _PyThreadState_DeleteCurrent(tstate); } /* Release the lock if necessary */ @@ -2614,7 +2657,7 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, return; } if (eval_frame != NULL) { - _Py_Executors_InvalidateAll(interp); + _Py_Executors_InvalidateAll(interp, 1); } RARE_EVENT_INC(set_eval_frame_func); interp->eval_frame = eval_frame; @@ -2645,9 +2688,8 @@ _PyInterpreterState_GetConfigCopy(PyConfig *config) const PyConfig* _Py_GetConfig(void) { - _PyRuntimeState *runtime = &_PyRuntime; assert(PyGILState_Check()); - PyThreadState *tstate = current_fast_get(runtime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); return _PyInterpreterState_GetConfig(tstate->interp); } @@ -2800,9 +2842,24 @@ tstate_mimalloc_bind(PyThreadState *tstate) // pools to keep Python objects from different interpreters separate. tld->segments.abandoned = &tstate->interp->mimalloc.abandoned_pool; + // Don't fill in the first N bytes up to ob_type in debug builds. We may + // access ob_tid and the refcount fields in the dict and list lock-less + // accesses, so they must remain valid for a while after deallocation. + size_t base_offset = offsetof(PyObject, ob_type); + if (_PyMem_DebugEnabled()) { + // The debug allocator adds two words at the beginning of each block. + base_offset += 2 * sizeof(size_t); + } + size_t debug_offsets[_Py_MIMALLOC_HEAP_COUNT] = { + [_Py_MIMALLOC_HEAP_OBJECT] = base_offset, + [_Py_MIMALLOC_HEAP_GC] = base_offset, + [_Py_MIMALLOC_HEAP_GC_PRE] = base_offset + 2 * sizeof(PyObject *), + }; + // Initialize each heap for (uint8_t i = 0; i < _Py_MIMALLOC_HEAP_COUNT; i++) { _mi_heap_init_ex(&mts->heaps[i], tld, _mi_arena_id_none(), false, i); + mts->heaps[i].debug_offset = (uint8_t)debug_offsets[i]; } // By default, object allocations use _Py_MIMALLOC_HEAP_OBJECT. diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 5f305aa00e08b9c..f87c53fb28fbead 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -89,7 +89,7 @@ int PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) { - PyObject *filename_obj; + PyObject *filename_obj = NULL; if (filename != NULL) { filename_obj = PyUnicode_DecodeFSDefault(filename); if (filename_obj == NULL) { @@ -97,9 +97,6 @@ PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, return -1; } } - else { - filename_obj = NULL; - } int res = _PyRun_AnyFileObject(fp, filename_obj, closeit, flags); Py_XDECREF(filename_obj); return res; diff --git a/Python/pytime.c b/Python/pytime.c index 77cb95f8feb179f..90ef2eeb546f7fd 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -1,5 +1,5 @@ #include "Python.h" -#include "pycore_time.h" // _PyTime_t +#include "pycore_time.h" // PyTime_t #include <time.h> // gmtime_r() #ifdef HAVE_SYS_TIME_H @@ -50,19 +50,19 @@ # error "time_t is not a two's complement integer type" #endif -#if _PyTime_MIN + _PyTime_MAX != -1 -# error "_PyTime_t is not a two's complement integer type" +#if PyTime_MIN + PyTime_MAX != -1 +# error "PyTime_t is not a two's complement integer type" #endif -static _PyTime_t -_PyTime_GCD(_PyTime_t x, _PyTime_t y) +static PyTime_t +_PyTime_GCD(PyTime_t x, PyTime_t y) { // Euclidean algorithm assert(x >= 1); assert(y >= 1); while (y != 0) { - _PyTime_t tmp = y; + PyTime_t tmp = y; y = x % y; x = tmp; } @@ -72,13 +72,13 @@ _PyTime_GCD(_PyTime_t x, _PyTime_t y) int -_PyTimeFraction_Set(_PyTimeFraction *frac, _PyTime_t numer, _PyTime_t denom) +_PyTimeFraction_Set(_PyTimeFraction *frac, PyTime_t numer, PyTime_t denom) { if (numer < 1 || denom < 1) { return -1; } - _PyTime_t gcd = _PyTime_GCD(numer, denom); + PyTime_t gcd = _PyTime_GCD(numer, denom); frac->numer = numer / gcd; frac->denom = denom / gcd; return 0; @@ -104,36 +104,20 @@ static void pytime_overflow(void) { PyErr_SetString(PyExc_OverflowError, - "timestamp too large to convert to C _PyTime_t"); + "timestamp too large to convert to C PyTime_t"); } -static inline _PyTime_t -pytime_from_nanoseconds(_PyTime_t t) -{ - // _PyTime_t is a number of nanoseconds - return t; -} - - -static inline _PyTime_t -pytime_as_nanoseconds(_PyTime_t t) -{ - // _PyTime_t is a number of nanoseconds: see pytime_from_nanoseconds() - return t; -} - - -// Compute t1 + t2. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. +// Compute t1 + t2. Clamp to [PyTime_MIN; PyTime_MAX] on overflow. static inline int -pytime_add(_PyTime_t *t1, _PyTime_t t2) +pytime_add(PyTime_t *t1, PyTime_t t2) { - if (t2 > 0 && *t1 > _PyTime_MAX - t2) { - *t1 = _PyTime_MAX; + if (t2 > 0 && *t1 > PyTime_MAX - t2) { + *t1 = PyTime_MAX; return -1; } - else if (t2 < 0 && *t1 < _PyTime_MIN - t2) { - *t1 = _PyTime_MIN; + else if (t2 < 0 && *t1 < PyTime_MIN - t2) { + *t1 = PyTime_MIN; return -1; } else { @@ -143,8 +127,8 @@ pytime_add(_PyTime_t *t1, _PyTime_t t2) } -_PyTime_t -_PyTime_Add(_PyTime_t t1, _PyTime_t t2) +PyTime_t +_PyTime_Add(PyTime_t t1, PyTime_t t2) { (void)pytime_add(&t1, t2); return t1; @@ -152,11 +136,11 @@ _PyTime_Add(_PyTime_t t1, _PyTime_t t2) static inline int -pytime_mul_check_overflow(_PyTime_t a, _PyTime_t b) +pytime_mul_check_overflow(PyTime_t a, PyTime_t b) { if (b != 0) { assert(b > 0); - return ((a < _PyTime_MIN / b) || (_PyTime_MAX / b < a)); + return ((a < PyTime_MIN / b) || (PyTime_MAX / b < a)); } else { return 0; @@ -164,13 +148,13 @@ pytime_mul_check_overflow(_PyTime_t a, _PyTime_t b) } -// Compute t * k. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. +// Compute t * k. Clamp to [PyTime_MIN; PyTime_MAX] on overflow. static inline int -pytime_mul(_PyTime_t *t, _PyTime_t k) +pytime_mul(PyTime_t *t, PyTime_t k) { assert(k >= 0); if (pytime_mul_check_overflow(*t, k)) { - *t = (*t >= 0) ? _PyTime_MAX : _PyTime_MIN; + *t = (*t >= 0) ? PyTime_MAX : PyTime_MIN; return -1; } else { @@ -180,20 +164,20 @@ pytime_mul(_PyTime_t *t, _PyTime_t k) } -// Compute t * k. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. -static inline _PyTime_t -_PyTime_Mul(_PyTime_t t, _PyTime_t k) +// Compute t * k. Clamp to [PyTime_MIN; PyTime_MAX] on overflow. +static inline PyTime_t +_PyTime_Mul(PyTime_t t, PyTime_t k) { (void)pytime_mul(&t, k); return t; } -_PyTime_t -_PyTimeFraction_Mul(_PyTime_t ticks, const _PyTimeFraction *frac) +PyTime_t +_PyTimeFraction_Mul(PyTime_t ticks, const _PyTimeFraction *frac) { - const _PyTime_t mul = frac->numer; - const _PyTime_t div = frac->denom; + const PyTime_t mul = frac->numer; + const PyTime_t div = frac->denom; if (div == 1) { // Fast-path taken by mach_absolute_time() with 1/1 time base. @@ -205,7 +189,7 @@ _PyTimeFraction_Mul(_PyTime_t ticks, const _PyTimeFraction *frac) (ticks * mul) / div == (ticks / div) * mul + (ticks % div) * mul / div */ - _PyTime_t intpart, remaining; + PyTime_t intpart, remaining; intpart = ticks / div; ticks %= div; remaining = _PyTime_Mul(ticks, mul) / div; @@ -247,17 +231,17 @@ _PyLong_FromTime_t(time_t t) } -// Convert _PyTime_t to time_t. +// Convert PyTime_t to time_t. // Return 0 on success. Return -1 and clamp the value on overflow. static int -_PyTime_AsTime_t(_PyTime_t t, time_t *t2) +_PyTime_AsTime_t(PyTime_t t, time_t *t2) { #if SIZEOF_TIME_T < _SIZEOF_PYTIME_T - if ((_PyTime_t)PY_TIME_T_MAX < t) { + if ((PyTime_t)PY_TIME_T_MAX < t) { *t2 = PY_TIME_T_MAX; return -1; } - if (t < (_PyTime_t)PY_TIME_T_MIN) { + if (t < (PyTime_t)PY_TIME_T_MIN) { *t2 = PY_TIME_T_MIN; return -1; } @@ -268,17 +252,17 @@ _PyTime_AsTime_t(_PyTime_t t, time_t *t2) #ifdef MS_WINDOWS -// Convert _PyTime_t to long. +// Convert PyTime_t to long. // Return 0 on success. Return -1 and clamp the value on overflow. static int -_PyTime_AsLong(_PyTime_t t, long *t2) +_PyTime_AsCLong(PyTime_t t, long *t2) { #if SIZEOF_LONG < _SIZEOF_PYTIME_T - if ((_PyTime_t)LONG_MAX < t) { + if ((PyTime_t)LONG_MAX < t) { *t2 = LONG_MAX; return -1; } - if (t < (_PyTime_t)LONG_MIN) { + if (t < (PyTime_t)LONG_MIN) { *t2 = LONG_MIN; return -1; } @@ -453,50 +437,42 @@ _PyTime_ObjectToTimeval(PyObject *obj, time_t *sec, long *usec, } -_PyTime_t +PyTime_t _PyTime_FromSeconds(int seconds) { /* ensure that integer overflow cannot happen, int type should have 32 - bits, whereas _PyTime_t type has at least 64 bits (SEC_TO_NS takes 30 + bits, whereas PyTime_t type has at least 64 bits (SEC_TO_NS takes 30 bits). */ - static_assert(INT_MAX <= _PyTime_MAX / SEC_TO_NS, "_PyTime_t overflow"); - static_assert(INT_MIN >= _PyTime_MIN / SEC_TO_NS, "_PyTime_t underflow"); + static_assert(INT_MAX <= PyTime_MAX / SEC_TO_NS, "PyTime_t overflow"); + static_assert(INT_MIN >= PyTime_MIN / SEC_TO_NS, "PyTime_t underflow"); - _PyTime_t t = (_PyTime_t)seconds; - assert((t >= 0 && t <= _PyTime_MAX / SEC_TO_NS) - || (t < 0 && t >= _PyTime_MIN / SEC_TO_NS)); + PyTime_t t = (PyTime_t)seconds; + assert((t >= 0 && t <= PyTime_MAX / SEC_TO_NS) + || (t < 0 && t >= PyTime_MIN / SEC_TO_NS)); t *= SEC_TO_NS; - return pytime_from_nanoseconds(t); -} - - -_PyTime_t -_PyTime_FromNanoseconds(_PyTime_t ns) -{ - return pytime_from_nanoseconds(ns); + return t; } -_PyTime_t -_PyTime_FromMicrosecondsClamp(_PyTime_t us) +PyTime_t +_PyTime_FromMicrosecondsClamp(PyTime_t us) { - _PyTime_t ns = _PyTime_Mul(us, US_TO_NS); - return pytime_from_nanoseconds(ns); + PyTime_t ns = _PyTime_Mul(us, US_TO_NS); + return ns; } int -_PyTime_FromNanosecondsObject(_PyTime_t *tp, PyObject *obj) +_PyTime_FromLong(PyTime_t *tp, PyObject *obj) { - if (!PyLong_Check(obj)) { PyErr_Format(PyExc_TypeError, "expect int, got %s", Py_TYPE(obj)->tp_name); return -1; } - static_assert(sizeof(long long) == sizeof(_PyTime_t), - "_PyTime_t is not long long"); + static_assert(sizeof(long long) == sizeof(PyTime_t), + "PyTime_t is not long long"); long long nsec = PyLong_AsLongLong(obj); if (nsec == -1 && PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_OverflowError)) { @@ -505,28 +481,28 @@ _PyTime_FromNanosecondsObject(_PyTime_t *tp, PyObject *obj) return -1; } - _PyTime_t t = (_PyTime_t)nsec; - *tp = pytime_from_nanoseconds(t); + PyTime_t t = (PyTime_t)nsec; + *tp = t; return 0; } #ifdef HAVE_CLOCK_GETTIME static int -pytime_fromtimespec(_PyTime_t *tp, const struct timespec *ts, int raise_exc) +pytime_fromtimespec(PyTime_t *tp, const struct timespec *ts, int raise_exc) { - _PyTime_t t, tv_nsec; + PyTime_t t, tv_nsec; - static_assert(sizeof(ts->tv_sec) <= sizeof(_PyTime_t), - "timespec.tv_sec is larger than _PyTime_t"); - t = (_PyTime_t)ts->tv_sec; + static_assert(sizeof(ts->tv_sec) <= sizeof(PyTime_t), + "timespec.tv_sec is larger than PyTime_t"); + t = (PyTime_t)ts->tv_sec; int res1 = pytime_mul(&t, SEC_TO_NS); tv_nsec = ts->tv_nsec; int res2 = pytime_add(&t, tv_nsec); - *tp = pytime_from_nanoseconds(t); + *tp = t; if (raise_exc && (res1 < 0 || res2 < 0)) { pytime_overflow(); @@ -536,7 +512,7 @@ pytime_fromtimespec(_PyTime_t *tp, const struct timespec *ts, int raise_exc) } int -_PyTime_FromTimespec(_PyTime_t *tp, const struct timespec *ts) +_PyTime_FromTimespec(PyTime_t *tp, const struct timespec *ts) { return pytime_fromtimespec(tp, ts, 1); } @@ -545,18 +521,18 @@ _PyTime_FromTimespec(_PyTime_t *tp, const struct timespec *ts) #ifndef MS_WINDOWS static int -pytime_fromtimeval(_PyTime_t *tp, struct timeval *tv, int raise_exc) +pytime_fromtimeval(PyTime_t *tp, struct timeval *tv, int raise_exc) { - static_assert(sizeof(tv->tv_sec) <= sizeof(_PyTime_t), - "timeval.tv_sec is larger than _PyTime_t"); - _PyTime_t t = (_PyTime_t)tv->tv_sec; + static_assert(sizeof(tv->tv_sec) <= sizeof(PyTime_t), + "timeval.tv_sec is larger than PyTime_t"); + PyTime_t t = (PyTime_t)tv->tv_sec; int res1 = pytime_mul(&t, SEC_TO_NS); - _PyTime_t usec = (_PyTime_t)tv->tv_usec * US_TO_NS; + PyTime_t usec = (PyTime_t)tv->tv_usec * US_TO_NS; int res2 = pytime_add(&t, usec); - *tp = pytime_from_nanoseconds(t); + *tp = t; if (raise_exc && (res1 < 0 || res2 < 0)) { pytime_overflow(); @@ -567,7 +543,7 @@ pytime_fromtimeval(_PyTime_t *tp, struct timeval *tv, int raise_exc) int -_PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv) +_PyTime_FromTimeval(PyTime_t *tp, struct timeval *tv) { return pytime_fromtimeval(tp, tv, 1); } @@ -575,7 +551,7 @@ _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv) static int -pytime_from_double(_PyTime_t *tp, double value, _PyTime_round_t round, +pytime_from_double(PyTime_t *tp, double value, _PyTime_round_t round, long unit_to_ns) { /* volatile avoids optimization changing how numbers are rounded */ @@ -587,19 +563,19 @@ pytime_from_double(_PyTime_t *tp, double value, _PyTime_round_t round, d = pytime_round(d, round); /* See comments in pytime_double_to_denominator */ - if (!((double)_PyTime_MIN <= d && d < -(double)_PyTime_MIN)) { + if (!((double)PyTime_MIN <= d && d < -(double)PyTime_MIN)) { pytime_time_t_overflow(); return -1; } - _PyTime_t ns = (_PyTime_t)d; + PyTime_t ns = (PyTime_t)d; - *tp = pytime_from_nanoseconds(ns); + *tp = ns; return 0; } static int -pytime_from_object(_PyTime_t *tp, PyObject *obj, _PyTime_round_t round, +pytime_from_object(PyTime_t *tp, PyObject *obj, _PyTime_round_t round, long unit_to_ns) { if (PyFloat_Check(obj)) { @@ -620,45 +596,44 @@ pytime_from_object(_PyTime_t *tp, PyObject *obj, _PyTime_round_t round, return -1; } - static_assert(sizeof(long long) <= sizeof(_PyTime_t), - "_PyTime_t is smaller than long long"); - _PyTime_t ns = (_PyTime_t)sec; + static_assert(sizeof(long long) <= sizeof(PyTime_t), + "PyTime_t is smaller than long long"); + PyTime_t ns = (PyTime_t)sec; if (pytime_mul(&ns, unit_to_ns) < 0) { pytime_overflow(); return -1; } - *tp = pytime_from_nanoseconds(ns); + *tp = ns; return 0; } } int -_PyTime_FromSecondsObject(_PyTime_t *tp, PyObject *obj, _PyTime_round_t round) +_PyTime_FromSecondsObject(PyTime_t *tp, PyObject *obj, _PyTime_round_t round) { return pytime_from_object(tp, obj, round, SEC_TO_NS); } int -_PyTime_FromMillisecondsObject(_PyTime_t *tp, PyObject *obj, _PyTime_round_t round) +_PyTime_FromMillisecondsObject(PyTime_t *tp, PyObject *obj, _PyTime_round_t round) { return pytime_from_object(tp, obj, round, MS_TO_NS); } double -_PyTime_AsSecondsDouble(_PyTime_t t) +PyTime_AsSecondsDouble(PyTime_t ns) { /* volatile avoids optimization changing how numbers are rounded */ volatile double d; - _PyTime_t ns = pytime_as_nanoseconds(t); if (ns % SEC_TO_NS == 0) { /* Divide using integers to avoid rounding issues on the integer part. 1e-9 cannot be stored exactly in IEEE 64-bit. */ - _PyTime_t secs = ns / SEC_TO_NS; + PyTime_t secs = ns / SEC_TO_NS; d = (double)secs; } else { @@ -670,18 +645,17 @@ _PyTime_AsSecondsDouble(_PyTime_t t) PyObject * -_PyTime_AsNanosecondsObject(_PyTime_t t) +_PyTime_AsLong(PyTime_t ns) { - _PyTime_t ns = pytime_as_nanoseconds(t); - static_assert(sizeof(long long) >= sizeof(_PyTime_t), - "_PyTime_t is larger than long long"); + static_assert(sizeof(long long) >= sizeof(PyTime_t), + "PyTime_t is larger than long long"); return PyLong_FromLongLong((long long)ns); } -_PyTime_t +PyTime_t _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round) { - _PyTime_t tp; + PyTime_t tp; if(pytime_from_double(&tp, seconds, round, SEC_TO_NS) < 0) { return -1; } @@ -689,14 +663,14 @@ _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round) } -static _PyTime_t -pytime_divide_round_up(const _PyTime_t t, const _PyTime_t k) +static PyTime_t +pytime_divide_round_up(const PyTime_t t, const PyTime_t k) { assert(k > 1); if (t >= 0) { // Don't use (t + k - 1) / k to avoid integer overflow - // if t is equal to _PyTime_MAX - _PyTime_t q = t / k; + // if t is equal to PyTime_MAX + PyTime_t q = t / k; if (t % k) { q += 1; } @@ -704,8 +678,8 @@ pytime_divide_round_up(const _PyTime_t t, const _PyTime_t k) } else { // Don't use (t - (k - 1)) / k to avoid integer overflow - // if t is equals to _PyTime_MIN. - _PyTime_t q = t / k; + // if t is equals to PyTime_MIN. + PyTime_t q = t / k; if (t % k) { q -= 1; } @@ -714,15 +688,15 @@ pytime_divide_round_up(const _PyTime_t t, const _PyTime_t k) } -static _PyTime_t -pytime_divide(const _PyTime_t t, const _PyTime_t k, +static PyTime_t +pytime_divide(const PyTime_t t, const PyTime_t k, const _PyTime_round_t round) { assert(k > 1); if (round == _PyTime_ROUND_HALF_EVEN) { - _PyTime_t x = t / k; - _PyTime_t r = t % k; - _PyTime_t abs_r = Py_ABS(r); + PyTime_t x = t / k; + PyTime_t r = t % k; + PyTime_t abs_r = Py_ABS(r); if (abs_r > k / 2 || (abs_r == k / 2 && (Py_ABS(x) & 1))) { if (t >= 0) { x++; @@ -759,17 +733,17 @@ pytime_divide(const _PyTime_t t, const _PyTime_t k, // Compute (t / k, t % k) in (pq, pr). // Make sure that 0 <= pr < k. // Return 0 on success. -// Return -1 on underflow and store (_PyTime_MIN, 0) in (pq, pr). +// Return -1 on underflow and store (PyTime_MIN, 0) in (pq, pr). static int -pytime_divmod(const _PyTime_t t, const _PyTime_t k, - _PyTime_t *pq, _PyTime_t *pr) +pytime_divmod(const PyTime_t t, const PyTime_t k, + PyTime_t *pq, PyTime_t *pr) { assert(k > 1); - _PyTime_t q = t / k; - _PyTime_t r = t % k; + PyTime_t q = t / k; + PyTime_t r = t % k; if (r < 0) { - if (q == _PyTime_MIN) { - *pq = _PyTime_MIN; + if (q == PyTime_MIN) { + *pq = PyTime_MIN; *pr = 0; return -1; } @@ -784,47 +758,36 @@ pytime_divmod(const _PyTime_t t, const _PyTime_t k, } -_PyTime_t -_PyTime_AsNanoseconds(_PyTime_t t) -{ - return pytime_as_nanoseconds(t); -} - - #ifdef MS_WINDOWS -_PyTime_t -_PyTime_As100Nanoseconds(_PyTime_t t, _PyTime_round_t round) +PyTime_t +_PyTime_As100Nanoseconds(PyTime_t ns, _PyTime_round_t round) { - _PyTime_t ns = pytime_as_nanoseconds(t); return pytime_divide(ns, NS_TO_100NS, round); } #endif -_PyTime_t -_PyTime_AsMicroseconds(_PyTime_t t, _PyTime_round_t round) +PyTime_t +_PyTime_AsMicroseconds(PyTime_t ns, _PyTime_round_t round) { - _PyTime_t ns = pytime_as_nanoseconds(t); return pytime_divide(ns, NS_TO_US, round); } -_PyTime_t -_PyTime_AsMilliseconds(_PyTime_t t, _PyTime_round_t round) +PyTime_t +_PyTime_AsMilliseconds(PyTime_t ns, _PyTime_round_t round) { - _PyTime_t ns = pytime_as_nanoseconds(t); return pytime_divide(ns, NS_TO_MS, round); } static int -pytime_as_timeval(_PyTime_t t, _PyTime_t *ptv_sec, int *ptv_usec, +pytime_as_timeval(PyTime_t ns, PyTime_t *ptv_sec, int *ptv_usec, _PyTime_round_t round) { - _PyTime_t ns = pytime_as_nanoseconds(t); - _PyTime_t us = pytime_divide(ns, US_TO_NS, round); + PyTime_t us = pytime_divide(ns, US_TO_NS, round); - _PyTime_t tv_sec, tv_usec; + PyTime_t tv_sec, tv_usec; int res = pytime_divmod(us, SEC_TO_US, &tv_sec, &tv_usec); *ptv_sec = tv_sec; *ptv_usec = (int)tv_usec; @@ -833,16 +796,16 @@ pytime_as_timeval(_PyTime_t t, _PyTime_t *ptv_sec, int *ptv_usec, static int -pytime_as_timeval_struct(_PyTime_t t, struct timeval *tv, +pytime_as_timeval_struct(PyTime_t t, struct timeval *tv, _PyTime_round_t round, int raise_exc) { - _PyTime_t tv_sec; + PyTime_t tv_sec; int tv_usec; int res = pytime_as_timeval(t, &tv_sec, &tv_usec, round); int res2; #ifdef MS_WINDOWS // On Windows, timeval.tv_sec type is long - res2 = _PyTime_AsLong(tv_sec, &tv->tv_sec); + res2 = _PyTime_AsCLong(tv_sec, &tv->tv_sec); #else res2 = _PyTime_AsTime_t(tv_sec, &tv->tv_sec); #endif @@ -860,24 +823,24 @@ pytime_as_timeval_struct(_PyTime_t t, struct timeval *tv, int -_PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) +_PyTime_AsTimeval(PyTime_t t, struct timeval *tv, _PyTime_round_t round) { return pytime_as_timeval_struct(t, tv, round, 1); } void -_PyTime_AsTimeval_clamp(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) +_PyTime_AsTimeval_clamp(PyTime_t t, struct timeval *tv, _PyTime_round_t round) { (void)pytime_as_timeval_struct(t, tv, round, 0); } int -_PyTime_AsTimevalTime_t(_PyTime_t t, time_t *p_secs, int *us, +_PyTime_AsTimevalTime_t(PyTime_t t, time_t *p_secs, int *us, _PyTime_round_t round) { - _PyTime_t secs; + PyTime_t secs; if (pytime_as_timeval(t, &secs, us, round) < 0) { pytime_time_t_overflow(); return -1; @@ -893,10 +856,9 @@ _PyTime_AsTimevalTime_t(_PyTime_t t, time_t *p_secs, int *us, #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) static int -pytime_as_timespec(_PyTime_t t, struct timespec *ts, int raise_exc) +pytime_as_timespec(PyTime_t ns, struct timespec *ts, int raise_exc) { - _PyTime_t ns = pytime_as_nanoseconds(t); - _PyTime_t tv_sec, tv_nsec; + PyTime_t tv_sec, tv_nsec; int res = pytime_divmod(ns, SEC_TO_NS, &tv_sec, &tv_nsec); int res2 = _PyTime_AsTime_t(tv_sec, &ts->tv_sec); @@ -913,21 +875,22 @@ pytime_as_timespec(_PyTime_t t, struct timespec *ts, int raise_exc) } void -_PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts) +_PyTime_AsTimespec_clamp(PyTime_t t, struct timespec *ts) { (void)pytime_as_timespec(t, ts, 0); } int -_PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) +_PyTime_AsTimespec(PyTime_t t, struct timespec *ts) { return pytime_as_timespec(t, ts, 1); } #endif +// N.B. If raise_exc=0, this may be called without the GIL. static int -py_get_system_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) +py_get_system_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { assert(info == NULL || raise_exc); @@ -941,8 +904,8 @@ py_get_system_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) /* 11,644,473,600,000,000,000: number of nanoseconds between the 1st january 1601 and the 1st january 1970 (369 years + 89 leap days). */ - _PyTime_t ns = large.QuadPart * 100 - 11644473600000000000; - *tp = pytime_from_nanoseconds(ns); + PyTime_t ns = large.QuadPart * 100 - 11644473600000000000; + *tp = ns; if (info) { DWORD timeAdjustment, timeIncrement; BOOL isTimeAdjustmentDisabled, ok; @@ -1037,10 +1000,10 @@ py_get_system_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) } -_PyTime_t -_PyTime_GetSystemClock(void) +PyTime_t +_PyTime_TimeUnchecked(void) { - _PyTime_t t; + PyTime_t t; if (py_get_system_clock(&t, NULL, 0) < 0) { // If clock_gettime(CLOCK_REALTIME) or gettimeofday() fails: // silently ignore the failure and return 0. @@ -1051,7 +1014,17 @@ _PyTime_GetSystemClock(void) int -_PyTime_GetSystemClockWithInfo(_PyTime_t *t, _Py_clock_info_t *info) +PyTime_Time(PyTime_t *result) +{ + if (py_get_system_clock(result, NULL, 1) < 0) { + *result = 0; + return -1; + } + return 0; +} + +int +_PyTime_TimeWithInfo(PyTime_t *t, _Py_clock_info_t *info) { return py_get_system_clock(t, info, 1); } @@ -1067,13 +1040,13 @@ py_mach_timebase_info(_PyTimeFraction *base, int raise) (void)mach_timebase_info(&timebase); // Check that timebase.numer and timebase.denom can be casted to - // _PyTime_t. In practice, timebase uses uint32_t, so casting cannot + // PyTime_t. In practice, timebase uses uint32_t, so casting cannot // overflow. At the end, only make sure that the type is uint32_t - // (_PyTime_t is 64-bit long). - Py_BUILD_ASSERT(sizeof(timebase.numer) <= sizeof(_PyTime_t)); - Py_BUILD_ASSERT(sizeof(timebase.denom) <= sizeof(_PyTime_t)); - _PyTime_t numer = (_PyTime_t)timebase.numer; - _PyTime_t denom = (_PyTime_t)timebase.denom; + // (PyTime_t is 64-bit long). + Py_BUILD_ASSERT(sizeof(timebase.numer) <= sizeof(PyTime_t)); + Py_BUILD_ASSERT(sizeof(timebase.denom) <= sizeof(PyTime_t)); + PyTime_t numer = (PyTime_t)timebase.numer; + PyTime_t denom = (PyTime_t)timebase.denom; // Known time bases: // @@ -1092,23 +1065,24 @@ py_mach_timebase_info(_PyTimeFraction *base, int raise) #endif +// N.B. If raise_exc=0, this may be called without the GIL. static int -py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) +py_get_monotonic_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { assert(info == NULL || raise_exc); #if defined(MS_WINDOWS) ULONGLONG ticks = GetTickCount64(); - static_assert(sizeof(ticks) <= sizeof(_PyTime_t), - "ULONGLONG is larger than _PyTime_t"); - _PyTime_t t; - if (ticks <= (ULONGLONG)_PyTime_MAX) { - t = (_PyTime_t)ticks; + static_assert(sizeof(ticks) <= sizeof(PyTime_t), + "ULONGLONG is larger than PyTime_t"); + PyTime_t t; + if (ticks <= (ULONGLONG)PyTime_MAX) { + t = (PyTime_t)ticks; } else { - // GetTickCount64() maximum is larger than _PyTime_t maximum: - // ULONGLONG is unsigned, whereas _PyTime_t is signed. - t = _PyTime_MAX; + // GetTickCount64() maximum is larger than PyTime_t maximum: + // ULONGLONG is unsigned, whereas PyTime_t is signed. + t = PyTime_MAX; } int res = pytime_mul(&t, MS_TO_NS); @@ -1151,16 +1125,14 @@ py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) uint64_t uticks = mach_absolute_time(); // unsigned => signed - assert(uticks <= (uint64_t)_PyTime_MAX); - _PyTime_t ticks = (_PyTime_t)uticks; + assert(uticks <= (uint64_t)PyTime_MAX); + PyTime_t ticks = (PyTime_t)uticks; - _PyTime_t ns = _PyTimeFraction_Mul(ticks, &base); - *tp = pytime_from_nanoseconds(ns); + PyTime_t ns = _PyTimeFraction_Mul(ticks, &base); + *tp = ns; #elif defined(__hpux) - hrtime_t time; - - time = gethrtime(); + hrtime_t time = gethrtime(); if (time == -1) { if (raise_exc) { PyErr_SetFromErrno(PyExc_OSError); @@ -1168,7 +1140,7 @@ py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) return -1; } - *tp = pytime_from_nanoseconds(time); + *tp = time; if (info) { info->implementation = "gethrtime()"; @@ -1216,10 +1188,10 @@ py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) } -_PyTime_t -_PyTime_GetMonotonicClock(void) +PyTime_t +_PyTime_MonotonicUnchecked(void) { - _PyTime_t t; + PyTime_t t; if (py_get_monotonic_clock(&t, NULL, 0) < 0) { // If mach_timebase_info(), clock_gettime() or gethrtime() fails: // silently ignore the failure and return 0. @@ -1230,7 +1202,18 @@ _PyTime_GetMonotonicClock(void) int -_PyTime_GetMonotonicClockWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +PyTime_Monotonic(PyTime_t *result) +{ + if (py_get_monotonic_clock(result, NULL, 1) < 0) { + *result = 0; + return -1; + } + return 0; +} + + +int +_PyTime_MonotonicWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { return py_get_monotonic_clock(tp, info, 1); } @@ -1250,8 +1233,8 @@ py_win_perf_counter_frequency(_PyTimeFraction *base, int raise) // Since Windows XP, frequency cannot be zero. assert(frequency >= 1); - Py_BUILD_ASSERT(sizeof(_PyTime_t) == sizeof(frequency)); - _PyTime_t denom = (_PyTime_t)frequency; + Py_BUILD_ASSERT(sizeof(PyTime_t) == sizeof(frequency)); + PyTime_t denom = (PyTime_t)frequency; // Known QueryPerformanceFrequency() values: // @@ -1268,8 +1251,9 @@ py_win_perf_counter_frequency(_PyTimeFraction *base, int raise) } +// N.B. If raise_exc=0, this may be called without the GIL. static int -py_get_win_perf_counter(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) +py_get_win_perf_counter(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { assert(info == NULL || raise_exc); @@ -1291,35 +1275,35 @@ py_get_win_perf_counter(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) QueryPerformanceCounter(&now); LONGLONG ticksll = now.QuadPart; - /* Make sure that casting LONGLONG to _PyTime_t cannot overflow, + /* Make sure that casting LONGLONG to PyTime_t cannot overflow, both types are signed */ - _PyTime_t ticks; + PyTime_t ticks; static_assert(sizeof(ticksll) <= sizeof(ticks), - "LONGLONG is larger than _PyTime_t"); - ticks = (_PyTime_t)ticksll; + "LONGLONG is larger than PyTime_t"); + ticks = (PyTime_t)ticksll; - _PyTime_t ns = _PyTimeFraction_Mul(ticks, &base); - *tp = pytime_from_nanoseconds(ns); + PyTime_t ns = _PyTimeFraction_Mul(ticks, &base); + *tp = ns; return 0; } #endif // MS_WINDOWS int -_PyTime_GetPerfCounterWithInfo(_PyTime_t *t, _Py_clock_info_t *info) +_PyTime_PerfCounterWithInfo(PyTime_t *t, _Py_clock_info_t *info) { #ifdef MS_WINDOWS return py_get_win_perf_counter(t, info, 1); #else - return _PyTime_GetMonotonicClockWithInfo(t, info); + return _PyTime_MonotonicWithInfo(t, info); #endif } -_PyTime_t -_PyTime_GetPerfCounter(void) +PyTime_t +_PyTime_PerfCounterUnchecked(void) { - _PyTime_t t; + PyTime_t t; int res; #ifdef MS_WINDOWS res = py_get_win_perf_counter(&t, NULL, 0); @@ -1335,6 +1319,25 @@ _PyTime_GetPerfCounter(void) } +int +PyTime_PerfCounter(PyTime_t *result) +{ + int res; +#ifdef MS_WINDOWS + res = py_get_win_perf_counter(result, NULL, 1); +#else + res = py_get_monotonic_clock(result, NULL, 1); +#endif + if (res < 0) { + // If py_win_perf_counter_frequency() or py_get_monotonic_clock() + // fails: silently ignore the failure and return 0. + *result = 0; + return -1; + } + return 0; +} + + int _PyTime_localtime(time_t t, struct tm *tm) { @@ -1402,17 +1405,17 @@ _PyTime_gmtime(time_t t, struct tm *tm) } -_PyTime_t -_PyDeadline_Init(_PyTime_t timeout) +PyTime_t +_PyDeadline_Init(PyTime_t timeout) { - _PyTime_t now = _PyTime_GetMonotonicClock(); + PyTime_t now = _PyTime_MonotonicUnchecked(); return _PyTime_Add(now, timeout); } -_PyTime_t -_PyDeadline_Get(_PyTime_t deadline) +PyTime_t +_PyDeadline_Get(PyTime_t deadline) { - _PyTime_t now = _PyTime_GetMonotonicClock(); + PyTime_t now = _PyTime_MonotonicUnchecked(); return deadline - now; } diff --git a/Python/qsbr.c b/Python/qsbr.c new file mode 100644 index 000000000000000..7f7ae03cf60d22f --- /dev/null +++ b/Python/qsbr.c @@ -0,0 +1,286 @@ +/* + * Implementation of safe memory reclamation scheme using + * quiescent states. + * + * This is dervied from the "GUS" safe memory reclamation technique + * in FreeBSD written by Jeffrey Roberson. It is heavily modified. Any bugs + * in this code are likely due to the modifications. + * + * The original copyright is preserved below. + * + * Copyright (c) 2019,2020 Jeffrey Roberson <jeff@FreeBSD.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "Python.h" +#include "pycore_initconfig.h" // _PyStatus_NO_MEMORY() +#include "pycore_lock.h" // PyMutex_Lock() +#include "pycore_qsbr.h" +#include "pycore_pystate.h" // _PyThreadState_GET() + + +// Wrap-around safe comparison. This is a holdover from the FreeBSD +// implementation, which uses 32-bit sequence numbers. We currently use 64-bit +// sequence numbers, so wrap-around is unlikely. +#define QSBR_LT(a, b) ((int64_t)((a)-(b)) < 0) +#define QSBR_LEQ(a, b) ((int64_t)((a)-(b)) <= 0) + +// Starting size of the array of qsbr thread states +#define MIN_ARRAY_SIZE 8 + +// For _Py_qsbr_deferred_advance(): the number of deferrals before advancing +// the write sequence. +#define QSBR_DEFERRED_LIMIT 10 + +// Allocate a QSBR thread state from the freelist +static struct _qsbr_thread_state * +qsbr_allocate(struct _qsbr_shared *shared) +{ + struct _qsbr_thread_state *qsbr = shared->freelist; + if (qsbr == NULL) { + return NULL; + } + shared->freelist = qsbr->freelist_next; + qsbr->freelist_next = NULL; + qsbr->shared = shared; + qsbr->allocated = true; + return qsbr; +} + +// Initialize (or reintialize) the freelist of QSBR thread states +static void +initialize_new_array(struct _qsbr_shared *shared) +{ + for (Py_ssize_t i = 0; i != shared->size; i++) { + struct _qsbr_thread_state *qsbr = &shared->array[i].qsbr; + if (qsbr->tstate != NULL) { + // Update the thread state pointer to its QSBR state + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)qsbr->tstate; + tstate->qsbr = qsbr; + } + if (!qsbr->allocated) { + // Push to freelist + qsbr->freelist_next = shared->freelist; + shared->freelist = qsbr; + } + } +} + +// Grow the array of QSBR thread states. Returns 0 on success, -1 on failure. +static int +grow_thread_array(struct _qsbr_shared *shared) +{ + Py_ssize_t new_size = shared->size * 2; + if (new_size < MIN_ARRAY_SIZE) { + new_size = MIN_ARRAY_SIZE; + } + + struct _qsbr_pad *array = PyMem_RawCalloc(new_size, sizeof(*array)); + if (array == NULL) { + return -1; + } + + struct _qsbr_pad *old = shared->array; + if (old != NULL) { + memcpy(array, shared->array, shared->size * sizeof(*array)); + } + + shared->array = array; + shared->size = new_size; + shared->freelist = NULL; + initialize_new_array(shared); + + PyMem_RawFree(old); + return 0; +} + +uint64_t +_Py_qsbr_advance(struct _qsbr_shared *shared) +{ + // NOTE: with 64-bit sequence numbers, we don't have to worry too much + // about the wr_seq getting too far ahead of rd_seq, but if we ever use + // 32-bit sequence numbers, we'll need to be more careful. + return _Py_atomic_add_uint64(&shared->wr_seq, QSBR_INCR) + QSBR_INCR; +} + +uint64_t +_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr) +{ + if (++qsbr->deferrals < QSBR_DEFERRED_LIMIT) { + return _Py_qsbr_shared_current(qsbr->shared) + QSBR_INCR; + } + qsbr->deferrals = 0; + return _Py_qsbr_advance(qsbr->shared); +} + +static uint64_t +qsbr_poll_scan(struct _qsbr_shared *shared) +{ + // Synchronize with store in _Py_qsbr_attach(). We need to ensure that + // the reads from each thread's sequence number are not reordered to see + // earlier "offline" states. + _Py_atomic_fence_seq_cst(); + + // Compute the minimum sequence number of all attached threads + uint64_t min_seq = _Py_atomic_load_uint64(&shared->wr_seq); + struct _qsbr_pad *array = shared->array; + for (Py_ssize_t i = 0, size = shared->size; i != size; i++) { + struct _qsbr_thread_state *qsbr = &array[i].qsbr; + + uint64_t seq = _Py_atomic_load_uint64(&qsbr->seq); + if (seq != QSBR_OFFLINE && QSBR_LT(seq, min_seq)) { + min_seq = seq; + } + } + + // Update the shared read sequence + uint64_t rd_seq = _Py_atomic_load_uint64(&shared->rd_seq); + if (QSBR_LT(rd_seq, min_seq)) { + // It's okay if the compare-exchange failed: another thread updated it + (void)_Py_atomic_compare_exchange_uint64(&shared->rd_seq, &rd_seq, min_seq); + rd_seq = min_seq; + } + + return rd_seq; +} + +bool +_Py_qsbr_poll(struct _qsbr_thread_state *qsbr, uint64_t goal) +{ + assert(_PyThreadState_GET()->state == _Py_THREAD_ATTACHED); + + uint64_t rd_seq = _Py_atomic_load_uint64(&qsbr->shared->rd_seq); + if (QSBR_LEQ(goal, rd_seq)) { + return true; + } + + rd_seq = qsbr_poll_scan(qsbr->shared); + return QSBR_LEQ(goal, rd_seq); +} + +void +_Py_qsbr_attach(struct _qsbr_thread_state *qsbr) +{ + assert(qsbr->seq == 0 && "already attached"); + + uint64_t seq = _Py_qsbr_shared_current(qsbr->shared); + _Py_atomic_store_uint64(&qsbr->seq, seq); // needs seq_cst +} + +void +_Py_qsbr_detach(struct _qsbr_thread_state *qsbr) +{ + assert(qsbr->seq != 0 && "already detached"); + + _Py_atomic_store_uint64_release(&qsbr->seq, QSBR_OFFLINE); +} + +Py_ssize_t +_Py_qsbr_reserve(PyInterpreterState *interp) +{ + struct _qsbr_shared *shared = &interp->qsbr; + + PyMutex_Lock(&shared->mutex); + // Try allocating from our internal freelist + struct _qsbr_thread_state *qsbr = qsbr_allocate(shared); + + // If there are no free entries, we pause all threads, grow the array, + // and update the pointers in PyThreadState to entries in the new array. + if (qsbr == NULL) { + _PyEval_StopTheWorld(interp); + if (grow_thread_array(shared) == 0) { + qsbr = qsbr_allocate(shared); + } + _PyEval_StartTheWorld(interp); + } + PyMutex_Unlock(&shared->mutex); + + if (qsbr == NULL) { + return -1; + } + + // Return an index rather than the pointer because the array may be + // resized and the pointer invalidated. + return (struct _qsbr_pad *)qsbr - shared->array; +} + +void +_Py_qsbr_register(_PyThreadStateImpl *tstate, PyInterpreterState *interp, + Py_ssize_t index) +{ + // Associate the QSBR state with the thread state + struct _qsbr_shared *shared = &interp->qsbr; + + PyMutex_Lock(&shared->mutex); + struct _qsbr_thread_state *qsbr = &interp->qsbr.array[index].qsbr; + assert(qsbr->allocated && qsbr->tstate == NULL); + qsbr->tstate = (PyThreadState *)tstate; + tstate->qsbr = qsbr; + PyMutex_Unlock(&shared->mutex); +} + +void +_Py_qsbr_unregister(_PyThreadStateImpl *tstate) +{ + struct _qsbr_thread_state *qsbr = tstate->qsbr; + struct _qsbr_shared *shared = qsbr->shared; + + assert(qsbr->seq == 0 && "thread state must be detached"); + + PyMutex_Lock(&shared->mutex); + assert(qsbr->allocated && qsbr->tstate == (PyThreadState *)tstate); + tstate->qsbr = NULL; + qsbr->tstate = NULL; + qsbr->allocated = false; + qsbr->freelist_next = shared->freelist; + shared->freelist = qsbr; + PyMutex_Unlock(&shared->mutex); +} + +void +_Py_qsbr_fini(PyInterpreterState *interp) +{ + struct _qsbr_shared *shared = &interp->qsbr; + PyMem_RawFree(shared->array); + shared->array = NULL; + shared->size = 0; + shared->freelist = NULL; +} + +void +_Py_qsbr_after_fork(_PyThreadStateImpl *tstate) +{ + struct _qsbr_thread_state *this_qsbr = tstate->qsbr; + struct _qsbr_shared *shared = this_qsbr->shared; + + _PyMutex_at_fork_reinit(&shared->mutex); + + for (Py_ssize_t i = 0; i != shared->size; i++) { + struct _qsbr_thread_state *qsbr = &shared->array[i].qsbr; + if (qsbr != this_qsbr && qsbr->allocated) { + qsbr->tstate = NULL; + qsbr->allocated = false; + qsbr->freelist_next = shared->freelist; + shared->freelist = qsbr; + } + } +} diff --git a/Python/specialize.c b/Python/specialize.c index a9efbe0453b94e2..f83d8a9ceb0282d 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -17,6 +17,7 @@ #include <stdlib.h> // rand() +extern const char *_PyUOpName(int index); /* For guidance on adding or extending families of instructions see * ./adaptive.md @@ -235,23 +236,24 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop); fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call); fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence); + fprintf(out, "Executors invalidated: %" PRIu64 "\n", stats->executors_invalidated); print_histogram(out, "Trace length", stats->trace_length_hist); print_histogram(out, "Trace run length", stats->trace_run_length_hist); print_histogram(out, "Optimized trace length", stats->optimized_trace_length_hist); + fprintf(out, "Optimization optimizer attempts: %" PRIu64 "\n", stats->optimizer_attempts); + fprintf(out, "Optimization optimizer successes: %" PRIu64 "\n", stats->optimizer_successes); + fprintf(out, "Optimization optimizer failure no memory: %" PRIu64 "\n", + stats->optimizer_failure_reason_no_memory); + const char* const* names; - for (int i = 0; i < 512; i++) { - if (i < 256) { - names = _PyOpcode_OpName; - } else { - names = _PyOpcode_uop_name; - } + for (int i = 0; i <= MAX_UOP_ID; i++) { if (stats->opcode[i].execution_count) { - fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", names[i], stats->opcode[i].execution_count); + fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].execution_count); } if (stats->opcode[i].miss) { - fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", names[i], stats->opcode[i].miss); + fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].miss); } } @@ -275,6 +277,8 @@ print_rare_event_stats(FILE *out, RareEventStats *stats) fprintf(out, "Rare event (set_eval_frame_func): %" PRIu64 "\n", stats->set_eval_frame_func); fprintf(out, "Rare event (builtin_dict): %" PRIu64 "\n", stats->builtin_dict); fprintf(out, "Rare event (func_modification): %" PRIu64 "\n", stats->func_modification); + fprintf(out, "Rare event (watched_dict_modification): %" PRIu64 "\n", stats->watched_dict_modification); + fprintf(out, "Rare event (watched_globals_modification): %" PRIu64 "\n", stats->watched_globals_modification); } static void @@ -540,6 +544,7 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_METHOD_WRAPPER 28 #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29 #define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30 +#define SPEC_FAIL_CALL_METACLASS 31 /* COMPARE_OP */ #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12 @@ -1757,6 +1762,10 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL); return -1; } + if (Py_TYPE(tp) != &PyType_Type) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_METACLASS); + return -1; + } if (tp->tp_new == PyBaseObject_Type.tp_new) { PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); if (type_get_version(tp, CALL) == 0) { diff --git a/Python/structmember.c b/Python/structmember.c index 7a5a6a49d231167..ba881d18a0973dd 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -2,6 +2,8 @@ /* Map C struct members to Python object attributes */ #include "Python.h" +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_long.h" // _PyLong_IsNegative() PyObject * @@ -197,45 +199,67 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) WARN("Truncation of value to int"); break; } - case Py_T_UINT:{ - unsigned long ulong_val = PyLong_AsUnsignedLong(v); - if ((ulong_val == (unsigned long)-1) && PyErr_Occurred()) { - /* XXX: For compatibility, accept negative int values - as well. */ - PyErr_Clear(); - ulong_val = PyLong_AsLong(v); - if ((ulong_val == (unsigned long)-1) && - PyErr_Occurred()) + case Py_T_UINT: { + /* XXX: For compatibility, accept negative int values + as well. */ + v = _PyNumber_Index(v); + if (v == NULL) { + return -1; + } + if (_PyLong_IsNegative((PyLongObject *)v)) { + long long_val = PyLong_AsLong(v); + Py_DECREF(v); + if (long_val == -1 && PyErr_Occurred()) { return -1; - *(unsigned int *)addr = (unsigned int)ulong_val; + } + *(unsigned int *)addr = (unsigned int)(unsigned long)long_val; WARN("Writing negative value into unsigned field"); - } else - *(unsigned int *)addr = (unsigned int)ulong_val; - if (ulong_val > UINT_MAX) - WARN("Truncation of value to unsigned int"); - break; } + else { + unsigned long ulong_val = PyLong_AsUnsignedLong(v); + Py_DECREF(v); + if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { + return -1; + } + *(unsigned int*)addr = (unsigned int)ulong_val; + if (ulong_val > UINT_MAX) { + WARN("Truncation of value to unsigned int"); + } + } + break; + } case Py_T_LONG:{ *(long*)addr = PyLong_AsLong(v); if ((*(long*)addr == -1) && PyErr_Occurred()) return -1; break; } - case Py_T_ULONG:{ - *(unsigned long*)addr = PyLong_AsUnsignedLong(v); - if ((*(unsigned long*)addr == (unsigned long)-1) - && PyErr_Occurred()) { - /* XXX: For compatibility, accept negative int values - as well. */ - PyErr_Clear(); - *(unsigned long*)addr = PyLong_AsLong(v); - if ((*(unsigned long*)addr == (unsigned long)-1) - && PyErr_Occurred()) + case Py_T_ULONG: { + /* XXX: For compatibility, accept negative int values + as well. */ + v = _PyNumber_Index(v); + if (v == NULL) { + return -1; + } + if (_PyLong_IsNegative((PyLongObject *)v)) { + long long_val = PyLong_AsLong(v); + Py_DECREF(v); + if (long_val == -1 && PyErr_Occurred()) { return -1; + } + *(unsigned long *)addr = (unsigned long)long_val; WARN("Writing negative value into unsigned field"); } - break; + else { + unsigned long ulong_val = PyLong_AsUnsignedLong(v); + Py_DECREF(v); + if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { + return -1; + } + *(unsigned long*)addr = ulong_val; } + break; + } case Py_T_PYSSIZET:{ *(Py_ssize_t*)addr = PyLong_AsSsize_t(v); if ((*(Py_ssize_t*)addr == (Py_ssize_t)-1) @@ -284,18 +308,30 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) return -1; break; } - case Py_T_ULONGLONG:{ - unsigned long long value; - /* ??? PyLong_AsLongLong accepts an int, but PyLong_AsUnsignedLongLong - doesn't ??? */ - if (PyLong_Check(v)) - *(unsigned long long*)addr = value = PyLong_AsUnsignedLongLong(v); - else - *(unsigned long long*)addr = value = PyLong_AsLong(v); - if ((value == (unsigned long long)-1) && PyErr_Occurred()) + case Py_T_ULONGLONG: { + v = _PyNumber_Index(v); + if (v == NULL) { return -1; - break; } + if (_PyLong_IsNegative((PyLongObject *)v)) { + long long_val = PyLong_AsLong(v); + Py_DECREF(v); + if (long_val == -1 && PyErr_Occurred()) { + return -1; + } + *(unsigned long long *)addr = (unsigned long long)(long long)long_val; + WARN("Writing negative value into unsigned field"); + } + else { + unsigned long long ulonglong_val = PyLong_AsUnsignedLongLong(v); + Py_DECREF(v); + if (ulonglong_val == (unsigned long long)-1 && PyErr_Occurred()) { + return -1; + } + *(unsigned long long*)addr = ulonglong_val; + } + break; + } default: PyErr_Format(PyExc_SystemError, "bad memberdescr type for %s", l->name); diff --git a/Python/symtable.c b/Python/symtable.c index 743029956e32faf..b69452bf77c5179 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -758,6 +758,8 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, { PyObject *k, *v; Py_ssize_t pos = 0; + int remove_dunder_class = 0; + while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { // skip comprehension parameter long comp_flags = PyLong_AS_LONG(v); @@ -779,6 +781,19 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, if (!existing) { // name does not exist in scope, copy from comprehension assert(scope != FREE || PySet_Contains(comp_free, k) == 1); + if (scope == FREE && ste->ste_type == ClassBlock && + _PyUnicode_EqualToASCIIString(k, "__class__")) { + // if __class__ is unbound in the enclosing class scope and free + // in the comprehension scope, it needs special handling; just + // letting it be marked as free in class scope will break due to + // drop_class_free + scope = GLOBAL_IMPLICIT; + only_flags &= ~DEF_FREE; + if (PySet_Discard(comp_free, k) < 0) { + return 0; + } + remove_dunder_class = 1; + } PyObject *v_flags = PyLong_FromLong(only_flags); if (v_flags == NULL) { return 0; @@ -803,6 +818,10 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, } } } + comp->ste_free = PySet_Size(comp_free) > 0; + if (remove_dunder_class && PyDict_DelItemString(comp->ste_symbols, "__class__") < 0) { + return 0; + } return 1; } @@ -1340,16 +1359,22 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, } static long -symtable_lookup(struct symtable *st, PyObject *name) +symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name) { PyObject *mangled = _Py_Mangle(st->st_private, name); if (!mangled) return 0; - long ret = _PyST_GetSymbol(st->st_cur, mangled); + long ret = _PyST_GetSymbol(ste, mangled); Py_DECREF(mangled); return ret; } +static long +symtable_lookup(struct symtable *st, PyObject *name) +{ + return symtable_lookup_entry(st, st->st_cur, name); +} + static int symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _symtable_entry *ste, int lineno, int col_offset, int end_lineno, int end_col_offset) @@ -1990,7 +2015,7 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) * binding conflict with iteration variables, otherwise skip it */ if (ste->ste_comprehension) { - long target_in_scope = _PyST_GetSymbol(ste, target_name); + long target_in_scope = symtable_lookup_entry(st, ste, target_name); if ((target_in_scope & DEF_COMP_ITER) && (target_in_scope & DEF_LOCAL)) { PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_CONFLICT, target_name); @@ -2006,7 +2031,7 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) /* If we find a FunctionBlock entry, add as GLOBAL/LOCAL or NONLOCAL/LOCAL */ if (ste->ste_type == FunctionBlock) { - long target_in_scope = _PyST_GetSymbol(ste, target_name); + long target_in_scope = symtable_lookup_entry(st, ste, target_name); if (target_in_scope & DEF_GLOBAL) { if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e))) VISIT_QUIT(st, 0); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index f558a00a6916ebc..1bfd031fdd26b21 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1878,7 +1878,15 @@ _PySys_GetSizeOf(PyObject *o) return (size_t)-1; } - return (size_t)size + _PyType_PreHeaderSize(Py_TYPE(o)); + size_t presize = 0; + if (!Py_IS_TYPE(o, &PyType_Type) || + PyType_HasFeature((PyTypeObject *)o, Py_TPFLAGS_HEAPTYPE)) + { + /* Add the size of the pre-header if "o" is not a static type */ + presize = _PyType_PreHeaderSize(Py_TYPE(o)); + } + + return (size_t)size + presize; } static PyObject * @@ -2119,6 +2127,22 @@ sys__clear_type_cache_impl(PyObject *module) Py_RETURN_NONE; } +/*[clinic input] +sys._clear_internal_caches + +Clear all internal performance-related caches. +[clinic start generated code]*/ + +static PyObject * +sys__clear_internal_caches_impl(PyObject *module) +/*[clinic end generated code: output=0ee128670a4966d6 input=253e741ca744f6e8]*/ +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _Py_Executors_InvalidateAll(interp, 0); + PyType_ClearCache(); + Py_RETURN_NONE; +} + /* Note that, for now, we do not have a per-interpreter equivalent for sys.is_finalizing(). */ @@ -2453,6 +2477,7 @@ static PyMethodDef sys_methods[] = { {"audit", _PyCFunction_CAST(sys_audit), METH_FASTCALL, audit_doc }, {"breakpointhook", _PyCFunction_CAST(sys_breakpointhook), METH_FASTCALL | METH_KEYWORDS, breakpointhook_doc}, + SYS__CLEAR_INTERNAL_CACHES_METHODDEF SYS__CLEAR_TYPE_CACHE_METHODDEF SYS__CURRENT_FRAMES_METHODDEF SYS__CURRENT_EXCEPTIONS_METHODDEF diff --git a/Python/thread.c b/Python/thread.c index fefae8391617f7d..b31d1dc5e770eff 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -20,8 +20,8 @@ // Define PY_TIMEOUT_MAX constant. #ifdef _POSIX_THREADS - // PyThread_acquire_lock_timed() uses _PyTime_FromNanoseconds(us * 1000), - // convert microseconds to nanoseconds. + // PyThread_acquire_lock_timed() uses (us * 1000) to convert microseconds + // to nanoseconds. # define PY_TIMEOUT_MAX_VALUE (LLONG_MAX / 1000) #elif defined (NT_THREADS) // WaitForSingleObject() accepts timeout in milliseconds in the range @@ -107,7 +107,7 @@ PyThread_ParseTimeoutArg(PyObject *arg, int blocking, PY_TIMEOUT_T *timeout_p) return -1; } - _PyTime_t timeout; + PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, arg, _PyTime_ROUND_TIMEOUT) < 0) { return -1; } @@ -132,14 +132,14 @@ PyThread_acquire_lock_timed_with_retries(PyThread_type_lock lock, PY_TIMEOUT_T timeout) { PyThreadState *tstate = _PyThreadState_GET(); - _PyTime_t endtime = 0; + PyTime_t endtime = 0; if (timeout > 0) { endtime = _PyDeadline_Init(timeout); } PyLockStatus r; do { - _PyTime_t microseconds; + PyTime_t microseconds; microseconds = _PyTime_AsMicroseconds(timeout, _PyTime_ROUND_CEILING); /* first a simple non-blocking try without releasing the GIL */ diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 14b9cddc24c0ec3..9dca833ff203ca9 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -1,4 +1,5 @@ -#include "pycore_interp.h" // _PyInterpreterState.threads.stacksize +#include "pycore_interp.h" // _PyInterpreterState.threads.stacksize +#include "pycore_time.h" // _PyTime_AsMicroseconds() /* This code implemented by Dag.Gruneau@elsa.preseco.comm.se */ /* Fast NonRecursiveMutex support by Yakov Markovitch, markovitch@iso.ru */ @@ -76,16 +77,16 @@ EnterNonRecursiveMutex(PNRMUTEX mutex, DWORD milliseconds) } } else if (milliseconds != 0) { /* wait at least until the deadline */ - _PyTime_t nanoseconds = _PyTime_FromNanoseconds((_PyTime_t)milliseconds * 1000000); - _PyTime_t deadline = _PyTime_Add(_PyTime_GetPerfCounter(), nanoseconds); + PyTime_t nanoseconds = (PyTime_t)milliseconds * (1000 * 1000); + PyTime_t deadline = _PyTime_Add(_PyTime_PerfCounterUnchecked(), nanoseconds); while (mutex->locked) { - _PyTime_t microseconds = _PyTime_AsMicroseconds(nanoseconds, + PyTime_t microseconds = _PyTime_AsMicroseconds(nanoseconds, _PyTime_ROUND_TIMEOUT); if (PyCOND_TIMEDWAIT(&mutex->cv, &mutex->cs, microseconds) < 0) { result = WAIT_FAILED; break; } - nanoseconds = deadline - _PyTime_GetPerfCounter(); + nanoseconds = deadline - _PyTime_PerfCounterUnchecked(); if (nanoseconds <= 0) { break; } @@ -242,10 +243,6 @@ PyThread_detach_thread(PyThread_handle_t handle) { return (CloseHandle(hThread) == 0); } -void -PyThread_update_thread_after_fork(PyThread_ident_t* ident, PyThread_handle_t* handle) { -} - /* * Return the thread Id instead of a handle. The Id is said to uniquely identify the * thread in the system @@ -444,16 +441,7 @@ PyThread_set_key_value(int key, void *value) void * PyThread_get_key_value(int key) { - /* because TLS is used in the Py_END_ALLOW_THREAD macro, - * it is necessary to preserve the windows error state, because - * it is assumed to be preserved across the call to the macro. - * Ideally, the macro should be fixed, but it is simpler to - * do it here. - */ - DWORD error = GetLastError(); - void *result = TlsGetValue(key); - SetLastError(error); - return result; + return TlsGetValue(key); } void @@ -525,14 +513,10 @@ void * PyThread_tss_get(Py_tss_t *key) { assert(key != NULL); - /* because TSS is used in the Py_END_ALLOW_THREAD macro, - * it is necessary to preserve the windows error state, because - * it is assumed to be preserved across the call to the macro. - * Ideally, the macro should be fixed, but it is simpler to - * do it here. - */ - DWORD error = GetLastError(); - void *result = TlsGetValue(key->_key); - SetLastError(error); - return result; + int err = GetLastError(); + void *r = TlsGetValue(key->_key); + if (r || !GetLastError()) { + SetLastError(err); + } + return r; } diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index fb3b79fc160502f..64cc60053e6cf7a 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -1,5 +1,6 @@ #include "pycore_interp.h" // _PyInterpreterState.threads.stacksize #include "pycore_pythread.h" // _POSIX_SEMAPHORES +#include "pycore_time.h" // _PyTime_FromMicrosecondsClamup() /* Posix threads interface */ @@ -149,16 +150,16 @@ _PyThread_cond_init(PyCOND_T *cond) void _PyThread_cond_after(long long us, struct timespec *abs) { - _PyTime_t timeout = _PyTime_FromMicrosecondsClamp(us); - _PyTime_t t; + PyTime_t timeout = _PyTime_FromMicrosecondsClamp(us); + PyTime_t t; #ifdef CONDATTR_MONOTONIC if (condattr_monotonic) { - t = _PyTime_GetMonotonicClock(); + t = _PyTime_MonotonicUnchecked(); } else #endif { - t = _PyTime_GetSystemClock(); + t = _PyTime_TimeUnchecked(); } t = _PyTime_Add(t, timeout); _PyTime_AsTimespec_clamp(t, abs); @@ -339,16 +340,6 @@ PyThread_detach_thread(PyThread_handle_t th) { return pthread_detach((pthread_t) th); } -void -PyThread_update_thread_after_fork(PyThread_ident_t* ident, PyThread_handle_t* handle) { - // The thread id might have been updated in the forked child - pthread_t th = pthread_self(); - *ident = (PyThread_ident_t) th; - *handle = (PyThread_handle_t) th; - assert(th == (pthread_t) *ident); - assert(th == (pthread_t) *handle); -} - /* XXX This implementation is considered (to quote Tim Peters) "inherently hosed" because: - It does not guarantee the promise that a non-zero integer is returned. @@ -491,31 +482,31 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, (void) error; /* silence unused-but-set-variable warning */ - _PyTime_t timeout; // relative timeout + PyTime_t timeout; // relative timeout if (microseconds >= 0) { // bpo-41710: PyThread_acquire_lock_timed() cannot report timeout // overflow to the caller, so clamp the timeout to - // [_PyTime_MIN, _PyTime_MAX]. + // [PyTime_MIN, PyTime_MAX]. // - // _PyTime_MAX nanoseconds is around 292.3 years. + // PyTime_MAX nanoseconds is around 292.3 years. // // _thread.Lock.acquire() and _thread.RLock.acquire() raise an // OverflowError if microseconds is greater than PY_TIMEOUT_MAX. timeout = _PyTime_FromMicrosecondsClamp(microseconds); } else { - timeout = _PyTime_FromNanoseconds(-1); + timeout = -1; } #ifdef HAVE_SEM_CLOCKWAIT struct timespec abs_timeout; // Local scope for deadline { - _PyTime_t deadline = _PyTime_Add(_PyTime_GetMonotonicClock(), timeout); + PyTime_t deadline = _PyTime_Add(_PyTime_MonotonicUnchecked(), timeout); _PyTime_AsTimespec_clamp(deadline, &abs_timeout); } #else - _PyTime_t deadline = 0; + PyTime_t deadline = 0; if (timeout > 0 && !intr_flag) { deadline = _PyDeadline_Init(timeout); } @@ -527,7 +518,7 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, status = fix_status(sem_clockwait(thelock, CLOCK_MONOTONIC, &abs_timeout)); #else - _PyTime_t abs_time = _PyTime_Add(_PyTime_GetSystemClock(), + PyTime_t abs_time = _PyTime_Add(_PyTime_TimeUnchecked(), timeout); struct timespec ts; _PyTime_AsTimespec_clamp(abs_time, &ts); diff --git a/Python/tier2_engine.md b/Python/tier2_engine.md new file mode 100644 index 000000000000000..df9f6c124509bdf --- /dev/null +++ b/Python/tier2_engine.md @@ -0,0 +1,150 @@ +# The tier 2 execution engine + +## General idea + +When execution in tier 1 becomes "hot", that is the counter for that point in +the code reaches some threshold, we create an executor and execute that +instead of the tier 1 bytecode. + +Since each executor must exit, we also track the "hotness" of those +exits and attach new executors to those exits. + +As the program executes, and the hot parts of the program get optimized, +a graph of executors forms. + +## Superblocks and Executors + +Once a point in the code has become hot enough, we want to optimize it. +Starting from that point we project the likely path of execution, +using information gathered by tier 1 to guide that projection to +form a "superblock", a mostly linear sequence of micro-ops. +Although mostly linear, it may include a single loop. + +We then optimize this superblock to form an optimized superblock, +which is equivalent but more efficient. + +A superblock is a representation of the code we want to execute, +but it is not in executable form. +The executable form is known as an executor. + +Executors are semantically equivalent to the superblock they are +created from, but are in a form that can be efficiently executable. + +There are two execution engines for executors, and two types of executors: +* The hardware which runs machine code executors created by the JIT compiler. +* The tier 2 interpreter runs bytecode executors. + +It would be very wasteful to support both a tier 2 interpreter and +JIT compiler in the same process. +For now, we will make the choice of engine a configuration option, +but we could make it a command line option in the future if that would prove useful. + + +### Tier 2 Interpreter + +For platforms without a JIT and for testing, we need an interpreter +for executors. It is similar in design to the tier 1 interpreter, but has a +different instruction set, and does not adapt. + +### JIT compiler + +The JIT compiler converts superblocks into machine code executors. +These have identical behavior to interpreted executors, except that +they consume more memory for the generated machine code and are a lot faster. + +## Transfering control + +There are three types of control transfer that we need to consider: +* Tier 1 to tier 2 +* Tier 2 to tier 1 +* One executor to another within tier 2 + +Since we expect the graph of executors to span most of the hot +part of the program, transfers from one executor to another should +be the most common. +Therefore, we want to make those transfers fast. + +### Tier 2 to tier 2 + +#### Cold exits + +All side exits start cold and most stay cold, but a few become +hot. We want to keep the memory consumption small for the many +cold exits, but those that become hot need to be fast. +However we cannot know in advance, which will be which. + +So that tier 2 to tier 2 transfers are fast for hot exits, +exits must be implemented as executors. In order to patch +executor exits when they get hot, a pointer to the current +executor must be passed to the exit executor. + +#### Handling reference counts + +There must be an implicit reference to the currently executing +executor, otherwise it might be freed. +Consequently, we must increment the reference count of an +executor just before executing it, and decrement it just after +executing it. + +We want to minimize the amount of data that is passed from +one executor to the next. In the JIT, this reduces the number +of arguments in the tailcall, freeing up registers for other uses. +It is less important in the interpreter, but following the same +design as the JIT simplifies debugging and is good for performance. + +Provided that we incref the new executor before executing it, we +can jump directly to the code of the executor, without needing +to pass a reference to that executor object. +However, we do need a reference to the previous executor, +so that it can be decref'd and for handling of cold exits. +To avoid messing up the JIT's register allocation, we pass a +reference to the previous executor in the thread state's +`previous_executor` field. + +#### The interpreter + +The tier 2 interpreter has a variable `current_executor` which +points to the currently live executor. When transfering from executor +`A` to executor `B` we do the following: +(Initially `current_executor` points to `A`, and the refcount of +`A` is elevated by one) + +1. Set the instruction pointer to start at the beginning of `B` +2. Increment the reference count of `B` +3. Start executing `B` + +We also make the first instruction in `B` do the following: +1. Set `current_executor` to point to `B` +2. Decrement the reference count of `A` (`A` is referenced by `tstate->previous_executor`) + +The net effect of the above is to safely decrement the refcount of `A`, +increment the refcount of `B` and set `current_executor` to point to `B`. + +#### In the JIT + +Transfering control from one executor to another is done via tailcalls. + +The compiled executor should do the same, except that there is no local +variable `current_executor`. + +### Tier 1 to tier 2 + +Since the executor doesn't know if the previous code was tier 1 or tier 2, +we need to make a transfer from tier 1 to tier 2 look like a tier 2 to tier 2 +transfer to the executor. + +We can then perform a tier 1 to tier 2 transfer by setting `current_executor` +to `None`, and then performing a tier 2 to tier 2 transfer as above. + +### Tier 2 to tier 1 + +Each micro-op that might exit to tier 1 contains a `target` value, +which is the offset of the tier 1 instruction to exit to in the +current code object. + +## Counters + +TO DO. +The implementation will change soon, so there is no point in +documenting it until then. + diff --git a/README.rst b/README.rst index fbfae16a7dbb0b1..fd3e6dd1771ae17 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.13.0 alpha 3 +This is Python version 3.13.0 alpha 4 ===================================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg @@ -161,15 +161,6 @@ For information about building Python's documentation, refer to `Doc/README.rst <https://github.com/python/cpython/blob/main/Doc/README.rst>`_. -Converting From Python 2.x to 3.x ---------------------------------- - -Significant backward incompatible changes were made for the release of Python -3.0, which may cause programs written for Python 2 to fail when run with Python -3. For more information about porting your code from Python 2 to Python 3, see -the `Porting HOWTO <https://docs.python.org/3/howto/pyporting.html>`_. - - Testing ------- diff --git a/Tools/README b/Tools/README index 9c4b6d86e990ba1..09bd6fb4798950c 100644 --- a/Tools/README +++ b/Tools/README @@ -10,8 +10,6 @@ c-analyzer Tools to check no new global variables have been added. cases_generator Tooling to generate interpreters. -ccbench A Python threads-based concurrency benchmark. (*) - clinic A preprocessor for CPython C files in order to automate the boilerplate involved with writing argument parsing code for "builtins". @@ -28,8 +26,6 @@ i18n Tools for internationalization. pygettext.py importbench A set of micro-benchmarks for various import scenarios. -iobench Benchmark for the new Python I/O system. (*) - msi Support for packaging Python as an MSI package on Windows. nuget Files for the NuGet package manager for .NET. @@ -45,9 +41,6 @@ scripts A number of useful single-file programs, e.g. run_tests.py ssl Scripts to generate ssl_data.h from OpenSSL sources, and run tests against multiple installations of OpenSSL and LibreSSL. -stringbench A suite of micro-benchmarks for various operations on - strings (both 8-bit and unicode). (*) - tz A script to dump timezone from /usr/share/zoneinfo. unicode Tools for generating unicodedata and codecs from unicode.org @@ -60,6 +53,4 @@ unittestgui A Tkinter based GUI test runner for unittest, with test wasm Config and helpers to facilitate cross compilation of CPython to WebAssembly (WASM). -(*) A generic benchmark suite is maintained separately at https://github.com/python/performance - Note: The pynche color editor has moved to https://gitlab.com/warsaw/pynche diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index aceb13f141cba40..201c81c4d14d793 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -53,13 +53,14 @@ class PackageFiles(typing.NamedTuple): # values to 'exclude' if we create new files within tracked # directories that aren't sourced from third-party packages. PACKAGE_TO_FILES = { - # NOTE: pip's entry in this structure is automatically generated in - # the 'discover_pip_sbom_package()' function below. "mpdecimal": PackageFiles( include=["Modules/_decimal/libmpdec/**"] ), "expat": PackageFiles( - include=["Modules/expat/**"] + include=["Modules/expat/**"], + exclude=[ + "Modules/expat/expat_config.h", + ] ), "macholib": PackageFiles( include=["Lib/ctypes/macholib/**"], @@ -124,268 +125,21 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]: return sorted([line.split()[-1] for line in git_check_ignore_lines if line.startswith("::")]) -def fetch_package_metadata_from_pypi(project: str, version: str, filename: str | None = None) -> tuple[str, str] | None: - """ - Fetches the SHA256 checksum and download location from PyPI. - If we're given a filename then we match with that, otherwise we use wheels. - """ - # Get pip's download location from PyPI. Check that the checksum is correct too. - try: - raw_text = urlopen(f"https://pypi.org/pypi/{project}/{version}/json").read() - release_metadata = json.loads(raw_text) - url: dict[str, typing.Any] - - # Look for a matching artifact filename and then check - # its remote checksum to the local one. - for url in release_metadata["urls"]: - # pip can only use Python-only dependencies, so there's - # no risk of picking the 'incorrect' wheel here. - if ( - (filename is None and url["packagetype"] == "bdist_wheel") - or (filename is not None and url["filename"] == filename) - ): - break - else: - raise ValueError(f"No matching filename on PyPI for '{filename}'") - - # Successfully found the download URL for the matching artifact. - download_url = url["url"] - checksum_sha256 = url["digests"]["sha256"] - return download_url, checksum_sha256 - - except (OSError, ValueError) as e: - # Fail if we're running in CI where we should have an internet connection. - error_if( - "CI" in os.environ, - f"Couldn't fetch metadata for project '{project}' from PyPI: {e}" - ) - return None - - -def find_ensurepip_pip_wheel() -> pathlib.Path | None: - """Try to find the pip wheel bundled in ensurepip. If missing return None""" - - ensurepip_bundled_dir = CPYTHON_ROOT_DIR / "Lib/ensurepip/_bundled" - - pip_wheels = [] - try: - for wheel_filename in os.listdir(ensurepip_bundled_dir): - if wheel_filename.startswith("pip-"): - pip_wheels.append(wheel_filename) - else: - print(f"Unexpected wheel in ensurepip: '{wheel_filename}'") - sys.exit(1) - - # Ignore this error, likely caused by downstream distributors - # deleting the 'ensurepip/_bundled' directory. - except FileNotFoundError: - pass - - if len(pip_wheels) == 0: - return None - elif len(pip_wheels) > 1: - print("Multiple pip wheels detected in 'Lib/ensurepip/_bundled'") - sys.exit(1) - # Otherwise return the one pip wheel. - return ensurepip_bundled_dir / pip_wheels[0] - - -def maybe_remove_pip_and_deps_from_sbom(sbom_data: dict[str, typing.Any]) -> None: - """ - Removes pip and its dependencies from the SBOM data - if the pip wheel is removed from ensurepip. This is done - by redistributors of Python and pip. - """ - - # If there's a wheel we don't remove anything. - if find_ensurepip_pip_wheel() is not None: - return - - # Otherwise we traverse the relationships - # to find dependent packages to remove. - sbom_pip_spdx_id = spdx_id("SPDXRef-PACKAGE-pip") - sbom_spdx_ids_to_remove = {sbom_pip_spdx_id} - - # Find all package SPDXIDs that pip depends on. - for sbom_relationship in sbom_data["relationships"]: - if ( - sbom_relationship["relationshipType"] == "DEPENDS_ON" - and sbom_relationship["spdxElementId"] == sbom_pip_spdx_id - ): - sbom_spdx_ids_to_remove.add(sbom_relationship["relatedSpdxElement"]) - - # Remove all the packages and relationships. - sbom_data["packages"] = [ - sbom_package for sbom_package in sbom_data["packages"] - if sbom_package["SPDXID"] not in sbom_spdx_ids_to_remove - ] - sbom_data["relationships"] = [ - sbom_relationship for sbom_relationship in sbom_data["relationships"] - if sbom_relationship["relatedSpdxElement"] not in sbom_spdx_ids_to_remove - ] - - -def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None: - """pip is a part of a packaging ecosystem (Python, surprise!) so it's actually - automatable to discover the metadata we need like the version and checksums - so let's do that on behalf of our friends at the PyPA. This function also - discovers vendored packages within pip and fetches their metadata. - """ - global PACKAGE_TO_FILES - - pip_wheel_filepath = find_ensurepip_pip_wheel() - if pip_wheel_filepath is None: - return # There's no pip wheel, nothing to discover. - - # Add the wheel filename to the list of files so the SBOM file - # and relationship generator can work its magic on the wheel too. - PACKAGE_TO_FILES["pip"] = PackageFiles( - include=[str(pip_wheel_filepath.relative_to(CPYTHON_ROOT_DIR))] - ) - - # Wheel filename format puts the version right after the project name. - pip_version = pip_wheel_filepath.name.split("-")[1] - pip_checksum_sha256 = hashlib.sha256( - pip_wheel_filepath.read_bytes() - ).hexdigest() - - pip_metadata = fetch_package_metadata_from_pypi( - project="pip", - version=pip_version, - filename=pip_wheel_filepath.name, - ) - # We couldn't fetch any metadata from PyPI, - # so we give up on verifying if we're not in CI. - if pip_metadata is None: - return - - pip_download_url, pip_actual_sha256 = pip_metadata - if pip_actual_sha256 != pip_checksum_sha256: - raise ValueError("Unexpected") - - # Parse 'pip/_vendor/vendor.txt' from the wheel for sub-dependencies. - with zipfile.ZipFile(pip_wheel_filepath) as whl: - vendor_txt_data = whl.read("pip/_vendor/vendor.txt").decode() - - # With this version regex we're assuming that pip isn't using pre-releases. - # If any version doesn't match we get a failure below, so we're safe doing this. - version_pin_re = re.compile(r"^([a-zA-Z0-9_.-]+)==([0-9.]*[0-9])$") - sbom_pip_dependency_spdx_ids = set() - for line in vendor_txt_data.splitlines(): - line = line.partition("#")[0].strip() # Strip comments and whitespace. - if not line: # Skip empty lines. - continue - - # Non-empty lines we must be able to match. - match = version_pin_re.match(line) - error_if(match is None, f"Couldn't parse line from pip vendor.txt: '{line}'") - assert match is not None # Make mypy happy. - - # Parse out and normalize the project name. - project_name, project_version = match.groups() - project_name = project_name.lower() - - # At this point if pip's metadata fetch succeeded we should - # expect this request to also succeed. - project_metadata = ( - fetch_package_metadata_from_pypi(project_name, project_version) - ) - assert project_metadata is not None - project_download_url, project_checksum_sha256 = project_metadata - - # Update our SBOM data with what we received from PyPI. - # Don't overwrite any existing values. - sbom_project_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{project_name}") - sbom_pip_dependency_spdx_ids.add(sbom_project_spdx_id) - for package in sbom_data["packages"]: - if package["SPDXID"] != sbom_project_spdx_id: - continue - - # Only thing missing from this blob is the `licenseConcluded`, - # that needs to be triaged by human maintainers if the list changes. - package.update({ - "SPDXID": sbom_project_spdx_id, - "name": project_name, - "versionInfo": project_version, - "downloadLocation": project_download_url, - "checksums": [ - {"algorithm": "SHA256", "checksumValue": project_checksum_sha256} - ], - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": f"pkg:pypi/{project_name}@{project_version}", - "referenceType": "purl", - }, - ], - "primaryPackagePurpose": "SOURCE" - }) - break - - PACKAGE_TO_FILES[project_name] = PackageFiles(include=None) - - # Remove pip from the existing SBOM packages if it's there - # and then overwrite its entry with our own generated one. - sbom_pip_spdx_id = spdx_id("SPDXRef-PACKAGE-pip") - sbom_data["packages"] = [ - sbom_package - for sbom_package in sbom_data["packages"] - if sbom_package["name"] != "pip" - ] - sbom_data["packages"].append( - { - "SPDXID": sbom_pip_spdx_id, - "name": "pip", - "versionInfo": pip_version, - "originator": "Organization: Python Packaging Authority", - "licenseConcluded": "MIT", - "downloadLocation": pip_download_url, - "checksums": [ - {"algorithm": "SHA256", "checksumValue": pip_checksum_sha256} - ], - "externalRefs": [ - { - "referenceCategory": "SECURITY", - "referenceLocator": f"cpe:2.3:a:pypa:pip:{pip_version}:*:*:*:*:*:*:*", - "referenceType": "cpe23Type", - }, - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": f"pkg:pypi/pip@{pip_version}", - "referenceType": "purl", - }, - ], - "primaryPackagePurpose": "SOURCE", - } - ) - for sbom_dep_spdx_id in sorted(sbom_pip_dependency_spdx_ids): - sbom_data["relationships"].append({ - "spdxElementId": sbom_pip_spdx_id, - "relatedSpdxElement": sbom_dep_spdx_id, - "relationshipType": "DEPENDS_ON" - }) - - def main() -> None: sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json" sbom_data = json.loads(sbom_path.read_bytes()) - # Check if pip should be removed if the wheel is missing. - # We can't reset the SBOM relationship data until checking this. - maybe_remove_pip_and_deps_from_sbom(sbom_data) - # We regenerate all of this information. Package information # should be preserved though since that is edited by humans. sbom_data["files"] = [] sbom_data["relationships"] = [] - # Insert pip's SBOM metadata from the wheel. - discover_pip_sbom_package(sbom_data) - # Ensure all packages in this tool are represented also in the SBOM file. + actual_names = {package["name"] for package in sbom_data["packages"]} + expected_names = set(PACKAGE_TO_FILES) error_if( - {package["name"] for package in sbom_data["packages"]} != set(PACKAGE_TO_FILES), - "Packages defined in SBOM tool don't match those defined in SBOM file.", + actual_names != expected_names, + f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}", ) # Make a bunch of assertions about the SBOM data to ensure it's consistent. @@ -422,8 +176,8 @@ def main() -> None: # License must be on the approved list for SPDX. license_concluded = package["licenseConcluded"] error_if( - license_concluded not in ALLOWED_LICENSE_EXPRESSIONS, - f"License identifier '{license_concluded}' not in SBOM tool allowlist" + license_concluded != "NOASSERTION", + f"License identifier must be 'NOASSERTION'" ) # We call 'sorted()' here a lot to avoid filesystem scan order issues. diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index 5dce4e042d1eb44..588dfda50658be3 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -34,6 +34,7 @@ '_testinternalcapi', '_testmultiphase', '_testsinglephase', + '_testexternalinspection', '_xxsubinterpreters', '_xxinterpchannels', '_xxinterpqueues', diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 85c437d521a15ad..83146622c74f941 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -278,6 +278,13 @@ def gen_ctypes_test(manifest, args, outfile): from _testcapi import get_feature_macros feature_macros = get_feature_macros() + + # Stable ABI is incompatible with Py_TRACE_REFS builds due to PyObject + # layout differences. + # See https://github.com/python/cpython/issues/88299#issuecomment-1113366226 + if feature_macros['Py_TRACE_REFS']: + raise unittest.SkipTest("incompatible with Py_TRACE_REFS.") + ctypes_test = import_module('ctypes') class TestStableABIAvailability(unittest.TestCase): @@ -308,16 +315,11 @@ def test_windows_feature_macros(self): {'function', 'data'}, include_abi_only=True, ) - optional_items = {} + feature_macros = list(manifest.select({'feature_macro'})) + optional_items = {m.name: [] for m in feature_macros} for item in items: - if item.name in ( - # Some symbols aren't exported on all platforms. - # This is a bug: https://bugs.python.org/issue44133 - 'PyModule_Create2', 'PyModule_FromDefAndSpec2', - ): - continue if item.ifdef: - optional_items.setdefault(item.ifdef, []).append(item.name) + optional_items[item.ifdef].append(item.name) else: write(f' "{item.name}",') write(")") @@ -328,7 +330,6 @@ def test_windows_feature_macros(self): write(f" {name!r},") write(" )") write("") - feature_macros = list(manifest.select({'feature_macro'})) feature_names = sorted(m.name for m in feature_macros) write(f"EXPECTED_FEATURE_MACROS = set({pprint.pformat(feature_names)})") diff --git a/Tools/build/verify_ensurepip_wheels.py b/Tools/build/verify_ensurepip_wheels.py index 29897425da6c036..a37da2f70757e58 100755 --- a/Tools/build/verify_ensurepip_wheels.py +++ b/Tools/build/verify_ensurepip_wheels.py @@ -14,7 +14,6 @@ from pathlib import Path from urllib.request import urlopen -PACKAGE_NAMES = ("pip",) ENSURE_PIP_ROOT = Path(__file__).parent.parent.parent / "Lib/ensurepip" WHEEL_DIR = ENSURE_PIP_ROOT / "_bundled" ENSURE_PIP_INIT_PY_TEXT = (ENSURE_PIP_ROOT / "__init__.py").read_text(encoding="utf-8") @@ -97,8 +96,5 @@ def verify_wheel(package_name: str) -> bool: if __name__ == "__main__": - exit_status = 0 - for package_name in PACKAGE_NAMES: - if not verify_wheel(package_name): - exit_status = 1 + exit_status = int(not verify_wheel("pip")) raise SystemExit(exit_status) diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index 781f9a4c8206c82..0c47470a0ecb7aa 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -7,17 +7,10 @@ set here=%~dp0 set rt_opts=-q -d set regrtest_args= set arm32_ssh= +set cmdline_args=%* +set cmdline_args=%cmdline_args:,=#COMMA#% -:CheckOpts -if "%1"=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-arm64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-arm32" (set rt_opts=%rt_opts% %1) & (set arm32_ssh=true) & shift & goto CheckOpts -if "%1"=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="+d" (set rt_opts=%rt_opts:-d=%) & shift & goto CheckOpts -if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts -if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts +call:CheckOpts %cmdline_args% if "%PROCESSOR_ARCHITECTURE%"=="ARM" if "%arm32_ssh%"=="true" goto NativeExecution if "%arm32_ssh%"=="true" goto :Arm32Ssh @@ -49,3 +42,16 @@ echo The test worker should have the SSH agent running. echo Also a key must be created with ssh-keygen and added to both the buildbot worker machine echo and the ARM32 worker device: see https://docs.microsoft.com/en-us/windows/iot-core/connect-your-device/ssh exit /b 127 + +:CheckOpts +set arg="%~1" +if %arg%=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="-arm64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="-arm32" (set rt_opts=%rt_opts% %1) & (set arm32_ssh=true) & shift & goto CheckOpts +if %arg%=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="+d" (set rt_opts=%rt_opts:-d=%) & shift & goto CheckOpts +if %arg%=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts +if NOT %arg%=="" (set regrtest_args=%regrtest_args% %arg:#COMMA#=,%) & shift & goto CheckOpts +goto:eof diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 444063d2148934b..12010f0e9c05491 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -83,9 +83,11 @@ def clean_lines(text): Python/frozen_modules/*.h Python/generated_cases.c.h Python/executor_cases.c.h +Python/optimizer_cases.c.h # not actually source Python/bytecodes.c +Python/optimizer_bytecodes.c # mimalloc Objects/mimalloc/*.c @@ -102,6 +104,7 @@ def clean_lines(text): # The problem with xmlparse.c is that something # has gone wrong where # we handle "maybe inline actual" # in Tools/c-analyzer/c_parser/parser/_global.py. +Modules/expat/internal.h Modules/expat/xmlparse.c ''') @@ -318,6 +321,7 @@ def clean_lines(text): _abs('Objects/stringlib/unicode_format.h'): (10_000, 400), _abs('Objects/typeobject.c'): (35_000, 200), _abs('Python/compile.c'): (20_000, 500), + _abs('Python/optimizer.c'): (100_000, 5_000), _abs('Python/parking_lot.c'): (40_000, 1000), _abs('Python/pylifecycle.c'): (500_000, 5000), _abs('Python/pystate.c'): (500_000, 5000), diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index e3a1b5d532bda2a..45119664af4362a 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -189,6 +189,7 @@ Objects/exceptions.c - _PyExc_ProcessLookupError - Objects/exceptions.c - _PyExc_TimeoutError - Objects/exceptions.c - _PyExc_EOFError - Objects/exceptions.c - _PyExc_RuntimeError - +Objects/exceptions.c - _PyExc_PythonFinalizationError - Objects/exceptions.c - _PyExc_RecursionError - Objects/exceptions.c - _PyExc_NotImplementedError - Objects/exceptions.c - _PyExc_NameError - @@ -197,6 +198,7 @@ Objects/exceptions.c - _PyExc_AttributeError - Objects/exceptions.c - _PyExc_SyntaxError - Objects/exceptions.c - _PyExc_IndentationError - Objects/exceptions.c - _PyExc_TabError - +Objects/exceptions.c - _PyExc_IncompleteInputError - Objects/exceptions.c - _PyExc_LookupError - Objects/exceptions.c - _PyExc_IndexError - Objects/exceptions.c - _PyExc_KeyError - @@ -253,6 +255,7 @@ Objects/exceptions.c - PyExc_ProcessLookupError - Objects/exceptions.c - PyExc_TimeoutError - Objects/exceptions.c - PyExc_EOFError - Objects/exceptions.c - PyExc_RuntimeError - +Objects/exceptions.c - PyExc_PythonFinalizationError - Objects/exceptions.c - PyExc_RecursionError - Objects/exceptions.c - PyExc_NotImplementedError - Objects/exceptions.c - PyExc_NameError - @@ -261,6 +264,7 @@ Objects/exceptions.c - PyExc_AttributeError - Objects/exceptions.c - PyExc_SyntaxError - Objects/exceptions.c - PyExc_IndentationError - Objects/exceptions.c - PyExc_TabError - +Objects/exceptions.c - PyExc_IncompleteInputError - Objects/exceptions.c - PyExc_LookupError - Objects/exceptions.c - PyExc_IndexError - Objects/exceptions.c - PyExc_KeyError - @@ -290,10 +294,10 @@ Objects/exceptions.c - PyExc_UnicodeWarning - Objects/exceptions.c - PyExc_BytesWarning - Objects/exceptions.c - PyExc_ResourceWarning - Objects/exceptions.c - PyExc_EncodingWarning - -Python/crossinterp.c - _PyExc_InterpreterError - -Python/crossinterp.c - _PyExc_InterpreterNotFoundError - -Python/crossinterp.c - PyExc_InterpreterError - -Python/crossinterp.c - PyExc_InterpreterNotFoundError - +Python/crossinterp_exceptions.h - _PyExc_InterpreterError - +Python/crossinterp_exceptions.h - _PyExc_InterpreterNotFoundError - +Python/crossinterp_exceptions.h - PyExc_InterpreterError - +Python/crossinterp_exceptions.h - PyExc_InterpreterNotFoundError - ##----------------------- ## singletons @@ -332,6 +336,7 @@ Modules/_testclinic.c - DeprStarNew - Modules/_testclinic.c - DeprKwdInit - Modules/_testclinic.c - DeprKwdInitNoInline - Modules/_testclinic.c - DeprKwdNew - +Modules/_testclinic.c - TestClass - ################################## diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index c75aff8c1723c10..0f212ecb528ac4e 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -361,6 +361,7 @@ Python/import.c - _PyImport_Inittab - Python/import.c - _PySys_ImplCacheTag - Python/intrinsics.c - _PyIntrinsics_UnaryFunctions - Python/intrinsics.c - _PyIntrinsics_BinaryFunctions - +Python/lock.c - TIME_TO_BE_FAIR_NS - Python/opcode_targets.h - opcode_targets - Python/perf_trampoline.c - _Py_perfmap_callbacks - Python/pyhash.c - PyHash_Func - @@ -382,6 +383,11 @@ Python/optimizer.c - _PyCounterOptimizer_Type - Python/optimizer.c - _PyUOpExecutor_Type - Python/optimizer.c - _PyUOpOptimizer_Type - Python/optimizer.c - _PyOptimizer_Default - +Python/optimizer.c - _ColdExit_Type - +Python/optimizer.c - COLD_EXITS - +Python/optimizer.c - Py_FatalErrorExecutor - +Python/optimizer.c - EMPTY_FILTER - +Python/optimizer.c - cold_exits_initialized - ##----------------------- ## test code @@ -734,6 +740,4 @@ Modules/expat/xmlrole.c - error - ## other Modules/_io/_iomodule.c - _PyIO_Module - Modules/_sqlite/module.c - _sqlite3module - -Python/optimizer_analysis.c - _Py_PartitionRootNode_Type - -Python/optimizer_analysis.c - _Py_UOpsAbstractInterpContext_Type - Modules/clinic/md5module.c.h _md5_md5 _keywords - diff --git a/Tools/cases_generator/README.md b/Tools/cases_generator/README.md index ed802e44f31ad5f..fb512c4646b851a 100644 --- a/Tools/cases_generator/README.md +++ b/Tools/cases_generator/README.md @@ -5,16 +5,33 @@ Documentation for the instruction definitions in `Python/bytecodes.c` What's currently here: +- `analyzer.py`: code for converting `AST` generated by `Parser` + to more high-level structure for easier interaction - `lexer.py`: lexer for C, originally written by Mark Shannon - `plexer.py`: OO interface on top of lexer.py; main class: `PLexer` -- `parsing.py`: Parser for instruction definition DSL; main class `Parser` -- `generate_cases.py`: driver script to read `Python/bytecodes.c` and +- `parsing.py`: Parser for instruction definition DSL; main class: `Parser` +- `parser.py` helper for interactions with `parsing.py` +- `tierN_generator.py`: a couple of driver scripts to read `Python/bytecodes.c` and write `Python/generated_cases.c.h` (and several other files) -- `analysis.py`: `Analyzer` class used to read the input files -- `flags.py`: abstractions related to metadata flags for instructions -- `formatting.py`: `Formatter` class used to write the output files -- `instructions.py`: classes to analyze and write instructions -- `stacking.py`: code to handle generalized stack effects +- `optimizer_generator.py`: reads `Python/bytecodes.c` and + `Python/optimizer_bytecodes.c` and writes + `Python/optimizer_cases.c.h` +- `stack.py`: code to handle generalized stack effects +- `cwriter.py`: code which understands tokens and how to format C code; + main class: `CWriter` +- `generators_common.py`: helpers for generators +- `opcode_id_generator.py`: generate a list of opcodes and write them to + `Include/opcode_ids.h` +- `opcode_metadata_generator.py`: reads the instruction definitions and + write the metadata to `Include/internal/pycore_opcode_metadata.h` +- `py_metadata_generator.py`: reads the instruction definitions and + write the metadata to `Lib/_opcode_metadata.py` +- `target_generator.py`: generate targets for computed goto dispatch and + write them to `Python/opcode_targets.h` +- `uop_id_generator.py`: generate a list of uop IDs and write them to + `Include/internal/pycore_uop_ids.h` +- `uop_metadata_generator.py`: reads the instruction definitions and + write the metadata to `Include/internal/pycore_uop_metadata.h` Note that there is some dummy C code at the top and bottom of `Python/bytecodes.c` diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index b80fa66e2a159ad..b0a15e6d87c2c6c 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -1,6 +1,7 @@ from dataclasses import dataclass, field import lexer import parser +import re from typing import Optional @@ -16,15 +17,16 @@ class Properties: needs_this: bool always_exits: bool stores_sp: bool - tier_one_only: bool uses_co_consts: bool uses_co_names: bool uses_locals: bool has_free: bool - + side_exit: bool pure: bool passthrough: bool - guard: bool + tier: int | None = None + oparg_and_1: bool = False + const_oparg: int = -1 def dump(self, indent: str) -> None: print(indent, end="") @@ -44,14 +46,13 @@ def from_list(properties: list["Properties"]) -> "Properties": needs_this=any(p.needs_this for p in properties), always_exits=any(p.always_exits for p in properties), stores_sp=any(p.stores_sp for p in properties), - tier_one_only=any(p.tier_one_only for p in properties), uses_co_consts=any(p.uses_co_consts for p in properties), uses_co_names=any(p.uses_co_names for p in properties), uses_locals=any(p.uses_locals for p in properties), has_free=any(p.has_free for p in properties), + side_exit=any(p.side_exit for p in properties), pure=all(p.pure for p in properties), passthrough=all(p.passthrough for p in properties), - guard=all(p.guard for p in properties), ) @@ -66,14 +67,13 @@ def from_list(properties: list["Properties"]) -> "Properties": needs_this=False, always_exits=False, stores_sp=False, - tier_one_only=False, uses_co_consts=False, uses_co_names=False, uses_locals=False, has_free=False, + side_exit=False, pure=False, passthrough=False, - guard=False, ) @@ -141,6 +141,8 @@ class Uop: properties: Properties _size: int = -1 implicitly_created: bool = False + replicated = 0 + replicates : "Uop | None" = None def dump(self, indent: str) -> None: print( @@ -271,17 +273,19 @@ def override_error( ) -def convert_stack_item(item: parser.StackEffect) -> StackItem: +def convert_stack_item(item: parser.StackEffect, replace_op_arg_1: str | None) -> StackItem: + cond = item.cond + if replace_op_arg_1 and OPARG_AND_1.match(item.cond): + cond = replace_op_arg_1 return StackItem( - item.name, item.type, item.cond, (item.size or "1"), type_prop=item.type_prop + item.name, item.type, cond, (item.size or "1") ) - -def analyze_stack(op: parser.InstDef) -> StackEffect: +def analyze_stack(op: parser.InstDef, replace_op_arg_1: str | None = None) -> StackEffect: inputs: list[StackItem] = [ - convert_stack_item(i) for i in op.inputs if isinstance(i, parser.StackEffect) + convert_stack_item(i, replace_op_arg_1) for i in op.inputs if isinstance(i, parser.StackEffect) ] - outputs: list[StackItem] = [convert_stack_item(i) for i in op.outputs] + outputs: list[StackItem] = [convert_stack_item(i, replace_op_arg_1) for i in op.outputs] for input, output in zip(inputs, outputs): if input.name == output.name: input.peek = output.peek = True @@ -306,6 +310,15 @@ def variable_used(node: parser.InstDef, name: str) -> bool: token.kind == "IDENTIFIER" and token.text == name for token in node.tokens ) +def tier_variable(node: parser.InstDef) -> int | None: + """Determine whether a tier variable is used in a node.""" + for token in node.tokens: + if token.kind == "ANNOTATION": + if token.text == "specializing": + return 1 + if re.fullmatch(r"tier\d", token.text): + return int(token.text[-1]) + return None def is_infallible(op: parser.InstDef) -> bool: return not ( @@ -444,6 +457,22 @@ def stack_effect_only_peeks(instr: parser.InstDef) -> bool: for s, other in zip(stack_inputs, instr.outputs) ) +OPARG_AND_1 = re.compile("\\(*oparg *& *1") + +def effect_depends_on_oparg_1(op: parser.InstDef) -> bool: + for effect in op.inputs: + if isinstance(effect, parser.CacheEffect): + continue + if not effect.cond: + continue + if OPARG_AND_1.match(effect.cond): + return True + for effect in op.outputs: + if not effect.cond: + continue + if OPARG_AND_1.match(effect.cond): + return True + return False def compute_properties(op: parser.InstDef) -> Properties: has_free = ( @@ -451,13 +480,24 @@ def compute_properties(op: parser.InstDef) -> Properties: or variable_used(op, "PyCell_GET") or variable_used(op, "PyCell_SET") ) + deopts_if = variable_used(op, "DEOPT_IF") + exits_if = variable_used(op, "EXIT_IF") + if deopts_if and exits_if: + tkn = op.tokens[0] + raise lexer.make_syntax_error( + "Op cannot contain both EXIT_IF and DEOPT_IF", + tkn.filename, + tkn.line, + tkn.column, + op.name, + ) infallible = is_infallible(op) - deopts = variable_used(op, "DEOPT_IF") passthrough = stack_effect_only_peeks(op) and infallible return Properties( escapes=makes_escaping_api_call(op), infallible=infallible, - deopts=deopts, + deopts=deopts_if or exits_if, + side_exit=exits_if, oparg=variable_used(op, "oparg"), jumps=variable_used(op, "JUMPBY"), eval_breaker=variable_used(op, "CHECK_EVAL_BREAKER"), @@ -465,7 +505,6 @@ def compute_properties(op: parser.InstDef) -> Properties: needs_this=variable_used(op, "this_instr"), always_exits=always_exits(op), stores_sp=variable_used(op, "SYNC_SP"), - tier_one_only=variable_used(op, "TIER_ONE_ONLY"), uses_co_consts=variable_used(op, "FRAME_CO_CONSTS"), uses_co_names=variable_used(op, "FRAME_CO_NAMES"), uses_locals=(variable_used(op, "GETLOCAL") or variable_used(op, "SETLOCAL")) @@ -473,12 +512,12 @@ def compute_properties(op: parser.InstDef) -> Properties: has_free=has_free, pure="pure" in op.annotations, passthrough=passthrough, - guard=passthrough and deopts, + tier=tier_variable(op), ) -def make_uop(name: str, op: parser.InstDef, inputs: list[parser.InputEffect]) -> Uop: - return Uop( +def make_uop(name: str, op: parser.InstDef, inputs: list[parser.InputEffect], uops: dict[str, Uop]) -> Uop: + result = Uop( name=name, context=op.context, annotations=op.annotations, @@ -487,6 +526,49 @@ def make_uop(name: str, op: parser.InstDef, inputs: list[parser.InputEffect]) -> body=op.block.tokens, properties=compute_properties(op), ) + if effect_depends_on_oparg_1(op) and "split" in op.annotations: + result.properties.oparg_and_1 = True + for bit in ("0", "1"): + name_x = name + "_" + bit + properties = compute_properties(op) + if properties.oparg: + # May not need oparg anymore + properties.oparg = any(token.text == "oparg" for token in op.block.tokens) + rep = Uop( + name=name_x, + context=op.context, + annotations=op.annotations, + stack=analyze_stack(op, bit), + caches=analyze_caches(inputs), + body=op.block.tokens, + properties=properties, + ) + rep.replicates = result + uops[name_x] = rep + for anno in op.annotations: + if anno.startswith("replicate"): + result.replicated = int(anno[10:-1]) + break + else: + return result + for oparg in range(result.replicated): + name_x = name + "_" + str(oparg) + properties = compute_properties(op) + properties.oparg = False + properties.const_oparg = oparg + rep = Uop( + name=name_x, + context=op.context, + annotations=op.annotations, + stack=analyze_stack(op), + caches=analyze_caches(inputs), + body=op.block.tokens, + properties=properties, + ) + rep.replicates = result + uops[name_x] = rep + + return result def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None: @@ -496,7 +578,7 @@ def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None: raise override_error( op.name, op.context, uops[op.name].context, op.tokens[0] ) - uops[op.name] = make_uop(op.name, op, op.inputs) + uops[op.name] = make_uop(op.name, op, op.inputs, uops) def add_instruction( @@ -523,7 +605,7 @@ def desugar_inst( uop_index = len(parts) # Place holder for the uop. parts.append(Skip(0)) - uop = make_uop("_" + inst.name, inst, op_inputs) + uop = make_uop("_" + inst.name, inst, op_inputs, uops) uop.implicitly_created = True uops[inst.name] = uop if uop_index < 0: diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 2fc2ab115321cf2..0b4b99c60768b5c 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -119,7 +119,10 @@ def replace_decrefs( out.emit(f"Py_DECREF({var.name}[_i]);\n") out.emit("}\n") elif var.condition: - out.emit(f"Py_XDECREF({var.name});\n") + if var.condition == "1": + out.emit(f"Py_DECREF({var.name});\n") + elif var.condition != "0": + out.emit(f"Py_XDECREF({var.name});\n") else: out.emit(f"Py_DECREF({var.name});\n") @@ -154,6 +157,7 @@ def replace_check_eval_breaker( REPLACEMENT_FUNCTIONS = { + "EXIT_IF": replace_deopt, "DEOPT_IF": replace_deopt, "ERROR_IF": replace_error, "DECREF_INPUTS": replace_decrefs, @@ -205,6 +209,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_EVAL_BREAK_FLAG") if p.deopts: flags.append("HAS_DEOPT_FLAG") + if p.side_exit: + flags.append("HAS_EXIT_FLAG") if not p.infallible: flags.append("HAS_ERROR_FLAG") if p.escapes: @@ -213,6 +219,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_PURE_FLAG") if p.passthrough: flags.append("HAS_PASSTHROUGH_FLAG") + if p.oparg_and_1: + flags.append("HAS_OPARG_AND_1_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/interpreter_definition.md b/Tools/cases_generator/interpreter_definition.md index e87aff43762b11d..889f58fc3e1a75d 100644 --- a/Tools/cases_generator/interpreter_definition.md +++ b/Tools/cases_generator/interpreter_definition.md @@ -109,10 +109,7 @@ and a piece of C code describing its semantics:: NAME [":" type] [ "if" "(" C-expression ")" ] type: - NAME ["*"] | type_prop - - type_prop: - "&" "(" NAME ["+" NAME] ")" + NAME ["*"] stream: NAME "/" size @@ -142,26 +139,7 @@ The following definitions may occur: The optional `type` in an `object` is the C type. It defaults to `PyObject *`. The objects before the "--" are the objects on top of the stack at the start of the instruction. Those after the "--" are the objects on top of the stack at the -end of the instruction. When prefixed by a `&`, the `type` production rule follows the -`type_prop` production rule. This indicates the type of the value is of that specific type -after the operation. In this case, the type may also contain 64-bit refinement information -that is fetched from a previously defined operand in the instruction header, such as -a type version tag. This follows the format `type + refinement`. The list of possible types -and their refinements are below. They obey the following predicates: - - -* `PYLONG_TYPE`: `Py_TYPE(val) == &PyLong_Type` -* `PYFLOAT_TYPE`: `Py_TYPE(val) == &PyFloat_Type` -* `PYUNICODE_TYPE`: `Py_TYPE(val) == &PYUNICODE_TYPE` -* `NULL_TYPE`: `val == NULL` -* `GUARD_TYPE_VERSION_TYPE`: `type->tp_version_tag == auxillary` -* `GUARD_DORV_VALUES_TYPE`: `_PyDictOrValues_IsValues(obj)` -* `GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_TYPE`: - `_PyDictOrValues_IsValues(obj) || _PyObject_MakeInstanceAttributesFromDict(obj, dorv)` -* `GUARD_KEYS_VERSION_TYPE`: `owner_heap_type->ht_cached_keys->dk_version == auxillary` -* `PYMETHOD_TYPE`: `Py_TYPE(val) == &PyMethod_Type` -* `PYFUNCTION_TYPE_VERSION_TYPE`: - `PyFunction_Check(callable) && func->func_version == auxillary && code->co_argcount == oparg + (self_or_null != NULL)` +end of the instruction. An `inst` without `stack_effect` is a transitional form to allow the original C code @@ -190,6 +168,7 @@ list of annotations and their meanings are as follows: * `override`. For external use by other interpreter definitions to override the current instruction definition. * `pure`. This instruction has no side effects. +* 'tierN'. This instruction only used by tier N interpreter. ### Special functions/macros diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 4f8d01c5492f51b..13aee94f2b957c6 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -222,6 +222,10 @@ def choice(*opts: str) -> str: "register", "replaced", "pure", + "split", + "replicate", + "tier1", + "tier2", } __all__ = [] diff --git a/Tools/cases_generator/opcode_id_generator.py b/Tools/cases_generator/opcode_id_generator.py index dbea3d0b622c87f..5a3009a5c04c27b 100644 --- a/Tools/cases_generator/opcode_id_generator.py +++ b/Tools/cases_generator/opcode_id_generator.py @@ -1,6 +1,6 @@ """Generate the list of opcode IDs. Reads the instruction definitions from bytecodes.c. -Writes the IDs to opcode._ids.h by default. +Writes the IDs to opcode_ids.h by default. """ import argparse diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 1826a0b645c3b86..ab597834a8892fa 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -1,6 +1,6 @@ -"""Generate uop metedata. +"""Generate opcode metadata. Reads the instruction definitions from bytecodes.c. -Writes the metadata to pycore_uop_metadata.h by default. +Writes the metadata to pycore_opcode_metadata.h by default. """ import argparse @@ -50,8 +50,10 @@ "DEOPT", "ERROR", "ESCAPES", + "EXIT", "PURE", "PASSTHROUGH", + "OPARG_AND_1", ] @@ -282,7 +284,7 @@ def is_viable_expansion(inst: Instruction) -> bool: continue if "replaced" in part.annotations: continue - if part.properties.tier_one_only or not part.is_viable(): + if part.properties.tier == 1 or not part.is_viable(): return False return True diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py new file mode 100644 index 000000000000000..d3ce4c8a25f5b85 --- /dev/null +++ b/Tools/cases_generator/optimizer_generator.py @@ -0,0 +1,237 @@ +"""Generate the cases for the tier 2 optimizer. +Reads the instruction definitions from bytecodes.c and optimizer_bytecodes.c +Writes the cases to optimizer_cases.c.h, which is #included in Python/optimizer_analysis.c. +""" + +import argparse +import os.path +import sys + +from analyzer import ( + Analysis, + Instruction, + Uop, + Part, + analyze_files, + Skip, + StackItem, + analysis_error, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + emit_tokens, + emit_to, + replace_sync_sp, +) +from cwriter import CWriter +from typing import TextIO, Iterator +from lexer import Token +from stack import Stack, SizeMismatch, UNUSED + +DEFAULT_OUTPUT = ROOT / "Python/optimizer_cases.c.h" +DEFAULT_ABSTRACT_INPUT = ROOT / "Python/optimizer_bytecodes.c" + + +def validate_uop(override: Uop, uop: Uop) -> None: + # To do + pass + + +def type_name(var: StackItem) -> str: + if var.is_array(): + return f"_Py_UopsSymbol **" + if var.type: + return var.type + return f"_Py_UopsSymbol *" + + +def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: + variables = {"unused"} + if not skip_inputs: + for var in reversed(uop.stack.inputs): + if var.name not in variables: + variables.add(var.name) + if var.condition: + out.emit(f"{type_name(var)}{var.name} = NULL;\n") + else: + out.emit(f"{type_name(var)}{var.name};\n") + for var in uop.stack.outputs: + if var.peek: + continue + if var.name not in variables: + variables.add(var.name) + if var.condition: + out.emit(f"{type_name(var)}{var.name} = NULL;\n") + else: + out.emit(f"{type_name(var)}{var.name};\n") + + +def decref_inputs( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + out.emit_at("", tkn) + + +def emit_default(out: CWriter, uop: Uop) -> None: + for i, var in enumerate(uop.stack.outputs): + if var.name != "unused" and not var.peek: + if var.is_array(): + out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + out.emit(f"{var.name}[_i] = sym_new_unknown(ctx);\n") + out.emit(f"if ({var.name}[_i] == NULL) goto out_of_space;\n") + out.emit("}\n") + elif var.name == "null": + out.emit(f"{var.name} = sym_new_null(ctx);\n") + out.emit(f"if ({var.name} == NULL) goto out_of_space;\n") + else: + out.emit(f"{var.name} = sym_new_unknown(ctx);\n") + out.emit(f"if ({var.name} == NULL) goto out_of_space;\n") + + +def write_uop( + override: Uop | None, + uop: Uop, + out: CWriter, + stack: Stack, + debug: bool, + skip_inputs: bool, +) -> None: + try: + prototype = override if override else uop + is_override = override is not None + out.start_line() + for var in reversed(prototype.stack.inputs): + res = stack.pop(var) + if not skip_inputs: + out.emit(res) + if not prototype.properties.stores_sp: + for i, var in enumerate(prototype.stack.outputs): + res = stack.push(var) + if not var.peek or is_override: + out.emit(res) + if debug: + args = [] + for var in prototype.stack.inputs: + if not var.peek or is_override: + args.append(var.name) + out.emit(f'DEBUG_PRINTF({", ".join(args)});\n') + if override: + for cache in uop.caches: + if cache.name != "unused": + if cache.size == 4: + type = cast = "PyObject *" + else: + type = f"uint{cache.size*16}_t " + cast = f"uint{cache.size*16}_t" + out.emit(f"{type}{cache.name} = ({cast})this_instr->operand;\n") + if override: + replacement_funcs = { + "DECREF_INPUTS": decref_inputs, + "SYNC_SP": replace_sync_sp, + } + emit_tokens(out, override, stack, None, replacement_funcs) + else: + emit_default(out, uop) + + if prototype.properties.stores_sp: + for i, var in enumerate(prototype.stack.outputs): + if not var.peek or is_override: + out.emit(stack.push(var)) + out.start_line() + stack.flush(out, cast_type="_Py_UopsSymbol *") + except SizeMismatch as ex: + raise analysis_error(ex.args[0], uop.body[0]) + + +SKIPS = ("_EXTENDED_ARG",) + + +def generate_abstract_interpreter( + filenames: list[str], + abstract: Analysis, + base: Analysis, + outfile: TextIO, + debug: bool, +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 2, False) + out.emit("\n") + base_uop_names = set([uop.name for uop in base.uops.values()]) + for abstract_uop_name in abstract.uops: + assert abstract_uop_name in base_uop_names,\ + f"All abstract uops should override base uops, but {abstract_uop_name} is not." + + for uop in base.uops.values(): + override: Uop | None = None + if uop.name in abstract.uops: + override = abstract.uops[uop.name] + validate_uop(override, uop) + if uop.properties.tier == 1: + continue + if uop.replicates: + continue + if uop.is_super(): + continue + if not uop.is_viable(): + out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n") + continue + out.emit(f"case {uop.name}: {{\n") + if override: + declare_variables(override, out, skip_inputs=False) + else: + declare_variables(uop, out, skip_inputs=True) + stack = Stack() + write_uop(override, uop, out, stack, debug, skip_inputs=(override is None)) + out.start_line() + out.emit("break;\n") + out.emit("}") + out.emit("\n\n") + + +def generate_tier2_abstract_from_files( + filenames: list[str], outfilename: str, debug: bool=False +) -> None: + assert len(filenames) == 2, "Need a base file and an abstract cases file." + base = analyze_files([filenames[0]]) + abstract = analyze_files([filenames[1]]) + with open(outfilename, "w") as outfile: + generate_abstract_interpreter(filenames, abstract, base, outfile, debug) + + +arg_parser = argparse.ArgumentParser( + description="Generate the code for the tier 2 interpreter.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + + +arg_parser.add_argument("input", nargs=1, help="Abstract interpreter definition file") + +arg_parser.add_argument( + "base", nargs=argparse.REMAINDER, help="The base instruction definition file(s)" +) + +arg_parser.add_argument("-d", "--debug", help="Insert debug calls", action="store_true") + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.base) == 0: + args.input.append(DEFAULT_INPUT) + args.input.append(DEFAULT_ABSTRACT_INPUT) + abstract = analyze_files(args.input) + base = analyze_files(args.base) + with open(args.output, "w") as outfile: + generate_abstract_interpreter(args.input, abstract, base, outfile, args.debug) diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 307919cb37ce1e1..0d54820e4e71fbe 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -75,11 +75,6 @@ class StackEffect(Node): size: str = "" # Optional `[size]` # Note: size cannot be combined with type or cond - # Optional `(type, refinement)` - type_prop: None | tuple[str, None | str] = field( - default_factory=lambda: None, init=True, compare=False, hash=False - ) - def __repr__(self) -> str: items = [self.name, self.type, self.cond, self.size] while items and items[-1] == "": @@ -184,7 +179,13 @@ def inst_header(self) -> InstHeader | None: # | annotation* op(NAME, (inputs -- outputs)) annotations = [] while anno := self.expect(lx.ANNOTATION): - annotations.append(anno.text) + if anno.text == "replicate": + self.require(lx.LPAREN) + times = self.require(lx.NUMBER) + self.require(lx.RPAREN) + annotations.append(f"replicate({times.text})") + else: + annotations.append(anno.text) tkn = self.expect(lx.INST) if not tkn: tkn = self.expect(lx.OP) @@ -260,25 +261,14 @@ def cache_effect(self) -> CacheEffect | None: @contextual def stack_effect(self) -> StackEffect | None: - # IDENTIFIER [':' [IDENTIFIER [TIMES]] ['&' '(' IDENTIFIER ['+' IDENTIFIER] ')']] ['if' '(' expression ')'] + # IDENTIFIER [':' IDENTIFIER [TIMES]] ['if' '(' expression ')'] # | IDENTIFIER '[' expression ']' if tkn := self.expect(lx.IDENTIFIER): type_text = "" - type_prop = None if self.expect(lx.COLON): - if i := self.expect(lx.IDENTIFIER): - type_text = i.text.strip() - if self.expect(lx.TIMES): - type_text += " *" - if self.expect(lx.AND): - consumed_bracket = self.expect(lx.LPAREN) is not None - type_prop_text = self.require(lx.IDENTIFIER).text.strip() - refinement = None - if self.expect(lx.PLUS): - refinement = self.require(lx.IDENTIFIER).text.strip() - type_prop = (type_prop_text, refinement) - if consumed_bracket: - self.require(lx.RPAREN) + type_text = self.require(lx.IDENTIFIER).text.strip() + if self.expect(lx.TIMES): + type_text += " *" cond_text = "" if self.expect(lx.IF): self.require(lx.LPAREN) @@ -295,7 +285,7 @@ def stack_effect(self) -> StackEffect | None: self.require(lx.RBRACKET) type_text = "PyObject **" size_text = size.text.strip() - return StackEffect(tkn.text, type_text, cond_text, size_text, type_prop) + return StackEffect(tkn.text, type_text, cond_text, size_text) return None @contextual diff --git a/Tools/cases_generator/py_metadata_generator.py b/Tools/cases_generator/py_metadata_generator.py index 43811fdacc8a9e1..0dbcd599f9d4d98 100644 --- a/Tools/cases_generator/py_metadata_generator.py +++ b/Tools/cases_generator/py_metadata_generator.py @@ -1,6 +1,6 @@ -"""Generate uop metedata. +"""Generate opcode metadata for Python. Reads the instruction definitions from bytecodes.c. -Writes the metadata to pycore_uop_metadata.h by default. +Writes the metadata to _opcode_metadata.py by default. """ import argparse diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index f62ece43c1be7fe..5aecac39aef5e2d 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -23,8 +23,12 @@ def maybe_parenthesize(sym: str) -> str: def var_size(var: StackItem) -> str: if var.condition: - # Special case simplification - if var.condition == "oparg & 1" and var.size == "1": + # Special case simplifications + if var.condition == "0": + return "0" + elif var.condition == "1": + return var.size + elif var.condition == "oparg & 1" and var.size == "1": return f"({var.condition})" else: return f"(({var.condition}) ? {var.size} : 0)" @@ -154,7 +158,12 @@ def pop(self, var: StackItem) -> str: f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}];" ) if var.condition: - return f"if ({var.condition}) {{ {assign} }}\n" + if var.condition == "1": + return f"{assign}\n" + elif var.condition == "0": + return "" + else: + return f"if ({var.condition}) {{ {assign} }}\n" return f"{assign}\n" def push(self, var: StackItem) -> str: @@ -168,14 +177,17 @@ def push(self, var: StackItem) -> str: self.top_offset.push(var) return "" - def flush(self, out: CWriter) -> None: + def flush(self, out: CWriter, cast_type: str = "PyObject *") -> None: out.start_line() for var in self.variables: if not var.peek: - cast = "(PyObject *)" if var.type else "" + cast = f"({cast_type})" if var.type else "" if var.name not in UNUSED and not var.is_array(): if var.condition: - out.emit(f"if ({var.condition}) ") + if var.condition == "0": + continue + elif var.condition != "1": + out.emit(f"if ({var.condition}) ") out.emit( f"stack_pointer[{self.base_offset.to_c()}] = {cast}{var.name};\n" ) diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 7897b89b2752a72..d8eed1078b09149 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -33,24 +33,29 @@ DEFAULT_OUTPUT = ROOT / "Python/executor_cases.c.h" +def declare_variable( + var: StackItem, uop: Uop, variables: set[str], out: CWriter +) -> None: + if var.name in variables: + return + type = var.type if var.type else "PyObject *" + variables.add(var.name) + if var.condition: + out.emit(f"{type}{var.name} = NULL;\n") + if uop.replicates: + # Replicas may not use all their conditional variables + # So avoid a compiler warning with a fake use + out.emit(f"(void){var.name};\n") + else: + out.emit(f"{type}{var.name};\n") + + def declare_variables(uop: Uop, out: CWriter) -> None: variables = {"unused"} for var in reversed(uop.stack.inputs): - if var.name not in variables: - type = var.type if var.type else "PyObject *" - variables.add(var.name) - if var.condition: - out.emit(f"{type}{var.name} = NULL;\n") - else: - out.emit(f"{type}{var.name};\n") + declare_variable(var, uop, variables, out) for var in uop.stack.outputs: - if var.name not in variables: - variables.add(var.name) - type = var.type if var.type else "PyObject *" - if var.condition: - out.emit(f"{type}{var.name} = NULL;\n") - else: - out.emit(f"{type}{var.name};\n") + declare_variable(var, uop, variables, out) def tier2_replace_error( @@ -98,9 +103,47 @@ def tier2_replace_deopt( out.emit(") goto deoptimize;\n") +def tier2_replace_exit_if( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("if ", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + out.emit(") goto side_exit;\n") + + +def tier2_replace_oparg( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + if not uop.name.endswith("_0") and not uop.name.endswith("_1"): + out.emit(tkn) + return + amp = next(tkn_iter) + if amp.text != "&": + out.emit(tkn) + out.emit(amp) + return + one = next(tkn_iter) + assert one.text == "1" + out.emit_at(uop.name[-1], tkn) + + TIER2_REPLACEMENT_FUNCTIONS = REPLACEMENT_FUNCTIONS.copy() TIER2_REPLACEMENT_FUNCTIONS["ERROR_IF"] = tier2_replace_error TIER2_REPLACEMENT_FUNCTIONS["DEOPT_IF"] = tier2_replace_deopt +TIER2_REPLACEMENT_FUNCTIONS["oparg"] = tier2_replace_oparg +TIER2_REPLACEMENT_FUNCTIONS["EXIT_IF"] = tier2_replace_exit_if def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None: @@ -108,6 +151,10 @@ def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None: out.start_line() if uop.properties.oparg: out.emit("oparg = CURRENT_OPARG();\n") + assert uop.properties.const_oparg < 0 + elif uop.properties.const_oparg >= 0: + out.emit(f"oparg = {uop.properties.const_oparg};\n") + out.emit(f"assert(oparg == CURRENT_OPARG());\n") for var in reversed(uop.stack.inputs): out.emit(stack.pop(var)) if not uop.properties.stores_sp: @@ -147,7 +194,10 @@ def generate_tier2( out = CWriter(outfile, 2, lines) out.emit("\n") for name, uop in analysis.uops.items(): - if uop.properties.tier_one_only: + if uop.properties.tier == 1: + continue + if uop.properties.oparg_and_1: + out.emit(f"/* {uop.name} is split on (oparg & 1) */\n\n") continue if uop.is_super(): continue diff --git a/Tools/cases_generator/uop_id_generator.py b/Tools/cases_generator/uop_id_generator.py index 633249f1c6b1feb..eb5e3f4a324735e 100644 --- a/Tools/cases_generator/uop_id_generator.py +++ b/Tools/cases_generator/uop_id_generator.py @@ -38,15 +38,17 @@ def generate_uop_ids( next_id += 1 PRE_DEFINED = {"_EXIT_TRACE", "_SET_IP"} - for uop in analysis.uops.values(): - if uop.name in PRE_DEFINED: + uops = [(uop.name, uop) for uop in analysis.uops.values()] + # Sort so that _BASE comes immediately before _BASE_0, etc. + for name, uop in sorted(uops): + if name in PRE_DEFINED: continue - if uop.properties.tier_one_only: + if uop.properties.tier == 1: continue - if uop.implicitly_created and not distinct_namespace: - out.emit(f"#define {uop.name} {uop.name[1:]}\n") + if uop.implicitly_created and not distinct_namespace and not uop.replicated: + out.emit(f"#define {name} {name[1:]}\n") else: - out.emit(f"#define {uop.name} {next_id}\n") + out.emit(f"#define {name} {next_id}\n") next_id += 1 out.emit(f"#define MAX_UOP_ID {next_id-1}\n") diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index d4f3a096d2acc18..72eed3041c55c9c 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -1,4 +1,4 @@ -"""Generate uop metedata. +"""Generate uop metadata. Reads the instruction definitions from bytecodes.c. Writes the metadata to pycore_uop_metadata.h by default. """ @@ -24,17 +24,24 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n") + out.emit("extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1];\n") out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") for uop in analysis.uops.values(): - if uop.is_viable() and not uop.properties.tier_one_only: + if uop.is_viable() and uop.properties.tier != 1: out.emit(f"[{uop.name}] = {cflags(uop.properties)},\n") + out.emit("};\n\n") + out.emit("const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = {\n") + for uop in analysis.uops.values(): + if uop.replicated: + out.emit(f"[{uop.name}] = {uop.replicated},\n") + out.emit("};\n\n") out.emit("const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {\n") for uop in sorted(analysis.uops.values(), key=lambda t: t.name): - if uop.is_viable() and not uop.properties.tier_one_only: + if uop.is_viable() and uop.properties.tier != 1: out.emit(f'[{uop.name}] = "{uop.name}",\n') out.emit("};\n") out.emit("#endif // NEED_OPCODE_METADATA\n\n") diff --git a/Tools/ccbench/ccbench.py b/Tools/ccbench/ccbench.py deleted file mode 100644 index d52701a82948daf..000000000000000 --- a/Tools/ccbench/ccbench.py +++ /dev/null @@ -1,606 +0,0 @@ -# This file should be kept compatible with both Python 2.6 and Python >= 3.0. - -from __future__ import division -from __future__ import print_function - -""" -ccbench, a Python concurrency benchmark. -""" - -import time -import os -import sys -import itertools -import threading -import subprocess -import socket -from optparse import OptionParser, SUPPRESS_HELP -import platform - -# Compatibility -try: - xrange -except NameError: - xrange = range - -try: - map = itertools.imap -except AttributeError: - pass - - -THROUGHPUT_DURATION = 2.0 - -LATENCY_PING_INTERVAL = 0.1 -LATENCY_DURATION = 2.0 - -BANDWIDTH_PACKET_SIZE = 1024 -BANDWIDTH_DURATION = 2.0 - - -def task_pidigits(): - """Pi calculation (Python)""" - _map = map - _count = itertools.count - _islice = itertools.islice - - def calc_ndigits(n): - # From http://shootout.alioth.debian.org/ - def gen_x(): - return _map(lambda k: (k, 4*k + 2, 0, 2*k + 1), _count(1)) - - def compose(a, b): - aq, ar, as_, at = a - bq, br, bs, bt = b - return (aq * bq, - aq * br + ar * bt, - as_ * bq + at * bs, - as_ * br + at * bt) - - def extract(z, j): - q, r, s, t = z - return (q*j + r) // (s*j + t) - - def pi_digits(): - z = (1, 0, 0, 1) - x = gen_x() - while 1: - y = extract(z, 3) - while y != extract(z, 4): - z = compose(z, next(x)) - y = extract(z, 3) - z = compose((10, -10*y, 0, 1), z) - yield y - - return list(_islice(pi_digits(), n)) - - return calc_ndigits, (50, ) - -def task_regex(): - """regular expression (C)""" - # XXX this task gives horrendous latency results. - import re - # Taken from the `inspect` module - pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)', re.MULTILINE) - with open(__file__, "r") as f: - arg = f.read(2000) - return pat.findall, (arg, ) - -def task_sort(): - """list sorting (C)""" - def list_sort(l): - l = l[::-1] - l.sort() - - return list_sort, (list(range(1000)), ) - -def task_compress_zlib(): - """zlib compression (C)""" - import zlib - with open(__file__, "rb") as f: - arg = f.read(5000) * 3 - - def compress(s): - zlib.decompress(zlib.compress(s, 5)) - return compress, (arg, ) - -def task_compress_bz2(): - """bz2 compression (C)""" - import bz2 - with open(__file__, "rb") as f: - arg = f.read(3000) * 2 - - def compress(s): - bz2.compress(s) - return compress, (arg, ) - -def task_hashing(): - """SHA1 hashing (C)""" - import hashlib - with open(__file__, "rb") as f: - arg = f.read(5000) * 30 - - def compute(s): - hashlib.sha1(s).digest() - return compute, (arg, ) - - -throughput_tasks = [task_pidigits, task_regex] -for mod in 'bz2', 'hashlib': - try: - globals()[mod] = __import__(mod) - except ImportError: - globals()[mod] = None - -# For whatever reasons, zlib gives irregular results, so we prefer bz2 or -# hashlib if available. -# (NOTE: hashlib releases the GIL from 2.7 and 3.1 onwards) -if bz2 is not None: - throughput_tasks.append(task_compress_bz2) -elif hashlib is not None: - throughput_tasks.append(task_hashing) -else: - throughput_tasks.append(task_compress_zlib) - -latency_tasks = throughput_tasks -bandwidth_tasks = [task_pidigits] - - -class TimedLoop: - def __init__(self, func, args): - self.func = func - self.args = args - - def __call__(self, start_time, min_duration, end_event, do_yield=False): - step = 20 - niters = 0 - duration = 0.0 - _time = time.time - _sleep = time.sleep - _func = self.func - _args = self.args - t1 = start_time - while True: - for i in range(step): - _func(*_args) - t2 = _time() - # If another thread terminated, the current measurement is invalid - # => return the previous one. - if end_event: - return niters, duration - niters += step - duration = t2 - start_time - if duration >= min_duration: - end_event.append(None) - return niters, duration - if t2 - t1 < 0.01: - # Minimize interference of measurement on overall runtime - step = step * 3 // 2 - elif do_yield: - # OS scheduling of Python threads is sometimes so bad that we - # have to force thread switching ourselves, otherwise we get - # completely useless results. - _sleep(0.0001) - t1 = t2 - - -def run_throughput_test(func, args, nthreads): - assert nthreads >= 1 - - # Warm up - func(*args) - - results = [] - loop = TimedLoop(func, args) - end_event = [] - - if nthreads == 1: - # Pure single-threaded performance, without any switching or - # synchronization overhead. - start_time = time.time() - results.append(loop(start_time, THROUGHPUT_DURATION, - end_event, do_yield=False)) - return results - - started = False - ready_cond = threading.Condition() - start_cond = threading.Condition() - ready = [] - - def run(): - with ready_cond: - ready.append(None) - ready_cond.notify() - with start_cond: - while not started: - start_cond.wait() - results.append(loop(start_time, THROUGHPUT_DURATION, - end_event, do_yield=True)) - - threads = [] - for i in range(nthreads): - threads.append(threading.Thread(target=run)) - for t in threads: - t.daemon = True - t.start() - # We don't want measurements to include thread startup overhead, - # so we arrange for timing to start after all threads are ready. - with ready_cond: - while len(ready) < nthreads: - ready_cond.wait() - with start_cond: - start_time = time.time() - started = True - start_cond.notify(nthreads) - for t in threads: - t.join() - - return results - -def run_throughput_tests(max_threads): - for task in throughput_tasks: - print(task.__doc__) - print() - func, args = task() - nthreads = 1 - baseline_speed = None - while nthreads <= max_threads: - results = run_throughput_test(func, args, nthreads) - # Taking the max duration rather than average gives pessimistic - # results rather than optimistic. - speed = sum(r[0] for r in results) / max(r[1] for r in results) - print("threads=%d: %d" % (nthreads, speed), end="") - if baseline_speed is None: - print(" iterations/s.") - baseline_speed = speed - else: - print(" ( %d %%)" % (speed / baseline_speed * 100)) - nthreads += 1 - print() - - -LAT_END = "END" - -def _sendto(sock, s, addr): - sock.sendto(s.encode('ascii'), addr) - -def _recv(sock, n): - return sock.recv(n).decode('ascii') - -def latency_client(addr, nb_pings, interval): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - _time = time.time - _sleep = time.sleep - def _ping(): - _sendto(sock, "%r\n" % _time(), addr) - # The first ping signals the parent process that we are ready. - _ping() - # We give the parent a bit of time to notice. - _sleep(1.0) - for i in range(nb_pings): - _sleep(interval) - _ping() - _sendto(sock, LAT_END + "\n", addr) - finally: - sock.close() - -def run_latency_client(**kwargs): - cmd_line = [sys.executable, '-E', os.path.abspath(__file__)] - cmd_line.extend(['--latclient', repr(kwargs)]) - return subprocess.Popen(cmd_line) #, stdin=subprocess.PIPE, - #stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - -def run_latency_test(func, args, nthreads): - # Create a listening socket to receive the pings. We use UDP which should - # be painlessly cross-platform. - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.bind(("127.0.0.1", 0)) - addr = sock.getsockname() - - interval = LATENCY_PING_INTERVAL - duration = LATENCY_DURATION - nb_pings = int(duration / interval) - - results = [] - threads = [] - end_event = [] - start_cond = threading.Condition() - started = False - if nthreads > 0: - # Warm up - func(*args) - - results = [] - loop = TimedLoop(func, args) - ready = [] - ready_cond = threading.Condition() - - def run(): - with ready_cond: - ready.append(None) - ready_cond.notify() - with start_cond: - while not started: - start_cond.wait() - loop(start_time, duration * 1.5, end_event, do_yield=False) - - for i in range(nthreads): - threads.append(threading.Thread(target=run)) - for t in threads: - t.daemon = True - t.start() - # Wait for threads to be ready - with ready_cond: - while len(ready) < nthreads: - ready_cond.wait() - - # Run the client and wait for the first ping(s) to arrive before - # unblocking the background threads. - chunks = [] - process = run_latency_client(addr=sock.getsockname(), - nb_pings=nb_pings, interval=interval) - s = _recv(sock, 4096) - _time = time.time - - with start_cond: - start_time = _time() - started = True - start_cond.notify(nthreads) - - while LAT_END not in s: - s = _recv(sock, 4096) - t = _time() - chunks.append((t, s)) - - # Tell the background threads to stop. - end_event.append(None) - for t in threads: - t.join() - process.wait() - sock.close() - - for recv_time, chunk in chunks: - # NOTE: it is assumed that a line sent by a client wasn't received - # in two chunks because the lines are very small. - for line in chunk.splitlines(): - line = line.strip() - if line and line != LAT_END: - send_time = eval(line) - assert isinstance(send_time, float) - results.append((send_time, recv_time)) - - return results - -def run_latency_tests(max_threads): - for task in latency_tasks: - print("Background CPU task:", task.__doc__) - print() - func, args = task() - nthreads = 0 - while nthreads <= max_threads: - results = run_latency_test(func, args, nthreads) - n = len(results) - # We print out milliseconds - lats = [1000 * (t2 - t1) for (t1, t2) in results] - #print(list(map(int, lats))) - avg = sum(lats) / n - dev = (sum((x - avg) ** 2 for x in lats) / n) ** 0.5 - print("CPU threads=%d: %d ms. (std dev: %d ms.)" % (nthreads, avg, dev), end="") - print() - #print(" [... from %d samples]" % n) - nthreads += 1 - print() - - -BW_END = "END" - -def bandwidth_client(addr, packet_size, duration): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.bind(("127.0.0.1", 0)) - local_addr = sock.getsockname() - _time = time.time - _sleep = time.sleep - def _send_chunk(msg): - _sendto(sock, ("%r#%s\n" % (local_addr, msg)).rjust(packet_size), addr) - # We give the parent some time to be ready. - _sleep(1.0) - try: - start_time = _time() - end_time = start_time + duration * 2.0 - i = 0 - while _time() < end_time: - _send_chunk(str(i)) - s = _recv(sock, packet_size) - assert len(s) == packet_size - i += 1 - _send_chunk(BW_END) - finally: - sock.close() - -def run_bandwidth_client(**kwargs): - cmd_line = [sys.executable, '-E', os.path.abspath(__file__)] - cmd_line.extend(['--bwclient', repr(kwargs)]) - return subprocess.Popen(cmd_line) #, stdin=subprocess.PIPE, - #stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - -def run_bandwidth_test(func, args, nthreads): - # Create a listening socket to receive the packets. We use UDP which should - # be painlessly cross-platform. - with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: - sock.bind(("127.0.0.1", 0)) - addr = sock.getsockname() - - duration = BANDWIDTH_DURATION - packet_size = BANDWIDTH_PACKET_SIZE - - results = [] - threads = [] - end_event = [] - start_cond = threading.Condition() - started = False - if nthreads > 0: - # Warm up - func(*args) - - results = [] - loop = TimedLoop(func, args) - ready = [] - ready_cond = threading.Condition() - - def run(): - with ready_cond: - ready.append(None) - ready_cond.notify() - with start_cond: - while not started: - start_cond.wait() - loop(start_time, duration * 1.5, end_event, do_yield=False) - - for i in range(nthreads): - threads.append(threading.Thread(target=run)) - for t in threads: - t.daemon = True - t.start() - # Wait for threads to be ready - with ready_cond: - while len(ready) < nthreads: - ready_cond.wait() - - # Run the client and wait for the first packet to arrive before - # unblocking the background threads. - process = run_bandwidth_client(addr=addr, - packet_size=packet_size, - duration=duration) - _time = time.time - # This will also wait for the parent to be ready - s = _recv(sock, packet_size) - remote_addr = eval(s.partition('#')[0]) - - with start_cond: - start_time = _time() - started = True - start_cond.notify(nthreads) - - n = 0 - first_time = None - while not end_event and BW_END not in s: - _sendto(sock, s, remote_addr) - s = _recv(sock, packet_size) - if first_time is None: - first_time = _time() - n += 1 - end_time = _time() - - end_event.append(None) - for t in threads: - t.join() - process.kill() - - return (n - 1) / (end_time - first_time) - -def run_bandwidth_tests(max_threads): - for task in bandwidth_tasks: - print("Background CPU task:", task.__doc__) - print() - func, args = task() - nthreads = 0 - baseline_speed = None - while nthreads <= max_threads: - results = run_bandwidth_test(func, args, nthreads) - speed = results - #speed = len(results) * 1.0 / results[-1][0] - print("CPU threads=%d: %.1f" % (nthreads, speed), end="") - if baseline_speed is None: - print(" packets/s.") - baseline_speed = speed - else: - print(" ( %d %%)" % (speed / baseline_speed * 100)) - nthreads += 1 - print() - - -def main(): - usage = "usage: %prog [-h|--help] [options]" - parser = OptionParser(usage=usage) - parser.add_option("-t", "--throughput", - action="store_true", dest="throughput", default=False, - help="run throughput tests") - parser.add_option("-l", "--latency", - action="store_true", dest="latency", default=False, - help="run latency tests") - parser.add_option("-b", "--bandwidth", - action="store_true", dest="bandwidth", default=False, - help="run I/O bandwidth tests") - parser.add_option("-i", "--interval", - action="store", type="int", dest="check_interval", default=None, - help="sys.setcheckinterval() value " - "(Python 3.8 and older)") - parser.add_option("-I", "--switch-interval", - action="store", type="float", dest="switch_interval", default=None, - help="sys.setswitchinterval() value " - "(Python 3.2 and newer)") - parser.add_option("-n", "--num-threads", - action="store", type="int", dest="nthreads", default=4, - help="max number of threads in tests") - - # Hidden option to run the pinging and bandwidth clients - parser.add_option("", "--latclient", - action="store", dest="latclient", default=None, - help=SUPPRESS_HELP) - parser.add_option("", "--bwclient", - action="store", dest="bwclient", default=None, - help=SUPPRESS_HELP) - - options, args = parser.parse_args() - if args: - parser.error("unexpected arguments") - - if options.latclient: - kwargs = eval(options.latclient) - latency_client(**kwargs) - return - - if options.bwclient: - kwargs = eval(options.bwclient) - bandwidth_client(**kwargs) - return - - if not options.throughput and not options.latency and not options.bandwidth: - options.throughput = options.latency = options.bandwidth = True - if options.check_interval: - sys.setcheckinterval(options.check_interval) - if options.switch_interval: - sys.setswitchinterval(options.switch_interval) - - print("== %s %s (%s) ==" % ( - platform.python_implementation(), - platform.python_version(), - platform.python_build()[0], - )) - # Processor identification often has repeated spaces - cpu = ' '.join(platform.processor().split()) - print("== %s %s on '%s' ==" % ( - platform.machine(), - platform.system(), - cpu, - )) - print() - - if options.throughput: - print("--- Throughput ---") - print() - run_throughput_tests(options.nthreads) - - if options.latency: - print("--- Latency ---") - print() - run_latency_tests(options.nthreads) - - if options.bandwidth: - print("--- I/O bandwidth ---") - print() - run_bandwidth_tests(options.nthreads) - -if __name__ == "__main__": - main() diff --git a/Tools/clinic/.ruff.toml b/Tools/clinic/.ruff.toml index cbb3a9a8f3a8c2c..c019572d0cb1864 100644 --- a/Tools/clinic/.ruff.toml +++ b/Tools/clinic/.ruff.toml @@ -1,5 +1,7 @@ target-version = "py310" fix = true + +[lint] select = [ "F", # Enable all pyflakes rules "UP", # Enable all pyupgrade rules by default diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 770878a3f8d2c78..7906e7c95d17ba1 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -116,11 +116,6 @@ def warn_or_fail( line_number: int | None = None, ) -> None: joined = " ".join([str(a) for a in args]) - if clinic: - if filename is None: - filename = clinic.filename - if getattr(clinic, 'block_parser', None) and (line_number is None): - line_number = clinic.block_parser.line_number error = ClinicError(joined, filename=filename, lineno=line_number) if fail: raise error @@ -143,75 +138,6 @@ def fail( warn_or_fail(*args, filename=filename, line_number=line_number, fail=True) -is_legal_c_identifier = re.compile('^[A-Za-z_][A-Za-z0-9_]*$').match - -def is_legal_py_identifier(s: str) -> bool: - return all(is_legal_c_identifier(field) for field in s.split('.')) - -# identifiers that are okay in Python but aren't a good idea in C. -# so if they're used Argument Clinic will add "_value" to the end -# of the name in C. -c_keywords = set(""" -asm auto break case char const continue default do double -else enum extern float for goto if inline int long -register return short signed sizeof static struct switch -typedef typeof union unsigned void volatile while -""".strip().split()) - -def ensure_legal_c_identifier(s: str) -> str: - # for now, just complain if what we're given isn't legal - if not is_legal_c_identifier(s): - fail("Illegal C identifier:", s) - # but if we picked a C keyword, pick something else - if s in c_keywords: - return s + "_value" - return s - - -def linear_format(s: str, **kwargs: str) -> str: - """ - Perform str.format-like substitution, except: - * The strings substituted must be on lines by - themselves. (This line is the "source line".) - * If the substitution text is empty, the source line - is removed in the output. - * If the field is not recognized, the original line - is passed unmodified through to the output. - * If the substitution text is not empty: - * Each line of the substituted text is indented - by the indent of the source line. - * A newline will be added to the end. - """ - lines = [] - for line in s.split('\n'): - indent, curly, trailing = line.partition('{') - if not curly: - lines.extend([line, "\n"]) - continue - - name, curly, trailing = trailing.partition('}') - if not curly or name not in kwargs: - lines.extend([line, "\n"]) - continue - - if trailing: - fail(f"Text found after {{{name}}} block marker! " - "It must be on a line by itself.") - if indent.strip(): - fail(f"Non-whitespace characters found before {{{name}}} block marker! " - "It must be on a line by itself.") - - value = kwargs[name] - if not value: - continue - - stripped = [line.rstrip() for line in value.split("\n")] - value = textwrap.indent("\n".join(stripped), indent) - lines.extend([value, "\n"]) - - return "".join(lines[:-1]) - - class CRenderData: def __init__(self) -> None: @@ -277,12 +203,12 @@ class Language(metaclass=abc.ABCMeta): checksum_line = "" def __init__(self, filename: str) -> None: - ... + self.filename = filename @abc.abstractmethod def render( self, - clinic: Clinic | None, + clinic: Clinic, signatures: Iterable[Module | Class | Function] ) -> str: ... @@ -635,7 +561,7 @@ def parse_line(self, line: str) -> None: def render( self, - clinic: Clinic | None, + clinic: Clinic, signatures: Iterable[Module | Class | Function] ) -> str: function = None @@ -833,8 +759,6 @@ def output_templates( if not p.is_optional(): min_kw_only = i - max_pos elif p.is_vararg(): - if vararg != self.NO_VARARG: - fail("Too many var args") pseudo_args += 1 vararg = i - 1 else: @@ -922,7 +846,8 @@ def parser_body( """) for field in preamble, *fields, finale: lines.append(field) - return linear_format("\n".join(lines), parser_declarations=declarations) + return libclinic.linear_format("\n".join(lines), + parser_declarations=declarations) fastcall = not new_or_init limited_capi = clinic.limited_capi @@ -960,7 +885,7 @@ def parser_body( return_error = ('return NULL;' if simple_return else 'goto exit;') parser_code = [libclinic.normalize_snippet(""" - if (nargs) {{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) {{ PyErr_SetString(PyExc_TypeError, "{name}() takes no arguments"); %s }} @@ -1577,7 +1502,7 @@ def render_option_group_parsing( {group_booleans} break; """ - s = linear_format(s, group_booleans=lines) + s = libclinic.linear_format(s, group_booleans=lines) s = s.format_map(d) out.append(s) @@ -1591,7 +1516,7 @@ def render_option_group_parsing( def render_function( self, - clinic: Clinic | None, + clinic: Clinic, f: Function | None ) -> str: if f is None or clinic is None: @@ -1736,9 +1661,9 @@ def render_function( for name, destination in clinic.destination_buffers.items(): template = templates[name] if has_option_groups: - template = linear_format(template, + template = libclinic.linear_format(template, option_group_parsing=template_dict['option_group_parsing']) - template = linear_format(template, + template = libclinic.linear_format(template, declarations=template_dict['declarations'], return_conversion=template_dict['return_conversion'], initializers=template_dict['initializers'], @@ -1751,10 +1676,8 @@ def render_function( # Only generate the "exit:" label # if we have any gotos - need_exit_label = "goto exit;" in template - template = linear_format(template, - exit_label="exit:" if need_exit_label else '' - ) + label = "exit:" if "goto exit;" in template else "" + template = libclinic.linear_format(template, exit_label=label) s = template.format_map(template_dict) @@ -1889,7 +1812,12 @@ def __next__(self) -> Block: raise StopIteration if self.dsl_name: - return_value = self.parse_clinic_block(self.dsl_name) + try: + return_value = self.parse_clinic_block(self.dsl_name) + except ClinicError as exc: + exc.filename = self.language.filename + exc.lineno = self.line_number + raise self.dsl_name = None self.first_block = False return return_value @@ -2222,7 +2150,6 @@ def __init__(self, clinic: Clinic) -> None: ... def parse(self, block: Block) -> None: ... -clinic: Clinic | None = None class Clinic: presets_text = """ @@ -2347,9 +2274,6 @@ def __init__( assert name in self.destination_buffers preset[name] = buffer - global clinic - clinic = self - def add_include(self, name: str, reason: str, *, condition: str | None = None) -> None: try: @@ -2500,12 +2424,12 @@ def parse_file( extension = os.path.splitext(filename)[1][1:] if not extension: - fail(f"Can't extract file type for file {filename!r}") + raise ClinicError(f"Can't extract file type for file {filename!r}") try: language = extensions[extension](filename) except KeyError: - fail(f"Can't identify file type for file {filename!r}") + raise ClinicError(f"Can't identify file type for file {filename!r}") with open(filename, encoding="utf-8") as f: raw = f.read() @@ -3005,7 +2929,7 @@ def __init__(self, unused: bool = False, **kwargs: Any ) -> None: - self.name = ensure_legal_c_identifier(name) + self.name = libclinic.ensure_legal_c_identifier(name) self.py_name = py_name self.unused = unused self.includes: list[Include] = [] @@ -4145,8 +4069,6 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st # mapping from arguments to format unit *and* registers the # legacy C converter for that format unit. # -ConverterKeywordDict = dict[str, TypeSet | bool] - def r(format_unit: str, *, accept: TypeSet, @@ -4162,7 +4084,7 @@ def r(format_unit: str, # # also don't add the converter for 's' because # the metaclass for CConverter adds it for us. - kwargs: ConverterKeywordDict = {} + kwargs: dict[str, Any] = {} if accept != {str}: kwargs['accept'] = accept if zeroes: @@ -4368,7 +4290,9 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st if (ptr == NULL) {{{{ goto exit; }}}} - PyBuffer_FillInfo(&{paramname}, {argname}, (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&{paramname}, {argname}, (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) {{{{ + goto exit; + }}}} }}}} else {{{{ /* any bytes-like object */ if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ @@ -4406,14 +4330,6 @@ def correct_name_for_self( return "PyTypeObject *", "type" raise AssertionError(f"Unhandled type of function f: {f.kind!r}") -def required_type_for_self_for_parser( - f: Function -) -> str | None: - type, _ = correct_name_for_self(f) - if f.kind in (METHOD_INIT, METHOD_NEW, STATIC_METHOD, CLASS_METHOD): - return type - return None - class self_converter(CConverter): """ @@ -4478,7 +4394,10 @@ def pre_render(self) -> None: @property def parser_type(self) -> str: assert self.type is not None - return required_type_for_self_for_parser(self.function) or self.type + if self.function.kind in {METHOD_INIT, METHOD_NEW, STATIC_METHOD, CLASS_METHOD}: + tp, _ = correct_name_for_self(self.function) + return tp + return self.type def render(self, parameter: Parameter, data: CRenderData) -> None: """ @@ -5069,6 +4988,7 @@ def parse(self, block: Block) -> None: self.state(line) except ClinicError as exc: exc.lineno = line_number + exc.filename = self.clinic.filename raise self.do_post_block_processing_cleanup(line_number) @@ -5076,7 +4996,8 @@ def parse(self, block: Block) -> None: if self.preserve_output: if block.output: - fail("'preserve' only works for blocks that don't produce any output!") + fail("'preserve' only works for blocks that don't produce any output!", + line_number=line_number) block.output = self.saved_output def in_docstring(self) -> bool: @@ -5124,8 +5045,7 @@ def state_dsl_start(self, line: str) -> None: self.next(self.state_modulename_name, line) - @staticmethod - def parse_function_names(line: str) -> FunctionNames: + def parse_function_names(self, line: str) -> FunctionNames: left, as_, right = line.partition(' as ') full_name = left.strip() c_basename = right.strip() @@ -5136,32 +5056,105 @@ def parse_function_names(line: str) -> FunctionNames: if fields[-1] == '__new__': fields.pop() c_basename = "_".join(fields) - if not is_legal_py_identifier(full_name): + if not libclinic.is_legal_py_identifier(full_name): fail(f"Illegal function name: {full_name!r}") - if not is_legal_c_identifier(c_basename): + if not libclinic.is_legal_c_identifier(c_basename): fail(f"Illegal C basename: {c_basename!r}") - return FunctionNames(full_name=full_name, c_basename=c_basename) + names = FunctionNames(full_name=full_name, c_basename=c_basename) + self.normalize_function_kind(names.full_name) + return names - def update_function_kind(self, fullname: str) -> None: + def normalize_function_kind(self, fullname: str) -> None: + # Fetch the method name and possibly class. fields = fullname.split('.') name = fields.pop() _, cls = self.clinic._module_and_class(fields) + + # Check special method requirements. if name in unsupported_special_methods: fail(f"{name!r} is a special method and cannot be converted to Argument Clinic!") - + if name == '__init__' and (self.kind is not CALLABLE or not cls): + fail(f"{name!r} must be a normal method; got '{self.kind}'!") + if name == '__new__' and (self.kind is not CLASS_METHOD or not cls): + fail("'__new__' must be a class method!") + if self.kind in {GETTER, SETTER} and not cls: + fail("@getter and @setter must be methods") + + # Normalise self.kind. if name == '__new__': - if (self.kind is CLASS_METHOD) and cls: - self.kind = METHOD_NEW - else: - fail("'__new__' must be a class method!") + self.kind = METHOD_NEW elif name == '__init__': - if (self.kind is CALLABLE) and cls: - self.kind = METHOD_INIT + self.kind = METHOD_INIT + + def resolve_return_converter( + self, full_name: str, forced_converter: str + ) -> CReturnConverter: + if forced_converter: + if self.kind in {GETTER, SETTER}: + fail(f"@{self.kind.name.lower()} method cannot define a return type") + ast_input = f"def x() -> {forced_converter}: pass" + try: + module_node = ast.parse(ast_input) + except SyntaxError: + fail(f"Badly formed annotation for {full_name!r}: {forced_converter!r}") + function_node = module_node.body[0] + assert isinstance(function_node, ast.FunctionDef) + try: + name, legacy, kwargs = self.parse_converter(function_node.returns) + if legacy: + fail(f"Legacy converter {name!r} not allowed as a return converter") + if name not in return_converters: + fail(f"No available return converter called {name!r}") + return return_converters[name](**kwargs) + except ValueError: + fail(f"Badly formed annotation for {full_name!r}: {forced_converter!r}") + + if self.kind is METHOD_INIT: + return init_return_converter() + return CReturnConverter() + + def parse_cloned_function(self, names: FunctionNames, existing: str) -> None: + full_name, c_basename = names + fields = [x.strip() for x in existing.split('.')] + function_name = fields.pop() + module, cls = self.clinic._module_and_class(fields) + parent = cls or module + + for existing_function in parent.functions: + if existing_function.name == function_name: + break + else: + print(f"{cls=}, {module=}, {existing=}", file=sys.stderr) + print(f"{(cls or module).functions=}", file=sys.stderr) + fail(f"Couldn't find existing function {existing!r}!") + + fields = [x.strip() for x in full_name.split('.')] + function_name = fields.pop() + module, cls = self.clinic._module_and_class(fields) + + overrides: dict[str, Any] = { + "name": function_name, + "full_name": full_name, + "module": module, + "cls": cls, + "c_basename": c_basename, + "docstring": "", + } + if not (existing_function.kind is self.kind and + existing_function.coexist == self.coexist): + # Allow __new__ or __init__ methods. + if existing_function.kind.new_or_init: + overrides["kind"] = self.kind + # Future enhancement: allow custom return converters + overrides["return_converter"] = CReturnConverter() else: - fail( - "'__init__' must be a normal method; " - f"got '{self.kind}'!" - ) + fail("'kind' of function and cloned function don't match! " + "(@classmethod/@staticmethod/@coexist)") + function = existing_function.copy(**overrides) + self.function = function + self.block.signatures.append(function) + (cls or module).functions.append(function) + self.next(self.state_function_docstring) def state_modulename_name(self, line: str) -> None: # looking for declaration, which establishes the leftmost column @@ -5186,111 +5179,56 @@ def state_modulename_name(self, line: str) -> None: # are we cloning? before, equals, existing = line.rpartition('=') if equals: - full_name, c_basename = self.parse_function_names(before) existing = existing.strip() - if is_legal_py_identifier(existing): + if libclinic.is_legal_py_identifier(existing): # we're cloning! - fields = [x.strip() for x in existing.split('.')] - function_name = fields.pop() - module, cls = self.clinic._module_and_class(fields) - - for existing_function in (cls or module).functions: - if existing_function.name == function_name: - break - else: - print(f"{cls=}, {module=}, {existing=}", file=sys.stderr) - print(f"{(cls or module).functions=}", file=sys.stderr) - fail(f"Couldn't find existing function {existing!r}!") - - fields = [x.strip() for x in full_name.split('.')] - function_name = fields.pop() - module, cls = self.clinic._module_and_class(fields) - - self.update_function_kind(full_name) - overrides: dict[str, Any] = { - "name": function_name, - "full_name": full_name, - "module": module, - "cls": cls, - "c_basename": c_basename, - "docstring": "", - } - if not (existing_function.kind is self.kind and - existing_function.coexist == self.coexist): - # Allow __new__ or __init__ methods. - if existing_function.kind.new_or_init: - overrides["kind"] = self.kind - # Future enhancement: allow custom return converters - overrides["return_converter"] = CReturnConverter() - else: - fail("'kind' of function and cloned function don't match! " - "(@classmethod/@staticmethod/@coexist)") - function = existing_function.copy(**overrides) - self.function = function - self.block.signatures.append(function) - (cls or module).functions.append(function) - self.next(self.state_function_docstring) - return + names = self.parse_function_names(before) + return self.parse_cloned_function(names, existing) line, _, returns = line.partition('->') returns = returns.strip() full_name, c_basename = self.parse_function_names(line) - - return_converter = None - if returns: - if self.kind in {GETTER, SETTER}: - fail(f"@{self.kind.name.lower()} method cannot define a return type") - ast_input = f"def x() -> {returns}: pass" - try: - module_node = ast.parse(ast_input) - except SyntaxError: - fail(f"Badly formed annotation for {full_name!r}: {returns!r}") - function_node = module_node.body[0] - assert isinstance(function_node, ast.FunctionDef) - try: - name, legacy, kwargs = self.parse_converter(function_node.returns) - if legacy: - fail(f"Legacy converter {name!r} not allowed as a return converter") - if name not in return_converters: - fail(f"No available return converter called {name!r}") - return_converter = return_converters[name](**kwargs) - except ValueError: - fail(f"Badly formed annotation for {full_name!r}: {returns!r}") + return_converter = self.resolve_return_converter(full_name, returns) fields = [x.strip() for x in full_name.split('.')] function_name = fields.pop() module, cls = self.clinic._module_and_class(fields) - if self.kind in {GETTER, SETTER}: - if not cls: - fail("@getter and @setter must be methods") - - self.update_function_kind(full_name) - if self.kind is METHOD_INIT and not return_converter: - return_converter = init_return_converter() - - if not return_converter: - return_converter = CReturnConverter() - - self.function = Function(name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename, - return_converter=return_converter, kind=self.kind, coexist=self.coexist, - critical_section=self.critical_section, - target_critical_section=self.target_critical_section) - self.block.signatures.append(self.function) - - # insert a self converter automatically - type, name = correct_name_for_self(self.function) - kwargs = {} - if cls and type == "PyObject *": - kwargs['type'] = cls.typedef - sc = self.function.self_converter = self_converter(name, name, self.function, **kwargs) - p_self = Parameter(name, inspect.Parameter.POSITIONAL_ONLY, - function=self.function, converter=sc) - self.function.parameters[name] = p_self - - (cls or module).functions.append(self.function) + func = Function( + name=function_name, + full_name=full_name, + module=module, + cls=cls, + c_basename=c_basename, + return_converter=return_converter, + kind=self.kind, + coexist=self.coexist, + critical_section=self.critical_section, + target_critical_section=self.target_critical_section + ) + self.add_function(func) + self.next(self.state_parameters_start) + def add_function(self, func: Function) -> None: + # Insert a self converter automatically. + tp, name = correct_name_for_self(func) + if func.cls and tp == "PyObject *": + func.self_converter = self_converter(name, name, func, + type=func.cls.typedef) + else: + func.self_converter = self_converter(name, name, func) + func.parameters[name] = Parameter( + name, + inspect.Parameter.POSITIONAL_ONLY, + function=func, + converter=func.self_converter + ) + + self.block.signatures.append(func) + self.function = func + (func.cls or func.module).functions.append(func) + # Now entering the parameters section. The rules, formally stated: # # * All lines must be indented with spaces only. @@ -5484,6 +5422,8 @@ def parse_parameter(self, line: str) -> None: f"invalid parameter declaration (**kwargs?): {line!r}") if function_args.vararg: + if any(p.is_vararg() for p in self.function.parameters.values()): + fail("Too many var args") is_vararg = True parameter = function_args.vararg else: @@ -6113,9 +6053,9 @@ def format_docstring(self) -> str: parameters = self.format_docstring_parameters(params) signature = self.format_docstring_signature(f, params) docstring = "\n".join(lines) - return linear_format(docstring, - signature=signature, - parameters=parameters).rstrip() + return libclinic.linear_format(docstring, + signature=signature, + parameters=parameters).rstrip() def check_remaining_star(self, lineno: int | None = None) -> None: assert isinstance(self.function, Function) @@ -6155,7 +6095,12 @@ def do_post_block_processing_cleanup(self, lineno: int) -> None: return self.check_remaining_star(lineno) - self.function.docstring = self.format_docstring() + try: + self.function.docstring = self.format_docstring() + except ClinicError as exc: + exc.lineno = lineno + exc.filename = self.clinic.filename + raise diff --git a/Tools/clinic/libclinic/__init__.py b/Tools/clinic/libclinic/__init__.py index 1b300b55acc21e6..738864a48c08d3f 100644 --- a/Tools/clinic/libclinic/__init__.py +++ b/Tools/clinic/libclinic/__init__.py @@ -9,12 +9,18 @@ docstring_for_c_string, format_escape, indent_all_lines, + linear_format, normalize_snippet, pprint_words, suffix_all_lines, wrap_declarations, wrapped_c_string_literal, ) +from .identifiers import ( + ensure_legal_c_identifier, + is_legal_c_identifier, + is_legal_py_identifier, +) from .utils import ( FormatCounterFormatter, compute_checksum, @@ -33,12 +39,18 @@ "docstring_for_c_string", "format_escape", "indent_all_lines", + "linear_format", "normalize_snippet", "pprint_words", "suffix_all_lines", "wrap_declarations", "wrapped_c_string_literal", + # Identifier helpers + "ensure_legal_c_identifier", + "is_legal_c_identifier", + "is_legal_py_identifier", + # Utility functions "FormatCounterFormatter", "compute_checksum", diff --git a/Tools/clinic/libclinic/formatting.py b/Tools/clinic/libclinic/formatting.py index 8b3ad7ba566bc84..873ece6210017a8 100644 --- a/Tools/clinic/libclinic/formatting.py +++ b/Tools/clinic/libclinic/formatting.py @@ -4,6 +4,8 @@ import textwrap from typing import Final +from libclinic import ClinicError + SIG_END_MARKER: Final = "--" @@ -171,3 +173,51 @@ def wrap_declarations(text: str, length: int = 78) -> str: lines.append(line.rstrip()) prefix = spaces return "\n".join(lines) + + +def linear_format(text: str, **kwargs: str) -> str: + """ + Perform str.format-like substitution, except: + * The strings substituted must be on lines by + themselves. (This line is the "source line".) + * If the substitution text is empty, the source line + is removed in the output. + * If the field is not recognized, the original line + is passed unmodified through to the output. + * If the substitution text is not empty: + * Each line of the substituted text is indented + by the indent of the source line. + * A newline will be added to the end. + """ + lines = [] + for line in text.split("\n"): + indent, curly, trailing = line.partition("{") + if not curly: + lines.extend([line, "\n"]) + continue + + name, curly, trailing = trailing.partition("}") + if not curly or name not in kwargs: + lines.extend([line, "\n"]) + continue + + if trailing: + raise ClinicError( + f"Text found after '{{{name}}}' block marker! " + "It must be on a line by itself." + ) + if indent.strip(): + raise ClinicError( + f"Non-whitespace characters found before '{{{name}}}' block marker! " + "It must be on a line by itself." + ) + + value = kwargs[name] + if not value: + continue + + stripped = [line.rstrip() for line in value.split("\n")] + value = textwrap.indent("\n".join(stripped), indent) + lines.extend([value, "\n"]) + + return "".join(lines[:-1]) diff --git a/Tools/clinic/libclinic/identifiers.py b/Tools/clinic/libclinic/identifiers.py new file mode 100644 index 000000000000000..d3b80bbcef3b2b2 --- /dev/null +++ b/Tools/clinic/libclinic/identifiers.py @@ -0,0 +1,31 @@ +import re +from .errors import ClinicError + + +is_legal_c_identifier = re.compile("^[A-Za-z_][A-Za-z0-9_]*$").match + + +def is_legal_py_identifier(identifier: str) -> bool: + return all(is_legal_c_identifier(field) for field in identifier.split(".")) + + +# Identifiers that are okay in Python but aren't a good idea in C. +# So if they're used Argument Clinic will add "_value" to the end +# of the name in C. +_c_keywords = frozenset(""" +asm auto break case char const continue default do double +else enum extern float for goto if inline int long +register return short signed sizeof static struct switch +typedef typeof union unsigned void volatile while +""".strip().split() +) + + +def ensure_legal_c_identifier(identifier: str) -> str: + # For now, just complain if what we're given isn't legal. + if not is_legal_c_identifier(identifier): + raise ClinicError(f"Illegal C identifier: {identifier}") + # But if we picked a C keyword, pick something else. + if identifier in _c_keywords: + return identifier + "_value" + return identifier diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 5ef55524c11be20..483f28b46dfec74 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -70,6 +70,14 @@ def _type_unsigned_int_ptr(): def _sizeof_void_p(): return gdb.lookup_type('void').pointer().sizeof +def _managed_dict_offset(): + # See pycore_object.h + pyobj = gdb.lookup_type("PyObject") + if any(field.name == "ob_ref_local" for field in pyobj.fields()): + return -1 * _sizeof_void_p() + else: + return -3 * _sizeof_void_p() + Py_TPFLAGS_MANAGED_DICT = (1 << 4) Py_TPFLAGS_HEAPTYPE = (1 << 9) @@ -457,7 +465,7 @@ def get_attr_dict(self): if dictoffset < 0: if int_from_int(typeobj.field('tp_flags')) & Py_TPFLAGS_MANAGED_DICT: assert dictoffset == -1 - dictoffset = -3 * _sizeof_void_p() + dictoffset = _managed_dict_offset() else: type_PyVarObject_ptr = gdb.lookup_type('PyVarObject').pointer() tsize = int_from_int(self._gdbval.cast(type_PyVarObject_ptr)['ob_size']) @@ -485,9 +493,8 @@ def get_keys_values(self): has_values = int_from_int(typeobj.field('tp_flags')) & Py_TPFLAGS_MANAGED_DICT if not has_values: return None - charptrptr_t = _type_char_ptr().pointer() - ptr = self._gdbval.cast(charptrptr_t) - 3 - char_ptr = ptr.dereference() + ptr = self._gdbval.cast(_type_char_ptr()) + _managed_dict_offset() + char_ptr = ptr.cast(_type_char_ptr().pointer()).dereference() if (int(char_ptr) & 1) == 0: return None char_ptr += 1 diff --git a/Tools/iobench/iobench.py b/Tools/iobench/iobench.py deleted file mode 100644 index 4017149ec91630f..000000000000000 --- a/Tools/iobench/iobench.py +++ /dev/null @@ -1,568 +0,0 @@ -import itertools -import os -import platform -import re -import sys -import time -from optparse import OptionParser - -out = sys.stdout - -TEXT_ENCODING = 'utf8' -NEWLINES = 'lf' - - -def text_open(fn, mode, encoding=None): - try: - return open(fn, mode, encoding=encoding or TEXT_ENCODING) - except TypeError: - return open(fn, mode) - - -def get_file_sizes(): - for s in ['20 KiB', '400 KiB', '10 MiB']: - size, unit = s.split() - size = int(size) * {'KiB': 1024, 'MiB': 1024 ** 2}[unit] - yield s.replace(' ', ''), size - - -def get_binary_files(): - return ((name + ".bin", size) for name, size in get_file_sizes()) - - -def get_text_files(): - return ((f"{name}-{TEXT_ENCODING}-{NEWLINES}.txt", size) - for name, size in get_file_sizes()) - - -def with_open_mode(mode): - def decorate(f): - f.file_open_mode = mode - return f - return decorate - - -def with_sizes(*sizes): - def decorate(f): - f.file_sizes = sizes - return f - return decorate - - -# Here begin the tests - -@with_open_mode("r") -@with_sizes("medium") -def read_bytewise(f): - """ read one unit at a time """ - f.seek(0) - while f.read(1): - pass - - -@with_open_mode("r") -@with_sizes("medium") -def read_small_chunks(f): - """ read 20 units at a time """ - f.seek(0) - while f.read(20): - pass - - -@with_open_mode("r") -@with_sizes("medium") -def read_big_chunks(f): - """ read 4096 units at a time """ - f.seek(0) - while f.read(4096): - pass - - -@with_open_mode("r") -@with_sizes("small", "medium", "large") -def read_whole_file(f): - """ read whole contents at once """ - f.seek(0) - while f.read(): - pass - - -@with_open_mode("rt") -@with_sizes("medium") -def read_lines(f): - """ read one line at a time """ - f.seek(0) - for line in f: - pass - - -@with_open_mode("r") -@with_sizes("medium") -def seek_forward_bytewise(f): - """ seek forward one unit at a time """ - f.seek(0, 2) - size = f.tell() - f.seek(0, 0) - for i in range(0, size - 1): - f.seek(i, 0) - - -@with_open_mode("r") -@with_sizes("medium") -def seek_forward_blockwise(f): - """ seek forward 1000 units at a time """ - f.seek(0, 2) - size = f.tell() - f.seek(0, 0) - for i in range(0, size - 1, 1000): - f.seek(i, 0) - - -@with_open_mode("rb") -@with_sizes("medium") -def read_seek_bytewise(f): - """ alternate read & seek one unit """ - f.seek(0) - while f.read(1): - f.seek(1, 1) - - -@with_open_mode("rb") -@with_sizes("medium") -def read_seek_blockwise(f): - """ alternate read & seek 1000 units """ - f.seek(0) - while f.read(1000): - f.seek(1000, 1) - - -@with_open_mode("w") -@with_sizes("small") -def write_bytewise(f, source): - """ write one unit at a time """ - for i in range(0, len(source)): - f.write(source[i:i+1]) - - -@with_open_mode("w") -@with_sizes("medium") -def write_small_chunks(f, source): - """ write 20 units at a time """ - for i in range(0, len(source), 20): - f.write(source[i:i+20]) - - -@with_open_mode("w") -@with_sizes("medium") -def write_medium_chunks(f, source): - """ write 4096 units at a time """ - for i in range(0, len(source), 4096): - f.write(source[i:i+4096]) - - -@with_open_mode("w") -@with_sizes("large") -def write_large_chunks(f, source): - """ write 1e6 units at a time """ - for i in range(0, len(source), 1000000): - f.write(source[i:i+1000000]) - - -@with_open_mode("w+") -@with_sizes("small") -def modify_bytewise(f, source): - """ modify one unit at a time """ - f.seek(0) - for i in range(0, len(source)): - f.write(source[i:i+1]) - - -@with_open_mode("w+") -@with_sizes("medium") -def modify_small_chunks(f, source): - """ modify 20 units at a time """ - f.seek(0) - for i in range(0, len(source), 20): - f.write(source[i:i+20]) - - -@with_open_mode("w+") -@with_sizes("medium") -def modify_medium_chunks(f, source): - """ modify 4096 units at a time """ - f.seek(0) - for i in range(0, len(source), 4096): - f.write(source[i:i+4096]) - - -@with_open_mode("wb+") -@with_sizes("medium") -def modify_seek_forward_bytewise(f, source): - """ alternate write & seek one unit """ - f.seek(0) - for i in range(0, len(source), 2): - f.write(source[i:i+1]) - f.seek(i+2) - - -@with_open_mode("wb+") -@with_sizes("medium") -def modify_seek_forward_blockwise(f, source): - """ alternate write & seek 1000 units """ - f.seek(0) - for i in range(0, len(source), 2000): - f.write(source[i:i+1000]) - f.seek(i+2000) - - -# XXX the 2 following tests don't work with py3k's text IO -@with_open_mode("wb+") -@with_sizes("medium") -def read_modify_bytewise(f, source): - """ alternate read & write one unit """ - f.seek(0) - for i in range(0, len(source), 2): - f.read(1) - f.write(source[i+1:i+2]) - - -@with_open_mode("wb+") -@with_sizes("medium") -def read_modify_blockwise(f, source): - """ alternate read & write 1000 units """ - f.seek(0) - for i in range(0, len(source), 2000): - f.read(1000) - f.write(source[i+1000:i+2000]) - - -read_tests = [ - read_bytewise, read_small_chunks, read_lines, read_big_chunks, - None, read_whole_file, None, - seek_forward_bytewise, seek_forward_blockwise, - read_seek_bytewise, read_seek_blockwise, -] - -write_tests = [ - write_bytewise, write_small_chunks, write_medium_chunks, write_large_chunks, -] - -modify_tests = [ - modify_bytewise, modify_small_chunks, modify_medium_chunks, - None, - modify_seek_forward_bytewise, modify_seek_forward_blockwise, - read_modify_bytewise, read_modify_blockwise, -] - - -def run_during(duration, func): - _t = time.time - n = 0 - start = os.times() - start_timestamp = _t() - real_start = start[4] or start_timestamp - while True: - func() - n += 1 - if _t() - start_timestamp > duration: - break - end = os.times() - real = (end[4] if start[4] else time.time()) - real_start - return n, real, sum(end[0:2]) - sum(start[0:2]) - - -def warm_cache(filename): - with open(filename, "rb") as f: - f.read() - - -def run_all_tests(options): - def print_label(filename, func): - name = re.split(r'[-.]', filename)[0] - out.write( - f"[{name.center(7)}] {func.__doc__.strip()}... ".ljust(52)) - out.flush() - - def print_results(size, n, real, cpu): - bw = n * float(size) / 1024 ** 2 / real - bw = ("%4d MiB/s" if bw > 100 else "%.3g MiB/s") % bw - out.write(bw.rjust(12) + "\n") - if cpu < 0.90 * real: - out.write(" warning: test above used only " - f"{cpu / real:%} CPU, " - "result may be flawed!\n") - - def run_one_test(name, size, open_func, test_func, *args): - mode = test_func.file_open_mode - print_label(name, test_func) - if "w" not in mode or "+" in mode: - warm_cache(name) - with open_func(name) as f: - n, real, cpu = run_during(1.5, lambda: test_func(f, *args)) - print_results(size, n, real, cpu) - - def run_test_family(tests, mode_filter, files, open_func, *make_args): - for test_func in tests: - if test_func is None: - out.write("\n") - continue - if mode_filter in test_func.file_open_mode: - continue - for s in test_func.file_sizes: - name, size = files[size_names[s]] - #name += file_ext - args = tuple(f(name, size) for f in make_args) - run_one_test(name, size, - open_func, test_func, *args) - - size_names = { - "small": 0, - "medium": 1, - "large": 2, - } - - print(f"Python {sys.version}") - print("Unicode: PEP 393") - print(platform.platform()) - binary_files = list(get_binary_files()) - text_files = list(get_text_files()) - if "b" in options: - print("Binary unit = one byte") - if "t" in options: - print(f"Text unit = one character ({TEXT_ENCODING}-decoded)") - - # Binary reads - if "b" in options and "r" in options: - print("\n** Binary input **\n") - run_test_family(read_tests, "t", binary_files, lambda fn: open(fn, "rb")) - - # Text reads - if "t" in options and "r" in options: - print("\n** Text input **\n") - run_test_family(read_tests, "b", text_files, lambda fn: text_open(fn, "r")) - - # Binary writes - if "b" in options and "w" in options: - print("\n** Binary append **\n") - - def make_test_source(name, size): - with open(name, "rb") as f: - return f.read() - run_test_family(write_tests, "t", binary_files, - lambda fn: open(os.devnull, "wb"), make_test_source) - - # Text writes - if "t" in options and "w" in options: - print("\n** Text append **\n") - - def make_test_source(name, size): - with text_open(name, "r") as f: - return f.read() - run_test_family(write_tests, "b", text_files, - lambda fn: text_open(os.devnull, "w"), make_test_source) - - # Binary overwrites - if "b" in options and "w" in options: - print("\n** Binary overwrite **\n") - - def make_test_source(name, size): - with open(name, "rb") as f: - return f.read() - run_test_family(modify_tests, "t", binary_files, - lambda fn: open(fn, "r+b"), make_test_source) - - # Text overwrites - if "t" in options and "w" in options: - print("\n** Text overwrite **\n") - - def make_test_source(name, size): - with text_open(name, "r") as f: - return f.read() - run_test_family(modify_tests, "b", text_files, - lambda fn: text_open(fn, "r+"), make_test_source) - - -def prepare_files(): - print("Preparing files...") - # Binary files - for name, size in get_binary_files(): - if os.path.isfile(name) and os.path.getsize(name) == size: - continue - with open(name, "wb") as f: - f.write(os.urandom(size)) - # Text files - chunk = [] - with text_open(__file__, "r", encoding='utf8') as f: - for line in f: - if line.startswith("# <iobench text chunk marker>"): - break - else: - raise RuntimeError( - f"Couldn't find chunk marker in {__file__} !") - if NEWLINES == "all": - it = itertools.cycle(["\n", "\r", "\r\n"]) - else: - it = itertools.repeat( - {"cr": "\r", "lf": "\n", "crlf": "\r\n"}[NEWLINES]) - chunk = "".join(line.replace("\n", next(it)) for line in f) - if isinstance(chunk, bytes): - chunk = chunk.decode('utf8') - chunk = chunk.encode(TEXT_ENCODING) - for name, size in get_text_files(): - if os.path.isfile(name) and os.path.getsize(name) == size: - continue - head = chunk * (size // len(chunk)) - tail = chunk[:size % len(chunk)] - # Adjust tail to end on a character boundary - while True: - try: - tail.decode(TEXT_ENCODING) - break - except UnicodeDecodeError: - tail = tail[:-1] - with open(name, "wb") as f: - f.write(head) - f.write(tail) - - -def main(): - global TEXT_ENCODING, NEWLINES - - usage = "usage: %prog [-h|--help] [options]" - parser = OptionParser(usage=usage) - parser.add_option("-b", "--binary", - action="store_true", dest="binary", default=False, - help="run binary I/O tests") - parser.add_option("-t", "--text", - action="store_true", dest="text", default=False, - help="run text I/O tests") - parser.add_option("-r", "--read", - action="store_true", dest="read", default=False, - help="run read tests") - parser.add_option("-w", "--write", - action="store_true", dest="write", default=False, - help="run write & modify tests") - parser.add_option("-E", "--encoding", - action="store", dest="encoding", default=None, - help=f"encoding for text tests (default: {TEXT_ENCODING})") - parser.add_option("-N", "--newlines", - action="store", dest="newlines", default='lf', - help="line endings for text tests " - "(one of: {lf (default), cr, crlf, all})") - parser.add_option("-m", "--io-module", - action="store", dest="io_module", default=None, - help="io module to test (default: builtin open())") - options, args = parser.parse_args() - if args: - parser.error("unexpected arguments") - NEWLINES = options.newlines.lower() - if NEWLINES not in ('lf', 'cr', 'crlf', 'all'): - parser.error(f"invalid 'newlines' option: {NEWLINES!r}") - - test_options = "" - if options.read: - test_options += "r" - if options.write: - test_options += "w" - elif not options.read: - test_options += "rw" - if options.text: - test_options += "t" - if options.binary: - test_options += "b" - elif not options.text: - test_options += "tb" - - if options.encoding: - TEXT_ENCODING = options.encoding - - if options.io_module: - globals()['open'] = __import__(options.io_module, {}, {}, ['open']).open - - prepare_files() - run_all_tests(test_options) - - -if __name__ == "__main__": - main() - - -# -- This part to exercise text reading. Don't change anything! -- -# <iobench text chunk marker> - -""" -1. -Gáttir allar, -áðr gangi fram, -um skoðask skyli, -um skyggnast skyli, -því at óvíst er at vita, -hvar óvinir -sitja á fleti fyrir. - -2. -Gefendr heilir! -Gestr er inn kominn, -hvar skal sitja sjá? -Mjök er bráðr, -sá er á bröndum skal -síns of freista frama. - -3. -Elds er þörf, -þeims inn er kominn -ok á kné kalinn; -matar ok váða -er manni þörf, -þeim er hefr um fjall farit. - -4. -Vatns er þörf, -þeim er til verðar kemr, -þerru ok þjóðlaðar, -góðs of æðis, -ef sér geta mætti, -orðs ok endrþögu. - -5. -Vits er þörf, -þeim er víða ratar; -dælt er heima hvat; -at augabragði verðr, -sá er ekki kann -ok með snotrum sitr. - -6. -At hyggjandi sinni -skyli-t maðr hræsinn vera, -heldr gætinn at geði; -þá er horskr ok þögull -kemr heimisgarða til, -sjaldan verðr víti vörum, -því at óbrigðra vin -fær maðr aldregi -en mannvit mikit. - -7. -Inn vari gestr, -er til verðar kemr, -þunnu hljóði þegir, -eyrum hlýðir, -en augum skoðar; -svá nýsisk fróðra hverr fyrir. - -8. -Hinn er sæll, -er sér of getr -lof ok líknstafi; -ódælla er við þat, -er maðr eiga skal -annars brjóstum í. -""" - -""" -C'est revenir tard, je le sens, sur un sujet trop rebattu et déjà presque oublié. Mon état, qui ne me permet plus aucun travail suivi, mon aversion pour le genre polémique, ont causé ma lenteur à écrire et ma répugnance à publier. J'aurais même tout à fait supprimé ces Lettres, ou plutôt je lie les aurais point écrites, s'il n'eût été question que de moi : Mais ma patrie ne m'est pas tellement devenue étrangère que je puisse voir tranquillement opprimer ses citoyens, surtout lorsqu'ils n'ont compromis leurs droits qu'en défendant ma cause. Je serais le dernier des hommes si dans une telle occasion j'écoutais un sentiment qui n'est plus ni douceur ni patience, mais faiblesse et lâcheté, dans celui qu'il empêche de remplir son devoir. -Rien de moins important pour le public, j'en conviens, que la matière de ces lettres. La constitution d'une petite République, le sort d'un petit particulier, l'exposé de quelques injustices, la réfutation de quelques sophismes ; tout cela n'a rien en soi d'assez considérable pour mériter beaucoup de lecteurs : mais si mes sujets sont petits mes objets sont grands, et dignes de l'attention de tout honnête homme. Laissons Genève à sa place, et Rousseau dans sa dépression ; mais la religion, mais la liberté, la justice ! voilà, qui que vous soyez, ce qui n'est pas au-dessous de vous. -Qu'on ne cherche pas même ici dans le style le dédommagement de l'aridité de la matière. Ceux que quelques traits heureux de ma plume ont si fort irrités trouveront de quoi s'apaiser dans ces lettres, L'honneur de défendre un opprimé eût enflammé mon coeur si j'avais parlé pour un autre. Réduit au triste emploi de me défendre moi-même, j'ai dû me borner à raisonner ; m'échauffer eût été m'avilir. J'aurai donc trouvé grâce en ce point devant ceux qui s'imaginent qu'il est essentiel à la vérité d'être dite froidement ; opinion que pourtant j'ai peine à comprendre. Lorsqu'une vive persuasion nous anime, le moyen d'employer un langage glacé ? Quand Archimède tout transporté courait nu dans les rues de Syracuse, en avait-il moins trouvé la vérité parce qu'il se passionnait pour elle ? Tout au contraire, celui qui la sent ne peut s'abstenir de l'adorer ; celui qui demeure froid ne l'a pas vue. -Quoi qu'il en soit, je prie les lecteurs de vouloir bien mettre à part mon beau style, et d'examiner seulement si je raisonne bien ou mal ; car enfin, de cela seul qu'un auteur s'exprime en bons termes, je ne vois pas comment il peut s'ensuivre que cet auteur ne sait ce qu'il dit. -""" diff --git a/Tools/jit/README.md b/Tools/jit/README.md new file mode 100644 index 000000000000000..04a6c0780bf9727 --- /dev/null +++ b/Tools/jit/README.md @@ -0,0 +1,46 @@ +The JIT Compiler +================ + +This version of CPython can be built with an experimental just-in-time compiler. While most everything you already know about building and using CPython is unchanged, you will probably need to install a compatible version of LLVM first. + +## Installing LLVM + +The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). + +LLVM version 16 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-16`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. + +It's easy to install all of the required tools: + +### Linux + +Install LLVM 16 on Ubuntu/Debian: + +```sh +wget https://apt.llvm.org/llvm.sh +chmod +x llvm.sh +sudo ./llvm.sh 16 +``` + +### macOS + +Install LLVM 16 with [Homebrew](https://brew.sh): + +```sh +brew install llvm@16 +``` + +Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. + +### Windows + +Install LLVM 16 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=16), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** + +## Building + +For `PCbuild`-based builds, pass the new `--experimental-jit` option to `build.bat`. + +For all other builds, pass the new `--enable-experimental-jit` option to `configure`. + +Otherwise, just configure and build as you normally would. Cross-compiling "just works", since the JIT is built for the host platform. + +[^why-llvm]: Clang is specifically needed because it's the only C compiler with support for guaranteed tail calls (`musttail`), which are required by CPython's continuation-passing-style approach to JIT compilation. Since LLVM also includes other functionalities we need (namely, object file parsing and disassembly), it's convenient to only support one toolchain at this time. diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py new file mode 100644 index 000000000000000..603bbef59ba2e63 --- /dev/null +++ b/Tools/jit/_llvm.py @@ -0,0 +1,99 @@ +"""Utilities for invoking LLVM tools.""" +import asyncio +import functools +import os +import re +import shlex +import subprocess +import typing + +_LLVM_VERSION = 16 +_LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\s+") + +_P = typing.ParamSpec("_P") +_R = typing.TypeVar("_R") +_C = typing.Callable[_P, typing.Awaitable[_R]] + + +def _async_cache(f: _C[_P, _R]) -> _C[_P, _R]: + cache = {} + lock = asyncio.Lock() + + @functools.wraps(f) + async def wrapper( + *args: _P.args, **kwargs: _P.kwargs # pylint: disable = no-member + ) -> _R: + async with lock: + if args not in cache: + cache[args] = await f(*args, **kwargs) + return cache[args] + + return wrapper + + +_CORES = asyncio.BoundedSemaphore(os.cpu_count() or 1) + + +async def _run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str | None: + command = [tool, *args] + async with _CORES: + if echo: + print(shlex.join(command)) + try: + process = await asyncio.create_subprocess_exec( + *command, stdout=subprocess.PIPE + ) + except FileNotFoundError: + return None + out, _ = await process.communicate() + if process.returncode: + raise RuntimeError(f"{tool} exited with return code {process.returncode}") + return out.decode() + + +@_async_cache +async def _check_tool_version(name: str, *, echo: bool = False) -> bool: + output = await _run(name, ["--version"], echo=echo) + return bool(output and _LLVM_VERSION_PATTERN.search(output)) + + +@_async_cache +async def _get_brew_llvm_prefix(*, echo: bool = False) -> str | None: + output = await _run("brew", ["--prefix", f"llvm@{_LLVM_VERSION}"], echo=echo) + return output and output.removesuffix("\n") + + +@_async_cache +async def _find_tool(tool: str, *, echo: bool = False) -> str | None: + # Unversioned executables: + path = tool + if await _check_tool_version(path, echo=echo): + return path + # Versioned executables: + path = f"{tool}-{_LLVM_VERSION}" + if await _check_tool_version(path, echo=echo): + return path + # Homebrew-installed executables: + prefix = await _get_brew_llvm_prefix(echo=echo) + if prefix is not None: + path = os.path.join(prefix, "bin", tool) + if await _check_tool_version(path, echo=echo): + return path + # Nothing found: + return None + + +async def maybe_run( + tool: str, args: typing.Iterable[str], echo: bool = False +) -> str | None: + """Run an LLVM tool if it can be found. Otherwise, return None.""" + path = await _find_tool(tool, echo=echo) + return path and await _run(path, args, echo=echo) + + +async def run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str: + """Run an LLVM tool if it can be found. Otherwise, raise RuntimeError.""" + output = await maybe_run(tool, args, echo=echo) + if output is None: + raise RuntimeError(f"Can't find {tool}-{_LLVM_VERSION}!") + return output diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py new file mode 100644 index 000000000000000..975ca650a13c1a8 --- /dev/null +++ b/Tools/jit/_schema.py @@ -0,0 +1,109 @@ +"""Schema for the JSON produced by llvm-readobj --elf-output-style=JSON.""" +import typing + +HoleKind: typing.TypeAlias = typing.Literal[ + "ARM64_RELOC_GOT_LOAD_PAGE21", + "ARM64_RELOC_GOT_LOAD_PAGEOFF12", + "ARM64_RELOC_UNSIGNED", + "IMAGE_REL_AMD64_ADDR64", + "IMAGE_REL_I386_DIR32", + "R_AARCH64_ABS64", + "R_AARCH64_ADR_GOT_PAGE", + "R_AARCH64_CALL26", + "R_AARCH64_JUMP26", + "R_AARCH64_LD64_GOT_LO12_NC", + "R_AARCH64_MOVW_UABS_G0_NC", + "R_AARCH64_MOVW_UABS_G1_NC", + "R_AARCH64_MOVW_UABS_G2_NC", + "R_AARCH64_MOVW_UABS_G3", + "R_X86_64_64", + "R_X86_64_GOTPCREL", + "R_X86_64_GOTPCRELX", + "R_X86_64_PC32", + "R_X86_64_REX_GOTPCRELX", + "X86_64_RELOC_BRANCH", + "X86_64_RELOC_GOT", + "X86_64_RELOC_GOT_LOAD", + "X86_64_RELOC_SIGNED", + "X86_64_RELOC_UNSIGNED", +] + + +class COFFRelocation(typing.TypedDict): + """A COFF object file relocation record.""" + + Type: dict[typing.Literal["Value"], HoleKind] + Symbol: str + Offset: int + + +class ELFRelocation(typing.TypedDict): + """An ELF object file relocation record.""" + + Addend: int + Offset: int + Symbol: dict[typing.Literal["Value"], str] + Type: dict[typing.Literal["Value"], HoleKind] + + +class MachORelocation(typing.TypedDict): + """A Mach-O object file relocation record.""" + + Offset: int + Section: typing.NotRequired[dict[typing.Literal["Value"], str]] + Symbol: typing.NotRequired[dict[typing.Literal["Value"], str]] + Type: dict[typing.Literal["Value"], HoleKind] + + +class _COFFSymbol(typing.TypedDict): + Name: str + Value: int + + +class _ELFSymbol(typing.TypedDict): + Name: dict[typing.Literal["Value"], str] + Value: int + + +class _MachOSymbol(typing.TypedDict): + Name: dict[typing.Literal["Value"], str] + Value: int + + +class COFFSection(typing.TypedDict): + """A COFF object file section.""" + + Characteristics: dict[ + typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]] + ] + Number: int + RawDataSize: int + Relocations: list[dict[typing.Literal["Relocation"], COFFRelocation]] + SectionData: typing.NotRequired[dict[typing.Literal["Bytes"], list[int]]] + Symbols: list[dict[typing.Literal["Symbol"], _COFFSymbol]] + + +class ELFSection(typing.TypedDict): + """An ELF object file section.""" + + Flags: dict[typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]]] + Index: int + Info: int + Relocations: list[dict[typing.Literal["Relocation"], ELFRelocation]] + SectionData: dict[typing.Literal["Bytes"], list[int]] + Symbols: list[dict[typing.Literal["Symbol"], _ELFSymbol]] + Type: dict[typing.Literal["Value"], str] + + +class MachOSection(typing.TypedDict): + """A Mach-O object file section.""" + + Address: int + Attributes: dict[typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]]] + Index: int + Name: dict[typing.Literal["Value"], str] + Relocations: typing.NotRequired[ + list[dict[typing.Literal["Relocation"], MachORelocation]] + ] + SectionData: typing.NotRequired[dict[typing.Literal["Bytes"], list[int]]] + Symbols: typing.NotRequired[list[dict[typing.Literal["Symbol"], _MachOSymbol]]] diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py new file mode 100644 index 000000000000000..71c678e04fbfd5a --- /dev/null +++ b/Tools/jit/_stencils.py @@ -0,0 +1,220 @@ +"""Core data structures for compiled code templates.""" +import dataclasses +import enum +import sys + +import _schema + + +@enum.unique +class HoleValue(enum.Enum): + """ + Different "base" values that can be patched into holes (usually combined with the + address of a symbol and/or an addend). + """ + + # The base address of the machine code for the current uop (exposed as _JIT_ENTRY): + CODE = enum.auto() + # The base address of the machine code for the next uop (exposed as _JIT_CONTINUE): + CONTINUE = enum.auto() + # The base address of the read-only data for this uop: + DATA = enum.auto() + # The address of the current executor (exposed as _JIT_EXECUTOR): + EXECUTOR = enum.auto() + # The base address of the "global" offset table located in the read-only data. + # Shouldn't be present in the final stencils, since these are all replaced with + # equivalent DATA values: + GOT = enum.auto() + # The current uop's oparg (exposed as _JIT_OPARG): + OPARG = enum.auto() + # The current uop's operand (exposed as _JIT_OPERAND): + OPERAND = enum.auto() + # The current uop's target (exposed as _JIT_TARGET): + TARGET = enum.auto() + # The base address of the machine code for the first uop (exposed as _JIT_TOP): + TOP = enum.auto() + # A hardcoded value of zero (used for symbol lookups): + ZERO = enum.auto() + + +@dataclasses.dataclass +class Hole: + """ + A "hole" in the stencil to be patched with a computed runtime value. + + Analogous to relocation records in an object file. + """ + + offset: int + kind: _schema.HoleKind + # Patch with this base value: + value: HoleValue + # ...plus the address of this symbol: + symbol: str | None + # ...plus this addend: + addend: int + # Convenience method: + replace = dataclasses.replace + + def as_c(self) -> str: + """Dump this hole as an initialization of a C Hole struct.""" + parts = [ + f"{self.offset:#x}", + f"HoleKind_{self.kind}", + f"HoleValue_{self.value.name}", + f"&{self.symbol}" if self.symbol else "NULL", + _format_addend(self.addend), + ] + return f"{{{', '.join(parts)}}}" + + +@dataclasses.dataclass +class Stencil: + """ + A contiguous block of machine code or data to be copied-and-patched. + + Analogous to a section or segment in an object file. + """ + + body: bytearray = dataclasses.field(default_factory=bytearray, init=False) + holes: list[Hole] = dataclasses.field(default_factory=list, init=False) + disassembly: list[str] = dataclasses.field(default_factory=list, init=False) + + def pad(self, alignment: int) -> None: + """Pad the stencil to the given alignment.""" + offset = len(self.body) + padding = -offset % alignment + self.disassembly.append(f"{offset:x}: {' '.join(['00'] * padding)}") + self.body.extend([0] * padding) + + def emit_aarch64_trampoline(self, hole: Hole) -> None: + """Even with the large code model, AArch64 Linux insists on 28-bit jumps.""" + base = len(self.body) + where = slice(hole.offset, hole.offset + 4) + instruction = int.from_bytes(self.body[where], sys.byteorder) + instruction &= 0xFC000000 + instruction |= ((base - hole.offset) >> 2) & 0x03FFFFFF + self.body[where] = instruction.to_bytes(4, sys.byteorder) + self.disassembly += [ + f"{base + 4 * 0: x}: d2800008 mov x8, #0x0", + f"{base + 4 * 0:016x}: R_AARCH64_MOVW_UABS_G0_NC {hole.symbol}", + f"{base + 4 * 1:x}: f2a00008 movk x8, #0x0, lsl #16", + f"{base + 4 * 1:016x}: R_AARCH64_MOVW_UABS_G1_NC {hole.symbol}", + f"{base + 4 * 2:x}: f2c00008 movk x8, #0x0, lsl #32", + f"{base + 4 * 2:016x}: R_AARCH64_MOVW_UABS_G2_NC {hole.symbol}", + f"{base + 4 * 3:x}: f2e00008 movk x8, #0x0, lsl #48", + f"{base + 4 * 3:016x}: R_AARCH64_MOVW_UABS_G3 {hole.symbol}", + f"{base + 4 * 4:x}: d61f0100 br x8", + ] + for code in [ + 0xD2800008.to_bytes(4, sys.byteorder), + 0xF2A00008.to_bytes(4, sys.byteorder), + 0xF2C00008.to_bytes(4, sys.byteorder), + 0xF2E00008.to_bytes(4, sys.byteorder), + 0xD61F0100.to_bytes(4, sys.byteorder), + ]: + self.body.extend(code) + for i, kind in enumerate( + [ + "R_AARCH64_MOVW_UABS_G0_NC", + "R_AARCH64_MOVW_UABS_G1_NC", + "R_AARCH64_MOVW_UABS_G2_NC", + "R_AARCH64_MOVW_UABS_G3", + ] + ): + self.holes.append(hole.replace(offset=base + 4 * i, kind=kind)) + + +@dataclasses.dataclass +class StencilGroup: + """ + Code and data corresponding to a given micro-opcode. + + Analogous to an entire object file. + """ + + code: Stencil = dataclasses.field(default_factory=Stencil, init=False) + data: Stencil = dataclasses.field(default_factory=Stencil, init=False) + symbols: dict[int | str, tuple[HoleValue, int]] = dataclasses.field( + default_factory=dict, init=False + ) + _got: dict[str, int] = dataclasses.field(default_factory=dict, init=False) + + def process_relocations(self, *, alignment: int = 1) -> None: + """Fix up all GOT and internal relocations for this stencil group.""" + self.code.pad(alignment) + self.data.pad(8) + for stencil in [self.code, self.data]: + holes = [] + for hole in stencil.holes: + if hole.value is HoleValue.GOT: + assert hole.symbol is not None + hole.value = HoleValue.DATA + hole.addend += self._global_offset_table_lookup(hole.symbol) + hole.symbol = None + elif hole.symbol in self.symbols: + hole.value, addend = self.symbols[hole.symbol] + hole.addend += addend + hole.symbol = None + elif ( + hole.kind in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26"} + and hole.value is HoleValue.ZERO + ): + self.code.emit_aarch64_trampoline(hole) + continue + holes.append(hole) + stencil.holes[:] = holes + self.code.pad(alignment) + self._emit_global_offset_table() + self.code.holes.sort(key=lambda hole: hole.offset) + self.data.holes.sort(key=lambda hole: hole.offset) + + def _global_offset_table_lookup(self, symbol: str) -> int: + return len(self.data.body) + self._got.setdefault(symbol, 8 * len(self._got)) + + def _emit_global_offset_table(self) -> None: + got = len(self.data.body) + for s, offset in self._got.items(): + if s in self.symbols: + value, addend = self.symbols[s] + symbol = None + else: + value, symbol = symbol_to_value(s) + addend = 0 + self.data.holes.append( + Hole(got + offset, "R_X86_64_64", value, symbol, addend) + ) + value_part = value.name if value is not HoleValue.ZERO else "" + if value_part and not symbol and not addend: + addend_part = "" + else: + addend_part = f"&{symbol}" if symbol else "" + addend_part += _format_addend(addend, signed=symbol is not None) + if value_part: + value_part += "+" + self.data.disassembly.append( + f"{len(self.data.body):x}: {value_part}{addend_part}" + ) + self.data.body.extend([0] * 8) + + +def symbol_to_value(symbol: str) -> tuple[HoleValue, str | None]: + """ + Convert a symbol name to a HoleValue and a symbol name. + + Some symbols (starting with "_JIT_") are special and are converted to their + own HoleValues. + """ + if symbol.startswith("_JIT_"): + try: + return HoleValue[symbol.removeprefix("_JIT_")], None + except KeyError: + pass + return HoleValue.ZERO, symbol + + +def _format_addend(addend: int, signed: bool = False) -> str: + addend %= 1 << 64 + if addend & (1 << 63): + addend -= 1 << 64 + return f"{addend:{'+#x' if signed else '#x'}}" diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py new file mode 100644 index 000000000000000..06dc4e7acc6c91c --- /dev/null +++ b/Tools/jit/_targets.py @@ -0,0 +1,435 @@ +"""Target-specific code generation, parsing, and processing.""" +import asyncio +import dataclasses +import hashlib +import json +import os +import pathlib +import re +import sys +import tempfile +import typing + +import _llvm +import _schema +import _stencils +import _writer + +if sys.version_info < (3, 11): + raise RuntimeError("Building the JIT compiler requires Python 3.11 or newer!") + +TOOLS_JIT_BUILD = pathlib.Path(__file__).resolve() +TOOLS_JIT = TOOLS_JIT_BUILD.parent +TOOLS = TOOLS_JIT.parent +CPYTHON = TOOLS.parent +PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" +TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" + + +_S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection) +_R = typing.TypeVar( + "_R", _schema.COFFRelocation, _schema.ELFRelocation, _schema.MachORelocation +) + + +@dataclasses.dataclass +class _Target(typing.Generic[_S, _R]): + triple: str + _: dataclasses.KW_ONLY + alignment: int = 1 + args: typing.Sequence[str] = () + prefix: str = "" + debug: bool = False + force: bool = False + verbose: bool = False + + def _compute_digest(self, out: pathlib.Path) -> str: + hasher = hashlib.sha256() + hasher.update(self.triple.encode()) + hasher.update(self.debug.to_bytes()) + # These dependencies are also reflected in _JITSources in regen.targets: + hasher.update(PYTHON_EXECUTOR_CASES_C_H.read_bytes()) + hasher.update((out / "pyconfig.h").read_bytes()) + for dirpath, _, filenames in sorted(os.walk(TOOLS_JIT)): + for filename in filenames: + hasher.update(pathlib.Path(dirpath, filename).read_bytes()) + return hasher.hexdigest() + + async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: + group = _stencils.StencilGroup() + args = ["--disassemble", "--reloc", f"{path}"] + output = await _llvm.maybe_run("llvm-objdump", args, echo=self.verbose) + if output is not None: + group.code.disassembly.extend( + line.expandtabs().strip() + for line in output.splitlines() + if not line.isspace() + ) + args = [ + "--elf-output-style=JSON", + "--expand-relocs", + # "--pretty-print", + "--section-data", + "--section-relocations", + "--section-symbols", + "--sections", + f"{path}", + ] + output = await _llvm.run("llvm-readobj", args, echo=self.verbose) + # --elf-output-style=JSON is only *slightly* broken on Mach-O... + output = output.replace("PrivateExtern\n", "\n") + output = output.replace("Extern\n", "\n") + # ...and also COFF: + output = output[output.index("[", 1, None) :] + output = output[: output.rindex("]", None, -1) + 1] + sections: list[dict[typing.Literal["Section"], _S]] = json.loads(output) + for wrapped_section in sections: + self._handle_section(wrapped_section["Section"], group) + assert group.symbols["_JIT_ENTRY"] == (_stencils.HoleValue.CODE, 0) + if group.data.body: + line = f"0: {str(bytes(group.data.body)).removeprefix('b')}" + group.data.disassembly.append(line) + group.process_relocations() + return group + + def _handle_section(self, section: _S, group: _stencils.StencilGroup) -> None: + raise NotImplementedError(type(self)) + + def _handle_relocation( + self, base: int, relocation: _R, raw: bytes + ) -> _stencils.Hole: + raise NotImplementedError(type(self)) + + async def _compile( + self, opname: str, c: pathlib.Path, tempdir: pathlib.Path + ) -> _stencils.StencilGroup: + o = tempdir / f"{opname}.o" + args = [ + f"--target={self.triple}", + "-DPy_BUILD_CORE", + "-D_DEBUG" if self.debug else "-DNDEBUG", + f"-D_JIT_OPCODE={opname}", + "-D_PyJIT_ACTIVE", + "-D_Py_JIT", + "-I.", + f"-I{CPYTHON / 'Include'}", + f"-I{CPYTHON / 'Include' / 'internal'}", + f"-I{CPYTHON / 'Include' / 'internal' / 'mimalloc'}", + f"-I{CPYTHON / 'Python'}", + "-O3", + "-c", + "-fno-asynchronous-unwind-tables", + "-fno-builtin", + # SET_FUNCTION_ATTRIBUTE on 32-bit Windows debug builds: + "-fno-jump-tables", + "-fno-plt", + # Don't make calls to weird stack-smashing canaries: + "-fno-stack-protector", + "-o", + f"{o}", + "-std=c11", + f"{c}", + *self.args, + ] + await _llvm.run("clang", args, echo=self.verbose) + return await self._parse(o) + + async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: + generated_cases = PYTHON_EXECUTOR_CASES_C_H.read_text() + opnames = sorted(re.findall(r"\n {8}case (\w+): \{\n", generated_cases)) + tasks = [] + with tempfile.TemporaryDirectory() as tempdir: + work = pathlib.Path(tempdir).resolve() + async with asyncio.TaskGroup() as group: + for opname in opnames: + coro = self._compile(opname, TOOLS_JIT_TEMPLATE_C, work) + tasks.append(group.create_task(coro, name=opname)) + return {task.get_name(): task.result() for task in tasks} + + def build(self, out: pathlib.Path, *, comment: str = "") -> None: + """Build jit_stencils.h in the given directory.""" + digest = f"// {self._compute_digest(out)}\n" + jit_stencils = out / "jit_stencils.h" + if ( + not self.force + and jit_stencils.exists() + and jit_stencils.read_text().startswith(digest) + ): + return + stencil_groups = asyncio.run(self._build_stencils()) + with jit_stencils.open("w") as file: + file.write(digest) + if comment: + file.write(f"// {comment}\n\n") + file.write("") + for line in _writer.dump(stencil_groups): + file.write(f"{line}\n") + + +class _COFF( + _Target[_schema.COFFSection, _schema.COFFRelocation] +): # pylint: disable = too-few-public-methods + def _handle_section( + self, section: _schema.COFFSection, group: _stencils.StencilGroup + ) -> None: + flags = {flag["Name"] for flag in section["Characteristics"]["Flags"]} + if "SectionData" in section: + section_data_bytes = section["SectionData"]["Bytes"] + else: + # Zeroed BSS data, seen with printf debugging calls: + section_data_bytes = [0] * section["RawDataSize"] + if "IMAGE_SCN_MEM_EXECUTE" in flags: + value = _stencils.HoleValue.CODE + stencil = group.code + elif "IMAGE_SCN_MEM_READ" in flags: + value = _stencils.HoleValue.DATA + stencil = group.data + else: + return + base = len(stencil.body) + group.symbols[section["Number"]] = value, base + stencil.body.extend(section_data_bytes) + for wrapped_symbol in section["Symbols"]: + symbol = wrapped_symbol["Symbol"] + offset = base + symbol["Value"] + name = symbol["Name"] + name = name.removeprefix(self.prefix) + group.symbols[name] = value, offset + for wrapped_relocation in section["Relocations"]: + relocation = wrapped_relocation["Relocation"] + hole = self._handle_relocation(base, relocation, stencil.body) + stencil.holes.append(hole) + + def _handle_relocation( + self, base: int, relocation: _schema.COFFRelocation, raw: bytes + ) -> _stencils.Hole: + match relocation: + case { + "Offset": offset, + "Symbol": s, + "Type": {"Value": "IMAGE_REL_AMD64_ADDR64" as kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + addend = int.from_bytes(raw[offset : offset + 8], "little") + case { + "Offset": offset, + "Symbol": s, + "Type": {"Value": "IMAGE_REL_I386_DIR32" as kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + addend = int.from_bytes(raw[offset : offset + 4], "little") + case _: + raise NotImplementedError(relocation) + return _stencils.Hole(offset, kind, value, symbol, addend) + + +class _ELF( + _Target[_schema.ELFSection, _schema.ELFRelocation] +): # pylint: disable = too-few-public-methods + def _handle_section( + self, section: _schema.ELFSection, group: _stencils.StencilGroup + ) -> None: + section_type = section["Type"]["Value"] + flags = {flag["Name"] for flag in section["Flags"]["Flags"]} + if section_type == "SHT_RELA": + assert "SHF_INFO_LINK" in flags, flags + assert not section["Symbols"] + value, base = group.symbols[section["Info"]] + if value is _stencils.HoleValue.CODE: + stencil = group.code + else: + assert value is _stencils.HoleValue.DATA + stencil = group.data + for wrapped_relocation in section["Relocations"]: + relocation = wrapped_relocation["Relocation"] + hole = self._handle_relocation(base, relocation, stencil.body) + stencil.holes.append(hole) + elif section_type == "SHT_PROGBITS": + if "SHF_ALLOC" not in flags: + return + if "SHF_EXECINSTR" in flags: + value = _stencils.HoleValue.CODE + stencil = group.code + else: + value = _stencils.HoleValue.DATA + stencil = group.data + group.symbols[section["Index"]] = value, len(stencil.body) + for wrapped_symbol in section["Symbols"]: + symbol = wrapped_symbol["Symbol"] + offset = len(stencil.body) + symbol["Value"] + name = symbol["Name"]["Value"] + name = name.removeprefix(self.prefix) + group.symbols[name] = value, offset + stencil.body.extend(section["SectionData"]["Bytes"]) + assert not section["Relocations"] + else: + assert section_type in { + "SHT_GROUP", + "SHT_LLVM_ADDRSIG", + "SHT_NULL", + "SHT_STRTAB", + "SHT_SYMTAB", + }, section_type + + def _handle_relocation( + self, base: int, relocation: _schema.ELFRelocation, raw: bytes + ) -> _stencils.Hole: + symbol: str | None + match relocation: + case { + "Addend": addend, + "Offset": offset, + "Symbol": {"Value": s}, + "Type": { + "Value": "R_AARCH64_ADR_GOT_PAGE" + | "R_AARCH64_LD64_GOT_LO12_NC" + | "R_X86_64_GOTPCREL" + | "R_X86_64_GOTPCRELX" + | "R_X86_64_REX_GOTPCRELX" as kind + }, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.HoleValue.GOT, s + case { + "Addend": addend, + "Offset": offset, + "Symbol": {"Value": s}, + "Type": {"Value": kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + case _: + raise NotImplementedError(relocation) + return _stencils.Hole(offset, kind, value, symbol, addend) + + +class _MachO( + _Target[_schema.MachOSection, _schema.MachORelocation] +): # pylint: disable = too-few-public-methods + def _handle_section( + self, section: _schema.MachOSection, group: _stencils.StencilGroup + ) -> None: + assert section["Address"] >= len(group.code.body) + assert "SectionData" in section + flags = {flag["Name"] for flag in section["Attributes"]["Flags"]} + name = section["Name"]["Value"] + name = name.removeprefix(self.prefix) + if "Debug" in flags: + return + if "SomeInstructions" in flags: + value = _stencils.HoleValue.CODE + stencil = group.code + start_address = 0 + group.symbols[name] = value, section["Address"] - start_address + else: + value = _stencils.HoleValue.DATA + stencil = group.data + start_address = len(group.code.body) + group.symbols[name] = value, len(group.code.body) + base = section["Address"] - start_address + group.symbols[section["Index"]] = value, base + stencil.body.extend( + [0] * (section["Address"] - len(group.code.body) - len(group.data.body)) + ) + stencil.body.extend(section["SectionData"]["Bytes"]) + assert "Symbols" in section + for wrapped_symbol in section["Symbols"]: + symbol = wrapped_symbol["Symbol"] + offset = symbol["Value"] - start_address + name = symbol["Name"]["Value"] + name = name.removeprefix(self.prefix) + group.symbols[name] = value, offset + assert "Relocations" in section + for wrapped_relocation in section["Relocations"]: + relocation = wrapped_relocation["Relocation"] + hole = self._handle_relocation(base, relocation, stencil.body) + stencil.holes.append(hole) + + def _handle_relocation( + self, base: int, relocation: _schema.MachORelocation, raw: bytes + ) -> _stencils.Hole: + symbol: str | None + match relocation: + case { + "Offset": offset, + "Symbol": {"Value": s}, + "Type": { + "Value": "ARM64_RELOC_GOT_LOAD_PAGE21" + | "ARM64_RELOC_GOT_LOAD_PAGEOFF12" as kind + }, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.HoleValue.GOT, s + addend = 0 + case { + "Offset": offset, + "Symbol": {"Value": s}, + "Type": {"Value": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.HoleValue.GOT, s + addend = ( + int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 + ) + case { + "Offset": offset, + "Section": {"Value": s}, + "Type": {"Value": "X86_64_RELOC_SIGNED" as kind}, + } | { + "Offset": offset, + "Symbol": {"Value": s}, + "Type": { + "Value": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind + }, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + addend = ( + int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 + ) + case { + "Offset": offset, + "Section": {"Value": s}, + "Type": {"Value": kind}, + } | { + "Offset": offset, + "Symbol": {"Value": s}, + "Type": {"Value": kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + addend = 0 + case _: + raise NotImplementedError(relocation) + return _stencils.Hole(offset, kind, value, symbol, addend) + + +def get_target(host: str) -> _COFF | _ELF | _MachO: + """Build a _Target for the given host "triple" and options.""" + if re.fullmatch(r"aarch64-apple-darwin.*", host): + args = ["-mcmodel=large"] + return _MachO(host, alignment=8, args=args, prefix="_") + if re.fullmatch(r"aarch64-.*-linux-gnu", host): + args = ["-mcmodel=large"] + return _ELF(host, alignment=8, args=args) + if re.fullmatch(r"i686-pc-windows-msvc", host): + args = ["-mcmodel=large"] + return _COFF(host, args=args, prefix="_") + if re.fullmatch(r"x86_64-apple-darwin.*", host): + return _MachO(host, prefix="_") + if re.fullmatch(r"x86_64-pc-windows-msvc", host): + args = ["-mcmodel=large"] + return _COFF(host, args=args) + if re.fullmatch(r"x86_64-.*-linux-gnu", host): + return _ELF(host) + raise ValueError(host) diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py new file mode 100644 index 000000000000000..8a2a42e75cfb9b3 --- /dev/null +++ b/Tools/jit/_writer.py @@ -0,0 +1,95 @@ +"""Utilities for writing StencilGroups out to a C header file.""" +import typing + +import _schema +import _stencils + + +def _dump_header() -> typing.Iterator[str]: + yield "typedef enum {" + for kind in typing.get_args(_schema.HoleKind): + yield f" HoleKind_{kind}," + yield "} HoleKind;" + yield "" + yield "typedef enum {" + for value in _stencils.HoleValue: + yield f" HoleValue_{value.name}," + yield "} HoleValue;" + yield "" + yield "typedef struct {" + yield " const uint64_t offset;" + yield " const HoleKind kind;" + yield " const HoleValue value;" + yield " const void *symbol;" + yield " const uint64_t addend;" + yield "} Hole;" + yield "" + yield "typedef struct {" + yield " const size_t body_size;" + yield " const unsigned char * const body;" + yield " const size_t holes_size;" + yield " const Hole * const holes;" + yield "} Stencil;" + yield "" + yield "typedef struct {" + yield " const Stencil code;" + yield " const Stencil data;" + yield "} StencilGroup;" + yield "" + + +def _dump_footer(opnames: typing.Iterable[str]) -> typing.Iterator[str]: + yield "#define INIT_STENCIL(STENCIL) { \\" + yield " .body_size = Py_ARRAY_LENGTH(STENCIL##_body) - 1, \\" + yield " .body = STENCIL##_body, \\" + yield " .holes_size = Py_ARRAY_LENGTH(STENCIL##_holes) - 1, \\" + yield " .holes = STENCIL##_holes, \\" + yield "}" + yield "" + yield "#define INIT_STENCIL_GROUP(OP) { \\" + yield " .code = INIT_STENCIL(OP##_code), \\" + yield " .data = INIT_STENCIL(OP##_data), \\" + yield "}" + yield "" + yield "static const StencilGroup stencil_groups[512] = {" + for opname in opnames: + yield f" [{opname}] = INIT_STENCIL_GROUP({opname})," + yield "};" + yield "" + yield "#define GET_PATCHES() { \\" + for value in _stencils.HoleValue: + yield f" [HoleValue_{value.name}] = (uint64_t)0xBADBADBADBADBADB, \\" + yield "}" + + +def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator[str]: + yield f"// {opname}" + for part, stencil in [("code", group.code), ("data", group.data)]: + for line in stencil.disassembly: + yield f"// {line}" + if stencil.body: + size = len(stencil.body) + 1 + yield f"static const unsigned char {opname}_{part}_body[{size}] = {{" + for i in range(0, len(stencil.body), 8): + row = " ".join(f"{byte:#04x}," for byte in stencil.body[i : i + 8]) + yield f" {row}" + yield "};" + else: + yield f"static const unsigned char {opname}_{part}_body[1];" + if stencil.holes: + size = len(stencil.holes) + 1 + yield f"static const Hole {opname}_{part}_holes[{size}] = {{" + for hole in stencil.holes: + yield f" {hole.as_c()}," + yield "};" + else: + yield f"static const Hole {opname}_{part}_holes[1];" + yield "" + + +def dump(groups: dict[str, _stencils.StencilGroup]) -> typing.Iterator[str]: + """Yield a JIT compiler line-by-line as a C header file.""" + yield from _dump_header() + for opname, group in groups.items(): + yield from _dump_stencil(opname, group) + yield from _dump_footer(groups) diff --git a/Tools/jit/build.py b/Tools/jit/build.py new file mode 100644 index 000000000000000..4d4ace14ebf26c4 --- /dev/null +++ b/Tools/jit/build.py @@ -0,0 +1,28 @@ +"""Build an experimental just-in-time compiler for CPython.""" +import argparse +import pathlib +import shlex +import sys + +import _targets + +if __name__ == "__main__": + comment = f"$ {shlex.join([sys.executable] + sys.argv)}" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "target", type=_targets.get_target, help="a PEP 11 target triple to compile for" + ) + parser.add_argument( + "-d", "--debug", action="store_true", help="compile for a debug build of Python" + ) + parser.add_argument( + "-f", "--force", action="store_true", help="force the entire JIT to be rebuilt" + ) + parser.add_argument( + "-v", "--verbose", action="store_true", help="echo commands as they are run" + ) + args = parser.parse_args() + args.target.debug = args.debug + args.target.force = args.force + args.target.verbose = args.verbose + args.target.build(pathlib.Path.cwd(), comment=comment) diff --git a/Tools/jit/mypy.ini b/Tools/jit/mypy.ini new file mode 100644 index 000000000000000..768d0028516abd0 --- /dev/null +++ b/Tools/jit/mypy.ini @@ -0,0 +1,5 @@ +[mypy] +files = Tools/jit +pretty = True +python_version = 3.11 +strict = True diff --git a/Tools/jit/template.c b/Tools/jit/template.c new file mode 100644 index 000000000000000..d79c6efb8f6de4e --- /dev/null +++ b/Tools/jit/template.c @@ -0,0 +1,118 @@ +#include "Python.h" + +#include "pycore_call.h" +#include "pycore_ceval.h" +#include "pycore_dict.h" +#include "pycore_emscripten_signal.h" +#include "pycore_intrinsics.h" +#include "pycore_jit.h" +#include "pycore_long.h" +#include "pycore_opcode_metadata.h" +#include "pycore_opcode_utils.h" +#include "pycore_range.h" +#include "pycore_setobject.h" +#include "pycore_sliceobject.h" + +#include "ceval_macros.h" + +#undef CURRENT_OPARG +#define CURRENT_OPARG() (_oparg) + +#undef CURRENT_OPERAND +#define CURRENT_OPERAND() (_operand) + +#undef DEOPT_IF +#define DEOPT_IF(COND, INSTNAME) \ + do { \ + if ((COND)) { \ + goto deoptimize; \ + } \ + } while (0) + +#undef ENABLE_SPECIALIZATION +#define ENABLE_SPECIALIZATION (0) + +#undef GOTO_ERROR +#define GOTO_ERROR(LABEL) \ + do { \ + goto LABEL ## _tier_two; \ + } while (0) + +#undef GOTO_TIER_TWO +#define GOTO_TIER_TWO(EXECUTOR) \ +do { \ + __attribute__((musttail)) \ + return ((jit_func)((EXECUTOR)->jit_code))(frame, stack_pointer, tstate); \ +} while (0) + +#undef GOTO_TIER_ONE +#define GOTO_TIER_ONE(TARGET) \ +do { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + return TARGET; \ +} while (0) + +#undef LOAD_IP +#define LOAD_IP(UNUSED) \ + do { \ + } while (0) + +#define PATCH_VALUE(TYPE, NAME, ALIAS) \ + extern void ALIAS; \ + TYPE NAME = (TYPE)(uint64_t)&ALIAS; + +#define PATCH_JUMP(ALIAS) \ + extern void ALIAS; \ + __attribute__((musttail)) \ + return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); + +_Py_CODEUNIT * +_JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate) +{ + // Locals that the instruction implementations expect to exist: + PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR) + int oparg; + int opcode = _JIT_OPCODE; + // Other stuff we need handy: + PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) + PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND) + PATCH_VALUE(uint32_t, _target, _JIT_TARGET) + // The actual instruction definitions (only one will be used): + if (opcode == _JUMP_TO_TOP) { + CHECK_EVAL_BREAKER(); + PATCH_JUMP(_JIT_TOP); + } + switch (opcode) { +#include "executor_cases.c.h" + default: + Py_UNREACHABLE(); + } + PATCH_JUMP(_JIT_CONTINUE); + // Labels that the instruction implementations expect to exist: +unbound_local_error_tier_two: + _PyEval_FormatExcCheckArg( + tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)); + goto error_tier_two; +pop_4_error_tier_two: + STACK_SHRINK(1); +pop_3_error_tier_two: + STACK_SHRINK(1); +pop_2_error_tier_two: + STACK_SHRINK(1); +pop_1_error_tier_two: + STACK_SHRINK(1); +error_tier_two: + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(NULL); +deoptimize: + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target); +side_exit: + { + _PyExitData *exit = ¤t_executor->exits[_target]; + Py_INCREF(exit->executor); + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_TWO(exit->executor); + } +} diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index b89f86a35d61153..c0a63b40ff4155f 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -3,5 +3,5 @@ mypy==1.8.0 # needed for peg_generator: -types-psutil==5.9.5.17 -types-setuptools==69.0.0.0 +types-psutil==5.9.5.20240106 +types-setuptools==69.0.0.20240125 diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index 0e6e16ae198162c..064731a236ee863 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.92.2 +hypothesis==6.97.4 diff --git a/Tools/scripts/sortperf.py b/Tools/scripts/sortperf.py new file mode 100644 index 000000000000000..b54681524ac1731 --- /dev/null +++ b/Tools/scripts/sortperf.py @@ -0,0 +1,196 @@ +""" +List sort performance test. + +To install `pyperf` you would need to: + + python3 -m pip install pyperf + +To run: + + python3 Tools/scripts/sortperf + +Options: + + * `benchmark` name to run + * `--rnd-seed` to set random seed + * `--size` to set the sorted list size + +Based on https://github.com/python/cpython/blob/963904335e579bfe39101adf3fd6a0cf705975ff/Lib/test/sortperf.py +""" + +from __future__ import annotations + +import argparse +import time +import random + + +# =============== +# Data generation +# =============== + +def _random_data(size: int, rand: random.Random) -> list[float]: + result = [rand.random() for _ in range(size)] + # Shuffle it a bit... + for i in range(10): + i = rand.randrange(size) + temp = result[:i] + del result[:i] + temp.reverse() + result.extend(temp) + del temp + assert len(result) == size + return result + + +def list_sort(size: int, rand: random.Random) -> list[float]: + return _random_data(size, rand) + + +def list_sort_descending(size: int, rand: random.Random) -> list[float]: + return list(reversed(list_sort_ascending(size, rand))) + + +def list_sort_ascending(size: int, rand: random.Random) -> list[float]: + return sorted(_random_data(size, rand)) + + +def list_sort_ascending_exchanged(size: int, rand: random.Random) -> list[float]: + result = list_sort_ascending(size, rand) + # Do 3 random exchanges. + for _ in range(3): + i1 = rand.randrange(size) + i2 = rand.randrange(size) + result[i1], result[i2] = result[i2], result[i1] + return result + + +def list_sort_ascending_random(size: int, rand: random.Random) -> list[float]: + assert size >= 10, "This benchmark requires size to be >= 10" + result = list_sort_ascending(size, rand) + # Replace the last 10 with random floats. + result[-10:] = [rand.random() for _ in range(10)] + return result + + +def list_sort_ascending_one_percent(size: int, rand: random.Random) -> list[float]: + result = list_sort_ascending(size, rand) + # Replace 1% of the elements at random. + for _ in range(size // 100): + result[rand.randrange(size)] = rand.random() + return result + + +def list_sort_duplicates(size: int, rand: random.Random) -> list[float]: + assert size >= 4 + result = list_sort_ascending(4, rand) + # Arrange for lots of duplicates. + result = result * (size // 4) + # Force the elements to be distinct objects, else timings can be + # artificially low. + return list(map(abs, result)) + + +def list_sort_equal(size: int, rand: random.Random) -> list[float]: + # All equal. Again, force the elements to be distinct objects. + return list(map(abs, [-0.519012] * size)) + + +def list_sort_worst_case(size: int, rand: random.Random) -> list[float]: + # This one looks like [3, 2, 1, 0, 0, 1, 2, 3]. It was a bad case + # for an older implementation of quicksort, which used the median + # of the first, last and middle elements as the pivot. + half = size // 2 + result = list(range(half - 1, -1, -1)) + result.extend(range(half)) + # Force to float, so that the timings are comparable. This is + # significantly faster if we leave them as ints. + return list(map(float, result)) + + +# ========= +# Benchmark +# ========= + +class Benchmark: + def __init__(self, name: str, size: int, seed: int) -> None: + self._name = name + self._size = size + self._seed = seed + self._random = random.Random(self._seed) + + def run(self, loops: int) -> float: + all_data = self._prepare_data(loops) + start = time.perf_counter() + + for data in all_data: + data.sort() # Benching this method! + + return time.perf_counter() - start + + def _prepare_data(self, loops: int) -> list[float]: + bench = BENCHMARKS[self._name] + return [bench(self._size, self._random)] * loops + + +def add_cmdline_args(cmd: list[str], args) -> None: + if args.benchmark: + cmd.append(args.benchmark) + cmd.append(f"--size={args.size}") + cmd.append(f"--rng-seed={args.rng_seed}") + + +def add_parser_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "benchmark", + choices=BENCHMARKS, + nargs="?", + help="Can be any of: {0}".format(", ".join(BENCHMARKS)), + ) + parser.add_argument( + "--size", + type=int, + default=DEFAULT_SIZE, + help=f"Size of the lists to sort (default: {DEFAULT_SIZE})", + ) + parser.add_argument( + "--rng-seed", + type=int, + default=DEFAULT_RANDOM_SEED, + help=f"Random number generator seed (default: {DEFAULT_RANDOM_SEED})", + ) + + +DEFAULT_SIZE = 1 << 14 +DEFAULT_RANDOM_SEED = 0 +BENCHMARKS = { + "list_sort": list_sort, + "list_sort_descending": list_sort_descending, + "list_sort_ascending": list_sort_ascending, + "list_sort_ascending_exchanged": list_sort_ascending_exchanged, + "list_sort_ascending_random": list_sort_ascending_random, + "list_sort_ascending_one_percent": list_sort_ascending_one_percent, + "list_sort_duplicates": list_sort_duplicates, + "list_sort_equal": list_sort_equal, + "list_sort_worst_case": list_sort_worst_case, +} + +if __name__ == "__main__": + # This needs `pyperf` 3rd party library: + import pyperf + + runner = pyperf.Runner(add_cmdline_args=add_cmdline_args) + add_parser_args(runner.argparser) + args = runner.parse_args() + + runner.metadata["description"] = "Test `list.sort()` with different data" + runner.metadata["list_sort_size"] = args.size + runner.metadata["list_sort_random_seed"] = args.rng_seed + + if args.benchmark: + benchmarks = (args.benchmark,) + else: + benchmarks = sorted(BENCHMARKS) + for bench in benchmarks: + benchmark = Benchmark(bench, args.size, args.rng_seed) + runner.bench_time_func(bench, benchmark.run) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 9b7e7b999ea7c7d..2925e096f4d95ef 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -11,6 +11,7 @@ import argparse import collections from collections.abc import KeysView +from dataclasses import dataclass from datetime import date import enum import functools @@ -21,6 +22,7 @@ from pathlib import Path import re import sys +import textwrap from typing import Any, Callable, TextIO, TypeAlias @@ -100,6 +102,10 @@ def load_raw_data(input: Path) -> RawData: file=sys.stderr, ) continue + # Hack to handle older data files where some uops + # are missing an underscore prefix in their name + if key.startswith("uops[") and key[5:6] != "_": + key = "uops[_" + key[5:] stats[key.strip()] += int(value) stats["__nfiles__"] += 1 @@ -115,6 +121,64 @@ def save_raw_data(data: RawData, json_output: TextIO): json.dump(data, json_output) +@dataclass(frozen=True) +class Doc: + text: str + doc: str + + def markdown(self) -> str: + return textwrap.dedent( + f""" + {self.text} + <details> + <summary>ⓘ</summary> + + {self.doc} + </details> + """ + ) + + +class Count(int): + def markdown(self) -> str: + return format(self, ",d") + + +@dataclass(frozen=True) +class Ratio: + num: int + den: int | None = None + percentage: bool = True + + def __float__(self): + if self.den == 0: + return 0.0 + elif self.den is None: + return self.num + else: + return self.num / self.den + + def markdown(self) -> str: + if self.den is None: + return "" + elif self.den == 0: + if self.num != 0: + return f"{self.num:,} / 0 !!" + return "" + elif self.percentage: + return f"{self.num / self.den:,.01%}" + else: + return f"{self.num / self.den:,.02f}" + + +class DiffRatio(Ratio): + def __init__(self, base: int | str, head: int | str): + if isinstance(base, str) or isinstance(head, str): + super().__init__(0, 0) + else: + super().__init__(head - base, base) + + class OpcodeStats: """ Manages the data related to specific set of opcodes, e.g. tier1 (with prefix @@ -387,19 +451,65 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: inner_loop = self._data["Optimization inner loop"] recursive_call = self._data["Optimization recursive call"] low_confidence = self._data["Optimization low confidence"] + executors_invalidated = self._data["Executors invalidated"] return { - "Optimization attempts": (attempts, None), - "Traces created": (created, attempts), - "Trace stack overflow": (trace_stack_overflow, attempts), - "Trace stack underflow": (trace_stack_underflow, attempts), - "Trace too long": (trace_too_long, attempts), - "Trace too short": (trace_too_short, attempts), - "Inner loop found": (inner_loop, attempts), - "Recursive call": (recursive_call, attempts), - "Low confidence": (low_confidence, attempts), - "Traces executed": (executed, None), - "Uops executed": (uops, executed), + Doc( + "Optimization attempts", + "The number of times a potential trace is identified. Specifically, this " + "occurs in the JUMP BACKWARD instruction when the counter reaches a " + "threshold.", + ): ( + attempts, + None, + ), + Doc( + "Traces created", "The number of traces that were successfully created." + ): (created, attempts), + Doc( + "Trace stack overflow", + "A trace is truncated because it would require more than 5 stack frames.", + ): (trace_stack_overflow, attempts), + Doc( + "Trace stack underflow", + "A potential trace is abandoned because it pops more frames than it pushes.", + ): (trace_stack_underflow, attempts), + Doc( + "Trace too long", + "A trace is truncated because it is longer than the instruction buffer.", + ): (trace_too_long, attempts), + Doc( + "Trace too short", + "A potential trace is abandoced because it it too short.", + ): (trace_too_short, attempts), + Doc( + "Inner loop found", "A trace is truncated because it has an inner loop" + ): (inner_loop, attempts), + Doc( + "Recursive call", + "A trace is truncated because it has a recursive call.", + ): (recursive_call, attempts), + Doc( + "Low confidence", + "A trace is abandoned because the likelihood of the jump to top being taken " + "is too low.", + ): (low_confidence, attempts), + Doc( + "Executors invalidated", + "The number of executors that were invalidated due to watched " + "dictionary changes.", + ): (executors_invalidated, created), + Doc("Traces executed", "The number of traces that were executed"): ( + executed, + None, + ), + Doc( + "Uops executed", + "The total number of uops (micro-operations) that were executed", + ): ( + uops, + executed, + ), } def get_histogram(self, prefix: str) -> list[tuple[int, int]]: @@ -415,52 +525,12 @@ def get_histogram(self, prefix: str) -> list[tuple[int, int]]: def get_rare_events(self) -> list[tuple[str, int]]: prefix = "Rare event " return [ - (key[len(prefix) + 1:-1], val) + (key[len(prefix) + 1 : -1].replace("_", " "), val) for key, val in self._data.items() if key.startswith(prefix) ] -class Count(int): - def markdown(self) -> str: - return format(self, ",d") - - -class Ratio: - def __init__(self, num: int, den: int | None, percentage: bool = True): - self.num = num - self.den = den - self.percentage = percentage - - def __float__(self): - if self.den == 0: - return 0.0 - elif self.den is None: - return self.num - else: - return self.num / self.den - - def markdown(self) -> str: - if self.den is None: - return "" - elif self.den == 0: - if self.num != 0: - return f"{self.num:,} / 0 !!" - return "" - elif self.percentage: - return f"{self.num / self.den:,.01%}" - else: - return f"{self.num / self.den:,.02f}" - - -class DiffRatio(Ratio): - def __init__(self, base: int | str, head: int | str): - if isinstance(base, str) or isinstance(head, str): - super().__init__(0, 0) - else: - super().__init__(head - base, base) - - class JoinMode(enum.Enum): # Join using the first column as a key SIMPLE = 0 @@ -568,13 +638,16 @@ def __init__( title: str = "", summary: str = "", part_iter=None, + *, comparative: bool = True, + doc: str = "", ): self.title = title if not summary: self.summary = title.lower() else: self.summary = summary + self.doc = textwrap.dedent(doc) if part_iter is None: part_iter = [] if isinstance(part_iter, list): @@ -620,7 +693,7 @@ def calc(stats: Stats) -> Rows: def execution_count_section() -> Section: return Section( "Execution counts", - "execution counts for all instructions", + "Execution counts for Tier 1 instructions.", [ Table( ("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"), @@ -628,6 +701,11 @@ def execution_count_section() -> Section: join_mode=JoinMode.CHANGE_ONE_COLUMN, ) ], + doc=""" + The "miss ratio" column shows the percentage of times the instruction + executed that it deoptimized. When this happens, the base unspecialized + instruction is not counted. + """, ) @@ -655,7 +733,7 @@ def calc_pair_count_table(stats: Stats) -> Rows: return Section( "Pair counts", - "Pair counts for top 100 pairs", + "Pair counts for top 100 Tier 1 instructions", [ Table( ("Pair", "Count:", "Self:", "Cumulative:"), @@ -663,6 +741,10 @@ def calc_pair_count_table(stats: Stats) -> Rows: ) ], comparative=False, + doc=""" + Pairs of specialized operations that deoptimize and are then followed by + the corresponding unspecialized instruction are not counted as pairs. + """, ) @@ -705,22 +787,33 @@ def iter_pre_succ_pairs_tables(base_stats: Stats, head_stats: Stats | None = Non return Section( "Predecessor/Successor Pairs", - "Top 5 predecessors and successors of each opcode", + "Top 5 predecessors and successors of each Tier 1 opcode.", iter_pre_succ_pairs_tables, comparative=False, + doc=""" + This does not include the unspecialized instructions that occur after a + specialized instruction deoptimizes. + """, ) def specialization_section() -> Section: def calc_specialization_table(opcode: str) -> RowCalculator: def calc(stats: Stats) -> Rows: + DOCS = { + "deferred": 'Lists the number of "deferred" (i.e. not specialized) instructions executed.', + "hit": "Specialized instructions that complete.", + "miss": "Specialized instructions that deopt.", + "deopt": "Specialized instructions that deopt.", + } + opcode_stats = stats.get_opcode_stats("opcode") total = opcode_stats.get_specialization_total(opcode) specialization_counts = opcode_stats.get_specialization_counts(opcode) return [ ( - f"{label:>12}", + Doc(label, DOCS[label]), Count(count), Ratio(count, total), ) @@ -790,7 +883,7 @@ def iter_specialization_tables(base_stats: Stats, head_stats: Stats | None = Non JoinMode.CHANGE, ), Table( - ("", "Count:", "Ratio:"), + ("Success", "Count:", "Ratio:"), calc_specialization_success_failure_table(opcode), JoinMode.CHANGE, ), @@ -804,7 +897,7 @@ def iter_specialization_tables(base_stats: Stats, head_stats: Stats | None = Non return Section( "Specialization stats", - "specialization stats by family", + "Specialization stats by family", iter_specialization_tables, ) @@ -822,19 +915,35 @@ def calc_specialization_effectiveness_table(stats: Stats) -> Rows: ) = opcode_stats.get_specialized_total_counts() return [ - ("Basic", Count(basic), Ratio(basic, total)), ( - "Not specialized", + Doc( + "Basic", + "Instructions that are not and cannot be specialized, e.g. `LOAD_FAST`.", + ), + Count(basic), + Ratio(basic, total), + ), + ( + Doc( + "Not specialized", + "Instructions that could be specialized but aren't, e.g. `LOAD_ATTR`, `BINARY_SLICE`.", + ), Count(not_specialized), Ratio(not_specialized, total), ), ( - "Specialized hits", + Doc( + "Specialized hits", + "Specialized instructions, e.g. `LOAD_ATTR_MODULE` that complete.", + ), Count(specialized_hits), Ratio(specialized_hits, total), ), ( - "Specialized misses", + Doc( + "Specialized misses", + "Specialized instructions, e.g. `LOAD_ATTR_MODULE` that deopt.", + ), Count(specialized_misses), Ratio(specialized_misses, total), ), @@ -879,7 +988,7 @@ def calc_misses_by_table(stats: Stats) -> Rows: ), Section( "Deferred by instruction", - "", + "Breakdown of deferred (not specialized) instruction counts by family", [ Table( ("Name", "Count:", "Ratio:"), @@ -890,7 +999,7 @@ def calc_misses_by_table(stats: Stats) -> Rows: ), Section( "Misses by instruction", - "", + "Breakdown of misses (specialized deopts) instruction counts by family", [ Table( ("Name", "Count:", "Ratio:"), @@ -900,6 +1009,10 @@ def calc_misses_by_table(stats: Stats) -> Rows: ], ), ], + doc=""" + All entries are execution counts. Should add up to the total number of + Tier 1 instructions executed. + """, ) @@ -922,6 +1035,13 @@ def calc_call_stats_table(stats: Stats) -> Rows: JoinMode.CHANGE, ) ], + doc=""" + This shows what fraction of calls to Python functions are inlined (i.e. + not having a call at the C level) and for those that are not, where the + call comes from. The various categories overlap. + + Also includes the count of frame objects created. + """, ) @@ -935,7 +1055,7 @@ def calc_object_stats_table(stats: Stats) -> Rows: return Section( "Object stats", - "allocations, frees and dict materializatons", + "Allocations, frees and dict materializatons", [ Table( ("", "Count:", "Ratio:"), @@ -943,6 +1063,16 @@ def calc_object_stats_table(stats: Stats) -> Rows: JoinMode.CHANGE, ) ], + doc=""" + Below, "allocations" means "allocations that are not from a freelist". + Total allocations = "Allocations from freelist" + "Allocations". + + "New values" is the number of values arrays created for objects with + managed dicts. + + The cache hit/miss numbers are for the MRO cache, split into dunder and + other names. + """, ) @@ -969,6 +1099,9 @@ def calc_gc_stats(stats: Stats) -> Rows: calc_gc_stats, ) ], + doc=""" + Collected/visits gives some measure of efficiency. + """, ) @@ -1074,7 +1207,19 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) def rare_event_section() -> Section: def calc_rare_event_table(stats: Stats) -> Table: - return [(x, Count(y)) for x, y in stats.get_rare_events()] + DOCS = { + "set class": "Setting an object's class, `obj.__class__ = ...`", + "set bases": "Setting the bases of a class, `cls.__bases__ = ...`", + "set eval frame func": ( + "Setting the PEP 523 frame eval function " + "`_PyInterpreterState_SetFrameEvalFunc()`" + ), + "builtin dict": "Modifying the builtins, `__builtins__.__dict__[var] = ...`", + "func modification": "Modifying a function, e.g. `func.__defaults__ = ...`, etc.", + "watched dict modification": "A watched dict has been modified", + "watched globals modification": "A watched `globals()` dict has been modified", + } + return [(Doc(x, DOCS[x]), Count(y)) for x, y in stats.get_rare_events()] return Section( "Rare events", @@ -1134,6 +1279,9 @@ def to_markdown(x): print("<details>", file=out) print("<summary>", obj.summary, "</summary>", file=out) print(file=out) + if obj.doc: + print(obj.doc, file=out) + if head_stats is not None and obj.comparative is False: print("Not included in comparative output.\n") else: @@ -1149,24 +1297,36 @@ def to_markdown(x): if len(rows) == 0: return - width = len(header) - header_line = "|" - under_line = "|" + alignments = [] for item in header: - under = "---" + if item.endswith(":"): + alignments.append("right") + else: + alignments.append("left") + + print("<table>", file=out) + print("<thead>", file=out) + print("<tr>", file=out) + for item, align in zip(header, alignments): if item.endswith(":"): item = item[:-1] - under += ":" - header_line += item + " | " - under_line += under + "|" - print(header_line, file=out) - print(under_line, file=out) + print(f'<th align="{align}">{item}</th>', file=out) + print("</tr>", file=out) + print("</thead>", file=out) + + print("<tbody>", file=out) for row in rows: - if len(row) != width: + if len(row) != len(header): raise ValueError( "Wrong number of elements in row '" + str(row) + "'" ) - print("|", " | ".join(to_markdown(i) for i in row), "|", file=out) + print("<tr>", file=out) + for col, align in zip(row, alignments): + print(f'<td align="{align}">{to_markdown(col)}</td>', file=out) + print("</tr>", file=out) + print("</tbody>", file=out) + + print("</table>", file=out) print(file=out) case list(): diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 120e3883adc795d..baa16102068aa08 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -47,8 +47,9 @@ OPENSSL_RECENT_VERSIONS = [ "1.1.1w", - "3.0.11", - "3.1.3", + "3.0.13", + "3.1.5", + "3.2.1", ] LIBRESSL_OLD_VERSIONS = [ diff --git a/Tools/stringbench/README b/Tools/stringbench/README deleted file mode 100644 index a271f12632afff3..000000000000000 --- a/Tools/stringbench/README +++ /dev/null @@ -1,68 +0,0 @@ -stringbench is a set of performance tests comparing byte string -operations with unicode operations. The two string implementations -are loosely based on each other and sometimes the algorithm for one is -faster than the other. - -These test set was started at the Need For Speed sprint in Reykjavik -to identify which string methods could be sped up quickly and to -identify obvious places for improvement. - -Here is an example of a benchmark - - -@bench('"Andrew".startswith("A")', 'startswith single character', 1000) -def startswith_single(STR): - s1 = STR("Andrew") - s2 = STR("A") - s1_startswith = s1.startswith - for x in _RANGE_1000: - s1_startswith(s2) - -The bench decorator takes three parameters. The first is a short -description of how the code works. In most cases this is Python code -snippet. It is not the code which is actually run because the real -code is hand-optimized to focus on the method being tested. - -The second parameter is a group title. All benchmarks with the same -group title are listed together. This lets you compare different -implementations of the same algorithm, such as "t in s" -vs. "s.find(t)". - -The last is a count. Each benchmark loops over the algorithm either -100 or 1000 times, depending on the algorithm performance. The output -time is the time per benchmark call so the reader needs a way to know -how to scale the performance. - -These parameters become function attributes. - - -Here is an example of the output - - -========== count newlines -38.54 41.60 92.7 ...text.with.2000.newlines.count("\n") (*100) -========== early match, single character -1.14 1.18 96.8 ("A"*1000).find("A") (*1000) -0.44 0.41 105.6 "A" in "A"*1000 (*1000) -1.15 1.17 98.1 ("A"*1000).index("A") (*1000) - -The first column is the run time in milliseconds for byte strings. -The second is the run time for unicode strings. The third is a -percentage; byte time / unicode time. It's the percentage by which -unicode is faster than byte strings. - -The last column contains the code snippet and the repeat count for the -internal benchmark loop. - -The times are computed with 'timeit.py' which repeats the test more -and more times until the total time takes over 0.2 seconds, returning -the best time for a single iteration. - -The final line of the output is the cumulative time for byte and -unicode strings, and the overall performance of unicode relative to -bytes. For example - -4079.83 5432.25 75.1 TOTAL - -However, this has no meaning as it evenly weights every test. - diff --git a/Tools/stringbench/stringbench.py b/Tools/stringbench/stringbench.py deleted file mode 100644 index 5d2b41463786261..000000000000000 --- a/Tools/stringbench/stringbench.py +++ /dev/null @@ -1,1482 +0,0 @@ - -# Various microbenchmarks comparing unicode and byte string performance -# Please keep this file both 2.x and 3.x compatible! - -import timeit -import itertools -import operator -import re -import sys -import datetime -import optparse - -VERSION = '2.0' - -def p(*args): - sys.stdout.write(' '.join(str(s) for s in args) + '\n') - -if sys.version_info >= (3,): - BYTES = bytes_from_str = lambda x: x.encode('ascii') - UNICODE = unicode_from_str = lambda x: x -else: - BYTES = bytes_from_str = lambda x: x - UNICODE = unicode_from_str = lambda x: x.decode('ascii') - -class UnsupportedType(TypeError): - pass - - -p('stringbench v%s' % VERSION) -p(sys.version) -p(datetime.datetime.now()) - -REPEAT = 1 -REPEAT = 3 -#REPEAT = 7 - -if __name__ != "__main__": - raise SystemExit("Must run as main program") - -parser = optparse.OptionParser() -parser.add_option("-R", "--skip-re", dest="skip_re", - action="store_true", - help="skip regular expression tests") -parser.add_option("-8", "--8-bit", dest="bytes_only", - action="store_true", - help="only do 8-bit string benchmarks") -parser.add_option("-u", "--unicode", dest="unicode_only", - action="store_true", - help="only do Unicode string benchmarks") - - -_RANGE_1000 = list(range(1000)) -_RANGE_100 = list(range(100)) -_RANGE_10 = list(range(10)) - -dups = {} -def bench(s, group, repeat_count): - def blah(f): - if f.__name__ in dups: - raise AssertionError("Multiple functions with same name: %r" % - (f.__name__,)) - dups[f.__name__] = 1 - f.comment = s - f.is_bench = True - f.group = group - f.repeat_count = repeat_count - return f - return blah - -def uses_re(f): - f.uses_re = True - -####### 'in' comparisons - -@bench('"A" in "A"*1000', "early match, single character", 1000) -def in_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - for x in _RANGE_1000: - s2 in s1 - -@bench('"B" in "A"*1000', "no match, single character", 1000) -def in_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - for x in _RANGE_1000: - s2 in s1 - - -@bench('"AB" in "AB"*1000', "early match, two characters", 1000) -def in_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - for x in _RANGE_1000: - s2 in s1 - -@bench('"BC" in "AB"*1000', "no match, two characters", 1000) -def in_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - for x in _RANGE_1000: - s2 in s1 - -@bench('"BC" in ("AB"*300+"C")', "late match, two characters", 1000) -def in_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - for x in _RANGE_1000: - s2 in s1 - -@bench('s="ABC"*33; (s+"E") in ((s+"D")*300+s+"E")', - "late match, 100 characters", 100) -def in_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*300 + m+e - s2 = m+e - for x in _RANGE_100: - s2 in s1 - -# Try with regex -@uses_re -@bench('s="ABC"*33; re.compile(s+"D").search((s+"D")*300+s+"E")', - "late match, 100 characters", 100) -def re_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*300 + m+e - s2 = m+e - pat = re.compile(s2) - search = pat.search - for x in _RANGE_100: - search(s1) - - -#### same tests as 'in' but use 'find' - -@bench('("A"*1000).find("A")', "early match, single character", 1000) -def find_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("A"*1000).find("B")', "no match, single character", 1000) -def find_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - - -@bench('("AB"*1000).find("AB")', "early match, two characters", 1000) -def find_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("AB"*1000).find("BC")', "no match, two characters", 1000) -def find_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("AB"*1000).find("CA")', "no match, two characters", 1000) -def find_test_no_match_two_character_bis(STR): - s1 = STR("AB" * 1000) - s2 = STR("CA") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("AB"*300+"C").find("BC")', "late match, two characters", 1000) -def find_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("AB"*300+"CA").find("CA")', "late match, two characters", 1000) -def find_test_slow_match_two_characters_bis(STR): - s1 = STR("AB" * 300+"CA") - s2 = STR("CA") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('s="ABC"*33; ((s+"D")*500+s+"E").find(s+"E")', - "late match, 100 characters", 100) -def find_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + m+e - s2 = m+e - s1_find = s1.find - for x in _RANGE_100: - s1_find(s2) - -@bench('s="ABC"*33; ((s+"D")*500+"E"+s).find("E"+s)', - "late match, 100 characters", 100) -def find_test_slow_match_100_characters_bis(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + e+m - s2 = e+m - s1_find = s1.find - for x in _RANGE_100: - s1_find(s2) - - -#### Same tests for 'rfind' - -@bench('("A"*1000).rfind("A")', "early match, single character", 1000) -def rfind_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("A"*1000).rfind("B")', "no match, single character", 1000) -def rfind_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - - -@bench('("AB"*1000).rfind("AB")', "early match, two characters", 1000) -def rfind_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("AB"*1000).rfind("BC")', "no match, two characters", 1000) -def rfind_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("AB"*1000).rfind("CA")', "no match, two characters", 1000) -def rfind_test_no_match_two_character_bis(STR): - s1 = STR("AB" * 1000) - s2 = STR("CA") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("C"+"AB"*300).rfind("CA")', "late match, two characters", 1000) -def rfind_test_slow_match_two_characters(STR): - s1 = STR("C" + "AB" * 300) - s2 = STR("CA") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("BC"+"AB"*300).rfind("BC")', "late match, two characters", 1000) -def rfind_test_slow_match_two_characters_bis(STR): - s1 = STR("BC" + "AB" * 300) - s2 = STR("BC") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('s="ABC"*33; ("E"+s+("D"+s)*500).rfind("E"+s)', - "late match, 100 characters", 100) -def rfind_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = e+m + (d+m)*500 - s2 = e+m - s1_rfind = s1.rfind - for x in _RANGE_100: - s1_rfind(s2) - -@bench('s="ABC"*33; (s+"E"+("D"+s)*500).rfind(s+"E")', - "late match, 100 characters", 100) -def rfind_test_slow_match_100_characters_bis(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = m+e + (d+m)*500 - s2 = m+e - s1_rfind = s1.rfind - for x in _RANGE_100: - s1_rfind(s2) - - -#### Now with index. -# Skip the ones which fail because that would include exception overhead. - -@bench('("A"*1000).index("A")', "early match, single character", 1000) -def index_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_index = s1.index - for x in _RANGE_1000: - s1_index(s2) - -@bench('("AB"*1000).index("AB")', "early match, two characters", 1000) -def index_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_index = s1.index - for x in _RANGE_1000: - s1_index(s2) - -@bench('("AB"*300+"C").index("BC")', "late match, two characters", 1000) -def index_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - s1_index = s1.index - for x in _RANGE_1000: - s1_index(s2) - -@bench('s="ABC"*33; ((s+"D")*500+s+"E").index(s+"E")', - "late match, 100 characters", 100) -def index_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + m+e - s2 = m+e - s1_index = s1.index - for x in _RANGE_100: - s1_index(s2) - - -#### Same for rindex - -@bench('("A"*1000).rindex("A")', "early match, single character", 1000) -def rindex_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_rindex = s1.rindex - for x in _RANGE_1000: - s1_rindex(s2) - -@bench('("AB"*1000).rindex("AB")', "early match, two characters", 1000) -def rindex_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_rindex = s1.rindex - for x in _RANGE_1000: - s1_rindex(s2) - -@bench('("C"+"AB"*300).rindex("CA")', "late match, two characters", 1000) -def rindex_test_slow_match_two_characters(STR): - s1 = STR("C" + "AB" * 300) - s2 = STR("CA") - s1_rindex = s1.rindex - for x in _RANGE_1000: - s1_rindex(s2) - -@bench('s="ABC"*33; ("E"+s+("D"+s)*500).rindex("E"+s)', - "late match, 100 characters", 100) -def rindex_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = e + m + (d+m)*500 - s2 = e + m - s1_rindex = s1.rindex - for x in _RANGE_100: - s1_rindex(s2) - - -#### Same for partition - -@bench('("A"*1000).partition("A")', "early match, single character", 1000) -def partition_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - -@bench('("A"*1000).partition("B")', "no match, single character", 1000) -def partition_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - - -@bench('("AB"*1000).partition("AB")', "early match, two characters", 1000) -def partition_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - -@bench('("AB"*1000).partition("BC")', "no match, two characters", 1000) -def partition_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - -@bench('("AB"*300+"C").partition("BC")', "late match, two characters", 1000) -def partition_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - -@bench('s="ABC"*33; ((s+"D")*500+s+"E").partition(s+"E")', - "late match, 100 characters", 100) -def partition_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + m+e - s2 = m+e - s1_partition = s1.partition - for x in _RANGE_100: - s1_partition(s2) - - -#### Same for rpartition - -@bench('("A"*1000).rpartition("A")', "early match, single character", 1000) -def rpartition_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - -@bench('("A"*1000).rpartition("B")', "no match, single character", 1000) -def rpartition_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - - -@bench('("AB"*1000).rpartition("AB")', "early match, two characters", 1000) -def rpartition_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - -@bench('("AB"*1000).rpartition("BC")', "no match, two characters", 1000) -def rpartition_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - -@bench('("C"+"AB"*300).rpartition("CA")', "late match, two characters", 1000) -def rpartition_test_slow_match_two_characters(STR): - s1 = STR("C" + "AB" * 300) - s2 = STR("CA") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - -@bench('s="ABC"*33; ("E"+s+("D"+s)*500).rpartition("E"+s)', - "late match, 100 characters", 100) -def rpartition_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = e + m + (d+m)*500 - s2 = e + m - s1_rpartition = s1.rpartition - for x in _RANGE_100: - s1_rpartition(s2) - - -#### Same for split(s, 1) - -@bench('("A"*1000).split("A", 1)', "early match, single character", 1000) -def split_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - -@bench('("A"*1000).split("B", 1)', "no match, single character", 1000) -def split_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - - -@bench('("AB"*1000).split("AB", 1)', "early match, two characters", 1000) -def split_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - -@bench('("AB"*1000).split("BC", 1)', "no match, two characters", 1000) -def split_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - -@bench('("AB"*300+"C").split("BC", 1)', "late match, two characters", 1000) -def split_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - -@bench('s="ABC"*33; ((s+"D")*500+s+"E").split(s+"E", 1)', - "late match, 100 characters", 100) -def split_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + m+e - s2 = m+e - s1_split = s1.split - for x in _RANGE_100: - s1_split(s2, 1) - - -#### Same for rsplit(s, 1) - -@bench('("A"*1000).rsplit("A", 1)', "early match, single character", 1000) -def rsplit_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - -@bench('("A"*1000).rsplit("B", 1)', "no match, single character", 1000) -def rsplit_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - - -@bench('("AB"*1000).rsplit("AB", 1)', "early match, two characters", 1000) -def rsplit_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - -@bench('("AB"*1000).rsplit("BC", 1)', "no match, two characters", 1000) -def rsplit_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - -@bench('("C"+"AB"*300).rsplit("CA", 1)', "late match, two characters", 1000) -def rsplit_test_slow_match_two_characters(STR): - s1 = STR("C" + "AB" * 300) - s2 = STR("CA") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - -@bench('s="ABC"*33; ("E"+s+("D"+s)*500).rsplit("E"+s, 1)', - "late match, 100 characters", 100) -def rsplit_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = e + m + (d+m)*500 - s2 = e + m - s1_rsplit = s1.rsplit - for x in _RANGE_100: - s1_rsplit(s2, 1) - - -#### Benchmark the operator-based methods - -@bench('"A"*10', "repeat 1 character 10 times", 1000) -def repeat_single_10_times(STR): - s = STR("A") - for x in _RANGE_1000: - s * 10 - -@bench('"A"*1000', "repeat 1 character 1000 times", 1000) -def repeat_single_1000_times(STR): - s = STR("A") - for x in _RANGE_1000: - s * 1000 - -@bench('"ABCDE"*10', "repeat 5 characters 10 times", 1000) -def repeat_5_10_times(STR): - s = STR("ABCDE") - for x in _RANGE_1000: - s * 10 - -@bench('"ABCDE"*1000', "repeat 5 characters 1000 times", 1000) -def repeat_5_1000_times(STR): - s = STR("ABCDE") - for x in _RANGE_1000: - s * 1000 - -# + for concat - -@bench('"Andrew"+"Dalke"', "concat two strings", 1000) -def concat_two_strings(STR): - s1 = STR("Andrew") - s2 = STR("Dalke") - for x in _RANGE_1000: - s1+s2 - -@bench('s1+s2+s3+s4+...+s20', "concat 20 strings of words length 4 to 15", - 1000) -def concat_many_strings(STR): - s1=STR('TIXSGYNREDCVBHJ') - s2=STR('PUMTLXBZVDO') - s3=STR('FVZNJ') - s4=STR('OGDXUW') - s5=STR('WEIMRNCOYVGHKB') - s6=STR('FCQTNMXPUZH') - s7=STR('TICZJYRLBNVUEAK') - s8=STR('REYB') - s9=STR('PWUOQ') - s10=STR('EQHCMKBS') - s11=STR('AEVDFOH') - s12=STR('IFHVD') - s13=STR('JGTCNLXWOHQ') - s14=STR('ITSKEPYLROZAWXF') - s15=STR('THEK') - s16=STR('GHPZFBUYCKMNJIT') - s17=STR('JMUZ') - s18=STR('WLZQMTB') - s19=STR('KPADCBW') - s20=STR('TNJHZQAGBU') - for x in _RANGE_1000: - (s1 + s2+ s3+ s4+ s5+ s6+ s7+ s8+ s9+s10+ - s11+s12+s13+s14+s15+s16+s17+s18+s19+s20) - - -#### Benchmark join - -def get_bytes_yielding_seq(STR, arg): - if STR is BYTES and sys.version_info >= (3,): - raise UnsupportedType - return STR(arg) - -@bench('"A".join("")', - "join empty string, with 1 character sep", 100) -def join_empty_single(STR): - sep = STR("A") - s2 = get_bytes_yielding_seq(STR, "") - sep_join = sep.join - for x in _RANGE_100: - sep_join(s2) - -@bench('"ABCDE".join("")', - "join empty string, with 5 character sep", 100) -def join_empty_5(STR): - sep = STR("ABCDE") - s2 = get_bytes_yielding_seq(STR, "") - sep_join = sep.join - for x in _RANGE_100: - sep_join(s2) - -@bench('"A".join("ABC..Z")', - "join string with 26 characters, with 1 character sep", 1000) -def join_alphabet_single(STR): - sep = STR("A") - s2 = get_bytes_yielding_seq(STR, "ABCDEFGHIJKLMnOPQRSTUVWXYZ") - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"ABCDE".join("ABC..Z")', - "join string with 26 characters, with 5 character sep", 1000) -def join_alphabet_5(STR): - sep = STR("ABCDE") - s2 = get_bytes_yielding_seq(STR, "ABCDEFGHIJKLMnOPQRSTUVWXYZ") - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"A".join(list("ABC..Z"))', - "join list of 26 characters, with 1 character sep", 1000) -def join_alphabet_list_single(STR): - sep = STR("A") - s2 = [STR(x) for x in "ABCDEFGHIJKLMnOPQRSTUVWXYZ"] - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"ABCDE".join(list("ABC..Z"))', - "join list of 26 characters, with 5 character sep", 1000) -def join_alphabet_list_five(STR): - sep = STR("ABCDE") - s2 = [STR(x) for x in "ABCDEFGHIJKLMnOPQRSTUVWXYZ"] - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"A".join(["Bob"]*100)', - "join list of 100 words, with 1 character sep", 1000) -def join_100_words_single(STR): - sep = STR("A") - s2 = [STR("Bob")]*100 - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"ABCDE".join(["Bob"]*100))', - "join list of 100 words, with 5 character sep", 1000) -def join_100_words_5(STR): - sep = STR("ABCDE") - s2 = [STR("Bob")]*100 - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -#### split tests - -@bench('("Here are some words. "*2).split()', "split whitespace (small)", 1000) -def whitespace_split(STR): - s = STR("Here are some words. "*2) - s_split = s.split - for x in _RANGE_1000: - s_split() - -@bench('("Here are some words. "*2).rsplit()', "split whitespace (small)", 1000) -def whitespace_rsplit(STR): - s = STR("Here are some words. "*2) - s_rsplit = s.rsplit - for x in _RANGE_1000: - s_rsplit() - -@bench('("Here are some words. "*2).split(None, 1)', - "split 1 whitespace", 1000) -def whitespace_split_1(STR): - s = STR("Here are some words. "*2) - s_split = s.split - N = None - for x in _RANGE_1000: - s_split(N, 1) - -@bench('("Here are some words. "*2).rsplit(None, 1)', - "split 1 whitespace", 1000) -def whitespace_rsplit_1(STR): - s = STR("Here are some words. "*2) - s_rsplit = s.rsplit - N = None - for x in _RANGE_1000: - s_rsplit(N, 1) - -@bench('("Here are some words. "*2).partition(" ")', - "split 1 whitespace", 1000) -def whitespace_partition(STR): - sep = STR(" ") - s = STR("Here are some words. "*2) - s_partition = s.partition - for x in _RANGE_1000: - s_partition(sep) - -@bench('("Here are some words. "*2).rpartition(" ")', - "split 1 whitespace", 1000) -def whitespace_rpartition(STR): - sep = STR(" ") - s = STR("Here are some words. "*2) - s_rpartition = s.rpartition - for x in _RANGE_1000: - s_rpartition(sep) - -human_text = """\ -Python is a dynamic object-oriented programming language that can be -used for many kinds of software development. It offers strong support -for integration with other languages and tools, comes with extensive -standard libraries, and can be learned in a few days. Many Python -programmers report substantial productivity gains and feel the language -encourages the development of higher quality, more maintainable code. - -Python runs on Windows, Linux/Unix, Mac OS X, Amiga, Palm -Handhelds, and Nokia mobile phones. Python has also been ported to the -Java and .NET virtual machines. - -Python is distributed under an OSI-approved open source license that -makes it free to use, even for commercial products. -"""*25 -human_text_bytes = bytes_from_str(human_text) -human_text_unicode = unicode_from_str(human_text) -def _get_human_text(STR): - if STR is UNICODE: - return human_text_unicode - if STR is BYTES: - return human_text_bytes - raise AssertionError - -@bench('human_text.split()', "split whitespace (huge)", 10) -def whitespace_split_huge(STR): - s = _get_human_text(STR) - s_split = s.split - for x in _RANGE_10: - s_split() - -@bench('human_text.rsplit()', "split whitespace (huge)", 10) -def whitespace_rsplit_huge(STR): - s = _get_human_text(STR) - s_rsplit = s.rsplit - for x in _RANGE_10: - s_rsplit() - - - -@bench('"this\\nis\\na\\ntest\\n".split("\\n")', "split newlines", 1000) -def newlines_split(STR): - s = STR("this\nis\na\ntest\n") - s_split = s.split - nl = STR("\n") - for x in _RANGE_1000: - s_split(nl) - - -@bench('"this\\nis\\na\\ntest\\n".rsplit("\\n")', "split newlines", 1000) -def newlines_rsplit(STR): - s = STR("this\nis\na\ntest\n") - s_rsplit = s.rsplit - nl = STR("\n") - for x in _RANGE_1000: - s_rsplit(nl) - -@bench('"this\\nis\\na\\ntest\\n".splitlines()', "split newlines", 1000) -def newlines_splitlines(STR): - s = STR("this\nis\na\ntest\n") - s_splitlines = s.splitlines - for x in _RANGE_1000: - s_splitlines() - -## split text with 2000 newlines - -def _make_2000_lines(): - import random - r = random.Random(100) - chars = list(map(chr, range(32, 128))) - i = 0 - while i < len(chars): - chars[i] = " " - i += r.randrange(9) - s = "".join(chars) - s = s*4 - words = [] - for i in range(2000): - start = r.randrange(96) - n = r.randint(5, 65) - words.append(s[start:start+n]) - return "\n".join(words)+"\n" - -_text_with_2000_lines = _make_2000_lines() -_text_with_2000_lines_bytes = bytes_from_str(_text_with_2000_lines) -_text_with_2000_lines_unicode = unicode_from_str(_text_with_2000_lines) -def _get_2000_lines(STR): - if STR is UNICODE: - return _text_with_2000_lines_unicode - if STR is BYTES: - return _text_with_2000_lines_bytes - raise AssertionError - - -@bench('"...text...".split("\\n")', "split 2000 newlines", 10) -def newlines_split_2000(STR): - s = _get_2000_lines(STR) - s_split = s.split - nl = STR("\n") - for x in _RANGE_10: - s_split(nl) - -@bench('"...text...".rsplit("\\n")', "split 2000 newlines", 10) -def newlines_rsplit_2000(STR): - s = _get_2000_lines(STR) - s_rsplit = s.rsplit - nl = STR("\n") - for x in _RANGE_10: - s_rsplit(nl) - -@bench('"...text...".splitlines()', "split 2000 newlines", 10) -def newlines_splitlines_2000(STR): - s = _get_2000_lines(STR) - s_splitlines = s.splitlines - for x in _RANGE_10: - s_splitlines() - - -## split text on "--" characters -@bench( - '"this--is--a--test--of--the--emergency--broadcast--system".split("--")', - "split on multicharacter separator (small)", 1000) -def split_multichar_sep_small(STR): - s = STR("this--is--a--test--of--the--emergency--broadcast--system") - s_split = s.split - pat = STR("--") - for x in _RANGE_1000: - s_split(pat) -@bench( - '"this--is--a--test--of--the--emergency--broadcast--system".rsplit("--")', - "split on multicharacter separator (small)", 1000) -def rsplit_multichar_sep_small(STR): - s = STR("this--is--a--test--of--the--emergency--broadcast--system") - s_rsplit = s.rsplit - pat = STR("--") - for x in _RANGE_1000: - s_rsplit(pat) - -## split dna text on "ACTAT" characters -@bench('dna.split("ACTAT")', - "split on multicharacter separator (dna)", 10) -def split_multichar_sep_dna(STR): - s = _get_dna(STR) - s_split = s.split - pat = STR("ACTAT") - for x in _RANGE_10: - s_split(pat) - -@bench('dna.rsplit("ACTAT")', - "split on multicharacter separator (dna)", 10) -def rsplit_multichar_sep_dna(STR): - s = _get_dna(STR) - s_rsplit = s.rsplit - pat = STR("ACTAT") - for x in _RANGE_10: - s_rsplit(pat) - - - -## split with limits - -GFF3_example = "\t".join([ - "I", "Genomic_canonical", "region", "357208", "396183", ".", "+", ".", - "ID=Sequence:R119;note=Clone R119%3B Genbank AF063007;Name=R119"]) - -@bench('GFF3_example.split("\\t")', "tab split", 1000) -def tab_split_no_limit(STR): - sep = STR("\t") - s = STR(GFF3_example) - s_split = s.split - for x in _RANGE_1000: - s_split(sep) - -@bench('GFF3_example.split("\\t", 8)', "tab split", 1000) -def tab_split_limit(STR): - sep = STR("\t") - s = STR(GFF3_example) - s_split = s.split - for x in _RANGE_1000: - s_split(sep, 8) - -@bench('GFF3_example.rsplit("\\t")', "tab split", 1000) -def tab_rsplit_no_limit(STR): - sep = STR("\t") - s = STR(GFF3_example) - s_rsplit = s.rsplit - for x in _RANGE_1000: - s_rsplit(sep) - -@bench('GFF3_example.rsplit("\\t", 8)', "tab split", 1000) -def tab_rsplit_limit(STR): - sep = STR("\t") - s = STR(GFF3_example) - s_rsplit = s.rsplit - for x in _RANGE_1000: - s_rsplit(sep, 8) - -#### Count characters - -@bench('...text.with.2000.newlines.count("\\n")', - "count newlines", 10) -def count_newlines(STR): - s = _get_2000_lines(STR) - s_count = s.count - nl = STR("\n") - for x in _RANGE_10: - s_count(nl) - -# Orchid sequences concatenated, from Biopython -_dna = """ -CGTAACAAGGTTTCCGTAGGTGAACCTGCGGAAGGATCATTGTTGAGATCACATAATAATTGATCGGGTT -AATCTGGAGGATCTGTTTACTTTGGTCACCCATGAGCATTTGCTGTTGAAGTGACCTAGAATTGCCATCG -AGCCTCCTTGGGAGCTTTCTTGTTGGCGAGATCTAAACCCTTGCCCGGCGCAGTTTTGCTCCAAGTCGTT -TGACACATAATTGGTGAAGGGGGTGGCATCCTTCCCTGACCCTCCCCCAACTATTTTTTTAACAACTCTC -AGCAACGGAGACTCAGTCTTCGGCAAATGCGATAAATGGTGTGAATTGCAGAATCCCGTGCACCATCGAG -TCTTTGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCACGCCTGCCTGGGCATTGCGAGTCATAT -CTCTCCCTTAACGAGGCTGTCCATACATACTGTTCAGCCGGTGCGGATGTGAGTTTGGCCCCTTGTTCTT -TGGTACGGGGGGTCTAAGAGCTGCATGGGCTTTTGATGGTCCTAAATACGGCAAGAGGTGGACGAACTAT -GCTACAACAAAATTGTTGTGCAGAGGCCCCGGGTTGTCGTATTAGATGGGCCACCGTAATCTGAAGACCC -TTTTGAACCCCATTGGAGGCCCATCAACCCATGATCAGTTGATGGCCATTTGGTTGCGACCCCAGGTCAG -GTGAGCAACAGCTGTCGTAACAAGGTTTCCGTAGGGTGAACTGCGGAAGGATCATTGTTGAGATCACATA -ATAATTGATCGAGTTAATCTGGAGGATCTGTTTACTTGGGTCACCCATGGGCATTTGCTGTTGAAGTGAC -CTAGATTTGCCATCGAGCCTCCTTGGGAGCATCCTTGTTGGCGATATCTAAACCCTCAATTTTTCCCCCA -ATCAAATTACACAAAATTGGTGGAGGGGGTGGCATTCTTCCCTTACCCTCCCCCAAATATTTTTTTAACA -ACTCTCAGCAACGGATATCTCAGCTCTTGCATCGATGAAGAACCCACCGAAATGCGATAAATGGTGTGAA -TTGCAGAATCCCGTGAACCATCGAGTCTTTGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCACG -CCTGCCTGGGCATTGCGAGTCATATCTCTCCCTTAACGAGGCTGTCCATACATACTGTTCAGCCGGTGCG -GATGTGAGTTTGGCCCCTTGTTCTTTGGTACGGGGGGTCTAAGAGATGCATGGGCTTTTGATGGTCCTAA -ATACGGCAAGAGGTGGACGAACTATGCTACAACAAAATTGTTGTGCAAAGGCCCCGGGTTGTCGTATAAG -ATGGGCCACCGATATCTGAAGACCCTTTTGGACCCCATTGGAGCCCATCAACCCATGTCAGTTGATGGCC -ATTCGTAACAAGGTTTCCGTAGGTGAACCTGCGGAAGGATCATTGTTGAGATCACATAATAATTGATCGA -GTTAATCTGGAGGATCTGTTTACTTGGGTCACCCATGGGCATTTGCTGTTGAAGTGACCTAGATTTGCCA -TCGAGCCTCCTTGGGAGCTTTCTTGTTGGCGATATCTAAACCCTTGCCCGGCAGAGTTTTGGGAATCCCG -TGAACCATCGAGTCTTTGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCACGCCTGCCTGGGCAT -TGCGAGTCATATCTCTCCCTTAACGAGGCTGTCCATACACACCTGTTCAGCCGGTGCGGATGTGAGTTTG -GCCCCTTGTTCTTTGGTACGGGGGGTCTAAGAGCTGCATGGGCTTTTGATGGTCCTAAATACGGCAAGAG -GTGGACGAACTATGCTACAACAAAATTGTTGTGCAAAGGCCCCGGGTTGTCGTATTAGATGGGCCACCAT -AATCTGAAGACCCTTTTGAACCCCATTGGAGGCCCATCAACCCATGATCAGTTGATGGCCATTTGGTTGC -GACCCAGTCAGGTGAGGGTAGGTGAACCTGCGGAAGGATCATTGTTGAGATCACATAATAATTGATCGAG -TTAATCTGGAGGATCTGTTTACTTTGGTCACCCATGGGCATTTGCTGTTGAAGTGACCTAGATTTGCCAT -CGAGCCTCCTTGGGAGCTTTCTTGTTGGCGAGATCTAAACCCTTGCCCGGCGGAGTTTGGCGCCAAGTCA -TATGACACATAATTGGTGAAGGGGGTGGCATCCTGCCCTGACCCTCCCCAAATTATTTTTTTAACAACTC -TCAGCAACGGATATCTCGGCTCTTGCATCGATGAAGAACGCAGCGAAATGCGATAAATGGTGTGAATTGC -AGAATCCCGTGAACCATCGAGTCTTTGGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCACGCCT -GCCTGGGCATTGGGAATCATATCTCTCCCCTAACGAGGCTATCCAAACATACTGTTCATCCGGTGCGGAT -GTGAGTTTGGCCCCTTGTTCTTTGGTACCGGGGGTCTAAGAGCTGCATGGGCATTTGATGGTCCTCAAAA -CGGCAAGAGGTGGACGAACTATGCCACAACAAAATTGTTGTCCCAAGGCCCCGGGTTGTCGTATTAGATG -GGCCACCGTAACCTGAAGACCCTTTTGAACCCCATTGGAGGCCCATCAACCCATGATCAGTTGATGACCA -TTTGTTGCGACCCCAGTCAGCTGAGCAACCCGCTGAGTGGAAGGTCATTGCCGATATCACATAATAATTG -ATCGAGTTAATCTGGAGGATCTGTTTACTTGGTCACCCATGAGCATTTGCTGTTGAAGTGACCTAGATTT -GCCATCGAGCCTCCTTGGGAGTTTTCTTGTTGGCGAGATCTAAACCCTTGCCCGGCGGAGTTGTGCGCCA -AGTCATATGACACATAATTGGTGAAGGGGGTGGCATCCTGCCCTGACCCTCCCCAAATTATTTTTTTAAC -AACTCTCAGCAACGGATATCTCGGCTCTTGCATCGATGAAGAACGCAGCGAAATGCGATAAATGGTGTGA -ATTGCAGAATCCCGTGAACCATCGAGTCTTTGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCAC -GCCTGCCTGGGCATTGCGAGTCATATCTCTCCCTTAACGAGGCTGTCCATACATACTGTTCATCCGGTGC -GGATGTGAGTTTGGCCCCTTGTTCTTTGGTACGGGGGGTCTAAGAGCTGCATGGGCATTTGATGGTCCTC -AAAACGGCAAGAGGTGGACGAACTATGCTACAACCAAATTGTTGTCCCAAGGCCCCGGGTTGTCGTATTA -GATGGGCCACCGTAACCTGAAGACCCTTTTGAACCCCATTGGAGGCCCATCAACCCATGATCAGTTGATG -ACCATGTGTTGCGACCCCAGTCAGCTGAGCAACGCGCTGAGCGTAACAAGGTTTCCGTAGGTGGACCTCC -GGGAGGATCATTGTTGAGATCACATAATAATTGATCGAGGTAATCTGGAGGATCTGCATATTTTGGTCAC -""" -_dna = "".join(_dna.splitlines()) -_dna = _dna * 25 -_dna_bytes = bytes_from_str(_dna) -_dna_unicode = unicode_from_str(_dna) - -def _get_dna(STR): - if STR is UNICODE: - return _dna_unicode - if STR is BYTES: - return _dna_bytes - raise AssertionError - -@bench('dna.count("AACT")', "count AACT substrings in DNA example", 10) -def count_aact(STR): - seq = _get_dna(STR) - seq_count = seq.count - needle = STR("AACT") - for x in _RANGE_10: - seq_count(needle) - -##### startswith and endswith - -@bench('"Andrew".startswith("A")', 'startswith single character', 1000) -def startswith_single(STR): - s1 = STR("Andrew") - s2 = STR("A") - s1_startswith = s1.startswith - for x in _RANGE_1000: - s1_startswith(s2) - -@bench('"Andrew".startswith("Andrew")', 'startswith multiple characters', - 1000) -def startswith_multiple(STR): - s1 = STR("Andrew") - s2 = STR("Andrew") - s1_startswith = s1.startswith - for x in _RANGE_1000: - s1_startswith(s2) - -@bench('"Andrew".startswith("Anders")', - 'startswith multiple characters - not!', 1000) -def startswith_multiple_not(STR): - s1 = STR("Andrew") - s2 = STR("Anders") - s1_startswith = s1.startswith - for x in _RANGE_1000: - s1_startswith(s2) - - -# endswith - -@bench('"Andrew".endswith("w")', 'endswith single character', 1000) -def endswith_single(STR): - s1 = STR("Andrew") - s2 = STR("w") - s1_endswith = s1.endswith - for x in _RANGE_1000: - s1_endswith(s2) - -@bench('"Andrew".endswith("Andrew")', 'endswith multiple characters', 1000) -def endswith_multiple(STR): - s1 = STR("Andrew") - s2 = STR("Andrew") - s1_endswith = s1.endswith - for x in _RANGE_1000: - s1_endswith(s2) - -@bench('"Andrew".endswith("Anders")', - 'endswith multiple characters - not!', 1000) -def endswith_multiple_not(STR): - s1 = STR("Andrew") - s2 = STR("Anders") - s1_endswith = s1.endswith - for x in _RANGE_1000: - s1_endswith(s2) - -#### Strip - -@bench('"Hello!\\n".strip()', 'strip terminal newline', 1000) -def terminal_newline_strip_right(STR): - s = STR("Hello!\n") - s_strip = s.strip - for x in _RANGE_1000: - s_strip() - -@bench('"Hello!\\n".rstrip()', 'strip terminal newline', 1000) -def terminal_newline_rstrip(STR): - s = STR("Hello!\n") - s_rstrip = s.rstrip - for x in _RANGE_1000: - s_rstrip() - -@bench('"\\nHello!".strip()', 'strip terminal newline', 1000) -def terminal_newline_strip_left(STR): - s = STR("\nHello!") - s_strip = s.strip - for x in _RANGE_1000: - s_strip() - -@bench('"\\nHello!\\n".strip()', 'strip terminal newline', 1000) -def terminal_newline_strip_both(STR): - s = STR("\nHello!\n") - s_strip = s.strip - for x in _RANGE_1000: - s_strip() - -@bench('"\\nHello!".rstrip()', 'strip terminal newline', 1000) -def terminal_newline_lstrip(STR): - s = STR("\nHello!") - s_lstrip = s.lstrip - for x in _RANGE_1000: - s_lstrip() - -@bench('s="Hello!\\n"; s[:-1] if s[-1]=="\\n" else s', - 'strip terminal newline', 1000) -def terminal_newline_if_else(STR): - s = STR("Hello!\n") - NL = STR("\n") - for x in _RANGE_1000: - s[:-1] if (s[-1] == NL) else s - - -# Strip multiple spaces or tabs - -@bench('"Hello\\t \\t".strip()', 'strip terminal spaces and tabs', 1000) -def terminal_space_strip(STR): - s = STR("Hello\t \t!") - s_strip = s.strip - for x in _RANGE_1000: - s_strip() - -@bench('"Hello\\t \\t".rstrip()', 'strip terminal spaces and tabs', 1000) -def terminal_space_rstrip(STR): - s = STR("Hello!\t \t") - s_rstrip = s.rstrip - for x in _RANGE_1000: - s_rstrip() - -@bench('"\\t \\tHello".rstrip()', 'strip terminal spaces and tabs', 1000) -def terminal_space_lstrip(STR): - s = STR("\t \tHello!") - s_lstrip = s.lstrip - for x in _RANGE_1000: - s_lstrip() - - -#### replace -@bench('"This is a test".replace(" ", "\\t")', 'replace single character', - 1000) -def replace_single_character(STR): - s = STR("This is a test!") - from_str = STR(" ") - to_str = STR("\t") - s_replace = s.replace - for x in _RANGE_1000: - s_replace(from_str, to_str) - -@uses_re -@bench('re.sub(" ", "\\t", "This is a test"', 'replace single character', - 1000) -def replace_single_character_re(STR): - s = STR("This is a test!") - pat = re.compile(STR(" ")) - to_str = STR("\t") - pat_sub = pat.sub - for x in _RANGE_1000: - pat_sub(to_str, s) - -@bench('"...text.with.2000.lines...replace("\\n", " ")', - 'replace single character, big string', 10) -def replace_single_character_big(STR): - s = _get_2000_lines(STR) - from_str = STR("\n") - to_str = STR(" ") - s_replace = s.replace - for x in _RANGE_10: - s_replace(from_str, to_str) - -@uses_re -@bench('re.sub("\\n", " ", "...text.with.2000.lines...")', - 'replace single character, big string', 10) -def replace_single_character_big_re(STR): - s = _get_2000_lines(STR) - pat = re.compile(STR("\n")) - to_str = STR(" ") - pat_sub = pat.sub - for x in _RANGE_10: - pat_sub(to_str, s) - - -@bench('dna.replace("ATC", "ATT")', - 'replace multiple characters, dna', 10) -def replace_multiple_characters_dna(STR): - seq = _get_dna(STR) - from_str = STR("ATC") - to_str = STR("ATT") - seq_replace = seq.replace - for x in _RANGE_10: - seq_replace(from_str, to_str) - -# This increases the character count -@bench('"...text.with.2000.newlines...replace("\\n", "\\r\\n")', - 'replace and expand multiple characters, big string', 10) -def replace_multiple_character_big(STR): - s = _get_2000_lines(STR) - from_str = STR("\n") - to_str = STR("\r\n") - s_replace = s.replace - for x in _RANGE_10: - s_replace(from_str, to_str) - - -# This decreases the character count -@bench('"When shall we three meet again?".replace("ee", "")', - 'replace/remove multiple characters', 1000) -def replace_multiple_character_remove(STR): - s = STR("When shall we three meet again?") - from_str = STR("ee") - to_str = STR("") - s_replace = s.replace - for x in _RANGE_1000: - s_replace(from_str, to_str) - - -big_s = "A" + ("Z"*128*1024) -big_s_bytes = bytes_from_str(big_s) -big_s_unicode = unicode_from_str(big_s) -def _get_big_s(STR): - if STR is UNICODE: return big_s_unicode - if STR is BYTES: return big_s_bytes - raise AssertionError - -# The older replace implementation counted all matches in -# the string even when it only needed to make one replacement. -@bench('("A" + ("Z"*128*1024)).replace("A", "BB", 1)', - 'quick replace single character match', 10) -def quick_replace_single_match(STR): - s = _get_big_s(STR) - from_str = STR("A") - to_str = STR("BB") - s_replace = s.replace - for x in _RANGE_10: - s_replace(from_str, to_str, 1) - -@bench('("A" + ("Z"*128*1024)).replace("AZZ", "BBZZ", 1)', - 'quick replace multiple character match', 10) -def quick_replace_multiple_match(STR): - s = _get_big_s(STR) - from_str = STR("AZZ") - to_str = STR("BBZZ") - s_replace = s.replace - for x in _RANGE_10: - s_replace(from_str, to_str, 1) - - -#### - -# CCP does a lot of this, for internationalisation of ingame messages. -_format = "The %(thing)s is %(place)s the %(location)s." -_format_dict = { "thing":"THING", "place":"PLACE", "location":"LOCATION", } -_format_bytes = bytes_from_str(_format) -_format_unicode = unicode_from_str(_format) -_format_dict_bytes = dict((bytes_from_str(k), bytes_from_str(v)) for (k,v) in _format_dict.items()) -_format_dict_unicode = dict((unicode_from_str(k), unicode_from_str(v)) for (k,v) in _format_dict.items()) - -def _get_format(STR): - if STR is UNICODE: - return _format_unicode - if STR is BYTES: - if sys.version_info >= (3,): - raise UnsupportedType - return _format_bytes - raise AssertionError - -def _get_format_dict(STR): - if STR is UNICODE: - return _format_dict_unicode - if STR is BYTES: - if sys.version_info >= (3,): - raise UnsupportedType - return _format_dict_bytes - raise AssertionError - -# Formatting. -@bench('"The %(k1)s is %(k2)s the %(k3)s."%{"k1":"x","k2":"y","k3":"z",}', - 'formatting a string type with a dict', 1000) -def format_with_dict(STR): - s = _get_format(STR) - d = _get_format_dict(STR) - for x in _RANGE_1000: - s % d - - -#### Upper- and lower- case conversion - -@bench('("Where in the world is Carmen San Deigo?"*10).lower()', - "case conversion -- rare", 1000) -def lower_conversion_rare(STR): - s = STR("Where in the world is Carmen San Deigo?"*10) - s_lower = s.lower - for x in _RANGE_1000: - s_lower() - -@bench('("WHERE IN THE WORLD IS CARMEN SAN DEIGO?"*10).lower()', - "case conversion -- dense", 1000) -def lower_conversion_dense(STR): - s = STR("WHERE IN THE WORLD IS CARMEN SAN DEIGO?"*10) - s_lower = s.lower - for x in _RANGE_1000: - s_lower() - - -@bench('("wHERE IN THE WORLD IS cARMEN sAN dEIGO?"*10).upper()', - "case conversion -- rare", 1000) -def upper_conversion_rare(STR): - s = STR("Where in the world is Carmen San Deigo?"*10) - s_upper = s.upper - for x in _RANGE_1000: - s_upper() - -@bench('("where in the world is carmen san deigo?"*10).upper()', - "case conversion -- dense", 1000) -def upper_conversion_dense(STR): - s = STR("where in the world is carmen san deigo?"*10) - s_upper = s.upper - for x in _RANGE_1000: - s_upper() - - -# end of benchmarks - -################# - -class BenchTimer(timeit.Timer): - def best(self, repeat=1): - for i in range(1, 10): - number = 10**i - x = self.timeit(number) - if x > 0.02: - break - times = [x] - for i in range(1, repeat): - times.append(self.timeit(number)) - return min(times) / number - -def main(): - (options, test_names) = parser.parse_args() - if options.bytes_only and options.unicode_only: - raise SystemExit("Only one of --8-bit and --unicode are allowed") - - bench_functions = [] - for (k,v) in globals().items(): - if hasattr(v, "is_bench"): - if test_names: - for name in test_names: - if name in v.group: - break - else: - # Not selected, ignore - continue - if options.skip_re and hasattr(v, "uses_re"): - continue - - bench_functions.append( (v.group, k, v) ) - bench_functions.sort() - - p("bytes\tunicode") - p("(in ms)\t(in ms)\t%\tcomment") - - bytes_total = uni_total = 0.0 - - for title, group in itertools.groupby(bench_functions, - operator.itemgetter(0)): - # Flush buffer before each group - sys.stdout.flush() - p("="*10, title) - for (_, k, v) in group: - if hasattr(v, "is_bench"): - bytes_time = 0.0 - bytes_time_s = " - " - if not options.unicode_only: - try: - bytes_time = BenchTimer("__main__.%s(__main__.BYTES)" % (k,), - "import __main__").best(REPEAT) - bytes_time_s = "%.2f" % (1000 * bytes_time) - bytes_total += bytes_time - except UnsupportedType: - bytes_time_s = "N/A" - uni_time = 0.0 - uni_time_s = " - " - if not options.bytes_only: - try: - uni_time = BenchTimer("__main__.%s(__main__.UNICODE)" % (k,), - "import __main__").best(REPEAT) - uni_time_s = "%.2f" % (1000 * uni_time) - uni_total += uni_time - except UnsupportedType: - uni_time_s = "N/A" - try: - average = bytes_time/uni_time - except (TypeError, ZeroDivisionError): - average = 0.0 - p("%s\t%s\t%.1f\t%s (*%d)" % ( - bytes_time_s, uni_time_s, 100.*average, - v.comment, v.repeat_count)) - - if bytes_total == uni_total == 0.0: - p("That was zippy!") - else: - try: - ratio = bytes_total/uni_total - except ZeroDivisionError: - ratio = 0.0 - p("%.2f\t%.2f\t%.1f\t%s" % ( - 1000*bytes_total, 1000*uni_total, 100.*ratio, - "TOTAL")) - -if __name__ == "__main__": - main() diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index beb857f69e40da1..23b38c8e93638ad 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -83,7 +83,7 @@ embuilder --pic build zlib bzip2 MINIMAL_PIC ``` -#### Compile a build Python interpreter +### Compile and build Python interpreter From within the container, run the following command: diff --git a/Tools/wasm/wasi-env b/Tools/wasm/wasi-env index 48908b02e60b960..e6c6fb2d8e47e75 100755 --- a/Tools/wasm/wasi-env +++ b/Tools/wasm/wasi-env @@ -55,7 +55,6 @@ if command -v ccache >/dev/null 2>&1; then CXX="ccache ${CXX}" fi -LDSHARED="${WASI_SDK_PATH}/bin/wasm-ld" AR="${WASI_SDK_PATH}/bin/llvm-ar" RANLIB="${WASI_SDK_PATH}/bin/ranlib" diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 46ecae74a9ecea7..1e75db5c7b83299 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -165,7 +165,7 @@ def wasi_sdk_env(context): wasi_sdk_path = context.wasi_sdk_path sysroot = wasi_sdk_path / "share" / "wasi-sysroot" env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++", - "LDSHARED": "wasm-ld", "AR": "llvm-ar", "RANLIB": "ranlib"} + "AR": "llvm-ar", "RANLIB": "ranlib"} for env_var, binary_name in list(env.items()): env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) diff --git a/aclocal.m4 b/aclocal.m4 index 09ae5d1aa8a608e..832aec19f48f178 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -150,6 +150,80 @@ AS_VAR_IF(CACHEVAR,yes, AS_VAR_POPDEF([CACHEVAR])dnl ])dnl AX_CHECK_COMPILE_FLAGS +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_check_define.html +# =========================================================================== +# +# SYNOPSIS +# +# AC_CHECK_DEFINE([symbol], [ACTION-IF-FOUND], [ACTION-IF-NOT]) +# AX_CHECK_DEFINE([includes],[symbol], [ACTION-IF-FOUND], [ACTION-IF-NOT]) +# +# DESCRIPTION +# +# Complements AC_CHECK_FUNC but it does not check for a function but for a +# define to exist. Consider a usage like: +# +# AC_CHECK_DEFINE(__STRICT_ANSI__, CFLAGS="$CFLAGS -D_XOPEN_SOURCE=500") +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim <guidod@gmx.de> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 11 + +AU_ALIAS([AC_CHECK_DEFINED], [AC_CHECK_DEFINE]) +AC_DEFUN([AC_CHECK_DEFINE],[ +AS_VAR_PUSHDEF([ac_var],[ac_cv_defined_$1])dnl +AC_CACHE_CHECK([for $1 defined], ac_var, +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[ + #ifdef $1 + int ok; + (void)ok; + #else + choke me + #endif +]])],[AS_VAR_SET(ac_var, yes)],[AS_VAR_SET(ac_var, no)])) +AS_IF([test AS_VAR_GET(ac_var) != "no"], [$2], [$3])dnl +AS_VAR_POPDEF([ac_var])dnl +]) + +AU_ALIAS([AX_CHECK_DEFINED], [AX_CHECK_DEFINE]) +AC_DEFUN([AX_CHECK_DEFINE],[ +AS_VAR_PUSHDEF([ac_var],[ac_cv_defined_$2_$1])dnl +AC_CACHE_CHECK([for $2 defined in $1], ac_var, +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <$1>]], [[ + #ifdef $2 + int ok; + (void)ok; + #else + choke me + #endif +]])],[AS_VAR_SET(ac_var, yes)],[AS_VAR_SET(ac_var, no)])) +AS_IF([test AS_VAR_GET(ac_var) != "no"], [$3], [$4])dnl +AS_VAR_POPDEF([ac_var])dnl +]) + +AC_DEFUN([AX_CHECK_FUNC], +[AS_VAR_PUSHDEF([ac_var], [ac_cv_func_$2])dnl +AC_CACHE_CHECK([for $2], ac_var, +dnl AC_LANG_FUNC_LINK_TRY +[AC_LINK_IFELSE([AC_LANG_PROGRAM([$1 + #undef $2 + char $2 ();],[ + char (*f) () = $2; + return f != $2; ])], + [AS_VAR_SET(ac_var, yes)], + [AS_VAR_SET(ac_var, no)])]) +AS_IF([test AS_VAR_GET(ac_var) = yes], [$3], [$4])dnl +AS_VAR_POPDEF([ac_var])dnl +])# AC_CHECK_FUNC + # =========================================================================== # https://www.gnu.org/software/autoconf-archive/ax_check_openssl.html # =========================================================================== diff --git a/config.sub b/config.sub index d74fb6deac942a4..1bb6a05dc11026c 100755 --- a/config.sub +++ b/config.sub @@ -1,14 +1,15 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright 1992-2021 Free Software Foundation, Inc. +# Copyright 1992-2024 Free Software Foundation, Inc. # shellcheck disable=SC2006,SC2268 # see below for rationale -timestamp='2021-08-14' +# Patched 2024-02-03 to include support for arm64_32 and iOS/tvOS/watchOS simulators +timestamp='2024-01-01' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or +# the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but @@ -76,13 +77,13 @@ Report bugs and patches to <config-patches@gnu.org>." version="\ GNU config.sub ($timestamp) -Copyright 1992-2021 Free Software Foundation, Inc. +Copyright 1992-2024 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." help=" -Try \`$me --help' for more information." +Try '$me --help' for more information." # Parse command line while test $# -gt 0 ; do @@ -130,7 +131,7 @@ IFS=$saved_IFS # Separate into logical components for further validation case $1 in *-*-*-*-*) - echo Invalid configuration \`"$1"\': more than four components >&2 + echo "Invalid configuration '$1': more than four components" >&2 exit 1 ;; *-*-*-*) @@ -145,7 +146,8 @@ case $1 in nto-qnx* | linux-* | uclinux-uclibc* \ | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \ | netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \ - | storm-chaos* | os2-emx* | rtmk-nova*) + | storm-chaos* | os2-emx* | rtmk-nova* | managarm-* \ + | windows-* ) basic_machine=$field1 basic_os=$maybe_os ;; @@ -943,7 +945,7 @@ $basic_machine EOF IFS=$saved_IFS ;; - # We use `pc' rather than `unknown' + # We use 'pc' rather than 'unknown' # because (1) that's what they normally are, and # (2) the word "unknown" tends to confuse beginning users. i*86 | x86_64) @@ -1020,6 +1022,11 @@ case $cpu-$vendor in ;; # Here we normalize CPU types with a missing or matching vendor + armh-unknown | armh-alt) + cpu=armv7l + vendor=alt + basic_os=${basic_os:-linux-gnueabihf} + ;; dpx20-unknown | dpx20-bull) cpu=rs6000 vendor=bull @@ -1070,7 +1077,7 @@ case $cpu-$vendor in pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) cpu=i586 ;; - pentiumpro-* | p6-* | 6x86-* | athlon-* | athalon_*-*) + pentiumpro-* | p6-* | 6x86-* | athlon-* | athlon_*-*) cpu=i686 ;; pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) @@ -1121,7 +1128,7 @@ case $cpu-$vendor in xscale-* | xscalee[bl]-*) cpu=`echo "$cpu" | sed 's/^xscale/arm/'` ;; - arm64-*) + arm64-* | aarch64le-* | arm64_32-*) cpu=aarch64 ;; @@ -1175,7 +1182,7 @@ case $cpu-$vendor in case $cpu in 1750a | 580 \ | a29k \ - | aarch64 | aarch64_be \ + | aarch64 | aarch64_be | aarch64c | arm64ec \ | abacus \ | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] \ | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] \ @@ -1194,50 +1201,29 @@ case $cpu-$vendor in | d10v | d30v | dlx | dsp16xx \ | e2k | elxsi | epiphany \ | f30[01] | f700 | fido | fr30 | frv | ft32 | fx80 \ + | javascript \ | h8300 | h8500 \ | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ | hexagon \ | i370 | i*86 | i860 | i960 | ia16 | ia64 \ | ip2k | iq2000 \ | k1om \ + | kvx \ | le32 | le64 \ | lm32 \ - | loongarch32 | loongarch64 | loongarchx32 \ + | loongarch32 | loongarch64 \ | m32c | m32r | m32rle \ | m5200 | m68000 | m680[012346]0 | m68360 | m683?2 | m68k \ | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x \ | m88110 | m88k | maxq | mb | mcore | mep | metag \ | microblaze | microblazeel \ - | mips | mipsbe | mipseb | mipsel | mipsle \ - | mips16 \ - | mips64 | mips64eb | mips64el \ - | mips64octeon | mips64octeonel \ - | mips64orion | mips64orionel \ - | mips64r5900 | mips64r5900el \ - | mips64vr | mips64vrel \ - | mips64vr4100 | mips64vr4100el \ - | mips64vr4300 | mips64vr4300el \ - | mips64vr5000 | mips64vr5000el \ - | mips64vr5900 | mips64vr5900el \ - | mipsisa32 | mipsisa32el \ - | mipsisa32r2 | mipsisa32r2el \ - | mipsisa32r3 | mipsisa32r3el \ - | mipsisa32r5 | mipsisa32r5el \ - | mipsisa32r6 | mipsisa32r6el \ - | mipsisa64 | mipsisa64el \ - | mipsisa64r2 | mipsisa64r2el \ - | mipsisa64r3 | mipsisa64r3el \ - | mipsisa64r5 | mipsisa64r5el \ - | mipsisa64r6 | mipsisa64r6el \ - | mipsisa64sb1 | mipsisa64sb1el \ - | mipsisa64sr71k | mipsisa64sr71kel \ - | mipsr5900 | mipsr5900el \ - | mipstx39 | mipstx39el \ + | mips* \ | mmix \ | mn10200 | mn10300 \ | moxie \ | mt \ | msp430 \ + | nanomips* \ | nds32 | nds32le | nds32be \ | nfp \ | nios | nios2 | nios2eb | nios2el \ @@ -1269,6 +1255,7 @@ case $cpu-$vendor in | ubicom32 \ | v70 | v850 | v850e | v850e1 | v850es | v850e2 | v850e2v3 \ | vax \ + | vc4 \ | visium \ | w65 \ | wasm32 | wasm64 \ @@ -1280,7 +1267,7 @@ case $cpu-$vendor in ;; *) - echo Invalid configuration \`"$1"\': machine \`"$cpu-$vendor"\' not recognized 1>&2 + echo "Invalid configuration '$1': machine '$cpu-$vendor' not recognized" 1>&2 exit 1 ;; esac @@ -1301,11 +1288,12 @@ esac # Decode manufacturer-specific aliases for certain operating systems. -if test x$basic_os != x +if test x"$basic_os" != x then -# First recognize some ad-hoc caes, or perhaps split kernel-os, or else just +# First recognize some ad-hoc cases, or perhaps split kernel-os, or else just # set os. +obj= case $basic_os in gnu/linux*) kernel=linux @@ -1336,6 +1324,10 @@ EOF kernel=linux os=`echo "$basic_os" | sed -e 's|linux|gnu|'` ;; + managarm*) + kernel=managarm + os=`echo "$basic_os" | sed -e 's|managarm|mlibc|'` + ;; *) kernel= os=$basic_os @@ -1501,10 +1493,16 @@ case $os in os=eabi ;; *) - os=elf + os= + obj=elf ;; esac ;; + aout* | coff* | elf* | pe*) + # These are machine code file formats, not OSes + obj=$os + os= + ;; *) # No normalization, but not necessarily accepted, that comes below. ;; @@ -1523,12 +1521,15 @@ else # system, and we'll never get to this point. kernel= +obj= case $cpu-$vendor in score-*) - os=elf + os= + obj=elf ;; spu-*) - os=elf + os= + obj=elf ;; *-acorn) os=riscix1.2 @@ -1538,28 +1539,35 @@ case $cpu-$vendor in os=gnu ;; arm*-semi) - os=aout + os= + obj=aout ;; c4x-* | tic4x-*) - os=coff + os= + obj=coff ;; c8051-*) - os=elf + os= + obj=elf ;; clipper-intergraph) os=clix ;; hexagon-*) - os=elf + os= + obj=elf ;; tic54x-*) - os=coff + os= + obj=coff ;; tic55x-*) - os=coff + os= + obj=coff ;; tic6x-*) - os=coff + os= + obj=coff ;; # This must come before the *-dec entry. pdp10-*) @@ -1581,19 +1589,24 @@ case $cpu-$vendor in os=sunos3 ;; m68*-cisco) - os=aout + os= + obj=aout ;; mep-*) - os=elf + os= + obj=elf ;; mips*-cisco) - os=elf + os= + obj=elf ;; - mips*-*) - os=elf + mips*-*|nanomips*-*) + os= + obj=elf ;; or32-*) - os=coff + os= + obj=coff ;; *-tti) # must be before sparc entry or we get the wrong os. os=sysv3 @@ -1602,7 +1615,8 @@ case $cpu-$vendor in os=sunos4.1.1 ;; pru-*) - os=elf + os= + obj=elf ;; *-be) os=beos @@ -1683,10 +1697,12 @@ case $cpu-$vendor in os=uxpv ;; *-rom68k) - os=coff + os= + obj=coff ;; *-*bug) - os=coff + os= + obj=coff ;; *-apple) os=macos @@ -1704,10 +1720,11 @@ esac fi -# Now, validate our (potentially fixed-up) OS. +# Now, validate our (potentially fixed-up) individual pieces (OS, OBJ). + case $os in # Sometimes we do "kernel-libc", so those need to count as OSes. - musl* | newlib* | relibc* | uclibc*) + llvm* | musl* | newlib* | relibc* | uclibc*) ;; # Likewise for "kernel-abi" eabi* | gnueabi*) @@ -1715,6 +1732,9 @@ case $os in # VxWorks passes extra cpu info in the 4th filed. simlinux | simwindows | spe) ;; + # See `case $cpu-$os` validation below + ghcjs) + ;; # Now accept the basic system types. # The portable systems comes first. # Each alternative MUST end in a * to match a version number. @@ -1723,7 +1743,7 @@ case $os in | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \ | sym* | plan9* | psp* | sim* | xray* | os68k* | v88r* \ | hiux* | abug | nacl* | netware* | windows* \ - | os9* | macos* | osx* | ios* \ + | os9* | macos* | osx* | ios* | tvos* | watchos* \ | mpw* | magic* | mmixware* | mon960* | lnews* \ | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \ | aos* | aros* | cloudabi* | sortix* | twizzler* \ @@ -1732,11 +1752,11 @@ case $os in | mirbsd* | netbsd* | dicos* | openedition* | ose* \ | bitrig* | openbsd* | secbsd* | solidbsd* | libertybsd* | os108* \ | ekkobsd* | freebsd* | riscix* | lynxos* | os400* \ - | bosx* | nextstep* | cxux* | aout* | elf* | oabi* \ - | ptx* | coff* | ecoff* | winnt* | domain* | vsta* \ + | bosx* | nextstep* | cxux* | oabi* \ + | ptx* | ecoff* | winnt* | domain* | vsta* \ | udi* | lites* | ieee* | go32* | aux* | hcos* \ | chorusrdb* | cegcc* | glidix* | serenity* \ - | cygwin* | msys* | pe* | moss* | proelf* | rtems* \ + | cygwin* | msys* | moss* | proelf* | rtems* \ | midipix* | mingw32* | mingw64* | mint* \ | uxpv* | beos* | mpeix* | udk* | moxiebox* \ | interix* | uwin* | mks* | rhapsody* | darwin* \ @@ -1748,49 +1768,119 @@ case $os in | skyos* | haiku* | rdos* | toppers* | drops* | es* \ | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \ | midnightbsd* | amdhsa* | unleashed* | emscripten* | wasi* \ - | nsk* | powerunix* | genode* | zvmoe* | qnx* | emx* | zephyr*) + | nsk* | powerunix* | genode* | zvmoe* | qnx* | emx* | zephyr* \ + | fiwix* | mlibc* | cos* | mbr* | ironclad* ) ;; # This one is extra strict with allowed versions sco3.2v2 | sco3.2v[4-9]* | sco5v6*) # Don't forget version if it is 3.2v4 or newer. ;; + # This refers to builds using the UEFI calling convention + # (which depends on the architecture) and PE file format. + # Note that this is both a different calling convention and + # different file format than that of GNU-EFI + # (x86_64-w64-mingw32). + uefi) + ;; none) ;; + kernel* | msvc* ) + # Restricted further below + ;; + '') + if test x"$obj" = x + then + echo "Invalid configuration '$1': Blank OS only allowed with explicit machine code file format" 1>&2 + fi + ;; *) - echo Invalid configuration \`"$1"\': OS \`"$os"\' not recognized 1>&2 + echo "Invalid configuration '$1': OS '$os' not recognized" 1>&2 + exit 1 + ;; +esac + +case $obj in + aout* | coff* | elf* | pe*) + ;; + '') + # empty is fine + ;; + *) + echo "Invalid configuration '$1': Machine code format '$obj' not recognized" 1>&2 + exit 1 + ;; +esac + +# Here we handle the constraint that a (synthetic) cpu and os are +# valid only in combination with each other and nowhere else. +case $cpu-$os in + # The "javascript-unknown-ghcjs" triple is used by GHC; we + # accept it here in order to tolerate that, but reject any + # variations. + javascript-ghcjs) + ;; + javascript-* | *-ghcjs) + echo "Invalid configuration '$1': cpu '$cpu' is not valid with os '$os$obj'" 1>&2 exit 1 ;; esac # As a final step for OS-related things, validate the OS-kernel combination # (given a valid OS), if there is a kernel. -case $kernel-$os in - linux-gnu* | linux-dietlibc* | linux-android* | linux-newlib* \ - | linux-musl* | linux-relibc* | linux-uclibc* ) +case $kernel-$os-$obj in + linux-gnu*- | linux-android*- | linux-dietlibc*- | linux-llvm*- \ + | linux-mlibc*- | linux-musl*- | linux-newlib*- \ + | linux-relibc*- | linux-uclibc*- ) + ;; + uclinux-uclibc*- ) ;; - uclinux-uclibc* ) + managarm-mlibc*- | managarm-kernel*- ) ;; - -dietlibc* | -newlib* | -musl* | -relibc* | -uclibc* ) + windows*-msvc*-) + ;; + -dietlibc*- | -llvm*- | -mlibc*- | -musl*- | -newlib*- | -relibc*- \ + | -uclibc*- ) # These are just libc implementations, not actual OSes, and thus # require a kernel. - echo "Invalid configuration \`$1': libc \`$os' needs explicit kernel." 1>&2 + echo "Invalid configuration '$1': libc '$os' needs explicit kernel." 1>&2 exit 1 ;; - kfreebsd*-gnu* | kopensolaris*-gnu*) + -kernel*- ) + echo "Invalid configuration '$1': '$os' needs explicit kernel." 1>&2 + exit 1 ;; - vxworks-simlinux | vxworks-simwindows | vxworks-spe) + *-kernel*- ) + echo "Invalid configuration '$1': '$kernel' does not support '$os'." 1>&2 + exit 1 ;; - nto-qnx*) + *-msvc*- ) + echo "Invalid configuration '$1': '$os' needs 'windows'." 1>&2 + exit 1 ;; - os2-emx) + kfreebsd*-gnu*- | kopensolaris*-gnu*-) ;; - *-eabi* | *-gnueabi*) + vxworks-simlinux- | vxworks-simwindows- | vxworks-spe-) ;; - -*) + nto-qnx*-) + ;; + os2-emx-) + ;; + *-eabi*- | *-gnueabi*-) + ;; + ios*-simulator- | tvos*-simulator- | watchos*-simulator- ) + ;; + none--*) + # None (no kernel, i.e. freestanding / bare metal), + # can be paired with an machine code file format + ;; + -*-) # Blank kernel with real OS is always fine. ;; - *-*) - echo "Invalid configuration \`$1': Kernel \`$kernel' not known to work with OS \`$os'." 1>&2 + --*) + # Blank kernel and OS with real machine code file format is always fine. + ;; + *-*-*) + echo "Invalid configuration '$1': Kernel '$kernel' not known to work with OS '$os'." 1>&2 exit 1 ;; esac @@ -1873,7 +1963,7 @@ case $vendor in ;; esac -echo "$cpu-$vendor-${kernel:+$kernel-}$os" +echo "$cpu-$vendor${kernel:+-$kernel}${os:+-$os}${obj:+-$obj}" exit # Local variables: diff --git a/configure b/configure index b1153df4d7ec523..f431c5dd15ec4a0 100755 --- a/configure +++ b/configure @@ -661,6 +661,8 @@ MODULE__XXTESTFUZZ_FALSE MODULE__XXTESTFUZZ_TRUE MODULE_XXSUBTYPE_FALSE MODULE_XXSUBTYPE_TRUE +MODULE__TESTEXTERNALINSPECTION_FALSE +MODULE__TESTEXTERNALINSPECTION_TRUE MODULE__TESTMULTIPHASE_FALSE MODULE__TESTMULTIPHASE_TRUE MODULE__TESTIMPORTMULTIPLE_FALSE @@ -834,7 +836,8 @@ LIBPL PY_ENABLE_SHARED PLATLIBDIR BINLIBDEST -LIBPYTHON +MODULE_LDFLAGS +MODULE_DEPS_SHARED EXT_SUFFIX ALT_SOABI SOABI @@ -920,6 +923,8 @@ LLVM_AR PROFILE_TASK DEF_MAKE_RULE DEF_MAKE_ALL_RULE +JIT_STENCILS_H +REGEN_JIT_COMMAND ABIFLAGS LN MKDIR_P @@ -967,10 +972,11 @@ LDFLAGS CFLAGS CC HAS_XCRUN +IOS_DEPLOYMENT_TARGET EXPORT_MACOSX_DEPLOYMENT_TARGET CONFIGURE_MACOSX_DEPLOYMENT_TARGET _PYTHON_HOST_PLATFORM -MACHDEP +INSTALLTARGETS FRAMEWORKINSTALLAPPSPREFIX FRAMEWORKUNIXTOOLSPREFIX FRAMEWORKPYTHONW @@ -978,6 +984,8 @@ FRAMEWORKALTINSTALLLAST FRAMEWORKALTINSTALLFIRST FRAMEWORKINSTALLLAST FRAMEWORKINSTALLFIRST +RESSRCDIR +PYTHONFRAMEWORKINSTALLNAMEPREFIX PYTHONFRAMEWORKINSTALLDIR PYTHONFRAMEWORKPREFIX PYTHONFRAMEWORKDIR @@ -987,6 +995,7 @@ LIPO_INTEL64_FLAGS LIPO_32BIT_FLAGS ARCH_RUN_32BIT UNIVERSALSDK +MACHDEP PKG_CONFIG_LIBDIR PKG_CONFIG_PATH PKG_CONFIG @@ -1074,6 +1083,7 @@ with_pydebug with_trace_refs enable_pystats with_assertions +enable_experimental_jit enable_optimizations with_lto enable_bolt @@ -1801,6 +1811,9 @@ Optional Features: --disable-gil enable experimental support for running without the GIL (default is no) --enable-pystats enable internal statistics gathering (default is no) + --enable-experimental-jit + build the experimental just-in-time compiler + (default is no) --enable-optimizations enable expensive, stable optimizations (PGO, etc.) (default is no) --enable-bolt enable usage of the llvm-bolt post-link optimizer @@ -3998,6 +4011,80 @@ if test "$with_pkg_config" = yes -a -z "$PKG_CONFIG"; then as_fn_error $? "pkg-config is required" "$LINENO" 5] fi +# Set name for machine-dependent library files + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking MACHDEP" >&5 +printf %s "checking MACHDEP... " >&6; } +if test -z "$MACHDEP" +then + # avoid using uname for cross builds + if test "$cross_compiling" = yes; then + # ac_sys_system and ac_sys_release are used for setting + # a lot of different things including 'define_xopen_source' + # in the case statement below. + case "$host" in + *-*-linux-android*) + ac_sys_system=Linux-android + ;; + *-*-linux*) + ac_sys_system=Linux + ;; + *-*-cygwin*) + ac_sys_system=Cygwin + ;; + *-apple-ios*) + ac_sys_system=iOS + ;; + *-*-vxworks*) + ac_sys_system=VxWorks + ;; + *-*-emscripten) + ac_sys_system=Emscripten + ;; + *-*-wasi) + ac_sys_system=WASI + ;; + *) + # for now, limit cross builds to known configurations + MACHDEP="unknown" + as_fn_error $? "cross build not supported for $host" "$LINENO" 5 + esac + ac_sys_release= + else + ac_sys_system=`uname -s` + if test "$ac_sys_system" = "AIX" \ + -o "$ac_sys_system" = "UnixWare" -o "$ac_sys_system" = "OpenUNIX"; then + ac_sys_release=`uname -v` + else + ac_sys_release=`uname -r` + fi + fi + ac_md_system=`echo $ac_sys_system | + tr -d '/ ' | tr '[A-Z]' '[a-z]'` + ac_md_release=`echo $ac_sys_release | + tr -d '/ ' | sed 's/^[A-Z]\.//' | sed 's/\..*//'` + MACHDEP="$ac_md_system$ac_md_release" + + case $MACHDEP in + aix*) MACHDEP="aix";; + linux*) MACHDEP="linux";; + cygwin*) MACHDEP="cygwin";; + darwin*) MACHDEP="darwin";; + '') MACHDEP="unknown";; + esac + + if test "$ac_sys_system" = "SunOS"; then + # For Solaris, there isn't an OS version specific macro defined + # in most compilers, so we define one here. + SUNOS_VERSION=`echo $ac_sys_release | sed -e 's!\.\(0-9\)$!.0\1!g' | tr -d '.'` + +printf "%s\n" "#define Py_SUNOS_VERSION $SUNOS_VERSION" >>confdefs.h + + fi +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: \"$MACHDEP\"" >&5 +printf "%s\n" "\"$MACHDEP\"" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-universalsdk" >&5 printf %s "checking for --enable-universalsdk... " >&6; } # Check whether --enable-universalsdk was given. @@ -4113,19 +4200,31 @@ then : enableval=$enable_framework; case $enableval in yes) + if test "$ac_sys_system" = "iOS"; then + as_fn_error $? "iOS builds must provide an explicit path for --enable-framework" "$LINENO" 5 + fi + enableval=/Library/Frameworks esac case $enableval in no) + if test "$ac_sys_system" = "iOS"; then + as_fn_error $? "iOS builds must use --enable-framework=<install path>" "$LINENO" 5 + fi + PYTHONFRAMEWORK= PYTHONFRAMEWORKDIR=no-framework PYTHONFRAMEWORKPREFIX= PYTHONFRAMEWORKINSTALLDIR= + PYTHONFRAMEWORKINSTALLNAMEPREFIX= + RESSRCDIR= FRAMEWORKINSTALLFIRST= FRAMEWORKINSTALLLAST= FRAMEWORKALTINSTALLFIRST= FRAMEWORKALTINSTALLLAST= FRAMEWORKPYTHONW= + INSTALLTARGETS="commoninstall bininstall maninstall" + if test "x${prefix}" = "xNONE"; then FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" else @@ -4136,79 +4235,112 @@ then : *) PYTHONFRAMEWORKPREFIX="${enableval}" PYTHONFRAMEWORKINSTALLDIR=$PYTHONFRAMEWORKPREFIX/$PYTHONFRAMEWORKDIR - FRAMEWORKINSTALLFIRST="frameworkinstallstructure" - FRAMEWORKALTINSTALLFIRST="frameworkinstallstructure " - FRAMEWORKINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools" - FRAMEWORKALTINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkaltinstallunixtools" - FRAMEWORKPYTHONW="frameworkpythonw" - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - - if test "x${prefix}" = "xNONE" ; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi + case $ac_sys_system in #( + Darwin) : + FRAMEWORKINSTALLFIRST="frameworkinstallversionedstructure" + FRAMEWORKALTINSTALLFIRST="frameworkinstallversionedstructure " + FRAMEWORKINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools" + FRAMEWORKALTINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkaltinstallunixtools" + FRAMEWORKPYTHONW="frameworkpythonw" + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + INSTALLTARGETS="commoninstall bininstall maninstall" + + if test "x${prefix}" = "xNONE" ; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi - case "${enableval}" in - /System*) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - if test "${prefix}" = "NONE" ; then - # See below - FRAMEWORKUNIXTOOLSPREFIX="/usr" - fi - ;; + case "${enableval}" in + /System*) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + if test "${prefix}" = "NONE" ; then + # See below + FRAMEWORKUNIXTOOLSPREFIX="/usr" + fi + ;; + + /Library*) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + ;; + + */Library/Frameworks) + MDIR="`dirname "${enableval}"`" + MDIR="`dirname "${MDIR}"`" + FRAMEWORKINSTALLAPPSPREFIX="${MDIR}/Applications" + + if test "${prefix}" = "NONE"; then + # User hasn't specified the + # --prefix option, but wants to install + # the framework in a non-default location, + # ensure that the compatibility links get + # installed relative to that prefix as well + # instead of in /usr/local. + FRAMEWORKUNIXTOOLSPREFIX="${MDIR}" + fi + ;; - /Library*) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - ;; + *) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + ;; + esac - */Library/Frameworks) - MDIR="`dirname "${enableval}"`" - MDIR="`dirname "${MDIR}"`" - FRAMEWORKINSTALLAPPSPREFIX="${MDIR}/Applications" - - if test "${prefix}" = "NONE"; then - # User hasn't specified the - # --prefix option, but wants to install - # the framework in a non-default location, - # ensure that the compatibility links get - # installed relative to that prefix as well - # instead of in /usr/local. - FRAMEWORKUNIXTOOLSPREFIX="${MDIR}" - fi - ;; + prefix=$PYTHONFRAMEWORKINSTALLDIR/Versions/$VERSION + PYTHONFRAMEWORKINSTALLNAMEPREFIX=${prefix} + RESSRCDIR=Mac/Resources/framework - *) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - ;; - esac + # Add files for Mac specific code to the list of output + # files: + ac_config_files="$ac_config_files Mac/Makefile" - prefix=$PYTHONFRAMEWORKINSTALLDIR/Versions/$VERSION + ac_config_files="$ac_config_files Mac/PythonLauncher/Makefile" - # Add files for Mac specific code to the list of output - # files: - ac_config_files="$ac_config_files Mac/Makefile" + ac_config_files="$ac_config_files Mac/Resources/framework/Info.plist" - ac_config_files="$ac_config_files Mac/PythonLauncher/Makefile" + ac_config_files="$ac_config_files Mac/Resources/app/Info.plist" - ac_config_files="$ac_config_files Mac/Resources/framework/Info.plist" + ;; + iOS) : + FRAMEWORKINSTALLFIRST="frameworkinstallunversionedstructure" + FRAMEWORKALTINSTALLFIRST="frameworkinstallunversionedstructure " + FRAMEWORKINSTALLLAST="frameworkinstallmobileheaders" + FRAMEWORKALTINSTALLLAST="frameworkinstallmobileheaders" + FRAMEWORKPYTHONW= + INSTALLTARGETS="libinstall inclinstall sharedinstall" - ac_config_files="$ac_config_files Mac/Resources/app/Info.plist" + prefix=$PYTHONFRAMEWORKPREFIX + PYTHONFRAMEWORKINSTALLNAMEPREFIX="@rpath/$PYTHONFRAMEWORKDIR" + RESSRCDIR=iOS/Resources - esac + ac_config_files="$ac_config_files iOS/Resources/Info.plist" + + ;; + *) + as_fn_error $? "Unknown platform for framework build" "$LINENO" 5 + ;; + esac + esac else $as_nop + if test "$ac_sys_system" = "iOS"; then + as_fn_error $? "iOS builds must use --enable-framework=<install path>" "$LINENO" 5 + fi + PYTHONFRAMEWORK= PYTHONFRAMEWORKDIR=no-framework PYTHONFRAMEWORKPREFIX= PYTHONFRAMEWORKINSTALLDIR= + PYTHONFRAMEWORKINSTALLNAMEPREFIX= + RESSRCDIR= FRAMEWORKINSTALLFIRST= FRAMEWORKINSTALLLAST= FRAMEWORKALTINSTALLFIRST= FRAMEWORKALTINSTALLLAST= FRAMEWORKPYTHONW= + INSTALLTARGETS="commoninstall bininstall maninstall" if test "x${prefix}" = "xNONE" ; then FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" else @@ -4233,79 +4365,11 @@ fi -printf "%s\n" "#define _PYTHONFRAMEWORK \"${PYTHONFRAMEWORK}\"" >>confdefs.h -# Set name for machine-dependent library files -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking MACHDEP" >&5 -printf %s "checking MACHDEP... " >&6; } -if test -z "$MACHDEP" -then - # avoid using uname for cross builds - if test "$cross_compiling" = yes; then - # ac_sys_system and ac_sys_release are used for setting - # a lot of different things including 'define_xopen_source' - # in the case statement below. - case "$host" in - *-*-linux-android*) - ac_sys_system=Linux-android - ;; - *-*-linux*) - ac_sys_system=Linux - ;; - *-*-cygwin*) - ac_sys_system=Cygwin - ;; - *-*-vxworks*) - ac_sys_system=VxWorks - ;; - *-*-emscripten) - ac_sys_system=Emscripten - ;; - *-*-wasi) - ac_sys_system=WASI - ;; - *) - # for now, limit cross builds to known configurations - MACHDEP="unknown" - as_fn_error $? "cross build not supported for $host" "$LINENO" 5 - esac - ac_sys_release= - else - ac_sys_system=`uname -s` - if test "$ac_sys_system" = "AIX" \ - -o "$ac_sys_system" = "UnixWare" -o "$ac_sys_system" = "OpenUNIX"; then - ac_sys_release=`uname -v` - else - ac_sys_release=`uname -r` - fi - fi - ac_md_system=`echo $ac_sys_system | - tr -d '/ ' | tr '[A-Z]' '[a-z]'` - ac_md_release=`echo $ac_sys_release | - tr -d '/ ' | sed 's/^[A-Z]\.//' | sed 's/\..*//'` - MACHDEP="$ac_md_system$ac_md_release" - - case $MACHDEP in - aix*) MACHDEP="aix";; - linux*) MACHDEP="linux";; - cygwin*) MACHDEP="cygwin";; - darwin*) MACHDEP="darwin";; - '') MACHDEP="unknown";; - esac - - if test "$ac_sys_system" = "SunOS"; then - # For Solaris, there isn't an OS version specific macro defined - # in most compilers, so we define one here. - SUNOS_VERSION=`echo $ac_sys_release | sed -e 's!\.\(0-9\)$!.0\1!g' | tr -d '.'` - -printf "%s\n" "#define Py_SUNOS_VERSION $SUNOS_VERSION" >>confdefs.h +printf "%s\n" "#define _PYTHONFRAMEWORK \"${PYTHONFRAMEWORK}\"" >>confdefs.h - fi -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: \"$MACHDEP\"" >&5 -printf "%s\n" "\"$MACHDEP\"" >&6; } if test "$cross_compiling" = yes; then @@ -4313,27 +4377,44 @@ if test "$cross_compiling" = yes; then *-*-linux*) case "$host_cpu" in arm*) - _host_cpu=arm + _host_ident=arm ;; *) - _host_cpu=$host_cpu + _host_ident=$host_cpu esac ;; *-*-cygwin*) - _host_cpu= + _host_ident= + ;; + *-apple-ios*) + _host_os=`echo $host | cut -d '-' -f3` + _host_device=`echo $host | cut -d '-' -f4` + _host_device=${_host_device:=os} + + IOS_DEPLOYMENT_TARGET=${_host_os:3} + IOS_DEPLOYMENT_TARGET=${IOS_DEPLOYMENT_TARGET:=12.0} + + case "$host_cpu" in + aarch64) + _host_ident=${IOS_DEPLOYMENT_TARGET}-arm64-iphone${_host_device} + ;; + *) + _host_ident=${IOS_DEPLOYMENT_TARGET}-$host_cpu-iphone${_host_device} + ;; + esac ;; *-*-vxworks*) - _host_cpu=$host_cpu + _host_ident=$host_cpu ;; wasm32-*-* | wasm64-*-*) - _host_cpu=$host_cpu + _host_ident=$host_cpu ;; *) # for now, limit cross builds to known configurations MACHDEP="unknown" as_fn_error $? "cross build not supported for $host" "$LINENO" 5 esac - _PYTHON_HOST_PLATFORM="$MACHDEP${_host_cpu:+-$_host_cpu}" + _PYTHON_HOST_PLATFORM="$MACHDEP${_host_ident:+-$_host_ident}" fi # Some systems cannot stand _XOPEN_SOURCE being defined at all; they @@ -4400,6 +4481,9 @@ printf "%s\n" "#define _BSD_SOURCE 1" >>confdefs.h define_xopen_source=no;; Darwin/[12][0-9].*) define_xopen_source=no;; + # On iOS, defining _POSIX_C_SOURCE also disables platform specific features. + iOS/*) + define_xopen_source=no;; # On QNX 6.3.2, defining _XOPEN_SOURCE prevents netdb.h from # defining NI_NUMERICHOST. QNX/6.3.2) @@ -4494,6 +4578,17 @@ case $host in #( ;; esac +case $ac_sys_system in #( + iOS) : + + as_fn_append CFLAGS " -mios-version-min=${IOS_DEPLOYMENT_TARGET}" + as_fn_append LDFLAGS " -mios-version-min=${IOS_DEPLOYMENT_TARGET}" + + ;; #( + *) : + ;; +esac + if test "$ac_sys_system" = "Darwin" then # Extract the first word of "xcrun", so it can be a program name with args. @@ -6756,6 +6851,8 @@ printf %s "checking for multiarch... " >&6; } case $ac_sys_system in #( Darwin*) : MULTIARCH="" ;; #( + iOS) : + MULTIARCH="" ;; #( FreeBSD*) : MULTIARCH="" ;; #( *) : @@ -6763,8 +6860,6 @@ case $ac_sys_system in #( ;; esac -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $MULTIARCH" >&5 -printf "%s\n" "$MULTIARCH" >&6; } if test x$PLATFORM_TRIPLET != x && test x$MULTIARCH != x; then if test x$PLATFORM_TRIPLET != x$MULTIARCH; then @@ -6774,6 +6869,16 @@ elif test x$PLATFORM_TRIPLET != x && test x$MULTIARCH = x; then MULTIARCH=$PLATFORM_TRIPLET fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $MULTIARCH" >&5 +printf "%s\n" "$MULTIARCH" >&6; } + +case $ac_sys_system in #( + iOS) : + SOABI_PLATFORM=`echo "$PLATFORM_TRIPLET" | cut -d '-' -f2` ;; #( + *) : + SOABI_PLATFORM=$PLATFORM_TRIPLET + ;; +esac if test x$MULTIARCH != x; then MULTIARCH_CPPFLAGS="-DMULTIARCH=\\\"$MULTIARCH\\\"" @@ -6799,6 +6904,8 @@ case $host/$ac_cv_cc_name in #( aarch64-*-linux-gnu/clang) : PY_SUPPORT_TIER=2 ;; #( powerpc64le-*-linux-gnu/gcc) : + PY_SUPPORT_TIER=2 ;; #( + wasm32-unknown-wasi/clang) : PY_SUPPORT_TIER=2 ;; #( x86_64-*-linux-gnu/clang) : PY_SUPPORT_TIER=2 ;; #( @@ -6811,11 +6918,11 @@ case $host/$ac_cv_cc_name in #( PY_SUPPORT_TIER=3 ;; #( s390x-*-linux-gnu/gcc) : PY_SUPPORT_TIER=3 ;; #( - wasm32-unknown-emscripten/clang) : + x86_64-*-freebsd*/clang) : PY_SUPPORT_TIER=3 ;; #( - wasm32-unknown-wasi/clang) : + aarch64-apple-ios*-simulator/clang) : PY_SUPPORT_TIER=3 ;; #( - x86_64-*-freebsd*/clang) : + aarch64-apple-ios*/clang) : PY_SUPPORT_TIER=3 ;; #( *) : PY_SUPPORT_TIER=0 @@ -7267,17 +7374,25 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking LDLIBRARY" >&5 printf %s "checking LDLIBRARY... " >&6; } -# MacOSX framework builds need more magic. LDLIBRARY is the dynamic +# Apple framework builds need more magic. LDLIBRARY is the dynamic # library that we build, but we do not want to link against it (we # will find it with a -framework option). For this reason there is an # extra variable BLDLIBRARY against which Python and the extension # modules are linked, BLDLIBRARY. This is normally the same as -# LDLIBRARY, but empty for MacOSX framework builds. +# LDLIBRARY, but empty for MacOSX framework builds. iOS does the same, +# but uses a non-versioned framework layout. if test "$enable_framework" then - LDLIBRARY='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' - RUNSHARED=DYLD_FRAMEWORK_PATH=`pwd`${DYLD_FRAMEWORK_PATH:+:${DYLD_FRAMEWORK_PATH}} + case $ac_sys_system in + Darwin) + LDLIBRARY='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)';; + iOS) + LDLIBRARY='$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK)';; + *) + as_fn_error $? "Unknown platform for framework build" "$LINENO" 5;; + esac BLDLIBRARY='' + RUNSHARED=DYLD_FRAMEWORK_PATH=`pwd`${DYLD_FRAMEWORK_PATH:+:${DYLD_FRAMEWORK_PATH}} else BLDLIBRARY='$(LDLIBRARY)' fi @@ -7290,66 +7405,74 @@ printf "%s\n" "#define Py_ENABLE_SHARED 1" >>confdefs.h case $ac_sys_system in CYGWIN*) - LDLIBRARY='libpython$(LDVERSION).dll.a' - DLLLIBRARY='libpython$(LDVERSION).dll' - ;; + LDLIBRARY='libpython$(LDVERSION).dll.a' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + DLLLIBRARY='libpython$(LDVERSION).dll' + ;; SunOS*) - LDLIBRARY='libpython$(LDVERSION).so' - BLDLIBRARY='-Wl,-R,$(LIBDIR) -L. -lpython$(LDVERSION)' - RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - INSTSONAME="$LDLIBRARY".$SOVERSION - if test "$with_pydebug" != yes - then - PY3LIBRARY=libpython3.so - fi - ;; + LDLIBRARY='libpython$(LDVERSION).so' + BLDLIBRARY='-Wl,-R,$(LIBDIR) -L. -lpython$(LDVERSION)' + RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + INSTSONAME="$LDLIBRARY".$SOVERSION + if test "$with_pydebug" != yes + then + PY3LIBRARY=libpython3.so + fi + ;; Linux*|GNU*|NetBSD*|FreeBSD*|DragonFly*|OpenBSD*|VxWorks*) - LDLIBRARY='libpython$(LDVERSION).so' - BLDLIBRARY='-L. -lpython$(LDVERSION)' - RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - INSTSONAME="$LDLIBRARY".$SOVERSION - if test "$with_pydebug" != yes - then - PY3LIBRARY=libpython3.so - fi - ;; + LDLIBRARY='libpython$(LDVERSION).so' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + INSTSONAME="$LDLIBRARY".$SOVERSION + if test "$with_pydebug" != yes + then + PY3LIBRARY=libpython3.so + fi + ;; hp*|HP*) - case `uname -m` in - ia64) - LDLIBRARY='libpython$(LDVERSION).so' - ;; - *) - LDLIBRARY='libpython$(LDVERSION).sl' - ;; - esac - BLDLIBRARY='-Wl,+b,$(LIBDIR) -L. -lpython$(LDVERSION)' - RUNSHARED=SHLIB_PATH=`pwd`${SHLIB_PATH:+:${SHLIB_PATH}} - ;; + case `uname -m` in + ia64) + LDLIBRARY='libpython$(LDVERSION).so' + ;; + *) + LDLIBRARY='libpython$(LDVERSION).sl' + ;; + esac + BLDLIBRARY='-Wl,+b,$(LIBDIR) -L. -lpython$(LDVERSION)' + RUNSHARED=SHLIB_PATH=`pwd`${SHLIB_PATH:+:${SHLIB_PATH}} + ;; Darwin*) - LDLIBRARY='libpython$(LDVERSION).dylib' - BLDLIBRARY='-L. -lpython$(LDVERSION)' - RUNSHARED=DYLD_LIBRARY_PATH=`pwd`${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}} - ;; + LDLIBRARY='libpython$(LDVERSION).dylib' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + RUNSHARED=DYLD_LIBRARY_PATH=`pwd`${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}} + ;; + iOS) + LDLIBRARY='libpython$(LDVERSION).dylib' + ;; AIX*) - LDLIBRARY='libpython$(LDVERSION).so' - RUNSHARED=LIBPATH=`pwd`${LIBPATH:+:${LIBPATH}} - ;; + LDLIBRARY='libpython$(LDVERSION).so' + RUNSHARED=LIBPATH=`pwd`${LIBPATH:+:${LIBPATH}} + ;; esac else # shared is disabled PY_ENABLE_SHARED=0 case $ac_sys_system in CYGWIN*) - BLDLIBRARY='$(LIBRARY)' - LDLIBRARY='libpython$(LDVERSION).dll.a' - ;; + BLDLIBRARY='$(LIBRARY)' + LDLIBRARY='libpython$(LDVERSION).dll.a' + ;; esac fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDLIBRARY" >&5 +printf "%s\n" "$LDLIBRARY" >&6; } if test "$cross_compiling" = yes; then - RUNSHARED= + RUNSHARED= fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 +printf %s "checking HOSTRUNNER... " >&6; } if test -z "$HOSTRUNNER" then @@ -7533,8 +7656,6 @@ fi esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 -printf %s "checking HOSTRUNNER... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $HOSTRUNNER" >&5 printf "%s\n" "$HOSTRUNNER" >&6; } @@ -7542,9 +7663,6 @@ if test -n "$HOSTRUNNER"; then PYTHON_FOR_BUILD="_PYTHON_HOSTRUNNER='$HOSTRUNNER' $PYTHON_FOR_BUILD" fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDLIBRARY" >&5 -printf "%s\n" "$LDLIBRARY" >&6; } - # LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable case $ac_sys_system/$ac_sys_emscripten_target in #( Emscripten/browser*) : @@ -7997,6 +8115,34 @@ else printf "%s\n" "no" >&6; } fi +# Check for --enable-experimental-jit: +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-experimental-jit" >&5 +printf %s "checking for --enable-experimental-jit... " >&6; } +# Check whether --enable-experimental-jit was given. +if test ${enable_experimental_jit+y} +then : + enableval=$enable_experimental_jit; +else $as_nop + enable_experimental_jit=no +fi + +if test "x$enable_experimental_jit" = xno +then : + +else $as_nop + as_fn_append CFLAGS_NODIST " -D_Py_JIT" + REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host" + JIT_STENCILS_H="jit_stencils.h" + if test "x$Py_DEBUG" = xtrue +then : + as_fn_append REGEN_JIT_COMMAND " --debug" +fi +fi + + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_experimental_jit" >&5 +printf "%s\n" "$enable_experimental_jit" >&6; } + # Enable optimization flags @@ -11151,42 +11297,22 @@ then : fi -# checks for typedefs - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_t in time.h" >&5 -printf %s "checking for clock_t in time.h... " >&6; } -if test ${ac_cv_clock_t_time_h+y} +# Check for clock_t in time.h. +ac_fn_c_check_type "$LINENO" "clock_t" "ac_cv_type_clock_t" "#include <time.h> +" +if test "x$ac_cv_type_clock_t" = xyes then : - printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include <time.h> +printf "%s\n" "#define HAVE_CLOCK_T 1" >>confdefs.h -_ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "clock_t" >/dev/null 2>&1 -then : - ac_cv_clock_t_time_h=yes -else $as_nop - ac_cv_clock_t_time_h=no -fi -rm -rf conftest* - - -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_clock_t_time_h" >&5 -printf "%s\n" "$ac_cv_clock_t_time_h" >&6; } -if test "x$ac_cv_clock_t_time_h" = xno -then : +else $as_nop printf "%s\n" "#define clock_t long" >>confdefs.h - fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for makedev" >&5 printf %s "checking for makedev... " >&6; } if test ${ac_cv_func_makedev+y} @@ -11388,6 +11514,7 @@ printf "%s\n" "#define size_t unsigned int" >>confdefs.h fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for uid_t in sys/types.h" >&5 printf %s "checking for uid_t in sys/types.h... " >&6; } if test ${ac_cv_type_uid_t+y} @@ -11427,12 +11554,16 @@ then : printf "%s\n" "#define HAVE_SSIZE_T 1" >>confdefs.h + fi ac_fn_c_check_type "$LINENO" "__uint128_t" "ac_cv_type___uint128_t" "$ac_includes_default" if test "x$ac_cv_type___uint128_t" = xyes then : +printf "%s\n" "#define HAVE___UINT128_T 1" >>confdefs.h + + printf "%s\n" "#define HAVE_GCC_UINT128_T 1" >>confdefs.h fi @@ -12552,6 +12683,7 @@ if test -z "$SHLIB_SUFFIX"; then esac ;; CYGWIN*) SHLIB_SUFFIX=.dll;; + iOS) SHLIB_SUFFIX=.dylib;; *) SHLIB_SUFFIX=.so;; esac fi @@ -12634,6 +12766,11 @@ then BLDSHARED="$LDSHARED" fi ;; + iOS/*) + LDSHARED='$(CC) -dynamiclib -F . -framework Python' + LDCXXSHARED='$(CXX) -dynamiclib -F . -framework Python' + BLDSHARED="$LDSHARED" + ;; Emscripten|WASI) LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; @@ -12722,7 +12859,6 @@ then then CCSHARED="-fPIC"; else CCSHARED="+z"; fi;; - Linux-android*) ;; Linux*|GNU*) CCSHARED="-fPIC";; Emscripten*|WASI*) if test "x$enable_wasm_dynamic_linking" = xyes @@ -12763,30 +12899,34 @@ then Linux-android*) LINKFORSHARED="-pie -Xlinker -export-dynamic";; Linux*|GNU*) LINKFORSHARED="-Xlinker -export-dynamic";; # -u libsys_s pulls in all symbols in libsys - Darwin/*) + Darwin/*|iOS/*) LINKFORSHARED="$extra_undefs -framework CoreFoundation" # Issue #18075: the default maximum stack size (8MBytes) is too # small for the default recursion limit. Increase the stack size # to ensure that tests don't crash - stack_size="1000000" # 16 MB - if test "$with_ubsan" = "yes" - then - # Undefined behavior sanitizer requires an even deeper stack - stack_size="4000000" # 64 MB - fi - - LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" + stack_size="1000000" # 16 MB + if test "$with_ubsan" = "yes" + then + # Undefined behavior sanitizer requires an even deeper stack + stack_size="4000000" # 64 MB + fi printf "%s\n" "#define THREAD_STACK_SIZE 0x$stack_size" >>confdefs.h - if test "$enable_framework" - then - LINKFORSHARED="$LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + if test $ac_sys_system = "Darwin"; then + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" + + if test "$enable_framework"; then + LINKFORSHARED="$LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + fi + LINKFORSHARED="$LINKFORSHARED" + elif test $ac_sys_system = "iOS"; then + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK)' fi - LINKFORSHARED="$LINKFORSHARED";; + ;; OpenUNIX*|UnixWare*) LINKFORSHARED="-Wl,-Bexport";; SCO_SV*) LINKFORSHARED="-Wl,-Bexport";; ReliantUNIX*) LINKFORSHARED="-W1 -Blargedynsym";; @@ -14175,6 +14315,10 @@ then : ctypes_malloc_closure=yes ;; #( + iOS) : + + ctypes_malloc_closure=yes + ;; #( sunos5) : as_fn_append LIBFFI_LIBS " -mimpure-text" ;; #( @@ -16021,24 +16165,47 @@ else # (e.g. gnu pth with pthread emulation) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _POSIX_THREADS in unistd.h" >&5 printf %s "checking for _POSIX_THREADS in unistd.h... " >&6; } - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _POSIX_THREADS defined in unistd.h" >&5 +printf %s "checking for _POSIX_THREADS defined in unistd.h... " >&6; } +if test ${ac_cv_defined__POSIX_THREADS_unistd_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include <unistd.h> -#ifdef _POSIX_THREADS -yes -#endif +int +main (void) +{ + + #ifdef _POSIX_THREADS + int ok; + (void)ok; + #else + choke me + #endif + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_defined__POSIX_THREADS_unistd_h=yes +else $as_nop + ac_cv_defined__POSIX_THREADS_unistd_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__POSIX_THREADS_unistd_h" >&5 +printf "%s\n" "$ac_cv_defined__POSIX_THREADS_unistd_h" >&6; } +if test $ac_cv_defined__POSIX_THREADS_unistd_h != "no" then : unistd_defines_pthreads=yes else $as_nop unistd_defines_pthreads=no fi -rm -rf conftest* - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $unistd_defines_pthreads" >&5 printf "%s\n" "$unistd_defines_pthreads" >&6; } @@ -16545,59 +16712,131 @@ printf %s "checking ipv6 stack type... " >&6; } do case $i in inria) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for IPV6_INRIA_VERSION defined in netinet/in.h" >&5 +printf %s "checking for IPV6_INRIA_VERSION defined in netinet/in.h... " >&6; } +if test ${ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <netinet/in.h> +int +main (void) +{ + + #ifdef IPV6_INRIA_VERSION + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h=yes +else $as_nop + ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h" >&5 +printf "%s\n" "$ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h" >&6; } +if test $ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h != "no" +then : + ipv6type=$i +fi + ;; + kame) + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __KAME__ defined in netinet/in.h" >&5 +printf %s "checking for __KAME__ defined in netinet/in.h... " >&6; } +if test ${ac_cv_defined___KAME___netinet_in_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ +#include <netinet/in.h> +int +main (void) +{ + + #ifdef __KAME__ + int ok; + (void)ok; + #else + choke me + #endif -#include <netinet/in.h> -#ifdef IPV6_INRIA_VERSION -yes -#endif + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i + ac_cv_defined___KAME___netinet_in_h=yes +else $as_nop + ac_cv_defined___KAME___netinet_in_h=no fi -rm -rf conftest* - - ;; - kame) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -#include <netinet/in.h> -#ifdef __KAME__ -yes -#endif -_ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___KAME___netinet_in_h" >&5 +printf "%s\n" "$ac_cv_defined___KAME___netinet_in_h" >&6; } +if test $ac_cv_defined___KAME___netinet_in_h != "no" then : - ipv6type=$i; - ipv6lib=inet6 - ipv6libdir=/usr/local/v6/lib - ipv6trylibc=yes + ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib + ipv6trylibc=yes fi -rm -rf conftest* - ;; linux-glibc) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __GLIBC__ defined in features.h" >&5 +printf %s "checking for __GLIBC__ defined in features.h... " >&6; } +if test ${ac_cv_defined___GLIBC___features_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include <features.h> -#if defined(__GLIBC__) && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 1) || (__GLIBC__ > 2)) -yes -#endif +int +main (void) +{ + + #ifdef __GLIBC__ + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6trylibc=yes + ac_cv_defined___GLIBC___features_h=yes +else $as_nop + ac_cv_defined___GLIBC___features_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___GLIBC___features_h" >&5 +printf "%s\n" "$ac_cv_defined___GLIBC___features_h" >&6; } +if test $ac_cv_defined___GLIBC___features_h != "no" +then : + ipv6type=$i + ipv6trylibc=yes fi -rm -rf conftest* - ;; linux-inet6) if test -d /usr/inet6; then @@ -16616,62 +16855,134 @@ rm -rf conftest* fi ;; toshiba) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _TOSHIBA_INET6 defined in sys/param.h" >&5 +printf %s "checking for _TOSHIBA_INET6 defined in sys/param.h... " >&6; } +if test ${ac_cv_defined__TOSHIBA_INET6_sys_param_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include <sys/param.h> -#ifdef _TOSHIBA_INET6 -yes -#endif +int +main (void) +{ + + #ifdef _TOSHIBA_INET6 + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_defined__TOSHIBA_INET6_sys_param_h=yes +else $as_nop + ac_cv_defined__TOSHIBA_INET6_sys_param_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__TOSHIBA_INET6_sys_param_h" >&5 +printf "%s\n" "$ac_cv_defined__TOSHIBA_INET6_sys_param_h" >&6; } +if test $ac_cv_defined__TOSHIBA_INET6_sys_param_h != "no" then : - ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib + ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib fi -rm -rf conftest* - ;; v6d) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __V6D__ defined in /usr/local/v6/include/sys/v6config.h" >&5 +printf %s "checking for __V6D__ defined in /usr/local/v6/include/sys/v6config.h... " >&6; } +if test ${ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include </usr/local/v6/include/sys/v6config.h> -#ifdef __V6D__ -yes -#endif +int +main (void) +{ + + #ifdef __V6D__ + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=v6; - ipv6libdir=/usr/local/v6/lib; - BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS" + ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h=yes +else $as_nop + ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h" >&5 +printf "%s\n" "$ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h" >&6; } +if test $ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h != "no" +then : + ipv6type=$i + ipv6lib=v6 + ipv6libdir=/usr/local/v6/lib + BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS" fi -rm -rf conftest* - ;; zeta) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _ZETA_MINAMI_INET6 defined in sys/param.h" >&5 +printf %s "checking for _ZETA_MINAMI_INET6 defined in sys/param.h... " >&6; } +if test ${ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include <sys/param.h> -#ifdef _ZETA_MINAMI_INET6 -yes -#endif +int +main (void) +{ + + #ifdef _ZETA_MINAMI_INET6 + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib + ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h=yes +else $as_nop + ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h" >&5 +printf "%s\n" "$ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h" >&6; } +if test $ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h != "no" +then : + ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib fi -rm -rf conftest* - ;; esac if test "$ipv6type" != "unknown"; then @@ -16683,16 +16994,18 @@ printf "%s\n" "$ipv6type" >&6; } fi if test "$ipv6" = "yes" -a "$ipv6lib" != "none"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking ipv6 library" >&5 +printf %s "checking ipv6 library... " >&6; } if test -d $ipv6libdir -a -f $ipv6libdir/lib$ipv6lib.a; then LIBS="-L$ipv6libdir -l$ipv6lib $LIBS" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: using lib$ipv6lib" >&5 -printf "%s\n" "$as_me: using lib$ipv6lib" >&6;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: lib$ipv6lib" >&5 +printf "%s\n" "lib$ipv6lib" >&6; } else if test "x$ipv6trylibc" = xyes then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: using libc" >&5 -printf "%s\n" "$as_me: using libc" >&6;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: libc" >&5 +printf "%s\n" "libc" >&6; } else $as_nop @@ -17426,12 +17739,6 @@ if test "x$ac_cv_func_getegid" = xyes then : printf "%s\n" "#define HAVE_GETEGID 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "getentropy" "ac_cv_func_getentropy" -if test "x$ac_cv_func_getentropy" = xyes -then : - printf "%s\n" "#define HAVE_GETENTROPY 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "geteuid" "ac_cv_func_geteuid" if test "x$ac_cv_func_geteuid" = xyes @@ -17444,6 +17751,12 @@ if test "x$ac_cv_func_getgid" = xyes then : printf "%s\n" "#define HAVE_GETGID 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "getgrent" "ac_cv_func_getgrent" +if test "x$ac_cv_func_getgrent" = xyes +then : + printf "%s\n" "#define HAVE_GETGRENT 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "getgrgid" "ac_cv_func_getgrgid" if test "x$ac_cv_func_getgrgid" = xyes @@ -17468,12 +17781,6 @@ if test "x$ac_cv_func_getgrouplist" = xyes then : printf "%s\n" "#define HAVE_GETGROUPLIST 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "getgroups" "ac_cv_func_getgroups" -if test "x$ac_cv_func_getgroups" = xyes -then : - printf "%s\n" "#define HAVE_GETGROUPS 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "gethostname" "ac_cv_func_gethostname" if test "x$ac_cv_func_gethostname" = xyes @@ -17606,6 +17913,12 @@ if test "x$ac_cv_func_getwd" = xyes then : printf "%s\n" "#define HAVE_GETWD 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "grantpt" "ac_cv_func_grantpt" +if test "x$ac_cv_func_grantpt" = xyes +then : + printf "%s\n" "#define HAVE_GRANTPT 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "if_nameindex" "ac_cv_func_if_nameindex" if test "x$ac_cv_func_if_nameindex" = xyes @@ -17792,6 +18105,12 @@ if test "x$ac_cv_func_posix_fallocate" = xyes then : printf "%s\n" "#define HAVE_POSIX_FALLOCATE 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "posix_openpt" "ac_cv_func_posix_openpt" +if test "x$ac_cv_func_posix_openpt" = xyes +then : + printf "%s\n" "#define HAVE_POSIX_OPENPT 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "posix_spawn" "ac_cv_func_posix_spawn" if test "x$ac_cv_func_posix_spawn" = xyes @@ -17828,6 +18147,18 @@ if test "x$ac_cv_func_preadv2" = xyes then : printf "%s\n" "#define HAVE_PREADV2 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "process_vm_readv" "ac_cv_func_process_vm_readv" +if test "x$ac_cv_func_process_vm_readv" = xyes +then : + printf "%s\n" "#define HAVE_PROCESS_VM_READV 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "pthread_cond_timedwait_relative_np" "ac_cv_func_pthread_cond_timedwait_relative_np" +if test "x$ac_cv_func_pthread_cond_timedwait_relative_np" = xyes +then : + printf "%s\n" "#define HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "pthread_condattr_setclock" "ac_cv_func_pthread_condattr_setclock" if test "x$ac_cv_func_pthread_condattr_setclock" = xyes @@ -17846,6 +18177,18 @@ if test "x$ac_cv_func_pthread_kill" = xyes then : printf "%s\n" "#define HAVE_PTHREAD_KILL 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "ptsname" "ac_cv_func_ptsname" +if test "x$ac_cv_func_ptsname" = xyes +then : + printf "%s\n" "#define HAVE_PTSNAME 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "ptsname_r" "ac_cv_func_ptsname_r" +if test "x$ac_cv_func_ptsname_r" = xyes +then : + printf "%s\n" "#define HAVE_PTSNAME_R 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "pwrite" "ac_cv_func_pwrite" if test "x$ac_cv_func_pwrite" = xyes @@ -18170,12 +18513,6 @@ if test "x$ac_cv_func_sysconf" = xyes then : printf "%s\n" "#define HAVE_SYSCONF 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "system" "ac_cv_func_system" -if test "x$ac_cv_func_system" = xyes -then : - printf "%s\n" "#define HAVE_SYSTEM 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "tcgetpgrp" "ac_cv_func_tcgetpgrp" if test "x$ac_cv_func_tcgetpgrp" = xyes @@ -18254,6 +18591,12 @@ if test "x$ac_cv_func_unlinkat" = xyes then : printf "%s\n" "#define HAVE_UNLINKAT 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "unlockpt" "ac_cv_func_unlockpt" +if test "x$ac_cv_func_unlockpt" = xyes +then : + printf "%s\n" "#define HAVE_UNLOCKPT 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "utimensat" "ac_cv_func_utimensat" if test "x$ac_cv_func_utimensat" = xyes @@ -18348,6 +18691,32 @@ fi fi +# iOS defines some system methods that can be linked (so they are +# found by configure), but either raise a compilation error (because the +# header definition prevents usage - autoconf doesn't use the headers), or +# raise an error if used at runtime. Force these symbols off. +if test "$ac_sys_system" != "iOS" ; then + ac_fn_c_check_func "$LINENO" "getentropy" "ac_cv_func_getentropy" +if test "x$ac_cv_func_getentropy" = xyes +then : + printf "%s\n" "#define HAVE_GETENTROPY 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "getgroups" "ac_cv_func_getgroups" +if test "x$ac_cv_func_getgroups" = xyes +then : + printf "%s\n" "#define HAVE_GETGROUPS 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "system" "ac_cv_func_system" +if test "x$ac_cv_func_system" = xyes +then : + printf "%s\n" "#define HAVE_SYSTEM 1" >>confdefs.h + +fi + +fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CC options needed to detect all undeclared functions" >&5 printf %s "checking for $CC options needed to detect all undeclared functions... " >&6; } if test ${ac_cv_c_undeclared_builtin_options+y} @@ -21643,6 +22012,10 @@ fi done +# On iOS, clock_settime can be linked (so it is found by +# configure), but it raises a runtime error if used because apps can't change +# the clock. Force the symbol off. +if test "$ac_sys_system" != "iOS" ; then for ac_func in clock_settime do : @@ -21653,7 +22026,7 @@ then : else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_settime in -lrt" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_settime in -lrt" >&5 printf %s "checking for clock_settime in -lrt... " >&6; } if test ${ac_cv_lib_rt_clock_settime+y} then : @@ -21691,7 +22064,7 @@ printf "%s\n" "$ac_cv_lib_rt_clock_settime" >&6; } if test "x$ac_cv_lib_rt_clock_settime" = xyes then : - printf "%s\n" "#define HAVE_CLOCK_SETTIME 1" >>confdefs.h + printf "%s\n" "#define HAVE_CLOCK_SETTIME 1" >>confdefs.h fi @@ -21700,6 +22073,7 @@ fi fi done +fi for ac_func in clock_nanosleep @@ -23824,7 +24198,7 @@ printf %s "checking ABIFLAGS... " >&6; } printf "%s\n" "$ABIFLAGS" >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking SOABI" >&5 printf %s "checking SOABI... " >&6; } -SOABI='cpython-'`echo $VERSION | tr -d .`${ABIFLAGS}${PLATFORM_TRIPLET:+-$PLATFORM_TRIPLET} +SOABI='cpython-'`echo $VERSION | tr -d .`${ABIFLAGS}${SOABI_PLATFORM:+-$SOABI_PLATFORM} { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $SOABI" >&5 printf "%s\n" "$SOABI" >&6; } @@ -23833,7 +24207,7 @@ printf "%s\n" "$SOABI" >&6; } if test "$Py_DEBUG" = 'true'; then # Similar to SOABI but remove "d" flag from ABIFLAGS - ALT_SOABI='cpython-'`echo $VERSION | tr -d .``echo $ABIFLAGS | tr -d d`${PLATFORM_TRIPLET:+-$PLATFORM_TRIPLET} + ALT_SOABI='cpython-'`echo $VERSION | tr -d .``echo $ABIFLAGS | tr -d d`${SOABI_PLATFORM:+-$SOABI_PLATFORM} printf "%s\n" "#define ALT_SOABI \"${ALT_SOABI}\"" >>confdefs.h @@ -23848,12 +24222,21 @@ LDVERSION='$(VERSION)$(ABIFLAGS)' { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDVERSION" >&5 printf "%s\n" "$LDVERSION" >&6; } -# On Android and Cygwin the shared libraries must be linked with libpython. +# Configure the flags and dependencies used when compiling shared modules + + +MODULE_DEPS_SHARED='$(MODULE_DEPS_STATIC) $(EXPORTSYMS)' +MODULE_LDFLAGS='' +# On Android and Cygwin the shared libraries must be linked with libpython. if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then - LIBPYTHON="-lpython${VERSION}${ABIFLAGS}" -else - LIBPYTHON='' + MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(LDLIBRARY)" + MODULE_LDFLAGS="\$(BLDLIBRARY)" +fi + +# On iOS the shared libraries must be linked with the Python framework +if test "$ac_sys_system" == "iOS"; then + MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(PYTHONFRAMEWORKDIR)/\$(PYTHONFRAMEWORK)" fi @@ -24849,6 +25232,7 @@ then : printf "%s\n" "#define HAVE_RL_COMPDISP_FUNC_T 1" >>confdefs.h + fi @@ -26584,24 +26968,28 @@ CPPFLAGS=$ac_save_cppflags { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for device files" >&5 printf "%s\n" "$as_me: checking for device files" >&6;} -if test "x$cross_compiling" = xyes; then - if test "${ac_cv_file__dev_ptmx+set}" != set; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptmx" >&5 +if test "$ac_sys_system" = "iOS" ; then + ac_cv_file__dev_ptmx=no + ac_cv_file__dev_ptc=no +else + if test "x$cross_compiling" = xyes; then + if test "${ac_cv_file__dev_ptmx+set}" != set; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptmx" >&5 printf %s "checking for /dev/ptmx... " >&6; } - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not set" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not set" >&5 printf "%s\n" "not set" >&6; } - as_fn_error $? "set ac_cv_file__dev_ptmx to yes/no in your CONFIG_SITE file when cross compiling" "$LINENO" 5 - fi - if test "${ac_cv_file__dev_ptc+set}" != set; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptc" >&5 + as_fn_error $? "set ac_cv_file__dev_ptmx to yes/no in your CONFIG_SITE file when cross compiling" "$LINENO" 5 + fi + if test "${ac_cv_file__dev_ptc+set}" != set; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptc" >&5 printf %s "checking for /dev/ptc... " >&6; } - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not set" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not set" >&5 printf "%s\n" "not set" >&6; } - as_fn_error $? "set ac_cv_file__dev_ptc to yes/no in your CONFIG_SITE file when cross compiling" "$LINENO" 5 + as_fn_error $? "set ac_cv_file__dev_ptc to yes/no in your CONFIG_SITE file when cross compiling" "$LINENO" 5 + fi fi -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptmx" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptmx" >&5 printf %s "checking for /dev/ptmx... " >&6; } if test ${ac_cv_file__dev_ptmx+y} then : @@ -26622,12 +27010,12 @@ then : fi -if test "x$ac_cv_file__dev_ptmx" = xyes; then + if test "x$ac_cv_file__dev_ptmx" = xyes; then printf "%s\n" "#define HAVE_DEV_PTMX 1" >>confdefs.h -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptc" >&5 + fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptc" >&5 printf %s "checking for /dev/ptc... " >&6; } if test ${ac_cv_file__dev_ptc+y} then : @@ -26648,10 +27036,11 @@ then : fi -if test "x$ac_cv_file__dev_ptc" = xyes; then + if test "x$ac_cv_file__dev_ptc" = xyes; then printf "%s\n" "#define HAVE_DEV_PTC 1" >>confdefs.h + fi fi if test $ac_sys_system = Darwin @@ -26671,6 +27060,9 @@ ac_fn_c_check_type "$LINENO" "socklen_t" "ac_cv_type_socklen_t" " if test "x$ac_cv_type_socklen_t" = xyes then : +printf "%s\n" "#define HAVE_SOCKLEN_T 1" >>confdefs.h + + else $as_nop printf "%s\n" "#define socklen_t int" >>confdefs.h @@ -28034,6 +28426,27 @@ case $ac_sys_system in #( ;; #( Darwin) : ;; #( + iOS) : + + + + py_cv_module__curses=n/a + py_cv_module__curses_panel=n/a + py_cv_module__gdbm=n/a + py_cv_module__multiprocessing=n/a + py_cv_module__posixshmem=n/a + py_cv_module__posixsubprocess=n/a + py_cv_module__scproxy=n/a + py_cv_module__tkinter=n/a + py_cv_module_grp=n/a + py_cv_module_nis=n/a + py_cv_module_readline=n/a + py_cv_module_pwd=n/a + py_cv_module_spwd=n/a + py_cv_module_syslog=n/a + py_cv_module_=n/a + + ;; #( CYGWIN*) : @@ -28901,7 +29314,8 @@ then : if true then : - if test "$ac_cv_func_getgrgid" = yes -o "$ac_cv_func_getgrgid_r" = yes + if test "$ac_cv_func_getgrent" = "yes" && + { test "$ac_cv_func_getgrgid" = "yes" || test "$ac_cv_func_getgrgid_r" = "yes"; } then : py_cv_module_grp=yes else $as_nop @@ -30430,6 +30844,44 @@ fi printf "%s\n" "$py_cv_module__testmultiphase" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testexternalinspection" >&5 +printf %s "checking for stdlib extension module _testexternalinspection... " >&6; } + if test "$py_cv_module__testexternalinspection" != "n/a" +then : + + if test "$TEST_MODULES" = yes +then : + if true +then : + py_cv_module__testexternalinspection=yes +else $as_nop + py_cv_module__testexternalinspection=missing +fi +else $as_nop + py_cv_module__testexternalinspection=disabled +fi + +fi + as_fn_append MODULE_BLOCK "MODULE__TESTEXTERNALINSPECTION_STATE=$py_cv_module__testexternalinspection$as_nl" + if test "x$py_cv_module__testexternalinspection" = xyes +then : + + + + +fi + if test "$py_cv_module__testexternalinspection" = yes; then + MODULE__TESTEXTERNALINSPECTION_TRUE= + MODULE__TESTEXTERNALINSPECTION_FALSE='#' +else + MODULE__TESTEXTERNALINSPECTION_TRUE='#' + MODULE__TESTEXTERNALINSPECTION_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__testexternalinspection" >&5 +printf "%s\n" "$py_cv_module__testexternalinspection" >&6; } + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module xxsubtype" >&5 printf %s "checking for stdlib extension module xxsubtype... " >&6; } if test "$py_cv_module_xxsubtype" != "n/a" @@ -31042,6 +31494,10 @@ if test -z "${MODULE__TESTMULTIPHASE_TRUE}" && test -z "${MODULE__TESTMULTIPHASE as_fn_error $? "conditional \"MODULE__TESTMULTIPHASE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__TESTEXTERNALINSPECTION_TRUE}" && test -z "${MODULE__TESTEXTERNALINSPECTION_FALSE}"; then + as_fn_error $? "conditional \"MODULE__TESTEXTERNALINSPECTION\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE_XXSUBTYPE_TRUE}" && test -z "${MODULE_XXSUBTYPE_FALSE}"; then as_fn_error $? "conditional \"MODULE_XXSUBTYPE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 @@ -31644,6 +32100,7 @@ do "Mac/PythonLauncher/Makefile") CONFIG_FILES="$CONFIG_FILES Mac/PythonLauncher/Makefile" ;; "Mac/Resources/framework/Info.plist") CONFIG_FILES="$CONFIG_FILES Mac/Resources/framework/Info.plist" ;; "Mac/Resources/app/Info.plist") CONFIG_FILES="$CONFIG_FILES Mac/Resources/app/Info.plist" ;; + "iOS/Resources/Info.plist") CONFIG_FILES="$CONFIG_FILES iOS/Resources/Info.plist" ;; "Makefile.pre") CONFIG_FILES="$CONFIG_FILES Makefile.pre" ;; "Misc/python.pc") CONFIG_FILES="$CONFIG_FILES Misc/python.pc" ;; "Misc/python-embed.pc") CONFIG_FILES="$CONFIG_FILES Misc/python-embed.pc" ;; diff --git a/configure.ac b/configure.ac index 9587e6d63499aac..a5abb06a8294139 100644 --- a/configure.ac +++ b/configure.ac @@ -307,6 +307,77 @@ if test "$with_pkg_config" = yes -a -z "$PKG_CONFIG"; then AC_MSG_ERROR([pkg-config is required])] fi +# Set name for machine-dependent library files +AC_ARG_VAR([MACHDEP], [name for machine-dependent library files]) +AC_MSG_CHECKING([MACHDEP]) +if test -z "$MACHDEP" +then + # avoid using uname for cross builds + if test "$cross_compiling" = yes; then + # ac_sys_system and ac_sys_release are used for setting + # a lot of different things including 'define_xopen_source' + # in the case statement below. + case "$host" in + *-*-linux-android*) + ac_sys_system=Linux-android + ;; + *-*-linux*) + ac_sys_system=Linux + ;; + *-*-cygwin*) + ac_sys_system=Cygwin + ;; + *-apple-ios*) + ac_sys_system=iOS + ;; + *-*-vxworks*) + ac_sys_system=VxWorks + ;; + *-*-emscripten) + ac_sys_system=Emscripten + ;; + *-*-wasi) + ac_sys_system=WASI + ;; + *) + # for now, limit cross builds to known configurations + MACHDEP="unknown" + AC_MSG_ERROR([cross build not supported for $host]) + esac + ac_sys_release= + else + ac_sys_system=`uname -s` + if test "$ac_sys_system" = "AIX" \ + -o "$ac_sys_system" = "UnixWare" -o "$ac_sys_system" = "OpenUNIX"; then + ac_sys_release=`uname -v` + else + ac_sys_release=`uname -r` + fi + fi + ac_md_system=`echo $ac_sys_system | + tr -d '[/ ]' | tr '[[A-Z]]' '[[a-z]]'` + ac_md_release=`echo $ac_sys_release | + tr -d '[/ ]' | sed 's/^[[A-Z]]\.//' | sed 's/\..*//'` + MACHDEP="$ac_md_system$ac_md_release" + + case $MACHDEP in + aix*) MACHDEP="aix";; + linux*) MACHDEP="linux";; + cygwin*) MACHDEP="cygwin";; + darwin*) MACHDEP="darwin";; + '') MACHDEP="unknown";; + esac + + if test "$ac_sys_system" = "SunOS"; then + # For Solaris, there isn't an OS version specific macro defined + # in most compilers, so we define one here. + SUNOS_VERSION=`echo $ac_sys_release | sed -e 's!\.\([0-9]\)$!.0\1!g' | tr -d '.'` + AC_DEFINE_UNQUOTED([Py_SUNOS_VERSION], [$SUNOS_VERSION], + [The version of SunOS/Solaris as reported by `uname -r' without the dot.]) + fi +fi +AC_MSG_RESULT(["$MACHDEP"]) + AC_MSG_CHECKING([for --enable-universalsdk]) AC_ARG_ENABLE([universalsdk], AS_HELP_STRING([--enable-universalsdk@<:@=SDKDIR@:>@], @@ -416,19 +487,31 @@ AC_ARG_ENABLE([framework], [ case $enableval in yes) + if test "$ac_sys_system" = "iOS"; then + AC_MSG_ERROR([iOS builds must provide an explicit path for --enable-framework]) + fi + enableval=/Library/Frameworks esac case $enableval in no) + if test "$ac_sys_system" = "iOS"; then + AC_MSG_ERROR([iOS builds must use --enable-framework=<install path>]) + fi + PYTHONFRAMEWORK= PYTHONFRAMEWORKDIR=no-framework PYTHONFRAMEWORKPREFIX= PYTHONFRAMEWORKINSTALLDIR= + PYTHONFRAMEWORKINSTALLNAMEPREFIX= + RESSRCDIR= FRAMEWORKINSTALLFIRST= FRAMEWORKINSTALLLAST= FRAMEWORKALTINSTALLFIRST= FRAMEWORKALTINSTALLLAST= FRAMEWORKPYTHONW= + INSTALLTARGETS="commoninstall bininstall maninstall" + if test "x${prefix}" = "xNONE"; then FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" else @@ -439,73 +522,105 @@ AC_ARG_ENABLE([framework], *) PYTHONFRAMEWORKPREFIX="${enableval}" PYTHONFRAMEWORKINSTALLDIR=$PYTHONFRAMEWORKPREFIX/$PYTHONFRAMEWORKDIR - FRAMEWORKINSTALLFIRST="frameworkinstallstructure" - FRAMEWORKALTINSTALLFIRST="frameworkinstallstructure " - FRAMEWORKINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools" - FRAMEWORKALTINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkaltinstallunixtools" - FRAMEWORKPYTHONW="frameworkpythonw" - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - - if test "x${prefix}" = "xNONE" ; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi - - case "${enableval}" in - /System*) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - if test "${prefix}" = "NONE" ; then - # See below - FRAMEWORKUNIXTOOLSPREFIX="/usr" - fi - ;; + case $ac_sys_system in #( + Darwin) : + FRAMEWORKINSTALLFIRST="frameworkinstallversionedstructure" + FRAMEWORKALTINSTALLFIRST="frameworkinstallversionedstructure " + FRAMEWORKINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools" + FRAMEWORKALTINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkaltinstallunixtools" + FRAMEWORKPYTHONW="frameworkpythonw" + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + INSTALLTARGETS="commoninstall bininstall maninstall" + + if test "x${prefix}" = "xNONE" ; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi - /Library*) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - ;; + case "${enableval}" in + /System*) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + if test "${prefix}" = "NONE" ; then + # See below + FRAMEWORKUNIXTOOLSPREFIX="/usr" + fi + ;; + + /Library*) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + ;; + + */Library/Frameworks) + MDIR="`dirname "${enableval}"`" + MDIR="`dirname "${MDIR}"`" + FRAMEWORKINSTALLAPPSPREFIX="${MDIR}/Applications" + + if test "${prefix}" = "NONE"; then + # User hasn't specified the + # --prefix option, but wants to install + # the framework in a non-default location, + # ensure that the compatibility links get + # installed relative to that prefix as well + # instead of in /usr/local. + FRAMEWORKUNIXTOOLSPREFIX="${MDIR}" + fi + ;; - */Library/Frameworks) - MDIR="`dirname "${enableval}"`" - MDIR="`dirname "${MDIR}"`" - FRAMEWORKINSTALLAPPSPREFIX="${MDIR}/Applications" - - if test "${prefix}" = "NONE"; then - # User hasn't specified the - # --prefix option, but wants to install - # the framework in a non-default location, - # ensure that the compatibility links get - # installed relative to that prefix as well - # instead of in /usr/local. - FRAMEWORKUNIXTOOLSPREFIX="${MDIR}" - fi - ;; + *) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + ;; + esac - *) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - ;; + prefix=$PYTHONFRAMEWORKINSTALLDIR/Versions/$VERSION + PYTHONFRAMEWORKINSTALLNAMEPREFIX=${prefix} + RESSRCDIR=Mac/Resources/framework + + # Add files for Mac specific code to the list of output + # files: + AC_CONFIG_FILES([Mac/Makefile]) + AC_CONFIG_FILES([Mac/PythonLauncher/Makefile]) + AC_CONFIG_FILES([Mac/Resources/framework/Info.plist]) + AC_CONFIG_FILES([Mac/Resources/app/Info.plist]) + ;; + iOS) : + FRAMEWORKINSTALLFIRST="frameworkinstallunversionedstructure" + FRAMEWORKALTINSTALLFIRST="frameworkinstallunversionedstructure " + FRAMEWORKINSTALLLAST="frameworkinstallmobileheaders" + FRAMEWORKALTINSTALLLAST="frameworkinstallmobileheaders" + FRAMEWORKPYTHONW= + INSTALLTARGETS="libinstall inclinstall sharedinstall" + + prefix=$PYTHONFRAMEWORKPREFIX + PYTHONFRAMEWORKINSTALLNAMEPREFIX="@rpath/$PYTHONFRAMEWORKDIR" + RESSRCDIR=iOS/Resources + + AC_CONFIG_FILES([iOS/Resources/Info.plist]) + ;; + *) + AC_MSG_ERROR([Unknown platform for framework build]) + ;; + esac esac - - prefix=$PYTHONFRAMEWORKINSTALLDIR/Versions/$VERSION - - # Add files for Mac specific code to the list of output - # files: - AC_CONFIG_FILES([Mac/Makefile]) - AC_CONFIG_FILES([Mac/PythonLauncher/Makefile]) - AC_CONFIG_FILES([Mac/Resources/framework/Info.plist]) - AC_CONFIG_FILES([Mac/Resources/app/Info.plist]) - esac ],[ + if test "$ac_sys_system" = "iOS"; then + AC_MSG_ERROR([iOS builds must use --enable-framework=<install path>]) + fi + PYTHONFRAMEWORK= PYTHONFRAMEWORKDIR=no-framework PYTHONFRAMEWORKPREFIX= PYTHONFRAMEWORKINSTALLDIR= + PYTHONFRAMEWORKINSTALLNAMEPREFIX= + RESSRCDIR= FRAMEWORKINSTALLFIRST= FRAMEWORKINSTALLLAST= FRAMEWORKALTINSTALLFIRST= FRAMEWORKALTINSTALLLAST= FRAMEWORKPYTHONW= + INSTALLTARGETS="commoninstall bininstall maninstall" if test "x${prefix}" = "xNONE" ; then FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" else @@ -519,6 +634,8 @@ AC_SUBST([PYTHONFRAMEWORKIDENTIFIER]) AC_SUBST([PYTHONFRAMEWORKDIR]) AC_SUBST([PYTHONFRAMEWORKPREFIX]) AC_SUBST([PYTHONFRAMEWORKINSTALLDIR]) +AC_SUBST([PYTHONFRAMEWORKINSTALLNAMEPREFIX]) +AC_SUBST([RESSRCDIR]) AC_SUBST([FRAMEWORKINSTALLFIRST]) AC_SUBST([FRAMEWORKINSTALLLAST]) AC_SUBST([FRAMEWORKALTINSTALLFIRST]) @@ -526,105 +643,56 @@ AC_SUBST([FRAMEWORKALTINSTALLLAST]) AC_SUBST([FRAMEWORKPYTHONW]) AC_SUBST([FRAMEWORKUNIXTOOLSPREFIX]) AC_SUBST([FRAMEWORKINSTALLAPPSPREFIX]) +AC_SUBST([INSTALLTARGETS]) AC_DEFINE_UNQUOTED([_PYTHONFRAMEWORK], ["${PYTHONFRAMEWORK}"], [framework name]) -# Set name for machine-dependent library files -AC_ARG_VAR([MACHDEP], [name for machine-dependent library files]) -AC_MSG_CHECKING([MACHDEP]) -if test -z "$MACHDEP" -then - # avoid using uname for cross builds - if test "$cross_compiling" = yes; then - # ac_sys_system and ac_sys_release are used for setting - # a lot of different things including 'define_xopen_source' - # in the case statement below. - case "$host" in - *-*-linux-android*) - ac_sys_system=Linux-android - ;; - *-*-linux*) - ac_sys_system=Linux - ;; - *-*-cygwin*) - ac_sys_system=Cygwin - ;; - *-*-vxworks*) - ac_sys_system=VxWorks - ;; - *-*-emscripten) - ac_sys_system=Emscripten - ;; - *-*-wasi) - ac_sys_system=WASI - ;; - *) - # for now, limit cross builds to known configurations - MACHDEP="unknown" - AC_MSG_ERROR([cross build not supported for $host]) - esac - ac_sys_release= - else - ac_sys_system=`uname -s` - if test "$ac_sys_system" = "AIX" \ - -o "$ac_sys_system" = "UnixWare" -o "$ac_sys_system" = "OpenUNIX"; then - ac_sys_release=`uname -v` - else - ac_sys_release=`uname -r` - fi - fi - ac_md_system=`echo $ac_sys_system | - tr -d '[/ ]' | tr '[[A-Z]]' '[[a-z]]'` - ac_md_release=`echo $ac_sys_release | - tr -d '[/ ]' | sed 's/^[[A-Z]]\.//' | sed 's/\..*//'` - MACHDEP="$ac_md_system$ac_md_release" - - case $MACHDEP in - aix*) MACHDEP="aix";; - linux*) MACHDEP="linux";; - cygwin*) MACHDEP="cygwin";; - darwin*) MACHDEP="darwin";; - '') MACHDEP="unknown";; - esac - - if test "$ac_sys_system" = "SunOS"; then - # For Solaris, there isn't an OS version specific macro defined - # in most compilers, so we define one here. - SUNOS_VERSION=`echo $ac_sys_release | sed -e 's!\.\([0-9]\)$!.0\1!g' | tr -d '.'` - AC_DEFINE_UNQUOTED([Py_SUNOS_VERSION], [$SUNOS_VERSION], - [The version of SunOS/Solaris as reported by `uname -r' without the dot.]) - fi -fi -AC_MSG_RESULT(["$MACHDEP"]) - AC_SUBST([_PYTHON_HOST_PLATFORM]) if test "$cross_compiling" = yes; then case "$host" in *-*-linux*) case "$host_cpu" in arm*) - _host_cpu=arm + _host_ident=arm ;; *) - _host_cpu=$host_cpu + _host_ident=$host_cpu esac ;; *-*-cygwin*) - _host_cpu= + _host_ident= + ;; + *-apple-ios*) + _host_os=`echo $host | cut -d '-' -f3` + _host_device=`echo $host | cut -d '-' -f4` + _host_device=${_host_device:=os} + + dnl IOS_DEPLOYMENT_TARGET is the minimum supported iOS version + IOS_DEPLOYMENT_TARGET=${_host_os:3} + IOS_DEPLOYMENT_TARGET=${IOS_DEPLOYMENT_TARGET:=12.0} + + case "$host_cpu" in + aarch64) + _host_ident=${IOS_DEPLOYMENT_TARGET}-arm64-iphone${_host_device} + ;; + *) + _host_ident=${IOS_DEPLOYMENT_TARGET}-$host_cpu-iphone${_host_device} + ;; + esac ;; *-*-vxworks*) - _host_cpu=$host_cpu + _host_ident=$host_cpu ;; wasm32-*-* | wasm64-*-*) - _host_cpu=$host_cpu + _host_ident=$host_cpu ;; *) # for now, limit cross builds to known configurations MACHDEP="unknown" AC_MSG_ERROR([cross build not supported for $host]) esac - _PYTHON_HOST_PLATFORM="$MACHDEP${_host_cpu:+-$_host_cpu}" + _PYTHON_HOST_PLATFORM="$MACHDEP${_host_ident:+-$_host_ident}" fi # Some systems cannot stand _XOPEN_SOURCE being defined at all; they @@ -690,6 +758,9 @@ case $ac_sys_system/$ac_sys_release in define_xopen_source=no;; Darwin/@<:@[12]@:>@@<:@0-9@:>@.*) define_xopen_source=no;; + # On iOS, defining _POSIX_C_SOURCE also disables platform specific features. + iOS/*) + define_xopen_source=no;; # On QNX 6.3.2, defining _XOPEN_SOURCE prevents netdb.h from # defining NI_NUMERICHOST. QNX/6.3.2) @@ -780,6 +851,15 @@ AS_CASE([$host], ], ) +dnl Add the compiler flag for the iOS minimum supported OS version. +AS_CASE([$ac_sys_system], + [iOS], [ + AS_VAR_APPEND([CFLAGS], [" -mios-version-min=${IOS_DEPLOYMENT_TARGET}"]) + AS_VAR_APPEND([LDFLAGS], [" -mios-version-min=${IOS_DEPLOYMENT_TARGET}"]) + AC_SUBST([IOS_DEPLOYMENT_TARGET]) + ], +) + if test "$ac_sys_system" = "Darwin" then dnl look for SDKROOT @@ -935,14 +1015,22 @@ else fi rm -f conftest.out +dnl On some platforms, using a true "triplet" for MULTIARCH would be redundant. +dnl For example, `arm64-apple-darwin` is redundant, because there isn't a +dnl non-Apple Darwin. Including the CPU architecture can also be potentially +dnl redundant - on macOS, for example, it's possible to do a single compile +dnl pass that includes multiple architectures, so it would be misleading for +dnl MULTIARCH (and thus the sysconfigdata module name) to include a single CPU +dnl architecture. PLATFORM_TRIPLET will be a pair or single value for these +dnl platforms. AC_MSG_CHECKING([for multiarch]) AS_CASE([$ac_sys_system], [Darwin*], [MULTIARCH=""], + [iOS], [MULTIARCH=""], [FreeBSD*], [MULTIARCH=""], [MULTIARCH=$($CC --print-multiarch 2>/dev/null)] ) AC_SUBST([MULTIARCH]) -AC_MSG_RESULT([$MULTIARCH]) if test x$PLATFORM_TRIPLET != x && test x$MULTIARCH != x; then if test x$PLATFORM_TRIPLET != x$MULTIARCH; then @@ -952,6 +1040,17 @@ elif test x$PLATFORM_TRIPLET != x && test x$MULTIARCH = x; then MULTIARCH=$PLATFORM_TRIPLET fi AC_SUBST([PLATFORM_TRIPLET]) +AC_MSG_RESULT([$MULTIARCH]) + +dnl Even if we *do* include the CPU architecture in the MULTIARCH value, some +dnl platforms don't need the CPU architecture in the SOABI tag. These platforms +dnl will have multiple sysconfig modules (one for each CPU architecture), but +dnl use a single "fat" binary at runtime. SOABI_PLATFORM is the component of +dnl the PLATFORM_TRIPLET that will be used in binary module extensions. +AS_CASE([$ac_sys_system], + [iOS], [SOABI_PLATFORM=`echo "$PLATFORM_TRIPLET" | cut -d '-' -f2`], + [SOABI_PLATFORM=$PLATFORM_TRIPLET] +) if test x$MULTIARCH != x; then MULTIARCH_CPPFLAGS="-DMULTIARCH=\\\"$MULTIARCH\\\"" @@ -973,15 +1072,16 @@ AS_CASE([$host/$ac_cv_cc_name], [aarch64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux ARM64, glibc, gcc+clang [aarch64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], [powerpc64le-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux on PPC64 little endian, glibc, gcc + [wasm32-unknown-wasi/clang], [PY_SUPPORT_TIER=2], dnl WebAssembly System Interface, clang [x86_64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], dnl Linux on AMD64, any vendor, glibc, clang [aarch64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=3], dnl Windows ARM64, MSVC [armv7l-*-linux-gnueabihf/gcc], [PY_SUPPORT_TIER=3], dnl ARMv7 LE with hardware floats, any vendor, glibc, gcc [powerpc64le-*-linux-gnu/clang], [PY_SUPPORT_TIER=3], dnl Linux on PPC64 little endian, glibc, clang [s390x-*-linux-gnu/gcc], [PY_SUPPORT_TIER=3], dnl Linux on 64bit s390x (big endian), glibc, gcc - [wasm32-unknown-emscripten/clang], [PY_SUPPORT_TIER=3], dnl WebAssembly Emscripten - [wasm32-unknown-wasi/clang], [PY_SUPPORT_TIER=3], dnl WebAssembly System Interface [x86_64-*-freebsd*/clang], [PY_SUPPORT_TIER=3], dnl FreeBSD on AMD64 + [aarch64-apple-ios*-simulator/clang], [PY_SUPPORT_TIER=3], dnl iOS Simulator on arm64 + [aarch64-apple-ios*/clang], [PY_SUPPORT_TIER=3], dnl iOS on ARM64 [PY_SUPPORT_TIER=0] ) @@ -1295,17 +1395,25 @@ fi AC_MSG_CHECKING([LDLIBRARY]) -# MacOSX framework builds need more magic. LDLIBRARY is the dynamic +# Apple framework builds need more magic. LDLIBRARY is the dynamic # library that we build, but we do not want to link against it (we # will find it with a -framework option). For this reason there is an # extra variable BLDLIBRARY against which Python and the extension # modules are linked, BLDLIBRARY. This is normally the same as -# LDLIBRARY, but empty for MacOSX framework builds. +# LDLIBRARY, but empty for MacOSX framework builds. iOS does the same, +# but uses a non-versioned framework layout. if test "$enable_framework" then - LDLIBRARY='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' - RUNSHARED=DYLD_FRAMEWORK_PATH=`pwd`${DYLD_FRAMEWORK_PATH:+:${DYLD_FRAMEWORK_PATH}} + case $ac_sys_system in + Darwin) + LDLIBRARY='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)';; + iOS) + LDLIBRARY='$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK)';; + *) + AC_MSG_ERROR([Unknown platform for framework build]);; + esac BLDLIBRARY='' + RUNSHARED=DYLD_FRAMEWORK_PATH=`pwd`${DYLD_FRAMEWORK_PATH:+:${DYLD_FRAMEWORK_PATH}} else BLDLIBRARY='$(LDLIBRARY)' fi @@ -1317,66 +1425,72 @@ if test $enable_shared = "yes"; then [Defined if Python is built as a shared library.]) case $ac_sys_system in CYGWIN*) - LDLIBRARY='libpython$(LDVERSION).dll.a' - DLLLIBRARY='libpython$(LDVERSION).dll' - ;; + LDLIBRARY='libpython$(LDVERSION).dll.a' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + DLLLIBRARY='libpython$(LDVERSION).dll' + ;; SunOS*) - LDLIBRARY='libpython$(LDVERSION).so' - BLDLIBRARY='-Wl,-R,$(LIBDIR) -L. -lpython$(LDVERSION)' - RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - INSTSONAME="$LDLIBRARY".$SOVERSION - if test "$with_pydebug" != yes - then - PY3LIBRARY=libpython3.so - fi - ;; + LDLIBRARY='libpython$(LDVERSION).so' + BLDLIBRARY='-Wl,-R,$(LIBDIR) -L. -lpython$(LDVERSION)' + RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + INSTSONAME="$LDLIBRARY".$SOVERSION + if test "$with_pydebug" != yes + then + PY3LIBRARY=libpython3.so + fi + ;; Linux*|GNU*|NetBSD*|FreeBSD*|DragonFly*|OpenBSD*|VxWorks*) - LDLIBRARY='libpython$(LDVERSION).so' - BLDLIBRARY='-L. -lpython$(LDVERSION)' - RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - INSTSONAME="$LDLIBRARY".$SOVERSION - if test "$with_pydebug" != yes - then - PY3LIBRARY=libpython3.so - fi - ;; + LDLIBRARY='libpython$(LDVERSION).so' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + INSTSONAME="$LDLIBRARY".$SOVERSION + if test "$with_pydebug" != yes + then + PY3LIBRARY=libpython3.so + fi + ;; hp*|HP*) - case `uname -m` in - ia64) - LDLIBRARY='libpython$(LDVERSION).so' - ;; - *) - LDLIBRARY='libpython$(LDVERSION).sl' - ;; - esac - BLDLIBRARY='-Wl,+b,$(LIBDIR) -L. -lpython$(LDVERSION)' - RUNSHARED=SHLIB_PATH=`pwd`${SHLIB_PATH:+:${SHLIB_PATH}} - ;; + case `uname -m` in + ia64) + LDLIBRARY='libpython$(LDVERSION).so' + ;; + *) + LDLIBRARY='libpython$(LDVERSION).sl' + ;; + esac + BLDLIBRARY='-Wl,+b,$(LIBDIR) -L. -lpython$(LDVERSION)' + RUNSHARED=SHLIB_PATH=`pwd`${SHLIB_PATH:+:${SHLIB_PATH}} + ;; Darwin*) - LDLIBRARY='libpython$(LDVERSION).dylib' - BLDLIBRARY='-L. -lpython$(LDVERSION)' - RUNSHARED=DYLD_LIBRARY_PATH=`pwd`${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}} - ;; + LDLIBRARY='libpython$(LDVERSION).dylib' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + RUNSHARED=DYLD_LIBRARY_PATH=`pwd`${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}} + ;; + iOS) + LDLIBRARY='libpython$(LDVERSION).dylib' + ;; AIX*) - LDLIBRARY='libpython$(LDVERSION).so' - RUNSHARED=LIBPATH=`pwd`${LIBPATH:+:${LIBPATH}} - ;; + LDLIBRARY='libpython$(LDVERSION).so' + RUNSHARED=LIBPATH=`pwd`${LIBPATH:+:${LIBPATH}} + ;; esac else # shared is disabled PY_ENABLE_SHARED=0 case $ac_sys_system in CYGWIN*) - BLDLIBRARY='$(LIBRARY)' - LDLIBRARY='libpython$(LDVERSION).dll.a' - ;; + BLDLIBRARY='$(LIBRARY)' + LDLIBRARY='libpython$(LDVERSION).dll.a' + ;; esac fi +AC_MSG_RESULT([$LDLIBRARY]) if test "$cross_compiling" = yes; then - RUNSHARED= + RUNSHARED= fi +AC_MSG_CHECKING([HOSTRUNNER]) AC_ARG_VAR([HOSTRUNNER], [Program to run CPython for the host platform]) if test -z "$HOSTRUNNER" then @@ -1422,7 +1536,6 @@ then ) fi AC_SUBST([HOSTRUNNER]) -AC_MSG_CHECKING([HOSTRUNNER]) AC_MSG_RESULT([$HOSTRUNNER]) if test -n "$HOSTRUNNER"; then @@ -1430,8 +1543,6 @@ if test -n "$HOSTRUNNER"; then PYTHON_FOR_BUILD="_PYTHON_HOSTRUNNER='$HOSTRUNNER' $PYTHON_FOR_BUILD" fi -AC_MSG_RESULT([$LDLIBRARY]) - # LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], [Emscripten/browser*], [LIBRARY_DEPS='$(PY3LIBRARY) $(WASM_STDLIB) python.html python.worker.js'], @@ -1579,6 +1690,28 @@ else AC_MSG_RESULT([no]) fi +# Check for --enable-experimental-jit: +AC_MSG_CHECKING([for --enable-experimental-jit]) +AC_ARG_ENABLE([experimental-jit], + [AS_HELP_STRING([--enable-experimental-jit], + [build the experimental just-in-time compiler (default is no)])], + [], + [enable_experimental_jit=no]) +AS_VAR_IF([enable_experimental_jit], + [no], + [], + [AS_VAR_APPEND([CFLAGS_NODIST], [" -D_Py_JIT"]) + AS_VAR_SET([REGEN_JIT_COMMAND], + ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"]) + AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) + AS_VAR_IF([Py_DEBUG], + [true], + [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])], + [])]) +AC_SUBST([REGEN_JIT_COMMAND]) +AC_SUBST([JIT_STENCILS_H]) +AC_MSG_RESULT([$enable_experimental_jit]) + # Enable optimization flags AC_SUBST([DEF_MAKE_ALL_RULE]) AC_SUBST([DEF_MAKE_RULE]) @@ -2762,15 +2895,11 @@ AC_CHECK_HEADERS( #endif ]) -# checks for typedefs -AC_CACHE_CHECK([for clock_t in time.h], [ac_cv_clock_t_time_h], [ - AC_EGREP_HEADER([clock_t], [time.h], [ac_cv_clock_t_time_h=yes], [ac_cv_clock_t_time_h=no]) -]) -dnl checks for "no" -AS_VAR_IF([ac_cv_clock_t_time_h], [no], [ - AC_DEFINE([clock_t], [long], - [Define to 'long' if <time.h> doesn't define.]) -]) +# Check for clock_t in time.h. +AC_CHECK_TYPES([clock_t], [], + [AC_DEFINE([clock_t], [long], + [Define to 'long' if <time.h> does not define clock_t.])], + [@%:@include <time.h>]) AC_CACHE_CHECK([for makedev], [ac_cv_func_makedev], [ AC_LINK_IFELSE([AC_LANG_PROGRAM([[ @@ -2845,12 +2974,10 @@ AC_DEFINE_UNQUOTED([RETSIGTYPE],[void],[assume C89 semantics that RETSIGTYPE is AC_TYPE_SIZE_T AC_TYPE_UID_T -AC_CHECK_TYPE([ssize_t], - AC_DEFINE([HAVE_SSIZE_T], [1], - [Define if your compiler provides ssize_t]), [], []) -AC_CHECK_TYPE([__uint128_t], - AC_DEFINE([HAVE_GCC_UINT128_T], [1], - [Define if your compiler provides __uint128_t]), [], []) +AC_CHECK_TYPES([ssize_t]) +AC_CHECK_TYPES([__uint128_t], + [AC_DEFINE([HAVE_GCC_UINT128_T], [1], + [Define if your compiler provides __uint128_t])]) # Sizes and alignments of various common basic types # ANSI C requires sizeof(char) == 1, so no need to check it @@ -3105,6 +3232,7 @@ if test -z "$SHLIB_SUFFIX"; then esac ;; CYGWIN*) SHLIB_SUFFIX=.dll;; + iOS) SHLIB_SUFFIX=.dylib;; *) SHLIB_SUFFIX=.so;; esac fi @@ -3185,6 +3313,11 @@ then BLDSHARED="$LDSHARED" fi ;; + iOS/*) + LDSHARED='$(CC) -dynamiclib -F . -framework Python' + LDCXXSHARED='$(CXX) -dynamiclib -F . -framework Python' + BLDSHARED="$LDSHARED" + ;; Emscripten|WASI) LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; @@ -3269,7 +3402,6 @@ then then CCSHARED="-fPIC"; else CCSHARED="+z"; fi;; - Linux-android*) ;; Linux*|GNU*) CCSHARED="-fPIC";; Emscripten*|WASI*) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ @@ -3305,30 +3437,34 @@ then Linux-android*) LINKFORSHARED="-pie -Xlinker -export-dynamic";; Linux*|GNU*) LINKFORSHARED="-Xlinker -export-dynamic";; # -u libsys_s pulls in all symbols in libsys - Darwin/*) + Darwin/*|iOS/*) LINKFORSHARED="$extra_undefs -framework CoreFoundation" # Issue #18075: the default maximum stack size (8MBytes) is too # small for the default recursion limit. Increase the stack size # to ensure that tests don't crash - stack_size="1000000" # 16 MB - if test "$with_ubsan" = "yes" - then - # Undefined behavior sanitizer requires an even deeper stack - stack_size="4000000" # 64 MB - fi + stack_size="1000000" # 16 MB + if test "$with_ubsan" = "yes" + then + # Undefined behavior sanitizer requires an even deeper stack + stack_size="4000000" # 64 MB + fi - LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" + AC_DEFINE_UNQUOTED([THREAD_STACK_SIZE], + [0x$stack_size], + [Custom thread stack size depending on chosen sanitizer runtimes.]) - AC_DEFINE_UNQUOTED([THREAD_STACK_SIZE], - [0x$stack_size], - [Custom thread stack size depending on chosen sanitizer runtimes.]) + if test $ac_sys_system = "Darwin"; then + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" - if test "$enable_framework" - then - LINKFORSHARED="$LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + if test "$enable_framework"; then + LINKFORSHARED="$LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + fi + LINKFORSHARED="$LINKFORSHARED" + elif test $ac_sys_system = "iOS"; then + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK)' fi - LINKFORSHARED="$LINKFORSHARED";; + ;; OpenUNIX*|UnixWare*) LINKFORSHARED="-Wl,-Bexport";; SCO_SV*) LINKFORSHARED="-Wl,-Bexport";; ReliantUNIX*) LINKFORSHARED="-W1 -Blargedynsym";; @@ -3702,6 +3838,9 @@ AS_VAR_IF([have_libffi], [yes], [ dnl when do we need USING_APPLE_OS_LIBFFI? ctypes_malloc_closure=yes ], + [iOS], [ + ctypes_malloc_closure=yes + ], [sunos5], [AS_VAR_APPEND([LIBFFI_LIBS], [" -mimpure-text"])] ) AS_VAR_IF([ctypes_malloc_closure], [yes], [ @@ -4188,13 +4327,9 @@ else # define _POSIX_THREADS in unistd.h. Some apparently don't # (e.g. gnu pth with pthread emulation) AC_MSG_CHECKING([for _POSIX_THREADS in unistd.h]) - AC_EGREP_CPP([yes], - [ -#include <unistd.h> -#ifdef _POSIX_THREADS -yes -#endif - ], unistd_defines_pthreads=yes, unistd_defines_pthreads=no) + AX_CHECK_DEFINE([unistd.h], [_POSIX_THREADS], + [unistd_defines_pthreads=yes], + [unistd_defines_pthreads=no]) AC_MSG_RESULT([$unistd_defines_pthreads]) AC_DEFINE([_REENTRANT]) @@ -4374,34 +4509,21 @@ if test "$ipv6" = yes -a "$cross_compiling" = no; then case $i in inria) dnl http://www.kame.net/ - AC_EGREP_CPP([yes], [ -#include <netinet/in.h> -#ifdef IPV6_INRIA_VERSION -yes -@%:@endif], - [ipv6type=$i]) + AX_CHECK_DEFINE([netinet/in.h], [IPV6_INRIA_VERSION], [ipv6type=$i]) ;; kame) dnl http://www.kame.net/ - AC_EGREP_CPP([yes], [ -#include <netinet/in.h> -#ifdef __KAME__ -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=inet6 - ipv6libdir=/usr/local/v6/lib - ipv6trylibc=yes]) + AX_CHECK_DEFINE([netinet/in.h], [__KAME__], + [ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib + ipv6trylibc=yes]) ;; linux-glibc) - dnl http://www.v6.linux.or.jp/ - AC_EGREP_CPP([yes], [ -#include <features.h> -#if defined(__GLIBC__) && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 1) || (__GLIBC__ > 2)) -yes -@%:@endif], - [ipv6type=$i; - ipv6trylibc=yes]) + dnl Advanced IPv6 support was added to glibc 2.1 in 1999. + AX_CHECK_DEFINE([features.h], [__GLIBC__], + [ipv6type=$i + ipv6trylibc=yes]) ;; linux-inet6) dnl http://www.v6.linux.or.jp/ @@ -4421,35 +4543,23 @@ yes fi ;; toshiba) - AC_EGREP_CPP([yes], [ -#include <sys/param.h> -#ifdef _TOSHIBA_INET6 -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib]) + AX_CHECK_DEFINE([sys/param.h], [_TOSHIBA_INET6], + [ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib]) ;; v6d) - AC_EGREP_CPP([yes], [ -#include </usr/local/v6/include/sys/v6config.h> -#ifdef __V6D__ -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=v6; - ipv6libdir=/usr/local/v6/lib; - BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS"]) + AX_CHECK_DEFINE([/usr/local/v6/include/sys/v6config.h], [__V6D__], + [ipv6type=$i + ipv6lib=v6 + ipv6libdir=/usr/local/v6/lib + BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS"]) ;; zeta) - AC_EGREP_CPP([yes], [ -#include <sys/param.h> -#ifdef _ZETA_MINAMI_INET6 -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib]) + AX_CHECK_DEFINE([sys/param.h], [_ZETA_MINAMI_INET6], + [ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib]) ;; esac if test "$ipv6type" != "unknown"; then @@ -4460,12 +4570,13 @@ yes fi if test "$ipv6" = "yes" -a "$ipv6lib" != "none"; then + AC_MSG_CHECKING([ipv6 library]) if test -d $ipv6libdir -a -f $ipv6libdir/lib$ipv6lib.a; then LIBS="-L$ipv6libdir -l$ipv6lib $LIBS" - AC_MSG_NOTICE([using lib$ipv6lib]) + AC_MSG_RESULT([lib$ipv6lib]) else AS_VAR_IF([ipv6trylibc], [yes], [ - AC_MSG_NOTICE([using libc]) + AC_MSG_RESULT([libc]) ], [ AC_MSG_ERROR([m4_normalize([ No $ipv6lib library found; cannot continue. @@ -4767,17 +4878,17 @@ AC_CHECK_FUNCS([ \ copy_file_range ctermid dup dup3 execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ - gai_strerror getegid getentropy geteuid getgid getgrgid getgrgid_r \ - getgrnam_r getgrouplist getgroups gethostname getitimer getloadavg getlogin \ + gai_strerror getegid geteuid getgid getgrent getgrgid getgrgid_r \ + getgrnam_r getgrouplist gethostname getitimer getloadavg getlogin \ getpeername getpgid getpid getppid getpriority _getpty \ getpwent getpwnam_r getpwuid getpwuid_r getresgid getresuid getrusage getsid getspent \ - getspnam getuid getwd if_nameindex initgroups kill killpg lchown linkat \ + getspnam getuid getwd grantpt if_nameindex initgroups kill killpg lchown linkat \ lockf lstat lutimes madvise mbrtowc memrchr mkdirat mkfifo mkfifoat \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ - pipe2 plock poll posix_fadvise posix_fallocate posix_spawn posix_spawnp \ + pipe2 plock poll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ posix_spawn_file_actions_addclosefrom_np \ - pread preadv preadv2 pthread_condattr_setclock pthread_init pthread_kill \ - pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ + pread preadv preadv2 process_vm_readv pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ + pthread_kill ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ rtpSpawn sched_get_priority_max sched_rr_get_interval sched_setaffinity \ sched_setparam sched_setscheduler sem_clockwait sem_getvalue sem_open \ sem_timedwait sem_unlink sendfile setegid seteuid setgid sethostname \ @@ -4785,8 +4896,8 @@ AC_CHECK_FUNCS([ \ setresuid setreuid setsid setuid setvbuf shutdown sigaction sigaltstack \ sigfillset siginterrupt sigpending sigrelse sigtimedwait sigwait \ sigwaitinfo snprintf splice strftime strlcpy strsignal symlinkat sync \ - sysconf system tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ - tmpnam tmpnam_r truncate ttyname umask uname unlinkat utimensat utimes vfork \ + sysconf tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ + tmpnam tmpnam_r truncate ttyname umask uname unlinkat unlockpt utimensat utimes vfork \ wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \ ]) @@ -4797,6 +4908,14 @@ if test "$MACHDEP" != linux; then AC_CHECK_FUNCS([lchmod]) fi +# iOS defines some system methods that can be linked (so they are +# found by configure), but either raise a compilation error (because the +# header definition prevents usage - autoconf doesn't use the headers), or +# raise an error if used at runtime. Force these symbols off. +if test "$ac_sys_system" != "iOS" ; then + AC_CHECK_FUNCS([getentropy getgroups system]) +fi + AC_CHECK_DECL([dirfd], [AC_DEFINE([HAVE_DIRFD], [1], [Define if you have the 'dirfd' function or macro.])], @@ -5097,11 +5216,16 @@ AC_CHECK_FUNCS([clock_getres], [], [ ]) ]) -AC_CHECK_FUNCS([clock_settime], [], [ - AC_CHECK_LIB([rt], [clock_settime], [ - AC_DEFINE([HAVE_CLOCK_SETTIME], [1]) - ]) -]) +# On iOS, clock_settime can be linked (so it is found by +# configure), but it raises a runtime error if used because apps can't change +# the clock. Force the symbol off. +if test "$ac_sys_system" != "iOS" ; then + AC_CHECK_FUNCS([clock_settime], [], [ + AC_CHECK_LIB([rt], [clock_settime], [ + AC_DEFINE([HAVE_CLOCK_SETTIME], [1]) + ]) + ]) +fi AC_CHECK_FUNCS([clock_nanosleep], [], [ AC_CHECK_LIB([rt], [clock_nanosleep], [ @@ -5803,7 +5927,7 @@ AC_SUBST([SOABI]) AC_MSG_CHECKING([ABIFLAGS]) AC_MSG_RESULT([$ABIFLAGS]) AC_MSG_CHECKING([SOABI]) -SOABI='cpython-'`echo $VERSION | tr -d .`${ABIFLAGS}${PLATFORM_TRIPLET:+-$PLATFORM_TRIPLET} +SOABI='cpython-'`echo $VERSION | tr -d .`${ABIFLAGS}${SOABI_PLATFORM:+-$SOABI_PLATFORM} AC_MSG_RESULT([$SOABI]) # Release build, debug build (Py_DEBUG), and trace refs build (Py_TRACE_REFS) @@ -5811,7 +5935,7 @@ AC_MSG_RESULT([$SOABI]) if test "$Py_DEBUG" = 'true'; then # Similar to SOABI but remove "d" flag from ABIFLAGS AC_SUBST([ALT_SOABI]) - ALT_SOABI='cpython-'`echo $VERSION | tr -d .``echo $ABIFLAGS | tr -d d`${PLATFORM_TRIPLET:+-$PLATFORM_TRIPLET} + ALT_SOABI='cpython-'`echo $VERSION | tr -d .``echo $ABIFLAGS | tr -d d`${SOABI_PLATFORM:+-$SOABI_PLATFORM} AC_DEFINE_UNQUOTED([ALT_SOABI], ["${ALT_SOABI}"], [Alternative SOABI used in debug build to load C extensions built in release mode]) fi @@ -5823,12 +5947,21 @@ AC_MSG_CHECKING([LDVERSION]) LDVERSION='$(VERSION)$(ABIFLAGS)' AC_MSG_RESULT([$LDVERSION]) +# Configure the flags and dependencies used when compiling shared modules +AC_SUBST([MODULE_DEPS_SHARED]) +AC_SUBST([MODULE_LDFLAGS]) +MODULE_DEPS_SHARED='$(MODULE_DEPS_STATIC) $(EXPORTSYMS)' +MODULE_LDFLAGS='' + # On Android and Cygwin the shared libraries must be linked with libpython. -AC_SUBST([LIBPYTHON]) if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then - LIBPYTHON="-lpython${VERSION}${ABIFLAGS}" -else - LIBPYTHON='' + MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(LDLIBRARY)" + MODULE_LDFLAGS="\$(BLDLIBRARY)" +fi + +# On iOS the shared libraries must be linked with the Python framework +if test "$ac_sys_system" == "iOS"; then + MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(PYTHONFRAMEWORKDIR)/\$(PYTHONFRAMEWORK)" fi @@ -6087,11 +6220,7 @@ AS_VAR_IF([with_readline], [no], [ ]) # in readline as well as newer editline (April 2023) - AC_CHECK_TYPE([rl_compdisp_func_t], - [AC_DEFINE([HAVE_RL_COMPDISP_FUNC_T], [1], - [Define if readline supports rl_compdisp_func_t])], - [], - [readline_includes]) + AC_CHECK_TYPES([rl_compdisp_func_t], [], [], [readline_includes]) m4_undefine([readline_includes]) ])dnl WITH_SAVE_ENV() @@ -6460,28 +6589,35 @@ CPPFLAGS=$ac_save_cppflags AC_MSG_NOTICE([checking for device files]) dnl NOTE: Inform user how to proceed with files when cross compiling. -if test "x$cross_compiling" = xyes; then - if test "${ac_cv_file__dev_ptmx+set}" != set; then - AC_MSG_CHECKING([for /dev/ptmx]) - AC_MSG_RESULT([not set]) - AC_MSG_ERROR([set ac_cv_file__dev_ptmx to yes/no in your CONFIG_SITE file when cross compiling]) - fi - if test "${ac_cv_file__dev_ptc+set}" != set; then - AC_MSG_CHECKING([for /dev/ptc]) - AC_MSG_RESULT([not set]) - AC_MSG_ERROR([set ac_cv_file__dev_ptc to yes/no in your CONFIG_SITE file when cross compiling]) +dnl iOS cross-compile builds are predictable; they won't ever +dnl have /dev/ptmx or /dev/ptc, so we can set them explicitly. +if test "$ac_sys_system" = "iOS" ; then + ac_cv_file__dev_ptmx=no + ac_cv_file__dev_ptc=no +else + if test "x$cross_compiling" = xyes; then + if test "${ac_cv_file__dev_ptmx+set}" != set; then + AC_MSG_CHECKING([for /dev/ptmx]) + AC_MSG_RESULT([not set]) + AC_MSG_ERROR([set ac_cv_file__dev_ptmx to yes/no in your CONFIG_SITE file when cross compiling]) + fi + if test "${ac_cv_file__dev_ptc+set}" != set; then + AC_MSG_CHECKING([for /dev/ptc]) + AC_MSG_RESULT([not set]) + AC_MSG_ERROR([set ac_cv_file__dev_ptc to yes/no in your CONFIG_SITE file when cross compiling]) + fi fi -fi -AC_CHECK_FILE([/dev/ptmx], [], []) -if test "x$ac_cv_file__dev_ptmx" = xyes; then - AC_DEFINE([HAVE_DEV_PTMX], [1], - [Define to 1 if you have the /dev/ptmx device file.]) -fi -AC_CHECK_FILE([/dev/ptc], [], []) -if test "x$ac_cv_file__dev_ptc" = xyes; then - AC_DEFINE([HAVE_DEV_PTC], [1], - [Define to 1 if you have the /dev/ptc device file.]) + AC_CHECK_FILE([/dev/ptmx], [], []) + if test "x$ac_cv_file__dev_ptmx" = xyes; then + AC_DEFINE([HAVE_DEV_PTMX], [1], + [Define to 1 if you have the /dev/ptmx device file.]) + fi + AC_CHECK_FILE([/dev/ptc], [], []) + if test "x$ac_cv_file__dev_ptc" = xyes; then + AC_DEFINE([HAVE_DEV_PTC], [1], + [Define to 1 if you have the /dev/ptc device file.]) + fi fi if test $ac_sys_system = Darwin @@ -6489,12 +6625,9 @@ then LIBS="$LIBS -framework CoreFoundation" fi -AC_CHECK_TYPE( - [socklen_t], [], - [AC_DEFINE( - [socklen_t], [int], - [Define to `int' if <sys/socket.h> does not define.] - )], [ +AC_CHECK_TYPES([socklen_t], [], + [AC_DEFINE([socklen_t], [int], + [Define to 'int' if <sys/socket.h> does not define.])], [ #ifdef HAVE_SYS_TYPES_H #include <sys/types.h> #endif @@ -7124,6 +7257,28 @@ AS_CASE([$ac_sys_system], [VxWorks*], [PY_STDLIB_MOD_SET_NA([_scproxy], [termios], [grp])], dnl The _scproxy module is available on macOS [Darwin], [], + [iOS], [ + dnl subprocess and multiprocessing are not supported (no fork syscall). + dnl curses and tkinter user interface are not available. + dnl gdbm and nis aren't available + dnl Stub implementations are provided for pwd, grp etc APIs + PY_STDLIB_MOD_SET_NA( + [_curses], + [_curses_panel], + [_gdbm], + [_multiprocessing], + [_posixshmem], + [_posixsubprocess], + [_scproxy], + [_tkinter], + [grp], + [nis], + [readline], + [pwd], + [spwd], + [syslog], + ) + ], [CYGWIN*], [PY_STDLIB_MOD_SET_NA([_scproxy])], [QNX*], [PY_STDLIB_MOD_SET_NA([_scproxy])], [FreeBSD*], [PY_STDLIB_MOD_SET_NA([_scproxy])], @@ -7293,7 +7448,9 @@ PY_STDLIB_MOD([_socket], -a "$ac_cv_header_netinet_in_h" = "yes"])) dnl platform specific extensions -PY_STDLIB_MOD([grp], [], [test "$ac_cv_func_getgrgid" = yes -o "$ac_cv_func_getgrgid_r" = yes]) +PY_STDLIB_MOD([grp], [], + [test "$ac_cv_func_getgrent" = "yes" && + { test "$ac_cv_func_getgrgid" = "yes" || test "$ac_cv_func_getgrgid_r" = "yes"; }]) PY_STDLIB_MOD([pwd], [], [test "$ac_cv_func_getpwuid" = yes -o "$ac_cv_func_getpwuid_r" = yes]) PY_STDLIB_MOD([resource], [], [test "$ac_cv_header_sys_resource_h" = yes]) PY_STDLIB_MOD([_scproxy], @@ -7391,6 +7548,7 @@ PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([_testexternalinspection], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([xxsubtype], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_ctypes_test], diff --git a/iOS/README.rst b/iOS/README.rst new file mode 100644 index 000000000000000..1043b5ecebbc0c7 --- /dev/null +++ b/iOS/README.rst @@ -0,0 +1,321 @@ +==================== +Python on iOS README +==================== + +:Authors: + Russell Keith-Magee (2023-11) + +This document provides a quick overview of some iOS specific features in the +Python distribution. + +These instructions are only needed if you're planning to compile Python for iOS +yourself. Most users should *not* need to do this. If you're looking to +experiment with writing an iOS app in Python, tools such as `BeeWare's Briefcase +<https://briefcase.readthedocs.io>`__ and `Kivy's Buildozer +<https://buildozer.readthedocs.io>`__ will provide a much more approachable +user experience. + +Compilers for building on iOS +============================= + +Building for iOS requires the use of Apple's Xcode tooling. It is strongly +recommended that you use the most recent stable release of Xcode. This will +require the use of the most (or second-most) recently released macOS version, +as Apple does not maintain Xcode for older macOS versions. The Xcode Command +Line Tools are not sufficient for iOS development; you need a *full* Xcode +install. + +If you want to run your code on the iOS simulator, you'll also need to install +an iOS Simulator Platform. You should be prompted to select an iOS Simulator +Platform when you first run Xcode. Alternatively, you can add an iOS Simulator +Platform by selecting an open the Platforms tab of the Xcode Settings panel. + +iOS specific arguments to configure +=================================== + +* ``--enable-framework=DIR`` + + This argument specifies the location where the Python.framework will be + installed. This argument is required for all iOS builds; a directory *must* + be specified. + +* ``--with-framework-name=NAME`` + + Specify the name for the Python framework; defaults to ``Python``. + +Building Python on iOS +====================== + +ABIs and Architectures +---------------------- + +iOS apps can be deployed on physical devices, and on the iOS simulator. Although +the API used on these devices is identical, the ABI is different - you need to +link against different libraries for an iOS device build (``iphoneos``) or an +iOS simulator build (``iphonesimulator``). + +Apple uses the ``XCframework`` format to allow specifying a single dependency +that supports multiple ABIs. An ``XCframework`` is a wrapper around multiple +ABI-specific frameworks that share a common API. + +iOS can also support different CPU architectures within each ABI. At present, +there is only a single supported architecture on physical devices - ARM64. +However, the *simulator* supports 2 architectures - ARM64 (for running on Apple +Silicon machines), and x86_64 (for running on older Intel-based machines). + +To support multiple CPU architectures on a single platform, Apple uses a "fat +binary" format - a single physical file that contains support for multiple +architectures. It is possible to compile and use a "thin" single architecture +version of a binary for testing purposes; however, the "thin" binary will not be +portable to machines using other architectures. + +Building a single-architecture framework +---------------------------------------- + +The Python build system will create a ``Python.framework`` that supports a +*single* ABI with a *single* architecture. Unlike macOS, iOS does not allow a +framework to contain non-library content, so the iOS build will produce a +``bin`` and ``lib`` folder in the same output folder as ``Python.framework``. +The ``lib`` folder will be needed at runtime to support the Python library. + +If you want to use Python in a real iOS project, you need to produce multiple +``Python.framework`` builds, one for each ABI and architecture. iOS builds of +Python *must* be constructed as framework builds. To support this, you must +provide the ``--enable-framework`` flag when configuring the build. The build +also requires the use of cross-compilation. The minimal commands for building +Python for the ARM64 iOS simulator will look something like:: + + $ export PATH="`pwd`/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" + $ ./configure \ + AR=arm64-apple-ios-simulator-ar \ + CC=arm64-apple-ios-simulator-clang \ + CPP=arm64-apple-ios-simulator-cpp \ + CXX=arm64-apple-ios-simulator-clang \ + --enable-framework=/path/to/install \ + --host=arm64-apple-ios-simulator \ + --build=arm64-apple-darwin \ + --with-build-python=/path/to/python.exe + $ make + $ make install + +In this invocation: + +* ``iOS/Resources/bin`` has been added to the path, providing some shims for the + compilers and linkers needed by the build. Xcode requires the use of ``xcrun`` + to invoke compiler tooling. However, if ``xcrun`` is pre-evaluated and the + result passed to ``configure``, these results can embed user- and + version-specific paths into the sysconfig data, which limits the portability + of the compiled Python. Alternatively, if ``xcrun`` is used *as* the compiler, + it requires that compiler variables like ``CC`` include spaces, which can + cause significant problems with many C configuration systems which assume that + ``CC`` will be a single executable. + + To work around this problem, the ``iOS/Resources/bin`` folder contains some + wrapper scripts that present as simple compilers and linkers, but wrap + underlying calls to ``xcrun``. This allows configure to use a ``CC`` + definition without spaces, and without user- or version-specific paths, while + retaining the ability to adapt to the local Xcode install. These scripts are + included in the ``bin`` directory of an iOS install. + + These scripts will, by default, use the currently active Xcode installation. + If you want to use a different Xcode installation, you can use + ``xcode-select`` to set a new default Xcode globally, or you can use the + ``DEVELOPER_DIR`` environment variable to specify an Xcode install. The + scripts will use the default ``iphoneos``/``iphonesimulator`` SDK version for + the select Xcode install; if you want to use a different SDK, you can set the + ``IOS_SDK_VERSION`` environment variable. (e.g, setting + ``IOS_SDK_VERSION=17.1`` would cause the scripts to use the ``iphoneos17.1`` + and ``iphonesimulator17.1`` SDKs, regardless of the Xcode default.) + + The path has also been cleared of any user customizations. A common source of + bugs is for tools like Homebrew to accidentally leak macOS binaries into an iOS + build. Resetting the path to a known "bare bones" value is the easiest way to + avoid these problems. + +* ``/path/to/install`` is the location where the final ``Python.framework`` will + be output. + +* ``--host`` is the architecture and ABI that you want to build, in GNU compiler + triple format. This will be one of: + + - ``arm64-apple-ios`` for ARM64 iOS devices. + - ``arm64-apple-ios-simulator`` for the iOS simulator running on Apple + Silicon devices. + - ``x86_64-apple-ios-simulator`` for the iOS simulator running on Intel + devices. + +* ``--build`` is the GNU compiler triple for the machine that will be running + the compiler. This is one of: + + - ``arm64-apple-darwin`` for Apple Silicon devices. + - ``x86_64-apple-darwin`` for Intel devices. + +* ``/path/to/python.exe`` is the path to a Python binary on the machine that + will be running the compiler. This is needed because the Python compilation + process involves running some Python code. On a normal desktop build of + Python, you can compile a python interpreter and then use that interpreter to + run Python code. However, the binaries produced for iOS won't run on macOS, so + you need to provide an external Python interpreter. This interpreter must be + the same version as the Python that is being compiled. To be completely safe, + this should be the *exact* same commit hash. However, the longer a Python + release has been stable, the more likely it is that this constraint can be + relaxed - the same micro version will often be sufficient. + +For a full CPython build, you also need to specify the paths to iOS builds of +the binary libraries that CPython depends on (XZ, BZip2, LibFFI and OpenSSL). +This can be done by defining the ``LIBLZMA_CFLAGS``, ``LIBLZMA_LIBS``, +``BZIP2_CFLAGS``, ``BZIP2_LIBS``, ``LIBFFI_CFLAGS``, and ``LIBFFI_LIBS`` +environment variables, and the ``--with-openssl`` configure option. Versions of +these libraries pre-compiled for iOS can be found in `this repository +<https://github.com/beeware/cpython-apple-source-deps/releases>`__. + +By default, Python will be compiled with an iOS deployment target (i.e., the +minimum supported iOS version) of 12.0. To specify a different deployment +target, provide the version number as part of the ``--host`` argument - for +example, ``--host=arm64-apple-ios15.4-simulator`` would compile an ARM64 +simulator build with a deployment target of 15.4. + +Merge thin frameworks into fat frameworks +----------------------------------------- + +Once you've built a ``Python.framework`` for each ABI and and architecture, you +must produce a "fat" framework for each ABI that contains all the architectures +for that ABI. + +The ``iphoneos`` build only needs to support a single architecture, so it can be +used without modification. + +If you only want to support a single simulator architecture, (e.g., only support +ARM64 simulators), you can use a single architecture ``Python.framework`` build. +However, if you want to create ``Python.xcframework`` that supports *all* +architectures, you'll need to merge the ``iphonesimulator`` builds for ARM64 and +x86_64 into a single "fat" framework. + +The "fat" framework can be constructed by performing a directory merge of the +content of the two "thin" ``Python.framework`` directories, plus the ``bin`` and +``lib`` folders for each thin framework. When performing this merge: + +* The pure Python standard library content is identical for each architecture, + except for a handful of platform-specific files (such as the ``sysconfig`` + module). Ensure that the "fat" framework has the union of all standard library + files. + +* Any binary files in the standard library, plus the main + ``libPython3.X.dylib``, can be merged using the ``lipo`` tool, provide by + Xcode:: + + $ lipo -create -output module.dylib path/to/x86_64/module.dylib path/to/arm64/module.dylib + +* The header files will be indentical on both architectures, except for + ``pyconfig.h``. Copy all the headers from one platform (say, arm64), rename + ``pyconfig.h`` to ``pyconfig-arm64.h``, and copy the ``pyconfig.h`` for the + other architecture into the merged header folder as ``pyconfig-x86_64.h``. + Then copy the ``iOS/Resources/pyconfig.h`` file from the CPython sources into + the merged headers folder. This will allow the two Python architectures to + share a common ``pyconfig.h`` header file. + +At this point, you should have 2 Python.framework folders - one for ``iphoneos``, +and one for ``iphonesimulator`` that is a merge of x86+64 and ARM64 content. + +Merge frameworks into an XCframework +------------------------------------ + +Now that we have 2 (potentially fat) ABI-specific frameworks, we can merge those +frameworks into a single ``XCframework``. + +The initial skeleton of an ``XCframework`` is built using:: + + xcodebuild -create-xcframework -output Python.xcframework -framework path/to/iphoneos/Python.framework -framework path/to/iphonesimulator/Python.framework + +Then, copy the ``bin`` and ``lib`` folders into the architecture-specific slices of +the XCframework:: + + cp path/to/iphoneos/bin Python.xcframework/ios-arm64 + cp path/to/iphoneos/lib Python.xcframework/ios-arm64 + + cp path/to/iphonesimulator/bin Python.xcframework/ios-arm64_x86-64-simulator + cp path/to/iphonesimulator/lib Python.xcframework/ios-arm64_x86-64-simulator + +Note that the name of the architecture-specific slice for the simulator will +depend on the CPU architecture that you build. + +Then, add symbolic links to "common" platform names for each slice:: + + ln -si ios-arm64 Python.xcframework/iphoneos + ln -si ios-arm64_x86-64-simulator Python.xcframework/iphonesimulator + +You now have a Python.xcframework that can be used in a project. + +Testing Python on iOS +===================== + +The ``iOS/testbed`` folder that contains an Xcode project that is able to run +the iOS test suite. This project converts the Python test suite into a single +test case in Xcode's XCTest framework. The single XCTest passes if the test +suite passes. + +To run the test suite, configure a Python build for an iOS simulator (i.e., +``--host=arm64-apple-ios-simulator`` or ``--host=x86_64-apple-ios-simulator`` +), setting the framework location to the testbed project:: + + --enable-framework="./iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator" + +Then run ``make all install testiOS``. This will build an iOS framework for your +chosen architecture, install the Python iOS framework into the testbed project, +and run the test suite on an "iPhone SE (3rd generation)" simulator. + +While the test suite is running, Xcode does not display any console output. +After showing some Xcode build commands, the console output will print ``Testing +started``, and then appear to stop. It will remain in this state until the test +suite completes. On a 2022 M1 MacBook Pro, the test suite takes approximately 12 +minutes to run; a couple of extra minutes is required to boot and prepare the +iOS simulator. + +On success, the test suite will exit and report successful completion of the +test suite. No output of the Python test suite will be displayed. + +On failure, the output of the Python test suite *will* be displayed. This will +show the details of the tests that failed. + +Debugging test failures +----------------------- + +The easiest way to diagnose a single test failure is to open the testbed project +in Xcode and run the tests from there using the "Product > Test" menu item. + +Running specific tests +^^^^^^^^^^^^^^^^^^^^^^ + +As the test suite is being executed on an iOS simulator, it is not possible to +pass in command line arguments to configure test suite operation. To work around +this limitation, the arguments that would normally be passed as command line +arguments are configured as a static string at the start of the XCTest method +``- (void)testPython`` in ``iOSTestbedTests.m``. To pass an argument to the test +suite, add a a string to the ``argv`` defintion. These arguments will be passed +to the test suite as if they had been passed to ``python -m test`` at the +command line. + +Disabling automated breakpoints +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, Xcode will inserts an automatic breakpoint whenever a signal is +raised. The Python test suite raises many of these signals as part of normal +operation; unless you are trying to diagnose an issue with signals, the +automatic breakpoints can be inconvenient. However, they can be disabled by +creating a symbolic breakpoint that is triggered at the start of the test run. + +Select "Debug > Breakpoints > Create Symbolic Breakpoint" from the Xcode menu, and +populate the new brewpoint with the following details: + +* **Name**: IgnoreSignals +* **Symbol**: UIApplicationMain +* **Action**: Add debugger commands for: + - ``process handle SIGINT -n true -p true -s false`` + - ``process handle SIGUSR1 -n true -p true -s false`` + - ``process handle SIGUSR2 -n true -p true -s false`` + - ``process handle SIGXFSZ -n true -p true -s false`` +* Check the "Automatically continue after evaluating" box. + +All other details can be left blank. When the process executes the +``UIApplicationMain`` entry point, the breakpoint will trigger, run the debugger +commands to disable the automatic breakpoints, and automatically resume. diff --git a/iOS/Resources/Info.plist.in b/iOS/Resources/Info.plist.in new file mode 100644 index 000000000000000..3ecdc894f0a285e --- /dev/null +++ b/iOS/Resources/Info.plist.in @@ -0,0 +1,34 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist SYSTEM "file://localhost/System/Library/DTDs/PropertyList.dtd"> +<plist version="0.9"> +<dict> + <key>CFBundleDevelopmentRegion</key> + <string>en</string> + <key>CFBundleExecutable</key> + <string>Python</string> + <key>CFBundleGetInfoString</key> + <string>Python Runtime and Library</string> + <key>CFBundleIdentifier</key> + <string>@PYTHONFRAMEWORKIDENTIFIER@</string> + <key>CFBundleInfoDictionaryVersion</key> + <string>6.0</string> + <key>CFBundleName</key> + <string>Python</string> + <key>CFBundlePackageType</key> + <string>FMWK</string> + <key>CFBundleShortVersionString</key> + <string>%VERSION%</string> + <key>CFBundleLongVersionString</key> + <string>%VERSION%, (c) 2001-2024 Python Software Foundation.</string> + <key>CFBundleSignature</key> + <string>????</string> + <key>CFBundleVersion</key> + <string>1</string> + <key>CFBundleSupportedPlatforms</key> + <array> + <string>iPhoneOS</string> + </array> + <key>MinimumOSVersion</key> + <string>@IOS_DEPLOYMENT_TARGET@</string> +</dict> +</plist> diff --git a/iOS/Resources/bin/arm64-apple-ios-ar b/iOS/Resources/bin/arm64-apple-ios-ar new file mode 100755 index 000000000000000..8122332b9c1de04 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} ar $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-clang b/iOS/Resources/bin/arm64-apple-ios-clang new file mode 100755 index 000000000000000..4d525751eba798c --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-cpp b/iOS/Resources/bin/arm64-apple-ios-cpp new file mode 100755 index 000000000000000..891bb25bb4318c0 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios -E $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-ar b/iOS/Resources/bin/arm64-apple-ios-simulator-ar new file mode 100755 index 000000000000000..74ed3bc6df1c2bf --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-clang b/iOS/Resources/bin/arm64-apple-ios-simulator-clang new file mode 100755 index 000000000000000..32574cad2844415 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-cpp b/iOS/Resources/bin/arm64-apple-ios-simulator-cpp new file mode 100755 index 000000000000000..6aaf6fbe188c326 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator -E $@ diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-ar b/iOS/Resources/bin/x86_64-apple-ios-simulator-ar new file mode 100755 index 000000000000000..74ed3bc6df1c2bf --- /dev/null +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar $@ diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang new file mode 100755 index 000000000000000..bcbe91f6061e160 --- /dev/null +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator $@ diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp b/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp new file mode 100755 index 000000000000000..e6a42d9b85dec7d --- /dev/null +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator -E $@ diff --git a/iOS/Resources/dylib-Info-template.plist b/iOS/Resources/dylib-Info-template.plist new file mode 100644 index 000000000000000..f652e272f71c88a --- /dev/null +++ b/iOS/Resources/dylib-Info-template.plist @@ -0,0 +1,26 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>CFBundleDevelopmentRegion</key> + <string>en</string> + <key>CFBundleExecutable</key> + <string></string> + <key>CFBundleIdentifier</key> + <string></string> + <key>CFBundleInfoDictionaryVersion</key> + <string>6.0</string> + <key>CFBundlePackageType</key> + <string>APPL</string> + <key>CFBundleShortVersionString</key> + <string>1.0</string> + <key>CFBundleSupportedPlatforms</key> + <array> + <string>iPhoneOS</string> + </array> + <key>MinimumOSVersion</key> + <string>12.0</string> + <key>CFBundleVersion</key> + <string>1</string> +</dict> +</plist> diff --git a/iOS/Resources/pyconfig.h b/iOS/Resources/pyconfig.h new file mode 100644 index 000000000000000..4acff2c60516377 --- /dev/null +++ b/iOS/Resources/pyconfig.h @@ -0,0 +1,7 @@ +#ifdef __arm64__ +#include "pyconfig-arm64.h" +#endif + +#ifdef __x86_64__ +#include "pyconfig-x86_64.h" +#endif diff --git a/pyconfig.h.in b/pyconfig.h.in index d8a9f68951afbd1..e28baef51d57373 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -157,6 +157,9 @@ /* Define to 1 if you have the `clock_settime' function. */ #undef HAVE_CLOCK_SETTIME +/* Define to 1 if the system has the type `clock_t'. */ +#undef HAVE_CLOCK_T + /* Define to 1 if you have the `closefrom' function. */ #undef HAVE_CLOSEFROM @@ -474,6 +477,9 @@ /* Define to 1 if you have the `getgid' function. */ #undef HAVE_GETGID +/* Define to 1 if you have the `getgrent' function. */ +#undef HAVE_GETGRENT + /* Define to 1 if you have the `getgrgid' function. */ #undef HAVE_GETGRGID @@ -601,6 +607,9 @@ bcopy. */ #undef HAVE_GLIBC_MEMMOVE_BUG +/* Define to 1 if you have the `grantpt' function. */ +#undef HAVE_GRANTPT + /* Define to 1 if you have the <grp.h> header file. */ #undef HAVE_GRP_H @@ -899,6 +908,9 @@ /* Define to 1 if you have the `posix_fallocate' function. */ #undef HAVE_POSIX_FALLOCATE +/* Define to 1 if you have the `posix_openpt' function. */ +#undef HAVE_POSIX_OPENPT + /* Define to 1 if you have the `posix_spawn' function. */ #undef HAVE_POSIX_SPAWN @@ -924,12 +936,19 @@ /* Define to 1 if you have the <process.h> header file. */ #undef HAVE_PROCESS_H +/* Define to 1 if you have the `process_vm_readv' function. */ +#undef HAVE_PROCESS_VM_READV + /* Define if your compiler supports function prototype */ #undef HAVE_PROTOTYPES /* Define to 1 if you have the `pthread_condattr_setclock' function. */ #undef HAVE_PTHREAD_CONDATTR_SETCLOCK +/* Define to 1 if you have the `pthread_cond_timedwait_relative_np' function. + */ +#undef HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP + /* Defined for Solaris 2.6 bug in pthread header. */ #undef HAVE_PTHREAD_DESTRUCTOR @@ -951,6 +970,12 @@ /* Define if platform requires stubbed pthreads support */ #undef HAVE_PTHREAD_STUBS +/* Define to 1 if you have the `ptsname' function. */ +#undef HAVE_PTSNAME + +/* Define to 1 if you have the `ptsname_r' function. */ +#undef HAVE_PTSNAME_R + /* Define to 1 if you have the <pty.h> header file. */ #undef HAVE_PTY_H @@ -990,7 +1015,7 @@ /* Define if you can turn off readline's signal handling. */ #undef HAVE_RL_CATCH_SIGNAL -/* Define if readline supports rl_compdisp_func_t */ +/* Define to 1 if the system has the type `rl_compdisp_func_t'. */ #undef HAVE_RL_COMPDISP_FUNC_T /* Define if you have readline 2.2 */ @@ -1176,13 +1201,16 @@ /* Define if you have the 'socketpair' function. */ #undef HAVE_SOCKETPAIR +/* Define to 1 if the system has the type `socklen_t'. */ +#undef HAVE_SOCKLEN_T + /* Define to 1 if you have the <spawn.h> header file. */ #undef HAVE_SPAWN_H /* Define to 1 if you have the `splice' function. */ #undef HAVE_SPLICE -/* Define if your compiler provides ssize_t */ +/* Define to 1 if the system has the type `ssize_t'. */ #undef HAVE_SSIZE_T /* Define to 1 if you have the `statvfs' function. */ @@ -1459,6 +1487,9 @@ /* Define to 1 if you have the `unlinkat' function. */ #undef HAVE_UNLINKAT +/* Define to 1 if you have the `unlockpt' function. */ +#undef HAVE_UNLOCKPT + /* Define to 1 if you have the `unshare' function. */ #undef HAVE_UNSHARE @@ -1546,6 +1577,9 @@ /* Define to 1 if you have the `_getpty' function. */ #undef HAVE__GETPTY +/* Define to 1 if the system has the type `__uint128_t'. */ +#undef HAVE___UINT128_T + /* Define to 1 if `major', `minor', and `makedev' are declared in <mkdev.h>. */ #undef MAJOR_IN_MKDEV @@ -1910,7 +1944,7 @@ /* Define on FreeBSD to activate all library features */ #undef __BSD_VISIBLE -/* Define to 'long' if <time.h> doesn't define. */ +/* Define to 'long' if <time.h> does not define clock_t. */ #undef clock_t /* Define to empty if `const' does not conform to ANSI C. */ @@ -1934,7 +1968,7 @@ /* Define to `unsigned int' if <sys/types.h> does not define. */ #undef size_t -/* Define to `int' if <sys/socket.h> does not define. */ +/* Define to 'int' if <sys/socket.h> does not define. */ #undef socklen_t /* Define to `int' if <sys/types.h> doesn't define. */