Merge branch 'main' into introduce-resuming-downloads

.github/workflows/ci.yml

@@ -149,17 +149,17 @@ jobs:
       - name: Run unit tests
         run: >-
           nox -s test-${{ matrix.python.key || matrix.python }} --
-          -m unit
+          tests/unit
           --verbose --numprocesses auto --showlocals
       - name: Run integration tests
         run: >-
-          nox -s test-${{ matrix.python.key || matrix.python }} --
-          -m integration
+          nox -s test-${{ matrix.python.key || matrix.python }} --no-install --
+          tests/functional
           --verbose --numprocesses auto --showlocals
           --durations=5
 
   tests-windows:
-    name: tests / ${{ matrix.python }} / ${{ matrix.os }} / ${{ matrix.group }}
+    name: tests / ${{ matrix.python }} / ${{ matrix.os }} / ${{ matrix.group.number }}
     runs-on: ${{ matrix.os }}-latest
 
     needs: [packaging, determine-changes]
@@ -180,50 +180,39 @@ jobs:
           # - "3.11"
           # - "3.12"
           - "3.13"
-        group: [1, 2]
+        group:
+          - { number: 1, pytest-filter: "not test_install" }
+          - { number: 2, pytest-filter: "test_install" }
 
     steps:
+      # The D: drive is significantly faster than the system C: drive.
+      # https://github.com/actions/runner-images/issues/8755
+      - name: Set TEMP to D:/Temp
+        run: |
+          mkdir "D:\\Temp"
+          echo "TEMP=D:\\Temp" >> $env:GITHUB_ENV
+
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
           allow-prereleases: true
 
-      # We use C:\Temp (which is already available on the worker)
-      # as a temporary directory for all of the tests because the
-      # default value (under the user dir) is more deeply nested
-      # and causes tests to fail with "path too long" errors.
       - run: pip install nox
-        env:
-          TEMP: "C:\\Temp"
 
       # Main check
-      - name: Run unit tests
-        if: matrix.group == 1
+      - name: Run unit tests (group 1)
+        if: matrix.group.number == 1
         run: >-
           nox -s test-${{ matrix.python }} --
-          -m unit
+          tests/unit
           --verbose --numprocesses auto --showlocals
-        env:
-          TEMP: "C:\\Temp"
 
-      - name: Run integration tests (group 1)
-        if: matrix.group == 1
+      - name: Run integration tests (group ${{ matrix.group.number }})
         run: >-
-          nox -s test-${{ matrix.python }} --
-          -m integration -k "not test_install"
-          --verbose --numprocesses auto --showlocals
-        env:
-          TEMP: "C:\\Temp"
-
-      - name: Run integration tests (group 2)
-        if: matrix.group == 2
-        run: >-
-          nox -s test-${{ matrix.python }} --
-          -m integration -k "test_install"
+          nox -s test-${{ matrix.python }} --no-install --
+          tests/functional -k "${{ matrix.group.pytest-filter }}"
           --verbose --numprocesses auto --showlocals
-        env:
-          TEMP: "C:\\Temp"
 
   tests-zipapp:
     name: tests / zipapp
@@ -251,7 +240,7 @@ jobs:
       - name: Run integration tests
         run: >-
           nox -s test-3.10 --
-          -m integration
+          tests/functional
           --verbose --numprocesses auto --showlocals
           --durations=5
           --use-zipapp

.pre-commit-config.yaml

@@ -35,7 +35,7 @@ repos:
     args: ["--pretty", "--show-error-codes"]
     additional_dependencies: [
         'keyring==24.2.0',
-        'nox==2023.4.22',
+        'nox==2024.03.02',
         'pytest',
         'types-docutils==0.20.0.3',
         'types-setuptools==68.2.0.0',

MANIFEST.in

@@ -7,21 +7,16 @@ include pyproject.toml
 
 include src/pip/_vendor/README.rst
 include src/pip/_vendor/vendor.txt
-include src/pip/_vendor/pyparsing/diagram/template.jinja2
 recursive-include  src/pip/_vendor *LICENSE*
 recursive-include  src/pip/_vendor *COPYING*
 
-include docs/docutils.conf
 include docs/requirements.txt
 
 exclude .git-blame-ignore-revs
-exclude .coveragerc
 exclude .mailmap
-exclude .appveyor.yml
 exclude .readthedocs.yml
 exclude .pre-commit-config.yaml
 exclude .readthedocs-custom-redirects.yml
-exclude tox.ini
 exclude noxfile.py
 
 recursive-include src/pip/_vendor *.pem
@@ -34,6 +29,5 @@ recursive-exclude src/pip/_vendor *.pyi
 prune .github
 prune docs/build
 prune news
-prune tasks
 prune tests
 prune tools

docs/html/topics/authentication.md

@@ -163,25 +163,6 @@ from the subprocess in which they run Pip. You won't know whether the keyring
 backend is waiting the user input or not in such situations.
 ```
 
-pip is conservative and does not query keyring at all when `--no-input` is used
-because the keyring might require user interaction such as prompting the user
-on the console. You can force keyring usage by passing `--force-keyring` or one
-of the following:
-
-```bash
-# possibly with --user, --global or --site
-$ pip config set global.force-keyring true
-# or
-$ export PIP_FORCE_KEYRING=1
-```
-
-```{warning}
-Be careful when doing this since it could cause tools such as pipx and Pipenv
-to appear to hang. They show their own progress indicator while hiding output
-from the subprocess in which they run Pip. You won't know whether the keyring
-backend is waiting the user input or not in such situations.
-```
-
 Note that `keyring` (the Python package) needs to be installed separately from
 pip. This can create a bootstrapping issue if you need the credentials stored in
 the keyring to download and install keyring.

news/12455.doc.rst

@@ -0,0 +1 @@
+Removed section about non-existing ``--force-keyring`` flag.

news/13132.feature.rst

@@ -0,0 +1 @@
+Optimize package collection by avoiding unnecessary URL parsing and other processing.

noxfile.py

@@ -19,6 +19,7 @@
 
 nox.options.reuse_existing_virtualenvs = True
 nox.options.sessions = ["lint"]
+nox.needs_version = ">=2024.03.02"  # for session.run_install()
 
 LOCATIONS = {
     "common-wheels": "tests/data/common_wheels",
@@ -44,7 +45,9 @@ def run_with_protected_pip(session: nox.Session, *arguments: str) -> None:
     env = {"VIRTUAL_ENV": session.virtualenv.location}
 
     command = ("python", LOCATIONS["protected-pip"]) + arguments
-    session.run(*command, env=env, silent=True)
+    # By using run_install(), these installation steps can be skipped when -R
+    # or --no-install is passed.
+    session.run_install(*command, env=env, silent=True)
 
 
 def should_update_common_wheels() -> bool:
@@ -84,8 +87,13 @@ def test(session: nox.Session) -> None:
         session.log(msg)
 
     # Build source distribution
+    # HACK: we want to skip building and installing pip when nox's --no-install
+    # flag is given (to save time when running tests back to back with different
+    # arguments), but unfortunately nox does not expose this configuration state
+    # yet. https://github.com/wntrblm/nox/issues/710
+    no_install = "-R" in sys.argv or "--no-install" in sys.argv
     sdist_dir = os.path.join(session.virtualenv.location, "sdist")
-    if os.path.exists(sdist_dir):
+    if not no_install and os.path.exists(sdist_dir):
         shutil.rmtree(sdist_dir, ignore_errors=True)
 
     run_with_protected_pip(session, "install", "build")
@@ -94,7 +102,7 @@ def test(session: nox.Session) -> None:
     # pip, so uninstall pip to force build to provision a known good version of pip.
     run_with_protected_pip(session, "uninstall", "pip", "-y")
     # fmt: off
-    session.run(
+    session.run_install(
         "python", "-I", "-m", "build", "--sdist", "--outdir", sdist_dir,
         silent=True,
     )
@@ -127,7 +135,6 @@ def test(session: nox.Session) -> None:
 
 @nox.session
 def docs(session: nox.Session) -> None:
-    session.install("-e", ".")
     session.install("-r", REQUIREMENTS["docs"])
 
     def get_sphinx_build_command(kind: str) -> List[str]:
@@ -143,6 +150,7 @@ def get_sphinx_build_command(kind: str) -> List[str]:
             "-c", "docs/html",  # see note above
             "-d", "docs/build/doctrees/" + kind,
             "-b", kind,
+            "--jobs", "auto",
             "docs/" + kind,
             "docs/build/" + kind,
         ]
@@ -154,7 +162,6 @@ def get_sphinx_build_command(kind: str) -> List[str]:
 
 @nox.session(name="docs-live")
 def docs_live(session: nox.Session) -> None:
-    session.install("-e", ".")
     session.install("-r", REQUIREMENTS["docs"], "sphinx-autobuild")
 
     session.run(
@@ -163,6 +170,7 @@ def docs_live(session: nox.Session) -> None:
         "-b=dirhtml",
         "docs/html",
         "docs/build/livehtml",
+        "--jobs=auto",
         *session.posargs,
     )
 

pyproject.toml

@@ -60,8 +60,6 @@ exclude = ["contrib", "docs", "tests*", "tasks"]
 "pip" = ["py.typed"]
 "pip._vendor" = ["vendor.txt"]
 "pip._vendor.certifi" = ["*.pem"]
-"pip._vendor.requests" = ["*.pem"]
-"pip._vendor.distlib._backport" = ["sysconfig.cfg"]
 "pip._vendor.distlib" = [
     "t32.exe",
     "t64.exe",
@@ -307,10 +305,7 @@ source0 = [
 ]
 
 [tool.coverage.report]
-exclude_lines = [
-    # We must re-state the default because the `exclude_lines` option overrides
-    # it.
-    "pragma: no cover",
+exclude_also = [
     # This excludes typing-specific code, which will be validated by mypy anyway.
     "if TYPE_CHECKING",
 ]

src/pip/_internal/models/link.py

@@ -170,12 +170,23 @@ def _ensure_quoted_url(url: str) -> str:
     and without double-quoting other characters.
     """
     # Split the URL into parts according to the general structure
-    # `scheme://netloc/path;parameters?query#fragment`.
-    result = urllib.parse.urlparse(url)
+    # `scheme://netloc/path?query#fragment`.
+    result = urllib.parse.urlsplit(url)
     # If the netloc is empty, then the URL refers to a local filesystem path.
     is_local_path = not result.netloc
     path = _clean_url_path(result.path, is_local_path=is_local_path)
-    return urllib.parse.urlunparse(result._replace(path=path))
+    return urllib.parse.urlunsplit(result._replace(path=path))
+
+
+def _absolute_link_url(base_url: str, url: str) -> str:
+    """
+    A faster implementation of urllib.parse.urljoin with a shortcut
+    for absolute http/https URLs.
+    """
+    if url.startswith(("https://", "http://")):
+        return url
+    else:
+        return urllib.parse.urljoin(base_url, url)
 
 
 @functools.total_ordering
@@ -185,6 +196,7 @@ class Link:
     __slots__ = [
         "_parsed_url",
         "_url",
+        "_path",
         "_hashes",
         "comes_from",
         "requires_python",
@@ -241,6 +253,8 @@ def __init__(
         # Store the url as a private attribute to prevent accidentally
         # trying to set a new value.
         self._url = url
+        # The .path property is hot, so calculate its value ahead of time.
+        self._path = urllib.parse.unquote(self._parsed_url.path)
 
         link_hash = LinkHash.find_hash_url_fragment(url)
         hashes_from_link = {} if link_hash is None else link_hash.as_dict()
@@ -270,7 +284,7 @@ def from_json(
         if file_url is None:
             return None
 
-        url = _ensure_quoted_url(urllib.parse.urljoin(page_url, file_url))
+        url = _ensure_quoted_url(_absolute_link_url(page_url, file_url))
         pyrequire = file_data.get("requires-python")
         yanked_reason = file_data.get("yanked")
         hashes = file_data.get("hashes", {})
@@ -322,7 +336,7 @@ def from_element(
         if not href:
             return None
 
-        url = _ensure_quoted_url(urllib.parse.urljoin(base_url, href))
+        url = _ensure_quoted_url(_absolute_link_url(base_url, href))
         pyrequire = anchor_attribs.get("data-requires-python")
         yanked_reason = anchor_attribs.get("data-yanked")
 
@@ -421,7 +435,7 @@ def netloc(self) -> str:
 
     @property
     def path(self) -> str:
-        return urllib.parse.unquote(self._parsed_url.path)
+        return self._path
 
     def splitext(self) -> Tuple[str, str]:
         return splitext(posixpath.basename(self.path.rstrip("/")))