Merge 97a447ba84 into a15dd75d98

* Update mypy to 1.6.1

* Fix mypy "Source file found twice under different module names" error

* Ignore type of intialized abstract class in tests

* Use more specific type ignore method-assign

* Type ignore for message.get_all

* Remove unused type ignore

* Add SizedBuffer type for xmlrpc.client.Transport subclass

* Add Self type for RequestHandlerClass in test

* Add type ignore for shutil.rmtree onexc handler

* Quote SizedBuffer

* Add news entry

* Remove no longer correct comment

* Update self import

* Also ignore type onerror=handler

* Update news entry

* Update news entry

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.x"
@@ -57,7 +57,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.x"
@@ -81,7 +81,7 @@ jobs:
       github.event_name != 'pull_request'
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.x"
@@ -112,7 +112,7 @@ jobs:
           - "3.12"
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
@@ -164,7 +164,7 @@ jobs:
         group: [1, 2]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
@@ -215,7 +215,7 @@ jobs:
       github.event_name != 'pull_request'
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

.github/workflows/lock-threads.yml

@@ -17,7 +17,7 @@ jobs:
     if: github.repository_owner == 'pypa'
     runs-on: ubuntu-latest
     steps:
-      - uses: dessant/lock-threads@v3
+      - uses: dessant/lock-threads@v4
         with:
           issue-inactive-days: '30'
           pr-inactive-days: '15'

.github/workflows/news-file.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           # `towncrier check` runs `git diff --name-only origin/main...`, which
           # needs a non-shallow clone.

.github/workflows/update-rtd-redirects.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     environment: RTD Deploys
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.11"

.pre-commit-config.yaml

@@ -22,26 +22,26 @@ repos:
   - id: black
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  # Ruff version.
-  rev: v0.0.287
+  rev: v0.1.4
   hooks:
     - id: ruff
+      args: [--fix, --exit-non-zero-on-fix]
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v0.961
+  rev: v1.6.1
   hooks:
   - id: mypy
     exclude: tests/data
     args: ["--pretty", "--show-error-codes"]
     additional_dependencies: [
-        'keyring==23.0.1',
-        'nox==2021.6.12',
+        'keyring==24.2.0',
+        'nox==2023.4.22',
         'pytest',
-        'types-docutils==0.18.3',
-        'types-setuptools==57.4.14',
-        'types-freezegun==1.1.9',
-        'types-six==1.16.15',
-        'types-pyyaml==6.0.12.2',
+        'types-docutils==0.20.0.3',
+        'types-setuptools==68.2.0.0',
+        'types-freezegun==1.1.10',
+        'types-six==1.16.21.9',
+        'types-pyyaml==6.0.12.12',
     ]
 
 - repo: https://github.com/pre-commit/pygrep-hooks

AUTHORS.txt

@@ -20,6 +20,7 @@ Albert-Guan
 albertg
 Alberto Sottile
 Aleks Bunin
+Ales Erjavec
 Alethea Flowers
 Alex Gaynor
 Alex Grönholm
@@ -30,6 +31,7 @@ Alex Stachowiak
 Alexander Shtyrov
 Alexandre Conrad
 Alexey Popravka
+Aleš Erjavec
 Alli
 Ami Fischman
 Ananya Maiti
@@ -196,9 +198,11 @@ David Runge
 David Tucker
 David Wales
 Davidovich
+ddelange
 Deepak Sharma
 Deepyaman Datta
 Denise Yu
+dependabot[bot]
 derwolfe
 Desetude
 Devesh Kumar Singh
@@ -312,6 +316,7 @@ Ilya Baryshev
 Inada Naoki
 Ionel Cristian Mărieș
 Ionel Maries Cristian
+Itamar Turner-Trauring
 Ivan Pozdeev
 Jacob Kim
 Jacob Walls
@@ -338,6 +343,7 @@ Jay Graves
 Jean-Christophe Fillion-Robin
 Jeff Barber
 Jeff Dairiki
+Jeff Widman
 Jelmer Vernooij
 jenix21
 Jeremy Stanley
@@ -367,6 +373,7 @@ Joseph Long
 Josh Bronson
 Josh Hansen
 Josh Schneier
+Joshua
 Juan Luis Cano Rodríguez
 Juanjo Bazán
 Judah Rand
@@ -397,6 +404,7 @@ KOLANICH
 kpinc
 Krishna Oza
 Kumar McMillan
+Kurt McKee
 Kyle Persohn
 lakshmanaram
 Laszlo Kiss-Kollar
@@ -413,6 +421,7 @@ lorddavidiii
 Loren Carvalho
 Lucas Cimon
 Ludovic Gasc
+Lukas Geiger
 Lukas Juhrich
 Luke Macken
 Luo Jiebin
@@ -529,6 +538,7 @@ Patrick Jenkins
 Patrick Lawson
 patricktokeeffe
 Patrik Kopkan
+Paul Ganssle
 Paul Kehrer
 Paul Moore
 Paul Nasrat
@@ -609,6 +619,7 @@ ryneeverett
 Sachi King
 Salvatore Rinchiera
 sandeepkiran-js
+Sander Van Balen
 Savio Jomton
 schlamar
 Scott Kitterman
@@ -621,6 +632,7 @@ SeongSoo Cho
 Sergey Vasilyev
 Seth Michael Larson
 Seth Woodworth
+Shahar Epstein
 Shantanu
 shireenrao
 Shivansh-007
@@ -648,6 +660,7 @@ Steve Kowalik
 Steven Myint
 Steven Silvester
 stonebig
+studioj
 Stéphane Bidoul
 Stéphane Bidoul (ACSONE)
 Stéphane Klein

NEWS.rst

@@ -9,13 +9,80 @@
 
 .. towncrier release notes start
 
+23.3.1 (2023-10-21)
+===================
+
+Bug Fixes
+---------
+
+- Handle a timezone indicator of Z when parsing dates in the self check. (`#12338 <https://github.com/pypa/pip/issues/12338>`_)
+- Fix bug where installing the same package at the same time with multiple pip processes could fail. (`#12361 <https://github.com/pypa/pip/issues/12361>`_)
+
+
+23.3 (2023-10-15)
+=================
+
+Process
+-------
+
+- Added reference to `vulnerability reporting guidelines <https://www.python.org/dev/security/>`_ to pip's security policy.
+
+Deprecations and Removals
+-------------------------
+
+- Drop a fallback to using SecureTransport on macOS. It was useful when pip detected OpenSSL older than 1.0.1, but the current pip does not support any Python version supporting such old OpenSSL versions. (`#12175 <https://github.com/pypa/pip/issues/12175>`_)
+
+Features
+--------
+
+- Improve extras resolution for multiple constraints on same base package. (`#11924 <https://github.com/pypa/pip/issues/11924>`_)
+- Improve use of datastructures to make candidate selection 1.6x faster. (`#12204 <https://github.com/pypa/pip/issues/12204>`_)
+- Allow ``pip install --dry-run`` to use platform and ABI overriding options. (`#12215 <https://github.com/pypa/pip/issues/12215>`_)
+- Add ``is_yanked`` boolean entry to the installation report (``--report``) to indicate whether the requirement was yanked from the index, but was still selected by pip conform to :pep:`592`. (`#12224 <https://github.com/pypa/pip/issues/12224>`_)
+
+Bug Fixes
+---------
+
+- Ignore errors in temporary directory cleanup (show a warning instead). (`#11394 <https://github.com/pypa/pip/issues/11394>`_)
+- Normalize extras according to :pep:`685` from package metadata in the resolver
+  for comparison. This ensures extras are correctly compared and merged as long
+  as the package providing the extra(s) is built with values normalized according
+  to the standard. Note, however, that this *does not* solve cases where the
+  package itself contains unnormalized extra values in the metadata. (`#11649 <https://github.com/pypa/pip/issues/11649>`_)
+- Prevent downloading sdists twice when :pep:`658` metadata is present. (`#11847 <https://github.com/pypa/pip/issues/11847>`_)
+- Include all requested extras in the install report (``--report``). (`#11924 <https://github.com/pypa/pip/issues/11924>`_)
+- Removed uses of ``datetime.datetime.utcnow`` from non-vendored code. (`#12005 <https://github.com/pypa/pip/issues/12005>`_)
+- Consistently report whether a dependency comes from an extra. (`#12095 <https://github.com/pypa/pip/issues/12095>`_)
+- Fix completion script for zsh (`#12166 <https://github.com/pypa/pip/issues/12166>`_)
+- Fix improper handling of the new onexc argument of ``shutil.rmtree()`` in Python 3.12. (`#12187 <https://github.com/pypa/pip/issues/12187>`_)
+- Filter out yanked links from the available versions error message: "(from versions: 1.0, 2.0, 3.0)" will not contain yanked versions conform PEP 592. The yanked versions (if any) will be mentioned in a separate error message. (`#12225 <https://github.com/pypa/pip/issues/12225>`_)
+- Fix crash when the git version number contains something else than digits and dots. (`#12280 <https://github.com/pypa/pip/issues/12280>`_)
+- Use ``-r=...`` instead of ``-r ...`` to specify references with Mercurial. (`#12306 <https://github.com/pypa/pip/issues/12306>`_)
+- Redact password from URLs in some additional places. (`#12350 <https://github.com/pypa/pip/issues/12350>`_)
+- pip uses less memory when caching large packages. As a result, there is a new on-disk cache format stored in a new directory ($PIP_CACHE_DIR/http-v2). (`#2984 <https://github.com/pypa/pip/issues/2984>`_)
+
+Vendored Libraries
+------------------
+
+- Upgrade certifi to 2023.7.22
+- Add truststore 0.8.0
+- Upgrade urllib3 to 1.26.17
+
+Improved Documentation
+----------------------
+
+- Document that ``pip search`` support has been removed from PyPI (`#12059 <https://github.com/pypa/pip/issues/12059>`_)
+- Clarify --prefer-binary in CLI and docs (`#12122 <https://github.com/pypa/pip/issues/12122>`_)
+- Document that using OS-provided Python can cause pip's test suite to report false failures. (`#12334 <https://github.com/pypa/pip/issues/12334>`_)
+
+
 23.2.1 (2023-07-22)
 ===================
 
 Bug Fixes
 ---------
 
-- Disable PEP 658 metadata fetching with the legacy resolver. (`#12156 <https://github.com/pypa/pip/issues/12156>`_)
+- Disable :pep:`658` metadata fetching with the legacy resolver. (`#12156 <https://github.com/pypa/pip/issues/12156>`_)
 
 
 23.2 (2023-07-15)
@@ -45,11 +112,11 @@ Bug Fixes
 ---------
 
 - Fix ``pip completion --zsh``. (`#11417 <https://github.com/pypa/pip/issues/11417>`_)
-- Prevent downloading files twice when PEP 658 metadata is present (`#11847 <https://github.com/pypa/pip/issues/11847>`_)
+- Prevent downloading files twice when :pep:`658` metadata is present (`#11847 <https://github.com/pypa/pip/issues/11847>`_)
 - Add permission check before configuration (`#11920 <https://github.com/pypa/pip/issues/11920>`_)
 - Fix deprecation warnings in Python 3.12 for usage of shutil.rmtree (`#11957 <https://github.com/pypa/pip/issues/11957>`_)
 - Ignore invalid or unreadable ``origin.json`` files in the cache of locally built wheels. (`#11985 <https://github.com/pypa/pip/issues/11985>`_)
-- Fix installation of packages with PEP658 metadata using non-canonicalized names (`#12038 <https://github.com/pypa/pip/issues/12038>`_)
+- Fix installation of packages with :pep:`658` metadata using non-canonicalized names (`#12038 <https://github.com/pypa/pip/issues/12038>`_)
 - Correctly parse ``dist-info-metadata`` values from JSON-format index data. (`#12042 <https://github.com/pypa/pip/issues/12042>`_)
 - Fail with an error if the ``--python`` option is specified after the subcommand name. (`#12067 <https://github.com/pypa/pip/issues/12067>`_)
 - Fix slowness when using ``importlib.metadata`` (the default way for pip to read metadata in Python 3.11+) and there is a large overlap between already installed and to-be-installed packages. (`#12079 <https://github.com/pypa/pip/issues/12079>`_)
@@ -220,7 +287,7 @@ Features
 
 - Change the hashes in the installation report to be a mapping. Emit the
   ``archive_info.hashes`` dictionary in ``direct_url.json``. (`#11312 <https://github.com/pypa/pip/issues/11312>`_)
-- Implement logic to read the ``EXTERNALLY-MANAGED`` file as specified in PEP 668.
+- Implement logic to read the ``EXTERNALLY-MANAGED`` file as specified in :pep:`668`.
   This allows a downstream Python distributor to prevent users from using pip to
   modify the externally managed environment. (`#11381 <https://github.com/pypa/pip/issues/11381>`_)
 - Enable the use of ``keyring`` found on ``PATH``. This allows ``keyring``
@@ -236,7 +303,7 @@ Bug Fixes
 - Use the "venv" scheme if available to obtain prefixed lib paths. (`#11598 <https://github.com/pypa/pip/issues/11598>`_)
 - Deprecated a historical ambiguity in how ``egg`` fragments in URL-style
   requirements are formatted and handled. ``egg`` fragments that do not look
-  like PEP 508 names now produce a deprecation warning. (`#11617 <https://github.com/pypa/pip/issues/11617>`_)
+  like :pep:`508` names now produce a deprecation warning. (`#11617 <https://github.com/pypa/pip/issues/11617>`_)
 - Fix scripts path in isolated build environment on Debian. (`#11623 <https://github.com/pypa/pip/issues/11623>`_)
 - Make ``pip show`` show the editable location if package is editable (`#11638 <https://github.com/pypa/pip/issues/11638>`_)
 - Stop checking that ``wheel`` is present when ``build-system.requires``

README.rst

@@ -3,9 +3,15 @@ pip - The Python Package Installer
 
 .. image:: https://img.shields.io/pypi/v/pip.svg
    :target: https://pypi.org/project/pip/
+   :alt: PyPI
+
+.. image:: https://img.shields.io/pypi/pyversions/pip
+   :target: https://pypi.org/project/pip
+   :alt: PyPI - Python Version
 
 .. image:: https://readthedocs.org/projects/pip/badge/?version=latest
    :target: https://pip.pypa.io/en/latest
+   :alt: Documentation
 
 pip is the `package installer`_ for Python. You can use pip to install packages from the `Python Package Index`_ and other indexes.
 
@@ -19,8 +25,6 @@ We release updates regularly, with a new version every 3 months. Find more detai
 * `Release notes`_
 * `Release process`_
 
-**Note**: pip 21.0, in January 2021, removed Python 2 support, per pip's `Python 2 support policy`_. Please migrate to Python 3.
-
 If you find bugs, need help, or want to talk to the developers, please use our mailing lists or chat rooms:
 
 * `Issue tracking`_
@@ -47,7 +51,6 @@ rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_.
 .. _Release process: https://pip.pypa.io/en/latest/development/release-process/
 .. _GitHub page: https://github.com/pypa/pip
 .. _Development documentation: https://pip.pypa.io/en/latest/development
-.. _Python 2 support policy: https://pip.pypa.io/en/latest/development/release-process/#python-2-support
 .. _Issue tracking: https://github.com/pypa/pip/issues
 .. _Discourse channel: https://discuss.python.org/c/packaging
 .. _User IRC: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa

docs/html/cli/pip_install.rst

@@ -45,8 +45,8 @@ When looking at the items to be installed, pip checks what type of item
 each is, in the following order:
 
 1. Project or archive URL.
-2. Local directory (which must contain a ``setup.py``, or pip will report
-   an error).
+2. Local directory (which must contain a ``pyproject.toml`` or ``setup.py``,
+   otherwise pip will report an error).
 3. Local file (a sdist or wheel format archive, following the naming
    conventions for those formats).
 4. A requirement, as specified in :pep:`440`.

docs/html/development/contributing.rst

@@ -112,7 +112,7 @@ the ``news/`` directory with the extension of ``.trivial.rst``. If you are on a
 POSIX like operating system, one can be added by running
 ``touch news/$(uuidgen).trivial.rst``. On Windows, the same result can be
 achieved in Powershell using ``New-Item "news/$([guid]::NewGuid()).trivial.rst"``.
-Core committers may also add a "trivial" label to the PR which will accomplish
+Core committers may also add a "skip news" label to the PR which will accomplish
 the same thing.
 
 Upgrading, removing, or adding a new vendored library gets a special mention

docs/html/development/getting-started.rst

@@ -73,7 +73,7 @@ pip's tests are written using the :pypi:`pytest` test framework and
 :mod:`unittest.mock`. :pypi:`nox` is used to automate the setup and execution
 of pip's tests.
 
-It is preferable to run the tests in parallel for better experience during development,
+It is preferable to run the tests in parallel for a better experience during development,
 since the tests can take a long time to finish when run sequentially.
 
 To run tests:
@@ -104,6 +104,15 @@ can select tests using the various ways that pytest provides:
     $ # Using keywords
     $ nox -s test-3.10 -- -k "install and not wheel"
 
+.. note::
+
+    When running pip's tests with OS distribution Python versions, be aware that some
+    functional tests may fail due to potential patches introduced by the distribution.
+    For all tests to pass consider:
+
+    - Installing Python from `python.org`_ or compile from source
+    - Or, using `pyenv`_ to assist with source compilation
+
 Running pip's entire test suite requires supported version control tools
 (subversion, bazaar, git, and mercurial) to be installed. If you are missing
 any of these VCS, those tests should be skipped automatically. You can also
@@ -114,6 +123,9 @@ explicitly tell pytest to skip those tests:
     $ nox -s test-3.10 -- -k "not svn"
     $ nox -s test-3.10 -- -k "not (svn or git)"
 
+.. _python.org: https://www.python.org/downloads/
+.. _pyenv: https://github.com/pyenv/pyenv
+
 
 Running Linters
 ===============

docs/html/development/release-process.rst

@@ -145,8 +145,8 @@ Creating a new release
 #. Push the tag created by ``prepare-release``.
 #. Regenerate the ``get-pip.py`` script in the `get-pip repository`_ (as
    documented there) and commit the results.
-#. Submit a Pull Request to `CPython`_ adding the new version of pip (and upgrading
-   setuptools) to ``Lib/ensurepip/_bundled``, removing the existing version, and
+#. Submit a Pull Request to `CPython`_ adding the new version of pip
+   to ``Lib/ensurepip/_bundled``, removing the existing version, and
    adjusting the versions listed in ``Lib/ensurepip/__init__.py``.
 
 

docs/html/topics/configuration.md

@@ -19,8 +19,8 @@ and how they are related to pip's various command line options.
 
 ## Configuration Files
 
-Configuration files can change the default values for command line option.
-They are written using a standard INI style configuration files.
+Configuration files can change the default values for command line options.
+The files are written using standard INI format.
 
 pip has 3 "levels" of configuration files:
 
@@ -28,11 +28,15 @@ pip has 3 "levels" of configuration files:
 - `user`: per-user configuration file.
 - `site`: per-environment configuration file; i.e. per-virtualenv.
 
+Additionally, environment variables can be specified which will override any of the above.
+
 ### Location
 
 pip's configuration files are located in fairly standard locations. This
 location is different on different operating systems, and has some additional
-complexity for backwards compatibility reasons.
+complexity for backwards compatibility reasons. Note that if user config files
+exist in both the legacy and current locations, values in the current file
+will override values in the legacy file.
 
 ```{tab} Unix
 
@@ -88,9 +92,10 @@ Site
 ### `PIP_CONFIG_FILE`
 
 Additionally, the environment variable `PIP_CONFIG_FILE` can be used to specify
-a configuration file that's loaded first, and whose values are overridden by
-the values set in the aforementioned files. Setting this to {any}`os.devnull`
-disables the loading of _all_ configuration files.
+a configuration file that's loaded last, and whose values override the values
+set in the aforementioned files. Setting this to {any}`os.devnull`
+disables the loading of _all_ configuration files. Note that if a file exists
+at the location that this is set to, the user config file will not be loaded.
 
 (config-precedence)=
 
@@ -99,10 +104,10 @@ disables the loading of _all_ configuration files.
 When multiple configuration files are found, pip combines them in the following
 order:
 
-- `PIP_CONFIG_FILE`, if given.
 - Global
 - User
 - Site
+- `PIP_CONFIG_FILE`, if given.
 
 Each file read overrides any values read from previous files, so if the
 global timeout is specified in both the global file and the per-user file

docs/pip_sphinxext.py

@@ -194,22 +194,17 @@ class PipReqFileOptionsReference(PipOptions):
             opt = option()
             opt_name = opt._long_opts[0]
             if opt._short_opts:
-                short_opt_name = "{}, ".format(opt._short_opts[0])
+                short_opt_name = f"{opt._short_opts[0]}, "
             else:
                 short_opt_name = ""
 
             if option in cmdoptions.general_group["options"]:
                 prefix = ""
             else:
-                prefix = "{}_".format(self.determine_opt_prefix(opt_name))
+                prefix = f"{self.determine_opt_prefix(opt_name)}_"
 
             self.view_list.append(
-                "*  :ref:`{short}{long}<{prefix}{opt_name}>`".format(
-                    short=short_opt_name,
-                    long=opt_name,
-                    prefix=prefix,
-                    opt_name=opt_name,
-                ),
+                f"*  :ref:`{short_opt_name}{opt_name}<{prefix}{opt_name}>`",
                 "\n",
             )
 

news/11394.bugfix.rst

@@ -1 +0,0 @@
-Ignore errors in temporary directory cleanup (show a warning instead).

news/11649.bugfix.rst

@@ -1,5 +0,0 @@
-Normalize extras according to :pep:`685` from package metadata in the resolver
-for comparison. This ensures extras are correctly compared and merged as long
-as the package providing the extra(s) is built with values normalized according
-to the standard. Note, however, that this *does not* solve cases where the
-package itself contains unnormalized extra values in the metadata.

news/11815.doc.rst

@@ -0,0 +1 @@
+Fix explanation of how PIP_CONFIG_FILE works

news/12005.bugfix.rst

@@ -1 +0,0 @@
-Removed uses of ``datetime.datetime.utcnow`` from non-vendored code.

news/12059.doc.rst

@@ -1 +0,0 @@
-Document that ``pip search`` support has been removed from PyPI

news/12122.doc.rst

@@ -1 +0,0 @@
-Clarify --prefer-binary in CLI and docs

news/12155.process.rst

@@ -1,6 +0,0 @@
-The metadata-fetching log message is moved to the VERBOSE level and now hidden
-by default. The more significant information in this message to most users are
-already available in surrounding logs (the package name and version of the
-metadata being fetched), while the URL to the exact metadata file is generally
-too long and clutters the output. The message can be brought back with
-``--verbose``.

news/12175.removal.rst

@@ -1 +0,0 @@
-Drop a fallback to using SecureTransport on macOS. It was useful when pip detected OpenSSL older than 1.0.1, but the current pip does not support any Python version supporting such old OpenSSL versions.

news/12183.trivial.rst

@@ -1 +0,0 @@
-Add test cases for some behaviors of ``install --dry-run`` and ``--use-feature=fast-deps``.

news/12187.bugfix.rst

@@ -1 +0,0 @@
-Fix improper handling of the new onexc argument of ``shutil.rmtree()`` in Python 3.12.

news/12191.bugfix.rst

@@ -1 +0,0 @@
-Prevent downloading sdists twice when PEP 658 metadata is present.

news/12194.trivial.rst

@@ -1 +0,0 @@
-Add lots of comments to the ``BuildTracker``.

news/12204.feature.rst

@@ -1 +0,0 @@
-Improve use of datastructures to make candidate selection 1.6x faster

news/12215.feature.rst

@@ -1 +0,0 @@
-Allow ``pip install --dry-run`` to use platform and ABI overriding options similar to ``--target``.

news/12224.feature.rst

@@ -1 +0,0 @@
-Add ``is_yanked`` boolean entry to the installation report (``--report``) to indicate whether the requirement was yanked from the index, but was still selected by pip conform to PEP 592.

news/12225.bugfix.rst

@@ -1 +0,0 @@
-Filter out yanked links from the available versions error message: "(from versions: 1.0, 2.0, 3.0)" will not contain yanked versions conform PEP 592. The yanked versions (if any) will be mentioned in a separate error message.

news/12254.process.rst

@@ -1 +0,0 @@
-Added reference to `vulnerability reporting guidelines <https://www.python.org/dev/security/>`_ to pip's security policy.

news/12389.bugfix.rst

@@ -0,0 +1 @@
+Update mypy to 1.6.1 and fix/ignore types

news/12390.trivial.rst

@@ -0,0 +1 @@
+Update ruff versions and config for dev

news/12393.trivial.rst

@@ -0,0 +1 @@
+Enforce and update code to use f-strings via Ruff rule UP032

news/12417.doc.rst

@@ -0,0 +1 @@
+Fix outdated pip install argument description in documentation.

news/2984.bugfix.rst

@@ -1 +0,0 @@
-pip uses less memory when caching large packages. As a result, there is a new on-disk cache format stored in a new directory ($PIP_CACHE_DIR/http-v2).

news/370392cf-52cd-402c-b402-06d2ff398f89.bugfix.rst

@@ -0,0 +1 @@
+Fix mercurial revision "parse error": use ``--rev={ref}`` instead of ``-r={ref}``

news/4A0C40FF-ABE1-48C7-954C-7C3EB229135F.trivial.rst

@@ -1 +0,0 @@
-Add ruff rules ASYNC,C4,C90,PERF,PLE,PLR for minor optimizations and to set upper limits on code complexity.

news/certifi.vendor.rst

@@ -1 +0,0 @@
-Upgrade certifi to 2023.7.22

news/truststore.vendor.rst

@@ -1 +0,0 @@
-Add truststore 0.8.0

noxfile.py

@@ -322,7 +322,7 @@ def build_release(session: nox.Session) -> None:
         )
 
     session.log("# Install dependencies")
-    session.install("setuptools", "wheel", "twine")
+    session.install("build", "twine")
 
     with release.isolated_temporary_checkout(session, version) as build_dir:
         session.log(
@@ -358,8 +358,7 @@ def build_dists(session: nox.Session) -> List[str]:
         )
 
     session.log("# Build distributions")
-    session.install("setuptools", "wheel")
-    session.run("python", "setup.py", "sdist", "bdist_wheel", silent=True)
+    session.run("python", "-m", "build", silent=True)
     produced_dists = glob.glob("dist/*")
 
     session.log(f"# Verify distributions: {', '.join(produced_dists)}")

pyproject.toml

@@ -84,8 +84,8 @@ ignore = [
     "B020",
     "B904", # Ruff enables opinionated warnings by default
     "B905", # Ruff enables opinionated warnings by default
-    "G202",
 ]
+target-version = "py37"
 line-length = 88
 select = [
     "ASYNC",
@@ -101,6 +101,8 @@ select = [
     "PLE",
     "PLR0",
     "W",
+    "RUF100",
+    "UP032",
 ]
 
 [tool.ruff.isort]

setup.cfg

@@ -1,39 +1,3 @@
-[isort]
-profile = black
-skip =
-    ./build,
-    .nox,
-    .tox,
-    .scratch,
-    _vendor,
-    data
-known_third_party =
-    pip._vendor
-
-[flake8]
-max-line-length = 88
-exclude =
-    ./build,
-    .nox,
-    .tox,
-    .scratch,
-    _vendor,
-    data
-enable-extensions = G
-extend-ignore =
-    G200, G202,
-    # black adds spaces around ':'
-    E203,
-    # using a cache
-    B019,
-    # reassigning variables in a loop
-    B020,
-per-file-ignores =
-    # G: The plugin logging-format treats every .log and .error as logging.
-    noxfile.py: G
-    # B011: Do not call assert False since python -O removes these calls
-    tests/*: B011
-
 [mypy]
 mypy_path = $MYPY_CONFIG_FILE_DIR/src
 

setup.py

@@ -77,7 +77,7 @@ setup(
     entry_points={
         "console_scripts": [
             "pip=pip._internal.cli.main:main",
-            "pip{}=pip._internal.cli.main:main".format(sys.version_info[0]),
+            f"pip{sys.version_info[0]}=pip._internal.cli.main:main",
             "pip{}.{}=pip._internal.cli.main:main".format(*sys.version_info[:2]),
         ],
     },

src/pip/__init__.py

@@ -1,6 +1,6 @@
 from typing import List, Optional
 
-__version__ = "23.3.dev0"
+__version__ = "24.0.dev0"
 
 
 def main(args: Optional[List[str]] = None) -> int:

src/pip/_internal/cli/base_command.py

@@ -181,7 +181,7 @@ class Command(CommandContextMixIn):
                     assert isinstance(status, int)
                     return status
                 except DiagnosticPipError as exc:
-                    logger.error("[present-rich] %s", exc)
+                    logger.error("%s", exc, extra={"rich": True})
                     logger.debug("Exception information:", exc_info=True)
 
                     return ERROR

src/pip/_internal/cli/cmdoptions.py

@@ -582,10 +582,7 @@ def _handle_python_version(
     """
     version_info, error_msg = _convert_python_version(value)
     if error_msg is not None:
-        msg = "invalid --python-version value: {!r}: {}".format(
-            value,
-            error_msg,
-        )
+        msg = f"invalid --python-version value: {value!r}: {error_msg}"
         raise_option_error(parser, option=option, msg=msg)
 
     parser.values.python_version = version_info
@@ -826,7 +823,7 @@ def _handle_config_settings(
 ) -> None:
     key, sep, val = value.partition("=")
     if sep != "=":
-        parser.error(f"Arguments to {opt_str} must be of the form KEY=VAL")  # noqa
+        parser.error(f"Arguments to {opt_str} must be of the form KEY=VAL")
     dest = getattr(parser.values, option.dest)
     if dest is None:
         dest = {}
@@ -921,13 +918,13 @@ def _handle_merge_hash(
         algo, digest = value.split(":", 1)
     except ValueError:
         parser.error(
-            "Arguments to {} must be a hash name "  # noqa
+            f"Arguments to {opt_str} must be a hash name "
             "followed by a value, like --hash=sha256:"
-            "abcde...".format(opt_str)
+            "abcde..."
         )
     if algo not in STRONG_HASHES:
         parser.error(
-            "Allowed hash algorithms for {} are {}.".format(  # noqa
+            "Allowed hash algorithms for {} are {}.".format(
                 opt_str, ", ".join(STRONG_HASHES)
             )
         )

src/pip/_internal/cli/parser.py

@@ -229,9 +229,9 @@ class ConfigOptionParser(CustomOptionParser):
                     val = strtobool(val)
                 except ValueError:
                     self.error(
-                        "{} is not a valid value for {} option, "  # noqa
+                        f"{val} is not a valid value for {key} option, "
                         "please specify a boolean value like yes/no, "
-                        "true/false or 1/0 instead.".format(val, key)
+                        "true/false or 1/0 instead."
                     )
             elif option.action == "count":
                 with suppress(ValueError):
@@ -240,10 +240,10 @@ class ConfigOptionParser(CustomOptionParser):
                     val = int(val)
                 if not isinstance(val, int) or val < 0:
                     self.error(
-                        "{} is not a valid value for {} option, "  # noqa
+                        f"{val} is not a valid value for {key} option, "
                         "please instead specify either a non-negative integer "
                         "or a boolean value like yes/no or false/true "
-                        "which is equivalent to 1/0.".format(val, key)
+                        "which is equivalent to 1/0."
                     )
             elif option.action == "append":
                 val = val.split()

src/pip/_internal/cli/req_command.py

@@ -265,7 +265,7 @@ class RequirementCommand(IndexGroupCommand):
         if "legacy-resolver" in options.deprecated_features_enabled:
             return "legacy"
 
-        return "2020-resolver"
+        return "resolvelib"
 
     @classmethod
     def make_requirement_preparer(
@@ -287,7 +287,7 @@ class RequirementCommand(IndexGroupCommand):
         legacy_resolver = False
 
         resolver_variant = cls.determine_resolver_variant(options)
-        if resolver_variant == "2020-resolver":
+        if resolver_variant == "resolvelib":
             lazy_wheel = "fast-deps" in options.features_enabled
             if lazy_wheel:
                 logger.warning(
@@ -349,7 +349,7 @@ class RequirementCommand(IndexGroupCommand):
         # The long import name and duplicated invocation is needed to convince
         # Mypy into correctly typechecking. Otherwise it would complain the
         # "Resolver" class being redefined.
-        if resolver_variant == "2020-resolver":
+        if resolver_variant == "resolvelib":
             import pip._internal.resolution.resolvelib.resolver
 
             return pip._internal.resolution.resolvelib.resolver.Resolver(

src/pip/_internal/commands/cache.py

@@ -175,7 +175,7 @@ class CacheCommand(Command):
             files += self._find_http_files(options)
         else:
             # Add the pattern to the log message
-            no_matching_msg += ' for pattern "{}"'.format(args[0])
+            no_matching_msg += f' for pattern "{args[0]}"'
 
         if not files:
             logger.warning(no_matching_msg)

src/pip/_internal/commands/completion.py

@@ -23,9 +23,18 @@ COMPLETION_SCRIPTS = {
     """,
     "zsh": """
         #compdef -P pip[0-9.]#
-        compadd $( COMP_WORDS="$words[*]" \\
-                   COMP_CWORD=$((CURRENT-1)) \\
-                   PIP_AUTO_COMPLETE=1 $words[1] 2>/dev/null )
+        __pip() {{
+          compadd $( COMP_WORDS="$words[*]" \\
+                     COMP_CWORD=$((CURRENT-1)) \\
+                     PIP_AUTO_COMPLETE=1 $words[1] 2>/dev/null )
+        }}
+        if [[ $zsh_eval_context[-1] == loadautofunc ]]; then
+          # autoload from fpath, call function directly
+          __pip "$@"
+        else
+          # eval/source/. command, register function for later
+          compdef __pip -P 'pip[0-9.]#'
+        fi
     """,
     "fish": """
         function __fish_complete_pip

src/pip/_internal/commands/configuration.py

@@ -242,17 +242,15 @@ class ConfigurationCommand(Command):
                 e.filename = editor
             raise
         except subprocess.CalledProcessError as e:
-            raise PipError(
-                "Editor Subprocess exited with exit code {}".format(e.returncode)
-            )
+            raise PipError(f"Editor Subprocess exited with exit code {e.returncode}")
 
     def _get_n_args(self, args: List[str], example: str, n: int) -> Any:
         """Helper to make sure the command got the right number of arguments"""
         if len(args) != n:
             msg = (
-                "Got unexpected number of arguments, expected {}. "
-                '(example: "{} config {}")'
-            ).format(n, get_prog(), example)
+                f"Got unexpected number of arguments, expected {n}. "
+                f'(example: "{get_prog()} config {example}")'
+            )
             raise PipError(msg)
 
         if n == 1:

src/pip/_internal/commands/debug.py

@@ -95,7 +95,7 @@ def show_actual_vendor_versions(vendor_txt_versions: Dict[str, str]) -> None:
         elif parse_version(actual_version) != parse_version(expected_version):
             extra_message = (
                 " (CONFLICT: vendor.txt suggests version should"
-                " be {})".format(expected_version)
+                f" be {expected_version})"
             )
         logger.info("%s==%s%s", module_name, actual_version, extra_message)
 
@@ -120,7 +120,7 @@ def show_tags(options: Values) -> None:
     if formatted_target:
         suffix = f" (target: {formatted_target})"
 
-    msg = "Compatible tags: {}{}".format(len(tags), suffix)
+    msg = f"Compatible tags: {len(tags)}{suffix}"
     logger.info(msg)
 
     if options.verbose < 1 and len(tags) > tag_limit:
@@ -134,16 +134,12 @@ def show_tags(options: Values) -> None:
             logger.info(str(tag))
 
         if tags_limited:
-            msg = (
-                "...\n[First {tag_limit} tags shown. Pass --verbose to show all.]"
-            ).format(tag_limit=tag_limit)
+            msg = f"...\n[First {tag_limit} tags shown. Pass --verbose to show all.]"
             logger.info(msg)
 
 
 def ca_bundle_info(config: Configuration) -> str:
-    # Ruff misidentifies config as a dict.
-    # Configuration does not have support the mapping interface.
-    levels = {key.split(".", 1)[0] for key, _ in config.items()}  # noqa: PERF102
+    levels = {key.split(".", 1)[0] for key, _ in config.items()}
     if not levels:
         return "Not specified"
 

src/pip/_internal/commands/index.py

@@ -128,12 +128,12 @@ class IndexCommand(IndexGroupCommand):
 
             if not versions:
                 raise DistributionNotFound(
-                    "No matching distribution found for {}".format(query)
+                    f"No matching distribution found for {query}"
                 )
 
             formatted_versions = [str(ver) for ver in sorted(versions, reverse=True)]
             latest = formatted_versions[0]
 
-        write_output("{} ({})".format(query, latest))
+        write_output(f"{query} ({latest})")
         write_output("Available versions: {}".format(", ".join(formatted_versions)))
         print_dist_installation_info(query, latest)

src/pip/_internal/commands/install.py

@@ -501,7 +501,7 @@ class InstallCommand(RequirementCommand):
                 show_traceback,
                 options.use_user_site,
             )
-            logger.error(message, exc_info=show_traceback)  # noqa
+            logger.error(message, exc_info=show_traceback)
 
             return ERROR
 
@@ -595,7 +595,7 @@ class InstallCommand(RequirementCommand):
                 "source of the following dependency conflicts."
             )
         else:
-            assert resolver_variant == "2020-resolver"
+            assert resolver_variant == "resolvelib"
             parts.append(
                 "pip's dependency resolver does not currently take into account "
                 "all the packages that are installed. This behaviour is the "
@@ -607,12 +607,8 @@ class InstallCommand(RequirementCommand):
             version = package_set[project_name][0]
             for dependency in missing[project_name]:
                 message = (
-                    "{name} {version} requires {requirement}, "
+                    f"{project_name} {version} requires {dependency[1]}, "
                     "which is not installed."
-                ).format(
-                    name=project_name,
-                    version=version,
-                    requirement=dependency[1],
                 )
                 parts.append(message)
 
@@ -628,7 +624,7 @@ class InstallCommand(RequirementCommand):
                     requirement=req,
                     dep_name=dep_name,
                     dep_version=dep_version,
-                    you=("you" if resolver_variant == "2020-resolver" else "you'll"),
+                    you=("you" if resolver_variant == "resolvelib" else "you'll"),
                 )
                 parts.append(message)
 

src/pip/_internal/configuration.py

@@ -59,8 +59,8 @@ def _disassemble_key(name: str) -> List[str]:
     if "." not in name:
         error_message = (
             "Key does not contain dot separated section and key. "
-            "Perhaps you wanted to use 'global.{}' instead?"
-        ).format(name)
+            f"Perhaps you wanted to use 'global.{name}' instead?"
+        )
         raise ConfigurationError(error_message)
     return name.split(".", 1)
 
@@ -327,33 +327,35 @@ class Configuration:
     def iter_config_files(self) -> Iterable[Tuple[Kind, List[str]]]:
         """Yields variant and configuration files associated with it.
 
-        This should be treated like items of a dictionary.
+        This should be treated like items of a dictionary. The order
+        here doesn't affect what gets overridden. That is controlled
+        by OVERRIDE_ORDER. However this does control the order they are
+        displayed to the user. It's probably most ergononmic to display
+        things in the same order as OVERRIDE_ORDER
         """
         # SMELL: Move the conditions out of this function
 
-        # environment variables have the lowest priority
-        config_file = os.environ.get("PIP_CONFIG_FILE", None)
-        if config_file is not None:
-            yield kinds.ENV, [config_file]
-        else:
-            yield kinds.ENV, []
-
+        env_config_file = os.environ.get("PIP_CONFIG_FILE", None)
         config_files = get_configuration_files()
 
-        # at the base we have any global configuration
         yield kinds.GLOBAL, config_files[kinds.GLOBAL]
 
-        # per-user configuration next
+        # per-user config is not loaded when env_config_file exists
         should_load_user_config = not self.isolated and not (
-            config_file and os.path.exists(config_file)
+            env_config_file and os.path.exists(env_config_file)
         )
         if should_load_user_config:
             # The legacy config file is overridden by the new config file
             yield kinds.USER, config_files[kinds.USER]
 
-        # finally virtualenv configuration first trumping others
+        # virtualenv config
         yield kinds.SITE, config_files[kinds.SITE]
 
+        if env_config_file is not None:
+            yield kinds.ENV, [env_config_file]
+        else:
+            yield kinds.ENV, []
+
     def get_values_in_config(self, variant: Kind) -> Dict[str, Any]:
         """Get values present in a config file"""
         return self._config[variant]

src/pip/_internal/distributions/sdist.py

@@ -6,7 +6,10 @@ from pip._internal.distributions.base import AbstractDistribution
 from pip._internal.exceptions import InstallationError
 from pip._internal.index.package_finder import PackageFinder
 from pip._internal.metadata import BaseDistribution
-from pip._internal.utils.subprocess import runner_with_spinner_message
+from pip._internal.utils.subprocess import (
+    log_backend_warnings,
+    runner_with_spinner_message,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -93,7 +96,7 @@ class SourceDistribution(AbstractDistribution):
             )
 
     def _get_build_requires_wheel(self) -> Iterable[str]:
-        with self.req.build_env:
+        with self.req.build_env, log_backend_warnings():
             runner = runner_with_spinner_message("Getting requirements to build wheel")
             backend = self.req.pep517_backend
             assert backend is not None
@@ -101,7 +104,7 @@ class SourceDistribution(AbstractDistribution):
                 return backend.get_requires_for_build_wheel()
 
     def _get_build_requires_editable(self) -> Iterable[str]:
-        with self.req.build_env:
+        with self.req.build_env, log_backend_warnings():
             runner = runner_with_spinner_message(
                 "Getting requirements to build editable"
             )

src/pip/_internal/exceptions.py

@@ -247,10 +247,7 @@ class NoneMetadataError(PipError):
     def __str__(self) -> str:
         # Use `dist` in the error message because its stringification
         # includes more information, like the version and location.
-        return "None {} metadata found for distribution: {}".format(
-            self.metadata_name,
-            self.dist,
-        )
+        return f"None {self.metadata_name} metadata found for distribution: {self.dist}"
 
 
 class UserInstallationInvalid(InstallationError):
@@ -594,7 +591,7 @@ class HashMismatch(HashError):
         self.gots = gots
 
     def body(self) -> str:
-        return "    {}:\n{}".format(self._requirement_name(), self._hash_comparison())
+        return f"    {self._requirement_name()}:\n{self._hash_comparison()}"
 
     def _hash_comparison(self) -> str:
         """
@@ -616,11 +613,9 @@ class HashMismatch(HashError):
         lines: List[str] = []
         for hash_name, expecteds in self.allowed.items():
             prefix = hash_then_or(hash_name)
-            lines.extend(
-                ("        Expected {} {}".format(next(prefix), e)) for e in expecteds
-            )
+            lines.extend((f"        Expected {next(prefix)} {e}") for e in expecteds)
             lines.append(
-                "             Got        {}\n".format(self.gots[hash_name].hexdigest())
+                f"             Got        {self.gots[hash_name].hexdigest()}\n"
             )
         return "\n".join(lines)
 

src/pip/_internal/index/package_finder.py

@@ -533,8 +533,8 @@ class CandidateEvaluator:
                 )
             except ValueError:
                 raise UnsupportedWheel(
-                    "{} is not a supported wheel for this platform. It "
-                    "can't be sorted.".format(wheel.filename)
+                    f"{wheel.filename} is not a supported wheel for this platform. It "
+                    "can't be sorted."
                 )
             if self._prefer_binary:
                 binary_preference = 1
@@ -939,9 +939,7 @@ class PackageFinder:
                 _format_versions(best_candidate_result.iter_all()),
             )
 
-            raise DistributionNotFound(
-                "No matching distribution found for {}".format(req)
-            )
+            raise DistributionNotFound(f"No matching distribution found for {req}")
 
         def _should_install_candidate(
             candidate: Optional[InstallationCandidate],

src/pip/_internal/locations/_distutils.py

@@ -56,8 +56,7 @@ def distutils_scheme(
         try:
             d.parse_config_files()
         except UnicodeDecodeError:
-            # Typeshed does not include find_config_files() for some reason.
-            paths = d.find_config_files()  # type: ignore
+            paths = d.find_config_files()
             logger.warning(
                 "Ignore distutils configs in %s due to encoding errors.",
                 ", ".join(os.path.basename(p) for p in paths),

src/pip/_internal/metadata/_json.py

@@ -64,10 +64,10 @@ def msg_to_json(msg: Message) -> Dict[str, Any]:
         key = json_name(field)
         if multi:
             value: Union[str, List[str]] = [
-                sanitise_header(v) for v in msg.get_all(field)
+                sanitise_header(v) for v in msg.get_all(field)  # type: ignore
             ]
         else:
-            value = sanitise_header(msg.get(field))
+            value = sanitise_header(msg.get(field))  # type: ignore
             if key == "keywords":
                 # Accept both comma-separated and space-separated
                 # forms, for better compatibility with old data.

src/pip/_internal/metadata/importlib/_envs.py

@@ -151,7 +151,8 @@ def _emit_egg_deprecation(location: Optional[str]) -> None:
     deprecated(
         reason=f"Loading egg at {location} is deprecated.",
         replacement="to use pip for package installation.",
-        gone_in="23.3",
+        gone_in="24.3",
+        issue=12330,
     )
 
 

src/pip/_internal/models/candidate.py

@@ -27,8 +27,4 @@ class InstallationCandidate(KeyBasedCompareMixin):
         )
 
     def __str__(self) -> str:
-        return "{!r} candidate (version {} at {})".format(
-            self.name,
-            self.version,
-            self.link,
-        )
+        return f"{self.name!r} candidate (version {self.version} at {self.link})"

src/pip/_internal/models/direct_url.py

@@ -31,9 +31,7 @@ def _get(
     value = d[key]
     if not isinstance(value, expected_type):
         raise DirectUrlValidationError(
-            "{!r} has unexpected type for {} (expected {})".format(
-                value, key, expected_type
-            )
+            f"{value!r} has unexpected type for {key} (expected {expected_type})"
         )
     return value
 

src/pip/_internal/models/format_control.py

@@ -33,9 +33,7 @@ class FormatControl:
         return all(getattr(self, k) == getattr(other, k) for k in self.__slots__)
 
     def __repr__(self) -> str:
-        return "{}({}, {})".format(
-            self.__class__.__name__, self.no_binary, self.only_binary
-        )
+        return f"{self.__class__.__name__}({self.no_binary}, {self.only_binary})"
 
     @staticmethod
     def handle_mutual_excludes(value: str, target: Set[str], other: Set[str]) -> None:

src/pip/_internal/models/link.py

@@ -368,9 +368,7 @@ class Link(KeyBasedCompareMixin):
         else:
             rp = ""
         if self.comes_from:
-            return "{} (from {}){}".format(
-                redact_auth_from_url(self._url), self.comes_from, rp
-            )
+            return f"{redact_auth_from_url(self._url)} (from {self.comes_from}){rp}"
         else:
             return redact_auth_from_url(str(self._url))
 

src/pip/_internal/network/cache.py

@@ -33,6 +33,18 @@ class SafeFileCache(SeparateBodyBaseCache):
     """
     A file based cache which is safe to use even when the target directory may
     not be accessible or writable.
+
+    There is a race condition when two processes try to write and/or read the
+    same entry at the same time, since each entry consists of two separate
+    files (https://github.com/psf/cachecontrol/issues/324).  We therefore have
+    additional logic that makes sure that both files to be present before
+    returning an entry; this fixes the read side of the race condition.
+
+    For the write side, we assume that the server will only ever return the
+    same data for the same URL, which ought to be the case for files pip is
+    downloading.  PyPI does not have a mechanism to swap out a wheel for
+    another wheel, for example.  If this assumption is not true, the
+    CacheControl issue will need to be fixed.
     """
 
     def __init__(self, directory: str) -> None:
@@ -49,9 +61,13 @@ class SafeFileCache(SeparateBodyBaseCache):
         return os.path.join(self.directory, *parts)
 
     def get(self, key: str) -> Optional[bytes]:
-        path = self._get_cache_path(key)
+        # The cache entry is only valid if both metadata and body exist.
+        metadata_path = self._get_cache_path(key)
+        body_path = metadata_path + ".body"
+        if not (os.path.exists(metadata_path) and os.path.exists(body_path)):
+            return None
         with suppressed_cache_errors():
-            with open(path, "rb") as f:
+            with open(metadata_path, "rb") as f:
                 return f.read()
 
     def _write(self, path: str, data: bytes) -> None:
@@ -77,9 +93,13 @@ class SafeFileCache(SeparateBodyBaseCache):
             os.remove(path + ".body")
 
     def get_body(self, key: str) -> Optional[BinaryIO]:
-        path = self._get_cache_path(key) + ".body"
+        # The cache entry is only valid if both metadata and body exist.
+        metadata_path = self._get_cache_path(key)
+        body_path = metadata_path + ".body"
+        if not (os.path.exists(metadata_path) and os.path.exists(body_path)):
+            return None
         with suppressed_cache_errors():
-            return open(path, "rb")
+            return open(body_path, "rb")
 
     def set_body(self, key: str, body: bytes) -> None:
         path = self._get_cache_path(key) + ".body"

src/pip/_internal/network/download.py

@@ -42,7 +42,7 @@ def _prepare_download(
     logged_url = redact_auth_from_url(url)
 
     if total_length:
-        logged_url = "{} ({})".format(logged_url, format_size(total_length))
+        logged_url = f"{logged_url} ({format_size(total_length)})"
 
     if is_from_cache(resp):
         logger.info("Using cached %s", logged_url)

src/pip/_internal/network/xmlrpc.py

@@ -13,6 +13,8 @@ from pip._internal.network.utils import raise_for_status
 if TYPE_CHECKING:
     from xmlrpc.client import _HostType, _Marshallable
 
+    from _typeshed import SizedBuffer
+
 logger = logging.getLogger(__name__)
 
 
@@ -33,7 +35,7 @@ class PipXmlrpcTransport(xmlrpc.client.Transport):
         self,
         host: "_HostType",
         handler: str,
-        request_body: bytes,
+        request_body: "SizedBuffer",
         verbose: bool = False,
     ) -> Tuple["_Marshallable", ...]:
         assert isinstance(host, str)

src/pip/_internal/operations/build/metadata.py

@@ -10,7 +10,10 @@ from pip._internal.exceptions import (
     InstallationSubprocessError,
     MetadataGenerationFailed,
 )
-from pip._internal.utils.subprocess import runner_with_spinner_message
+from pip._internal.utils.subprocess import (
+    log_backend_warnings,
+    runner_with_spinner_message,
+)
 from pip._internal.utils.temp_dir import TempDirectory
 
 
@@ -25,7 +28,7 @@ def generate_metadata(
 
     metadata_dir = metadata_tmpdir.path
 
-    with build_env:
+    with build_env, log_backend_warnings():
         # Note that BuildBackendHookCaller implements a fallback for
         # prepare_metadata_for_build_wheel, so we don't have to
         # consider the possibility that this hook doesn't exist.

src/pip/_internal/operations/build/metadata_editable.py

@@ -10,7 +10,10 @@ from pip._internal.exceptions import (
     InstallationSubprocessError,
     MetadataGenerationFailed,
 )
-from pip._internal.utils.subprocess import runner_with_spinner_message
+from pip._internal.utils.subprocess import (
+    log_backend_warnings,
+    runner_with_spinner_message,
+)
 from pip._internal.utils.temp_dir import TempDirectory
 
 
@@ -25,7 +28,7 @@ def generate_editable_metadata(
 
     metadata_dir = metadata_tmpdir.path
 
-    with build_env:
+    with build_env, log_backend_warnings():
         # Note that BuildBackendHookCaller implements a fallback for
         # prepare_metadata_for_build_wheel/editable, so we don't have to
         # consider the possibility that this hook doesn't exist.

src/pip/_internal/operations/build/metadata_legacy.py

@@ -12,7 +12,7 @@ from pip._internal.exceptions import (
     MetadataGenerationFailed,
 )
 from pip._internal.utils.setuptools_build import make_setuptools_egg_info_args
-from pip._internal.utils.subprocess import call_subprocess
+from pip._internal.utils.subprocess import call_subprocess, log_backend_warnings
 from pip._internal.utils.temp_dir import TempDirectory
 
 logger = logging.getLogger(__name__)
@@ -58,7 +58,7 @@ def generate_metadata(
         no_user_config=isolated,
     )
 
-    with build_env:
+    with build_env, log_backend_warnings():
         with open_spinner("Preparing metadata (setup.py)") as spinner:
             try:
                 call_subprocess(

src/pip/_internal/operations/check.py

@@ -168,7 +168,7 @@ def warn_legacy_versions_and_specifiers(package_set: PackageSet) -> None:
                     f"release a version with a conforming version number"
                 ),
                 issue=12063,
-                gone_in="23.3",
+                gone_in="24.0",
             )
         for dep in package_details.dependencies:
             if any(isinstance(spec, LegacySpecifier) for spec in dep.specifier):
@@ -183,5 +183,5 @@ def warn_legacy_versions_and_specifiers(package_set: PackageSet) -> None:
                         f"release a version with a conforming dependency specifiers"
                     ),
                     issue=12063,
-                    gone_in="23.3",
+                    gone_in="24.0",
                 )

src/pip/_internal/operations/install/editable_legacy.py

@@ -6,7 +6,7 @@ from typing import Optional, Sequence
 from pip._internal.build_env import BuildEnvironment
 from pip._internal.utils.logging import indent_log
 from pip._internal.utils.setuptools_build import make_setuptools_develop_args
-from pip._internal.utils.subprocess import call_subprocess
+from pip._internal.utils.subprocess import call_subprocess, log_backend_warnings
 
 logger = logging.getLogger(__name__)
 
@@ -38,7 +38,7 @@ def install_editable(
     )
 
     with indent_log():
-        with build_env:
+        with build_env, log_backend_warnings():
             call_subprocess(
                 args,
                 command_desc="python setup.py develop",

src/pip/_internal/operations/install/wheel.py

@@ -164,16 +164,14 @@ def message_about_scripts_not_on_PATH(scripts: Sequence[str]) -> Optional[str]:
     for parent_dir, dir_scripts in warn_for.items():
         sorted_scripts: List[str] = sorted(dir_scripts)
         if len(sorted_scripts) == 1:
-            start_text = "script {} is".format(sorted_scripts[0])
+            start_text = f"script {sorted_scripts[0]} is"
         else:
             start_text = "scripts {} are".format(
                 ", ".join(sorted_scripts[:-1]) + " and " + sorted_scripts[-1]
             )
 
         msg_lines.append(
-            "The {} installed in '{}' which is not on PATH.".format(
-                start_text, parent_dir
-            )
+            f"The {start_text} installed in '{parent_dir}' which is not on PATH."
         )
 
     last_line_fmt = (
@@ -321,9 +319,7 @@ def get_console_script_specs(console: Dict[str, str]) -> List[str]:
             scripts_to_generate.append("pip = " + pip_script)
 
         if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall":
-            scripts_to_generate.append(
-                "pip{} = {}".format(sys.version_info[0], pip_script)
-            )
+            scripts_to_generate.append(f"pip{sys.version_info[0]} = {pip_script}")
 
         scripts_to_generate.append(f"pip{get_major_minor_version()} = {pip_script}")
         # Delete any other versioned pip entry points
@@ -336,9 +332,7 @@ def get_console_script_specs(console: Dict[str, str]) -> List[str]:
             scripts_to_generate.append("easy_install = " + easy_install_script)
 
         scripts_to_generate.append(
-            "easy_install-{} = {}".format(
-                get_major_minor_version(), easy_install_script
-            )
+            f"easy_install-{get_major_minor_version()} = {easy_install_script}"
         )
         # Delete any other versioned easy_install entry points
         easy_install_ep = [
@@ -408,10 +402,10 @@ class ScriptFile:
 class MissingCallableSuffix(InstallationError):
     def __init__(self, entry_point: str) -> None:
         super().__init__(
-            "Invalid script entry point: {} - A callable "
+            f"Invalid script entry point: {entry_point} - A callable "
             "suffix is required. Cf https://packaging.python.org/"
             "specifications/entry-points/#use-for-scripts for more "
-            "information.".format(entry_point)
+            "information."
         )
 
 
@@ -712,7 +706,7 @@ def req_error_context(req_description: str) -> Generator[None, None, None]:
     try:
         yield
     except InstallationError as e:
-        message = "For req: {}. {}".format(req_description, e.args[0])
+        message = f"For req: {req_description}. {e.args[0]}"
         raise InstallationError(message) from e
 
 

src/pip/_internal/operations/prepare.py

@@ -47,6 +47,7 @@ from pip._internal.utils.misc import (
     display_path,
     hash_file,
     hide_url,
+    redact_auth_from_requirement,
 )
 from pip._internal.utils.temp_dir import TempDirectory
 from pip._internal.utils.unpacking import unpack_file
@@ -277,7 +278,7 @@ class RequirementPreparer:
             information = str(display_path(req.link.file_path))
         else:
             message = "Collecting %s"
-            information = str(req.req or req)
+            information = redact_auth_from_requirement(req.req) if req.req else str(req)
 
         # If we used req.req, inject requirement source if available (this
         # would already be included if we used req directly)
@@ -602,8 +603,8 @@ class RequirementPreparer:
                 )
             except NetworkConnectionError as exc:
                 raise InstallationError(
-                    "Could not install requirement {} because of HTTP "
-                    "error {} for URL {}".format(req, exc, link)
+                    f"Could not install requirement {req} because of HTTP "
+                    f"error {exc} for URL {link}"
                 )
         else:
             file_path = self._downloaded[link.url]
@@ -683,9 +684,9 @@ class RequirementPreparer:
         with indent_log():
             if self.require_hashes:
                 raise InstallationError(
-                    "The editable requirement {} cannot be installed when "
+                    f"The editable requirement {req} cannot be installed when "
                     "requiring hashes, because there is no single file to "
-                    "hash.".format(req)
+                    "hash."
                 )
             req.ensure_has_source_dir(self.src_dir)
             req.update_editable()
@@ -713,7 +714,7 @@ class RequirementPreparer:
         assert req.satisfied_by, "req should have been satisfied but isn't"
         assert skip_reason is not None, (
             "did not get skip reason skipped but req.satisfied_by "
-            "is set to {}".format(req.satisfied_by)
+            f"is set to {req.satisfied_by}"
         )
         logger.info(
             "Requirement %s: %s (%s)", skip_reason, req, req.satisfied_by.version

src/pip/_internal/req/constructors.py

@@ -8,10 +8,11 @@ These are meant to be used elsewhere within pip to create instances of
 InstallRequirement.
 """
 
+import copy
 import logging
 import os
 import re
-from typing import Dict, List, Optional, Set, Tuple, Union
+from typing import Collection, Dict, List, Optional, Set, Tuple, Union
 
 from pip._vendor.packaging.markers import Marker
 from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
@@ -57,6 +58,31 @@ def convert_extras(extras: Optional[str]) -> Set[str]:
     return get_requirement("placeholder" + extras.lower()).extras
 
 
+def _set_requirement_extras(req: Requirement, new_extras: Set[str]) -> Requirement:
+    """
+    Returns a new requirement based on the given one, with the supplied extras. If the
+    given requirement already has extras those are replaced (or dropped if no new extras
+    are given).
+    """
+    match: Optional[re.Match[str]] = re.fullmatch(
+        # see https://peps.python.org/pep-0508/#complete-grammar
+        r"([\w\t .-]+)(\[[^\]]*\])?(.*)",
+        str(req),
+        flags=re.ASCII,
+    )
+    # ireq.req is a valid requirement so the regex should always match
+    assert (
+        match is not None
+    ), f"regex match on requirement {req} failed, this should never happen"
+    pre: Optional[str] = match.group(1)
+    post: Optional[str] = match.group(3)
+    assert (
+        pre is not None and post is not None
+    ), f"regex group selection for requirement {req} failed, this should never happen"
+    extras: str = "[%s]" % ",".join(sorted(new_extras)) if new_extras else ""
+    return Requirement(f"{pre}{extras}{post}")
+
+
 def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]:
     """Parses an editable requirement into:
         - a requirement name
@@ -436,7 +462,7 @@ def install_req_from_req_string(
         raise InstallationError(
             "Packages installed from PyPI cannot depend on packages "
             "which are not also hosted on PyPI.\n"
-            "{} depends on {} ".format(comes_from.name, req)
+            f"{comes_from.name} depends on {req} "
         )
 
     return InstallRequirement(
@@ -504,3 +530,47 @@ def install_req_from_link_and_ireq(
         config_settings=ireq.config_settings,
         user_supplied=ireq.user_supplied,
     )
+
+
+def install_req_drop_extras(ireq: InstallRequirement) -> InstallRequirement:
+    """
+    Creates a new InstallationRequirement using the given template but without
+    any extras. Sets the original requirement as the new one's parent
+    (comes_from).
+    """
+    return InstallRequirement(
+        req=(
+            _set_requirement_extras(ireq.req, set()) if ireq.req is not None else None
+        ),
+        comes_from=ireq,
+        editable=ireq.editable,
+        link=ireq.link,
+        markers=ireq.markers,
+        use_pep517=ireq.use_pep517,
+        isolated=ireq.isolated,
+        global_options=ireq.global_options,
+        hash_options=ireq.hash_options,
+        constraint=ireq.constraint,
+        extras=[],
+        config_settings=ireq.config_settings,
+        user_supplied=ireq.user_supplied,
+        permit_editable_wheels=ireq.permit_editable_wheels,
+    )
+
+
+def install_req_extend_extras(
+    ireq: InstallRequirement,
+    extras: Collection[str],
+) -> InstallRequirement:
+    """
+    Returns a copy of an installation requirement with some additional extras.
+    Makes a shallow copy of the ireq object.
+    """
+    result = copy.copy(ireq)
+    result.extras = {*ireq.extras, *extras}
+    result.req = (
+        _set_requirement_extras(ireq.req, result.extras)
+        if ireq.req is not None
+        else None
+    )
+    return result

src/pip/_internal/req/req_install.py

@@ -49,10 +49,14 @@ from pip._internal.utils.misc import (
     display_path,
     hide_url,
     is_installable_dir,
+    redact_auth_from_requirement,
     redact_auth_from_url,
 )
 from pip._internal.utils.packaging import safe_extra
-from pip._internal.utils.subprocess import runner_with_spinner_message
+from pip._internal.utils.subprocess import (
+    log_backend_warnings,
+    runner_with_spinner_message,
+)
 from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
 from pip._internal.utils.unpacking import unpack_file
 from pip._internal.utils.virtualenv import running_under_virtualenv
@@ -188,9 +192,9 @@ class InstallRequirement:
 
     def __str__(self) -> str:
         if self.req:
-            s = str(self.req)
+            s = redact_auth_from_requirement(self.req)
             if self.link:
-                s += " from {}".format(redact_auth_from_url(self.link.url))
+                s += f" from {redact_auth_from_url(self.link.url)}"
         elif self.link:
             s = redact_auth_from_url(self.link.url)
         else:
@@ -220,7 +224,7 @@ class InstallRequirement:
         attributes = vars(self)
         names = sorted(attributes)
 
-        state = ("{}={!r}".format(attr, attributes[attr]) for attr in sorted(names))
+        state = (f"{attr}={attributes[attr]!r}" for attr in sorted(names))
         return "<{name} object: {{{state}}}>".format(
             name=self.__class__.__name__,
             state=", ".join(state),
@@ -238,7 +242,7 @@ class InstallRequirement:
         if not self.use_pep517:
             return False
         assert self.pep517_backend
-        with self.build_env:
+        with self.build_env, log_backend_warnings():
             runner = runner_with_spinner_message(
                 "Checking if build backend supports build_editable"
             )
@@ -514,7 +518,7 @@ class InstallRequirement:
                         "to use --use-pep517 or add a "
                         "pyproject.toml file to the project"
                     ),
-                    gone_in="23.3",
+                    gone_in="24.0",
                 )
             self.use_pep517 = False
             return
@@ -753,8 +757,8 @@ class InstallRequirement:
 
         if os.path.exists(archive_path):
             response = ask_path_exists(
-                "The file {} exists. (i)gnore, (w)ipe, "
-                "(b)ackup, (a)bort ".format(display_path(archive_path)),
+                f"The file {display_path(archive_path)} exists. (i)gnore, (w)ipe, "
+                "(b)ackup, (a)bort ",
                 ("i", "w", "b", "a"),
             )
             if response == "i":
@@ -904,7 +908,7 @@ def check_legacy_setup_py_options(
             reason="--build-option and --global-option are deprecated.",
             issue=11859,
             replacement="to use --config-settings",
-            gone_in="23.3",
+            gone_in="24.0",
         )
         logger.warning(
             "Implying --no-binary=:all: due to the presence of "

src/pip/_internal/req/req_set.py

@@ -99,7 +99,7 @@ class RequirementSet:
                         "or contact the package author to fix the version number"
                     ),
                     issue=12063,
-                    gone_in="23.3",
+                    gone_in="24.0",
                 )
             for dep in req.get_dist().iter_dependencies():
                 if any(isinstance(spec, LegacySpecifier) for spec in dep.specifier):
@@ -115,5 +115,5 @@ class RequirementSet:
                             "or contact the package author to fix the version number"
                         ),
                         issue=12063,
-                        gone_in="23.3",
+                        gone_in="24.0",
                     )

src/pip/_internal/req/req_uninstall.py

@@ -71,16 +71,16 @@ def uninstallation_paths(dist: BaseDistribution) -> Generator[str, None, None]:
 
     entries = dist.iter_declared_entries()
     if entries is None:
-        msg = "Cannot uninstall {dist}, RECORD file not found.".format(dist=dist)
+        msg = f"Cannot uninstall {dist}, RECORD file not found."
         installer = dist.installer
         if not installer or installer == "pip":
-            dep = "{}=={}".format(dist.raw_name, dist.version)
+            dep = f"{dist.raw_name}=={dist.version}"
             msg += (
                 " You might be able to recover from this via: "
-                "'pip install --force-reinstall --no-deps {}'.".format(dep)
+                f"'pip install --force-reinstall --no-deps {dep}'."
             )
         else:
-            msg += " Hint: The package was installed by {}.".format(installer)
+            msg += f" Hint: The package was installed by {installer}."
         raise UninstallationError(msg)
 
     for entry in entries:

src/pip/_internal/resolution/legacy/resolver.py

@@ -231,9 +231,7 @@ class Resolver(BaseResolver):
             tags = compatibility_tags.get_supported()
             if requirement_set.check_supported_wheels and not wheel.supported(tags):
                 raise InstallationError(
-                    "{} is not a supported wheel on this platform.".format(
-                        wheel.filename
-                    )
+                    f"{wheel.filename} is not a supported wheel on this platform."
                 )
 
         # This next bit is really a sanity check.
@@ -287,9 +285,9 @@ class Resolver(BaseResolver):
         )
         if does_not_satisfy_constraint:
             raise InstallationError(
-                "Could not satisfy constraints for '{}': "
+                f"Could not satisfy constraints for '{install_req.name}': "
                 "installation from path or url cannot be "
-                "constrained to a version".format(install_req.name)
+                "constrained to a version"
             )
         # If we're now installing a constraint, mark the existing
         # object for real installation.
@@ -398,9 +396,9 @@ class Resolver(BaseResolver):
                 # "UnicodeEncodeError: 'ascii' codec can't encode character"
                 # in Python 2 when the reason contains non-ascii characters.
                 "The candidate selected for download or install is a "
-                "yanked version: {candidate}\n"
-                "Reason for being yanked: {reason}"
-            ).format(candidate=best_candidate, reason=reason)
+                f"yanked version: {best_candidate}\n"
+                f"Reason for being yanked: {reason}"
+            )
             logger.warning(msg)
 
         return link

src/pip/_internal/resolution/resolvelib/candidates.py

@@ -159,10 +159,7 @@ class _InstallRequirementBackedCandidate(Candidate):
         return f"{self.name} {self.version}"
 
     def __repr__(self) -> str:
-        return "{class_name}({link!r})".format(
-            class_name=self.__class__.__name__,
-            link=str(self._link),
-        )
+        return f"{self.__class__.__name__}({str(self._link)!r})"
 
     def __hash__(self) -> int:
         return hash((self.__class__, self._link))
@@ -240,7 +237,7 @@ class _InstallRequirementBackedCandidate(Candidate):
     def iter_dependencies(self, with_requires: bool) -> Iterable[Optional[Requirement]]:
         requires = self.dist.iter_dependencies() if with_requires else ()
         for r in requires:
-            yield self._factory.make_requirement_from_spec(str(r), self._ireq)
+            yield from self._factory.make_requirements_from_spec(str(r), self._ireq)
         yield self._factory.make_requires_python_requirement(self.dist.requires_python)
 
     def get_install_requirement(self) -> Optional[InstallRequirement]:
@@ -354,10 +351,7 @@ class AlreadyInstalledCandidate(Candidate):
         return str(self.dist)
 
     def __repr__(self) -> str:
-        return "{class_name}({distribution!r})".format(
-            class_name=self.__class__.__name__,
-            distribution=self.dist,
-        )
+        return f"{self.__class__.__name__}({self.dist!r})"
 
     def __hash__(self) -> int:
         return hash((self.__class__, self.name, self.version))
@@ -392,7 +386,7 @@ class AlreadyInstalledCandidate(Candidate):
         if not with_requires:
             return
         for r in self.dist.iter_dependencies():
-            yield self._factory.make_requirement_from_spec(str(r), self._ireq)
+            yield from self._factory.make_requirements_from_spec(str(r), self._ireq)
 
     def get_install_requirement(self) -> Optional[InstallRequirement]:
         return None
@@ -427,7 +421,17 @@ class ExtrasCandidate(Candidate):
         self,
         base: BaseCandidate,
         extras: FrozenSet[str],
+        *,
+        comes_from: Optional[InstallRequirement] = None,
     ) -> None:
+        """
+        :param comes_from: the InstallRequirement that led to this candidate if it
+            differs from the base's InstallRequirement. This will often be the
+            case in the sense that this candidate's requirement has the extras
+            while the base's does not. Unlike the InstallRequirement backed
+            candidates, this requirement is used solely for reporting purposes,
+            it does not do any leg work.
+        """
         self.base = base
         self.extras = frozenset(canonicalize_name(e) for e in extras)
         # If any extras are requested in their non-normalized forms, keep track
@@ -438,17 +442,14 @@ class ExtrasCandidate(Candidate):
         # TODO: Remove this attribute when packaging is upgraded to support the
         # marker comparison logic specified in PEP 685.
         self._unnormalized_extras = extras.difference(self.extras)
+        self._comes_from = comes_from if comes_from is not None else self.base._ireq
 
     def __str__(self) -> str:
         name, rest = str(self.base).split(" ", 1)
         return "{}[{}] {}".format(name, ",".join(self.extras), rest)
 
     def __repr__(self) -> str:
-        return "{class_name}(base={base!r}, extras={extras!r})".format(
-            class_name=self.__class__.__name__,
-            base=self.base,
-            extras=self.extras,
-        )
+        return f"{self.__class__.__name__}(base={self.base!r}, extras={self.extras!r})"
 
     def __hash__(self) -> int:
         return hash((self.base, self.extras))
@@ -543,11 +544,11 @@ class ExtrasCandidate(Candidate):
 
         valid_extras = self._calculate_valid_requested_extras()
         for r in self.base.dist.iter_dependencies(valid_extras):
-            requirement = factory.make_requirement_from_spec(
-                str(r), self.base._ireq, valid_extras
+            yield from factory.make_requirements_from_spec(
+                str(r),
+                self._comes_from,
+                valid_extras,
             )
-            if requirement:
-                yield requirement
 
     def get_install_requirement(self) -> Optional[InstallRequirement]:
         # We don't return anything here, because we always

src/pip/_internal/resolution/resolvelib/factory.py

@@ -62,6 +62,7 @@ from .requirements import (
     ExplicitRequirement,
     RequiresPythonRequirement,
     SpecifierRequirement,
+    SpecifierWithoutExtrasRequirement,
     UnsatisfiableRequirement,
 )
 
@@ -141,12 +142,14 @@ class Factory:
         self,
         base: BaseCandidate,
         extras: FrozenSet[str],
+        *,
+        comes_from: Optional[InstallRequirement] = None,
     ) -> ExtrasCandidate:
         cache_key = (id(base), frozenset(canonicalize_name(e) for e in extras))
         try:
             candidate = self._extras_candidate_cache[cache_key]
         except KeyError:
-            candidate = ExtrasCandidate(base, extras)
+            candidate = ExtrasCandidate(base, extras, comes_from=comes_from)
             self._extras_candidate_cache[cache_key] = candidate
         return candidate
 
@@ -163,7 +166,7 @@ class Factory:
             self._installed_candidate_cache[dist.canonical_name] = base
         if not extras:
             return base
-        return self._make_extras_candidate(base, extras)
+        return self._make_extras_candidate(base, extras, comes_from=template)
 
     def _make_candidate_from_link(
         self,
@@ -225,7 +228,7 @@ class Factory:
 
         if not extras:
             return base
-        return self._make_extras_candidate(base, extras)
+        return self._make_extras_candidate(base, extras, comes_from=template)
 
     def _iter_found_candidates(
         self,
@@ -387,16 +390,21 @@ class Factory:
             if ireq is not None:
                 ireqs.append(ireq)
 
-        # If the current identifier contains extras, add explicit candidates
-        # from entries from extra-less identifier.
+        # If the current identifier contains extras, add requires and explicit
+        # candidates from entries from extra-less identifier.
         with contextlib.suppress(InvalidRequirement):
             parsed_requirement = get_requirement(identifier)
-            explicit_candidates.update(
-                self._iter_explicit_candidates_from_base(
-                    requirements.get(parsed_requirement.name, ()),
-                    frozenset(parsed_requirement.extras),
-                ),
-            )
+            if parsed_requirement.name != identifier:
+                explicit_candidates.update(
+                    self._iter_explicit_candidates_from_base(
+                        requirements.get(parsed_requirement.name, ()),
+                        frozenset(parsed_requirement.extras),
+                    ),
+                )
+                for req in requirements.get(parsed_requirement.name, []):
+                    _, ireq = req.get_candidate_lookup()
+                    if ireq is not None:
+                        ireqs.append(ireq)
 
         # Add explicit candidates from constraints. We only do this if there are
         # known ireqs, which represent requirements not already explicit. If
@@ -439,37 +447,49 @@ class Factory:
             and all(req.is_satisfied_by(c) for req in requirements[identifier])
         )
 
-    def _make_requirement_from_install_req(
+    def _make_requirements_from_install_req(
         self, ireq: InstallRequirement, requested_extras: Iterable[str]
-    ) -> Optional[Requirement]:
+    ) -> Iterator[Requirement]:
+        """
+        Returns requirement objects associated with the given InstallRequirement. In
+        most cases this will be a single object but the following special cases exist:
+            - the InstallRequirement has markers that do not apply -> result is empty
+            - the InstallRequirement has both a constraint and extras -> result is split
+                in two requirement objects: one with the constraint and one with the
+                extra. This allows centralized constraint handling for the base,
+                resulting in fewer candidate rejections.
+        """
         if not ireq.match_markers(requested_extras):
             logger.info(
                 "Ignoring %s: markers '%s' don't match your environment",
                 ireq.name,
                 ireq.markers,
             )
-            return None
-        if not ireq.link:
-            return SpecifierRequirement(ireq)
-        self._fail_if_link_is_unsupported_wheel(ireq.link)
-        cand = self._make_candidate_from_link(
-            ireq.link,
-            extras=frozenset(ireq.extras),
-            template=ireq,
-            name=canonicalize_name(ireq.name) if ireq.name else None,
-            version=None,
-        )
-        if cand is None:
-            # There's no way we can satisfy a URL requirement if the underlying
-            # candidate fails to build. An unnamed URL must be user-supplied, so
-            # we fail eagerly. If the URL is named, an unsatisfiable requirement
-            # can make the resolver do the right thing, either backtrack (and
-            # maybe find some other requirement that's buildable) or raise a
-            # ResolutionImpossible eventually.
-            if not ireq.name:
-                raise self._build_failures[ireq.link]
-            return UnsatisfiableRequirement(canonicalize_name(ireq.name))
-        return self.make_requirement_from_candidate(cand)
+        elif not ireq.link:
+            if ireq.extras and ireq.req is not None and ireq.req.specifier:
+                yield SpecifierWithoutExtrasRequirement(ireq)
+            yield SpecifierRequirement(ireq)
+        else:
+            self._fail_if_link_is_unsupported_wheel(ireq.link)
+            cand = self._make_candidate_from_link(
+                ireq.link,
+                extras=frozenset(ireq.extras),
+                template=ireq,
+                name=canonicalize_name(ireq.name) if ireq.name else None,
+                version=None,
+            )
+            if cand is None:
+                # There's no way we can satisfy a URL requirement if the underlying
+                # candidate fails to build. An unnamed URL must be user-supplied, so
+                # we fail eagerly. If the URL is named, an unsatisfiable requirement
+                # can make the resolver do the right thing, either backtrack (and
+                # maybe find some other requirement that's buildable) or raise a
+                # ResolutionImpossible eventually.
+                if not ireq.name:
+                    raise self._build_failures[ireq.link]
+                yield UnsatisfiableRequirement(canonicalize_name(ireq.name))
+            else:
+                yield self.make_requirement_from_candidate(cand)
 
     def collect_root_requirements(
         self, root_ireqs: List[InstallRequirement]
@@ -490,15 +510,27 @@ class Factory:
                 else:
                     collected.constraints[name] = Constraint.from_ireq(ireq)
             else:
-                req = self._make_requirement_from_install_req(
-                    ireq,
-                    requested_extras=(),
+                reqs = list(
+                    self._make_requirements_from_install_req(
+                        ireq,
+                        requested_extras=(),
+                    )
                 )
-                if req is None:
+                if not reqs:
                     continue
-                if ireq.user_supplied and req.name not in collected.user_requested:
-                    collected.user_requested[req.name] = i
-                collected.requirements.append(req)
+                template = reqs[0]
+                if ireq.user_supplied and template.name not in collected.user_requested:
+                    collected.user_requested[template.name] = i
+                collected.requirements.extend(reqs)
+        # Put requirements with extras at the end of the root requires. This does not
+        # affect resolvelib's picking preference but it does affect its initial criteria
+        # population: by putting extras at the end we enable the candidate finder to
+        # present resolvelib with a smaller set of candidates to resolvelib, already
+        # taking into account any non-transient constraints on the associated base. This
+        # means resolvelib will have fewer candidates to visit and reject.
+        # Python's list sort is stable, meaning relative order is kept for objects with
+        # the same key.
+        collected.requirements.sort(key=lambda r: r.name != r.project_name)
         return collected
 
     def make_requirement_from_candidate(
@@ -506,14 +538,23 @@ class Factory:
     ) -> ExplicitRequirement:
         return ExplicitRequirement(candidate)
 
-    def make_requirement_from_spec(
+    def make_requirements_from_spec(
         self,
         specifier: str,
         comes_from: Optional[InstallRequirement],
         requested_extras: Iterable[str] = (),
-    ) -> Optional[Requirement]:
+    ) -> Iterator[Requirement]:
+        """
+        Returns requirement objects associated with the given specifier. In most cases
+        this will be a single object but the following special cases exist:
+            - the specifier has markers that do not apply -> result is empty
+            - the specifier has both a constraint and extras -> result is split
+                in two requirement objects: one with the constraint and one with the
+                extra. This allows centralized constraint handling for the base,
+                resulting in fewer candidate rejections.
+        """
         ireq = self._make_install_req_from_spec(specifier, comes_from)
-        return self._make_requirement_from_install_req(ireq, requested_extras)
+        return self._make_requirements_from_install_req(ireq, requested_extras)
 
     def make_requires_python_requirement(
         self,
@@ -712,8 +753,8 @@ class Factory:
             info = "the requested packages"
 
         msg = (
-            "Cannot install {} because these package versions "
-            "have conflicting dependencies.".format(info)
+            f"Cannot install {info} because these package versions "
+            "have conflicting dependencies."
         )
         logger.critical(msg)
         msg = "\nThe conflict is caused by:"

src/pip/_internal/resolution/resolvelib/requirements.py

@@ -1,6 +1,7 @@
 from pip._vendor.packaging.specifiers import SpecifierSet
 from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
 
+from pip._internal.req.constructors import install_req_drop_extras
 from pip._internal.req.req_install import InstallRequirement
 
 from .base import Candidate, CandidateLookup, Requirement, format_name
@@ -14,10 +15,7 @@ class ExplicitRequirement(Requirement):
         return str(self.candidate)
 
     def __repr__(self) -> str:
-        return "{class_name}({candidate!r})".format(
-            class_name=self.__class__.__name__,
-            candidate=self.candidate,
-        )
+        return f"{self.__class__.__name__}({self.candidate!r})"
 
     @property
     def project_name(self) -> NormalizedName:
@@ -43,16 +41,13 @@ class SpecifierRequirement(Requirement):
     def __init__(self, ireq: InstallRequirement) -> None:
         assert ireq.link is None, "This is a link, not a specifier"
         self._ireq = ireq
-        self._extras = frozenset(canonicalize_name(e) for e in ireq.extras)
+        self._extras = frozenset(canonicalize_name(e) for e in self._ireq.extras)
 
     def __str__(self) -> str:
         return str(self._ireq.req)
 
     def __repr__(self) -> str:
-        return "{class_name}({requirement!r})".format(
-            class_name=self.__class__.__name__,
-            requirement=str(self._ireq.req),
-        )
+        return f"{self.__class__.__name__}({str(self._ireq.req)!r})"
 
     @property
     def project_name(self) -> NormalizedName:
@@ -92,6 +87,18 @@ class SpecifierRequirement(Requirement):
         return spec.contains(candidate.version, prereleases=True)
 
 
+class SpecifierWithoutExtrasRequirement(SpecifierRequirement):
+    """
+    Requirement backed by an install requirement on a base package.
+    Trims extras from its install requirement if there are any.
+    """
+
+    def __init__(self, ireq: InstallRequirement) -> None:
+        assert ireq.link is None, "This is a link, not a specifier"
+        self._ireq = install_req_drop_extras(ireq)
+        self._extras = frozenset(canonicalize_name(e) for e in self._ireq.extras)
+
+
 class RequiresPythonRequirement(Requirement):
     """A requirement representing Requires-Python metadata."""
 
@@ -103,10 +110,7 @@ class RequiresPythonRequirement(Requirement):
         return f"Python {self.specifier}"
 
     def __repr__(self) -> str:
-        return "{class_name}({specifier!r})".format(
-            class_name=self.__class__.__name__,
-            specifier=str(self.specifier),
-        )
+        return f"{self.__class__.__name__}({str(self.specifier)!r})"
 
     @property
     def project_name(self) -> NormalizedName:
@@ -142,10 +146,7 @@ class UnsatisfiableRequirement(Requirement):
         return f"{self._name} (unavailable)"
 
     def __repr__(self) -> str:
-        return "{class_name}({name!r})".format(
-            class_name=self.__class__.__name__,
-            name=str(self._name),
-        )
+        return f"{self.__class__.__name__}({str(self._name)!r})"
 
     @property
     def project_name(self) -> NormalizedName:

src/pip/_internal/resolution/resolvelib/resolver.py

@@ -1,3 +1,4 @@
+import contextlib
 import functools
 import logging
 import os
@@ -11,6 +12,7 @@ from pip._vendor.resolvelib.structs import DirectedGraph
 from pip._internal.cache import WheelCache
 from pip._internal.index.package_finder import PackageFinder
 from pip._internal.operations.prepare import RequirementPreparer
+from pip._internal.req.constructors import install_req_extend_extras
 from pip._internal.req.req_install import InstallRequirement
 from pip._internal.req.req_set import RequirementSet
 from pip._internal.resolution.base import BaseResolver, InstallRequirementProvider
@@ -19,6 +21,7 @@ from pip._internal.resolution.resolvelib.reporter import (
     PipDebuggingReporter,
     PipReporter,
 )
+from pip._internal.utils.packaging import get_requirement
 
 from .base import Candidate, Requirement
 from .factory import Factory
@@ -101,9 +104,24 @@ class Resolver(BaseResolver):
             raise error from e
 
         req_set = RequirementSet(check_supported_wheels=check_supported_wheels)
-        for candidate in result.mapping.values():
+        # process candidates with extras last to ensure their base equivalent is
+        # already in the req_set if appropriate.
+        # Python's sort is stable so using a binary key function keeps relative order
+        # within both subsets.
+        for candidate in sorted(
+            result.mapping.values(), key=lambda c: c.name != c.project_name
+        ):
             ireq = candidate.get_install_requirement()
             if ireq is None:
+                if candidate.name != candidate.project_name:
+                    # extend existing req's extras
+                    with contextlib.suppress(KeyError):
+                        req = req_set.get_requirement(candidate.project_name)
+                        req_set.add_named_requirement(
+                            install_req_extend_extras(
+                                req, get_requirement(candidate.name).extras
+                            )
+                        )
                 continue
 
             # Check if there is already an installation under the same name,

src/pip/_internal/self_outdated_check.py

@@ -39,6 +39,15 @@ def _get_statefile_name(key: str) -> str:
     return name
 
 
+def _convert_date(isodate: str) -> datetime.datetime:
+    """Convert an ISO format string to a date.
+
+    Handles the format 2020-01-22T14:24:01Z (trailing Z)
+    which is not supported by older versions of fromisoformat.
+    """
+    return datetime.datetime.fromisoformat(isodate.replace("Z", "+00:00"))
+
+
 class SelfCheckState:
     def __init__(self, cache_dir: str) -> None:
         self._state: Dict[str, Any] = {}
@@ -73,7 +82,7 @@ class SelfCheckState:
             return None
 
         # Determine if we need to refresh the state
-        last_check = datetime.datetime.fromisoformat(self._state["last_check"])
+        last_check = _convert_date(self._state["last_check"])
         time_since_last_check = current_time - last_check
         if time_since_last_check > _WEEK:
             return None
@@ -233,7 +242,7 @@ def pip_self_version_check(session: PipSession, options: optparse.Values) -> Non
             ),
         )
         if upgrade_prompt is not None:
-            logger.warning("[present-rich] %s", upgrade_prompt)
+            logger.warning("%s", upgrade_prompt, extra={"rich": True})
     except Exception:
         logger.warning("There was an error checking the latest version of pip.")
         logger.debug("See below for error", exc_info=True)

src/pip/_internal/utils/logging.py

@@ -155,8 +155,8 @@ class RichPipStreamHandler(RichHandler):
 
         # If we are given a diagnostic error to present, present it with indentation.
         assert isinstance(record.args, tuple)
-        if record.msg == "[present-rich] %s" and len(record.args) == 1:
-            rich_renderable = record.args[0]
+        if getattr(record, "rich", False):
+            (rich_renderable,) = record.args
             assert isinstance(
                 rich_renderable, (ConsoleRenderable, RichCast, str)
             ), f"{rich_renderable} is not rich-console-renderable"

src/pip/_internal/utils/misc.py

@@ -35,6 +35,7 @@ from typing import (
     cast,
 )
 
+from pip._vendor.packaging.requirements import Requirement
 from pip._vendor.pyproject_hooks import BuildBackendHookCaller
 from pip._vendor.tenacity import retry, stop_after_delay, wait_fixed
 
@@ -76,11 +77,7 @@ def get_pip_version() -> str:
     pip_pkg_dir = os.path.join(os.path.dirname(__file__), "..", "..")
     pip_pkg_dir = os.path.abspath(pip_pkg_dir)
 
-    return "pip {} from {} (python {})".format(
-        __version__,
-        pip_pkg_dir,
-        get_major_minor_version(),
-    )
+    return f"pip {__version__} from {pip_pkg_dir} (python {get_major_minor_version()})"
 
 
 def normalize_version_info(py_version_info: Tuple[int, ...]) -> Tuple[int, int, int]:
@@ -144,9 +141,9 @@ def rmtree(
     )
     if sys.version_info >= (3, 12):
         # See https://docs.python.org/3.12/whatsnew/3.12.html#shutil.
-        shutil.rmtree(dir, onexc=handler)
+        shutil.rmtree(dir, onexc=handler)  # type: ignore
     else:
-        shutil.rmtree(dir, onerror=handler)
+        shutil.rmtree(dir, onerror=handler)  # type: ignore
 
 
 def _onerror_ignore(*_args: Any) -> None:
@@ -278,13 +275,13 @@ def strtobool(val: str) -> int:
 
 def format_size(bytes: float) -> str:
     if bytes > 1000 * 1000:
-        return "{:.1f} MB".format(bytes / 1000.0 / 1000)
+        return f"{bytes / 1000.0 / 1000:.1f} MB"
     elif bytes > 10 * 1000:
-        return "{} kB".format(int(bytes / 1000))
+        return f"{int(bytes / 1000)} kB"
     elif bytes > 1000:
-        return "{:.1f} kB".format(bytes / 1000.0)
+        return f"{bytes / 1000.0:.1f} kB"
     else:
-        return "{} bytes".format(int(bytes))
+        return f"{int(bytes)} bytes"
 
 
 def tabulate(rows: Iterable[Iterable[Any]]) -> Tuple[List[str], List[int]]:
@@ -521,9 +518,7 @@ def redact_netloc(netloc: str) -> str:
     else:
         user = urllib.parse.quote(user)
         password = ":****"
-    return "{user}{password}@{netloc}".format(
-        user=user, password=password, netloc=netloc
-    )
+    return f"{user}{password}@{netloc}"
 
 
 def _transform_url(
@@ -578,13 +573,20 @@ def redact_auth_from_url(url: str) -> str:
     return _transform_url(url, _redact_netloc)[0]
 
 
+def redact_auth_from_requirement(req: Requirement) -> str:
+    """Replace the password in a given requirement url with ****."""
+    if not req.url:
+        return str(req)
+    return str(req).replace(req.url, redact_auth_from_url(req.url))
+
+
 class HiddenText:
     def __init__(self, secret: str, redacted: str) -> None:
         self.secret = secret
         self.redacted = redacted
 
     def __repr__(self) -> str:
-        return "<HiddenText {!r}>".format(str(self))
+        return f"<HiddenText {str(self)!r}>"
 
     def __str__(self) -> str:
         return self.redacted

src/pip/_internal/utils/subprocess.py

@@ -1,18 +1,24 @@
+import contextlib
 import logging
 import os
 import shlex
 import subprocess
+import warnings
 from typing import (
     TYPE_CHECKING,
     Any,
     Callable,
+    Generator,
     Iterable,
     List,
     Mapping,
     Optional,
+    TextIO,
+    Type,
     Union,
 )
 
+from pip._vendor.pyproject_hooks import BuildBackendWarning
 from pip._vendor.rich.markup import escape
 
 from pip._internal.cli.spinners import SpinnerInterface, open_spinner
@@ -209,7 +215,7 @@ def call_subprocess(
                 output_lines=all_output if not showing_subprocess else None,
             )
             if log_failed_cmd:
-                subprocess_logger.error("[present-rich] %s", error)
+                subprocess_logger.error("%s", error, extra={"rich": True})
                 subprocess_logger.verbose(
                     "[bold magenta]full command[/]: [blue]%s[/]",
                     escape(format_command_args(cmd)),
@@ -258,3 +264,24 @@ def runner_with_spinner_message(message: str) -> Callable[..., None]:
             )
 
     return runner
+
+
+@contextlib.contextmanager
+def log_backend_warnings() -> Generator[None, None, None]:
+    def showwarning(
+        message: Warning | str,
+        category: Type[Warning],
+        filename: str,
+        lineno: int,
+        file: TextIO | None = None,
+        line: str | None = None,
+    ) -> None:
+        if category == BuildBackendWarning:
+            subprocess_logger.warning(message)
+
+    try:
+        original_showwarning = warnings.showwarning
+        warnings.showwarning = showwarning
+        yield
+    finally:
+        warnings.showwarning = original_showwarning

src/pip/_internal/utils/wheel.py

@@ -28,7 +28,7 @@ def parse_wheel(wheel_zip: ZipFile, name: str) -> Tuple[str, Message]:
         metadata = wheel_metadata(wheel_zip, info_dir)
         version = wheel_version(metadata)
     except UnsupportedWheel as e:
-        raise UnsupportedWheel("{} has an invalid wheel, {}".format(name, str(e)))
+        raise UnsupportedWheel(f"{name} has an invalid wheel, {str(e)}")
 
     check_compatibility(version, name)
 
@@ -60,9 +60,7 @@ def wheel_dist_info_dir(source: ZipFile, name: str) -> str:
     canonical_name = canonicalize_name(name)
     if not info_dir_name.startswith(canonical_name):
         raise UnsupportedWheel(
-            ".dist-info directory {!r} does not start with {!r}".format(
-                info_dir, canonical_name
-            )
+            f".dist-info directory {info_dir!r} does not start with {canonical_name!r}"
         )
 
     return info_dir

src/pip/_internal/vcs/git.py

@@ -101,7 +101,7 @@ class Git(VersionControl):
         if not match:
             logger.warning("Can't parse git version: %s", version)
             return ()
-        return tuple(int(c) for c in match.groups())
+        return (int(match.group(1)), int(match.group(2)))
 
     @classmethod
     def get_current_branch(cls, location: str) -> Optional[str]:

src/pip/_internal/vcs/mercurial.py

@@ -31,7 +31,7 @@ class Mercurial(VersionControl):
 
     @staticmethod
     def get_base_rev_args(rev: str) -> List[str]:
-        return ["-r", rev]
+        return [f"--rev={rev}"]
 
     def fetch_new(
         self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int

src/pip/_internal/vcs/versioncontrol.py

@@ -405,9 +405,9 @@ class VersionControl:
         scheme, netloc, path, query, frag = urllib.parse.urlsplit(url)
         if "+" not in scheme:
             raise ValueError(
-                "Sorry, {!r} is a malformed VCS url. "
+                f"Sorry, {url!r} is a malformed VCS url. "
                 "The format is <vcs>+<protocol>://<url>, "
-                "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url)
+                "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp"
             )
         # Remove the vcs prefix.
         scheme = scheme.split("+", 1)[1]
@@ -417,9 +417,9 @@ class VersionControl:
             path, rev = path.rsplit("@", 1)
             if not rev:
                 raise InstallationError(
-                    "The URL {!r} has an empty revision (after @) "
+                    f"The URL {url!r} has an empty revision (after @) "
                     "which is not supported. Include a revision after @ "
-                    "or remove @ from the URL.".format(url)
+                    "or remove @ from the URL."
                 )
         url = urllib.parse.urlunsplit((scheme, netloc, path, query, ""))
         return url, rev, user_pass
@@ -566,7 +566,7 @@ class VersionControl:
             self.name,
             url,
         )
-        response = ask_path_exists("What to do?  {}".format(prompt[0]), prompt[1])
+        response = ask_path_exists(f"What to do?  {prompt[0]}", prompt[1])
 
         if response == "a":
             sys.exit(-1)