diff --git a/.mailmap b/.mailmap index 29f9ec039..c8f94a9d8 100644 --- a/.mailmap +++ b/.mailmap @@ -21,6 +21,8 @@ Endoh Takanao Erik M. Bray Gabriel de Perthuis Hsiaoming Yang +Hugo van Kemenade Hugo +Hugo van Kemenade hugovk Igor Kuzmitshov Ilya Baryshev Jakub Stasiak diff --git a/NEWS.rst b/NEWS.rst index e4f63f0e8..4fd3dfdc7 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -360,7 +360,7 @@ Features - When installing a git URL that refers to a commit that is not available locally after git clone, attempt to fetch it from the remote. (`#8815 `_) - Include http subdirectory in ``pip cache info`` and ``pip cache purge`` commands. (`#8892 `_) -- Cache package listings on index packages so they are guarenteed to stay stable +- Cache package listings on index packages so they are guaranteed to stay stable during a pip command session. This also improves performance when a index page is accessed multiple times during the command session. (`#8905 `_) - New resolver: Tweak resolution logic to improve user experience when @@ -432,7 +432,7 @@ Features and considered good enough. (`#8023 `_) - Improve error message friendliness when an environment has packages with corrupted metadata. (`#8676 `_) -- Cache package listings on index packages so they are guarenteed to stay stable +- Cache package listings on index packages so they are guaranteed to stay stable during a pip command session. This also improves performance when a index page is accessed multiple times during the command session. (`#8905 `_) - New resolver: Tweak resolution logic to improve user experience when diff --git a/docs/html/cli/index.md b/docs/html/cli/index.md index f608da521..a3497c308 100644 --- a/docs/html/cli/index.md +++ b/docs/html/cli/index.md @@ -17,6 +17,7 @@ pip pip_install pip_uninstall pip_list +pip_show pip_freeze pip_check ``` @@ -34,7 +35,6 @@ pip_hash :maxdepth: 1 :caption: Package Index information -pip_show pip_search ``` diff --git a/docs/html/cli/pip_install.rst b/docs/html/cli/pip_install.rst index 9ebb6e3f7..6bc15349d 100644 --- a/docs/html/cli/pip_install.rst +++ b/docs/html/cli/pip_install.rst @@ -574,62 +574,14 @@ overridden by using ``--cert`` option or by using ``PIP_CERT``, Caching ------- -Starting with v6.0, pip provides an on-by-default cache which functions -similarly to that of a web browser. While the cache is on by default and is -designed do the right thing by default you can disable the cache and always -access PyPI by utilizing the ``--no-cache-dir`` option. - -When making any HTTP request pip will first check its local cache to determine -if it has a suitable response stored for that request which has not expired. If -it does then it simply returns that response and doesn't make the request. - -If it has a response stored, but it has expired, then it will attempt to make a -conditional request to refresh the cache which will either return an empty -response telling pip to simply use the cached item (and refresh the expiration -timer) or it will return a whole new response which pip can then store in the -cache. - -While this cache attempts to minimize network activity, it does not prevent -network access altogether. If you want a local install solution that -circumvents accessing PyPI, see :ref:`Installing from local packages`. - -The default location for the cache directory depends on the operating system: - -Unix - :file:`~/.cache/pip` and it respects the ``XDG_CACHE_HOME`` directory. -macOS - :file:`~/Library/Caches/pip`. -Windows - :file:`\\pip\\Cache` - -Run ``pip cache dir`` to show the cache directory and see :ref:`pip cache` to -inspect and manage pip’s cache. - +This is now covered in :doc:`../topics/caching` .. _`Wheel cache`: Wheel Cache ^^^^^^^^^^^ -pip will read from the subdirectory ``wheels`` within the pip cache directory -and use any packages found there. This is disabled via the same -``--no-cache-dir`` option that disables the HTTP cache. The internal structure -of that is not part of the pip API. As of 7.0, pip makes a subdirectory for -each sdist that wheels are built from and places the resulting wheels inside. - -As of version 20.0, pip also caches wheels when building from an immutable Git -reference (i.e. a commit hash). - -pip attempts to choose the best wheels from those built in preference to -building a new wheel. Note that this means when a package has both optional -C extensions and builds ``py`` tagged wheels when the C extension can't be built -that pip will not attempt to build a better wheel for Pythons that would have -supported it, once any generic wheel is built. To correct this, make sure that -the wheels are built with Python specific tags - e.g. pp on PyPy. - -When no wheels are found for an sdist, pip will attempt to build a wheel -automatically and insert it into the wheel cache. - +This is now covered in :doc:`../topics/caching` .. _`hash-checking mode`: diff --git a/docs/html/conf.py b/docs/html/conf.py index 2a4387a35..64fddeffc 100644 --- a/docs/html/conf.py +++ b/docs/html/conf.py @@ -30,7 +30,7 @@ extensions = [ # General information about the project. project = "pip" -copyright = "2008-2020, PyPA" +copyright = "The pip developers" # Find the version and release information. # We have a single source of truth for our version number: pip's __init__.py file. @@ -52,6 +52,10 @@ with open(file_with_version) as f: print("pip version:", version) print("pip release:", release) +# -- Options for myst-parser ---------------------------------------------------------- + +myst_enable_extensions = ["deflist"] + # -- Options for smartquotes ---------------------------------------------------------- # Disable the conversion of dashes so that long options like "--find-links" won't diff --git a/docs/html/cookbook.rst b/docs/html/cookbook.rst deleted file mode 100644 index efd76af15..000000000 --- a/docs/html/cookbook.rst +++ /dev/null @@ -1,7 +0,0 @@ -:orphan: - -======== -Cookbook -======== - -This content is now covered in the :doc:`User Guide ` diff --git a/docs/html/getting-started.md b/docs/html/getting-started.md new file mode 100644 index 000000000..42ac2c934 --- /dev/null +++ b/docs/html/getting-started.md @@ -0,0 +1,104 @@ +# Getting Started + +To get started with using pip, you should [install Python] on your system. + +[install Python]: https://realpython.com/installing-python/ + +## Ensure you have a working pip + +As a first step, you should check that you have a working Python with pip +installed. This can be done by running the following commands and making +sure that the output looks similar. + +```{pip-cli} +$ python --version +Python 3.N.N +$ pip --version +pip X.Y.Z from ... (python 3.N.N) +``` + +If that worked, congratulations! You have a working pip in your environment. + +If you got output that does not look like the sample above, please read +the {doc}`installation` page. It provides guidance on how to install pip +within a Python environment that doesn't have it. + +## Common tasks + +### Install a package + +```{pip-cli} +$ pip install sampleproject +[...] +Successfully installed sampleproject +``` + +By default, pip will fetch packages from [Python Package Index][PyPI], a +repository of software for the Python programming language where anyone can +upload packages. + +[PyPI]: https://pypi.org/ + +### Install a package from GitHub + +```{pip-cli} +$ pip install git+https://github.com/pypa/sampleproject.git@main +[...] +Successfully installed sampleproject +``` + +See {ref}`VCS Support` for more information about this syntax. + +### Install a package from a distribution file + +pip can install directly from distribution files as well. They come in 2 forms: + +- {term}`source distribution ` (usually shortened to "sdist") +- {term}`wheel distribution ` (usually shortened to "wheel") + +```{pip-cli} +$ pip install sampleproject-1.0.tar.gz +[...] +Successfully installed sampleproject +$ pip install sampleproject-1.0-py3-none-any.whl +[...] +Successfully installed sampleproject +``` + +### Install multiple packages using a requirements file + +Many Python projects use {file}`requirements.txt` files, to specify the +list of packages that need to be installed for the project to run. To install +the packages listed in that file, you can run: + +```{pip-cli} +$ pip install -r requirements.txt +[...] +Successfully installed sampleproject +``` + +### Upgrade a package + +```{pip-cli} +$ pip install --upgrade sampleproject +Uninstalling sampleproject: + [...] +Proceed (y/n)? y +Successfully uninstalled sampleproject +``` + +### Uninstall a package + +```{pip-cli} +$ pip uninstall sampleproject +Uninstalling sampleproject: + [...] +Proceed (y/n)? y +Successfully uninstalled sampleproject +``` + +## Next Steps + +It is recommended to learn about what virtual environments are and how to use +them. This is covered in the ["Installing Packages"](pypug:tutorials/installing-packages) +tutorial on packaging.python.org. diff --git a/docs/html/index.md b/docs/html/index.md index a84c2665d..9ab5df298 100644 --- a/docs/html/index.md +++ b/docs/html/index.md @@ -10,9 +10,10 @@ install packages from the [Python Package Index][pypi] and other indexes. ```{toctree} :hidden: -quickstart -installing +getting-started +installation user_guide +topics/index cli/index ``` @@ -29,7 +30,7 @@ GitHub If you want to learn about how to use pip, check out the following resources: -- [Quickstart](quickstart) +- [Getting Started](getting-started) - [Python Packaging User Guide](https://packaging.python.org) If you find bugs, need help, or want to talk to the developers, use our mailing diff --git a/docs/html/installation.md b/docs/html/installation.md new file mode 100644 index 000000000..e389a8fa4 --- /dev/null +++ b/docs/html/installation.md @@ -0,0 +1,80 @@ +# Installation + +Usually, pip is automatically installed if you are: + +- working in a + {ref}`virtual environment ` +- using Python downloaded from [python.org](https://www.python.org) +- using Python that has not been modified by a redistributor to remove + {mod}`ensurepip` + +## Supported Methods + +If your Python environment does not have pip installed, there are 2 mechanisms +to install pip supported directly by pip's maintainers: + +- [`ensurepip`](#using-ensurepip) +- [`get-pip.py`](#using-get-pip-py) + +### `ensurepip` + +Python comes with an {mod}`ensurepip` module[^python], which can install pip in +a Python environment. + +```{pip-cli} +$ python -m ensurepip --upgrade +``` + +More details about how {mod}`ensurepip` works and how it can be used, is +available in the standard library documentation. + +### `get-pip.py` + +This is a Python script that uses some bootstrapping logic to install +pip. + +- Download the script, from . +- Open a terminal/command prompt, `cd` to the folder containing the + `get-pip.py` file and run: + + ```{pip-cli} + $ python get-pip.py + ``` + +More details about this script can be found in [pypa/get-pip]'s README. + +[pypa/get-pip]: https://github.com/pypa/get-pip + +## Alternative Methods + +Depending on how you installed Python, there might be other mechanisms +available to you for installing pip such as +{ref}`using Linux package managers `. + +These mechanisms are provided by redistributors of pip, who may have modified +pip to change its behaviour. This has been a frequent source of user confusion, +since it causes a mismatch between documented behaviour in this documentation +and how pip works after those modifications. + +If you face issues when using Python and pip installed using these mechanisms, +it is recommended to request for support from the relevant provider (eg: Linux +distro community, cloud provider support channels, etc). + +(compatibility-requirements)= + +## Compatibility + +The current version of pip works on: + +- Windows, Linux and MacOS. +- CPython 3.6, 3.7, 3.8, 3.9 and latest PyPy3. + +pip is tested to work on the latest patch version of the Python interpreter, +for each of the minor versions listed above. Previous patch versions are +supported on a best effort approach. + +pip's maintainers do not provide support for users on older versions of Python, +and these users should request for support from the relevant provider +(eg: Linux distro community, cloud provider support channels, etc). + +[^python]: The `ensurepip` module was added to the Python standard library in Python 3.4. diff --git a/docs/html/installing.rst b/docs/html/installing.rst index 95b21899d..e8d86f344 100644 --- a/docs/html/installing.rst +++ b/docs/html/installing.rst @@ -1,230 +1,11 @@ -.. _`Installation`: +:orphan: -============ -Installation -============ +.. meta:: -Do I need to install pip? -========================= + :http-equiv=refresh: 3; url=../installation/ -pip is already installed if you are using Python 2 >=2.7.9 or Python 3 >=3.4 -downloaded from `python.org `_ or if you are working -in a :ref:`Virtual Environment ` -created by :ref:`pypug:virtualenv` or :ref:`venv `. Just make sure -to :ref:`upgrade pip `. +This page has moved +=================== -Use the following command to check whether pip is installed: - -.. tab:: Unix/macOS - - .. code-block:: console - - $ python -m pip --version - pip X.Y.Z from .../site-packages/pip (python X.Y) - -.. tab:: Windows - - .. code-block:: console - - C:\> py -m pip --version - pip X.Y.Z from ...\site-packages\pip (python X.Y) - -Using Linux Package Managers -============================ - -.. warning:: - - If you installed Python from a package manager on Linux, you should always - install pip for that Python installation using the same source. - -See `pypug:Installing pip/setuptools/wheel with Linux Package Managers `_ -in the Python Packaging User Guide. - -Here are ways to contact a few Linux package maintainers if you run into -problems: - -* `Deadsnakes PPA `_ -* `Debian Python Team `_ (for general - issues related to ``apt``) -* `Red Hat Bugzilla `_ - -pip developers do not have control over how Linux distributions handle pip -installations, and are unable to provide solutions to related issues in -general. - -Using ensurepip -=============== - -Python >=3.4 can self-bootstrap pip with the built-in -:ref:`ensurepip ` module. Refer to the standard library -documentation for more details. Make sure to :ref:`upgrade pip ` -after ``ensurepip`` installs pip. - -See the `Using Linux Package Managers`_ section if your Python reports -``No module named ensurepip`` on Debian and derived systems (e.g. Ubuntu). - - -.. _`get-pip`: - -Installing with get-pip.py -========================== - -.. warning:: - - Be cautious if you are using a Python install that is managed by your operating - system or another package manager. ``get-pip.py`` does not coordinate with - those tools, and may leave your system in an inconsistent state. - -To manually install pip, securely [1]_ download ``get-pip.py`` by following -this link: `get-pip.py -`_. Alternatively, use ``curl``:: - - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py - -Then run the following command in the folder where you -have downloaded ``get-pip.py``: - -.. tab:: Unix/macOS - - .. code-block:: shell - - python get-pip.py - -.. tab:: Windows - - .. code-block:: shell - - py get-pip.py - -``get-pip.py`` also installs :ref:`pypug:setuptools` [2]_ and :ref:`pypug:wheel` -if they are not already. :ref:`pypug:setuptools` is required to install -:term:`source distributions `. Both are -required in order to build a :ref:`Wheel cache` (which improves installation -speed), although neither are required to install pre-built :term:`wheels -`. - -.. note:: - - The get-pip.py script is supported on the same python version as pip. - For the now unsupported Python 2.6, alternate script is available - `here `__. - - -get-pip.py options ------------------- - -.. option:: --no-setuptools - - If set, do not attempt to install :ref:`pypug:setuptools` - -.. option:: --no-wheel - - If set, do not attempt to install :ref:`pypug:wheel` - - -``get-pip.py`` allows :ref:`pip install options ` and the :ref:`general options `. Below are -some examples: - -Install from local copies of pip and setuptools: - -.. tab:: Unix/macOS - - .. code-block:: shell - - python get-pip.py --no-index --find-links=/local/copies - -.. tab:: Windows - - .. code-block:: shell - - py get-pip.py --no-index --find-links=/local/copies - -Install to the user site [3]_: - -.. tab:: Unix/macOS - - .. code-block:: shell - - python get-pip.py --user - -.. tab:: Windows - - .. code-block:: shell - - py get-pip.py --user - -Install behind a proxy: - -.. tab:: Unix/macOS - - .. code-block:: shell - - python get-pip.py --proxy="http://[user:passwd@]proxy.server:port" - -.. tab:: Windows - - .. code-block:: shell - - py get-pip.py --proxy="http://[user:passwd@]proxy.server:port" - -``get-pip.py`` can also be used to install a specified combination of ``pip``, -``setuptools``, and ``wheel`` using the same requirements syntax as pip: - -.. tab:: Unix/macOS - - .. code-block:: shell - - python get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 - -.. tab:: Windows - - .. code-block:: shell - - py get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 - -.. _`Upgrading pip`: - -Upgrading pip -============= - -.. tab:: Unix/macOS - - .. code-block:: shell - - python -m pip install -U pip - -.. tab:: Windows - - .. code-block:: shell - - py -m pip install -U pip - - -.. _compatibility-requirements: - -Python and OS Compatibility -=========================== - -pip works with CPython versions 3.6, 3.7, 3.8, 3.9 and also PyPy. - -This means pip works on the latest patch version of each of these minor -versions. Previous patch versions are supported on a best effort approach. - -pip works on Unix/Linux, macOS, and Windows. - - ----- - -.. [1] "Secure" in this context means using a modern browser or a - tool like ``curl`` that verifies SSL certificates when downloading from - https URLs. - -.. [2] Beginning with pip v1.5.1, ``get-pip.py`` stopped requiring setuptools to - be installed first. - -.. [3] The pip developers are considering making ``--user`` the default for all - installs, including ``get-pip.py`` installs of pip, but at this time, - ``--user`` installs for pip itself, should not be considered to be fully - tested or endorsed. For discussion, see `Issue 1668 - `_. +You should be redirected automatically in 3 seconds. If that didn't +work, here's a link: :doc:`installation` diff --git a/docs/html/logic.rst b/docs/html/logic.rst deleted file mode 100644 index 189169a8c..000000000 --- a/docs/html/logic.rst +++ /dev/null @@ -1,7 +0,0 @@ -:orphan: - -================ -Internal Details -================ - -This content is now covered in the :doc:`Architecture section `. diff --git a/docs/html/quickstart.rst b/docs/html/quickstart.rst index 96602a7b3..4385f4a73 100644 --- a/docs/html/quickstart.rst +++ b/docs/html/quickstart.rst @@ -1,136 +1,11 @@ -========== -Quickstart -========== +:orphan: -First, :doc:`install pip `. +.. meta:: -Install a package from `PyPI`_: + :http-equiv=refresh: 3; url=../getting-started/ -.. tab:: Unix/macOS +This page has moved +=================== - .. code-block:: console - - $ python -m pip install SomePackage - [...] - Successfully installed SomePackage - -.. tab:: Windows - - .. code-block:: console - - C:\> py -m pip install SomePackage - [...] - Successfully installed SomePackage - - -Install a package that's already been downloaded from `PyPI`_ or -obtained from elsewhere. This is useful if the target machine does not have a -network connection: - -.. tab:: Unix/macOS - - .. code-block:: console - - $ python -m pip install SomePackage-1.0-py2.py3-none-any.whl - [...] - Successfully installed SomePackage - -.. tab:: Windows - - .. code-block:: console - - C:\> py -m pip install SomePackage-1.0-py2.py3-none-any.whl - [...] - Successfully installed SomePackage - -Show what files were installed: - -.. tab:: Unix/macOS - - .. code-block:: console - - $ python -m pip show --files SomePackage - Name: SomePackage - Version: 1.0 - Location: /my/env/lib/pythonx.x/site-packages - Files: - ../somepackage/__init__.py - [...] - -.. tab:: Windows - - .. code-block:: console - - C:\> py -m pip show --files SomePackage - Name: SomePackage - Version: 1.0 - Location: /my/env/lib/pythonx.x/site-packages - Files: - ../somepackage/__init__.py - [...] - -List what packages are outdated: - -.. tab:: Unix/macOS - - .. code-block:: console - - $ python -m pip list --outdated - SomePackage (Current: 1.0 Latest: 2.0) - -.. tab:: Windows - - .. code-block:: console - - C:\> py -m pip list --outdated - SomePackage (Current: 1.0 Latest: 2.0) - -Upgrade a package: - -.. tab:: Unix/macOS - - .. code-block:: console - - $ python -m pip install --upgrade SomePackage - [...] - Found existing installation: SomePackage 1.0 - Uninstalling SomePackage: - Successfully uninstalled SomePackage - Running setup.py install for SomePackage - Successfully installed SomePackage - -.. tab:: Windows - - .. code-block:: console - - C:\> py -m pip install --upgrade SomePackage - [...] - Found existing installation: SomePackage 1.0 - Uninstalling SomePackage: - Successfully uninstalled SomePackage - Running setup.py install for SomePackage - Successfully installed SomePackage - -Uninstall a package: - -.. tab:: Unix/macOS - - .. code-block:: console - - $ python -m pip uninstall SomePackage - Uninstalling SomePackage: - /my/env/lib/pythonx.x/site-packages/somepackage - Proceed (y/n)? y - Successfully uninstalled SomePackage - -.. tab:: Windows - - .. code-block:: console - - C:\> py -m pip uninstall SomePackage - Uninstalling SomePackage: - /my/env/lib/pythonx.x/site-packages/somepackage - Proceed (y/n)? y - Successfully uninstalled SomePackage - -.. _PyPI: https://pypi.org/ +You should be redirected automatically in 3 seconds. If that didn't +work, here's a link: :doc:`getting-started` diff --git a/docs/html/topics/authentication.md b/docs/html/topics/authentication.md new file mode 100644 index 000000000..981aab5ab --- /dev/null +++ b/docs/html/topics/authentication.md @@ -0,0 +1,83 @@ +# Authentication + +## Basic HTTP authentication + +pip supports basic HTTP-based authentication credentials. This is done by +providing the username (and optionally password) in the URL: + +``` +https://username:password@pypi.company.com/simple +``` + +For indexes that only require single-part authentication tokens, provide the +token as the "username" and do not provide a password: + +``` +https://0123456789abcdef@pypi.company.com/simple +``` + +### Percent-encoding special characters + +```{versionadded} 10.0 +``` + +Certain special characters are not valid in the credential part of a URL. +If the user or password part of your login credentials contain any of these +[special characters][reserved-chars], then they must be percent-encoded. As an +example, for a user with username `user` and password `he//o` accessing a +repository at `pypi.company.com/simple`, the URL with credentials would look +like: + +``` +https://user:he%2F%2Fo@pypi.company.com/simple +``` + +[reserved-chars]: https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_reserved_characters + +## netrc support + +pip supports loading credentials from a user's `.netrc` file. If no credentials +are part of the URL, pip will attempt to get authentication credentials for the +URL's hostname from the user's `.netrc` file. This behaviour comes from the +underlying use of {pypi}`requests`, which in turn delegates it to the +[Python standard library's `netrc` module][netrc-std-lib]. + +```{note} +As mentioned in the [standard library documentation for netrc][netrc-std-lib], +only ASCII characters are allowed in `.netrc` files. Whitespace and +non-printable characters are not allowed in passwords. +``` + +Below is an example `.netrc`, for the host `example.com`, with a user named +`daniel`, using the password `qwerty`: + +``` +machine example.com +login daniel +password qwerty +``` + +More information about the `.netrc` file format can be found in the GNU [`ftp` +man pages][netrc-docs]. + +[netrc-docs]: https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html +[netrc-std-lib]: https://docs.python.org/3/library/netrc.html + +## Keyring Support + +pip supports loading credentials stored in your keyring using the +{pypi}`keyring` library. + +```bash +$ pip install keyring # install keyring from PyPI +$ echo "your-password" | keyring set pypi.company.com your-username +$ pip install your-package --index-url https://pypi.company.com/ +``` + +Note that `keyring` (the Python package) needs to be installed separately from +pip. This can create a bootstrapping issue if you need the credentials stored in +the keyring to download and install keyring. + +It is, thus, expected that users that wish to use pip's keyring support have +some mechanism for downloading and installing {pypi}`keyring` in their Python +environment. diff --git a/docs/html/topics/caching.md b/docs/html/topics/caching.md new file mode 100644 index 000000000..0f4dfe9b9 --- /dev/null +++ b/docs/html/topics/caching.md @@ -0,0 +1,86 @@ +# Caching + +```{versionadded} 6.0 +``` + +pip provides an on-by-default caching, designed to reduce the amount of time +spent on duplicate downloads and builds. + +## What is cached + +### HTTP responses + +This cache functions like a web browser cache. + +When making any HTTP request, pip will first check its local cache to determine +if it has a suitable response stored for that request which has not expired. If +it does then it returns that response and doesn't re-download the content. + +If it has a response stored but it has expired, then it will attempt to make a +conditional request to refresh the cache which will either return an empty +response telling pip to simply use the cached item (and refresh the expiration +timer) or it will return a whole new response which pip can then store in the +cache. + +While this cache attempts to minimize network activity, it does not prevent +network access altogether. If you want a local install solution that +circumvents accessing PyPI, see {ref}`Installing from local packages`. + +### Locally built wheels + +pip attempts to use wheels from its local wheel cache whenever possible. + +This means that if there is a cached wheel for the same version of a specific +package name, pip will use that wheel instead of rebuilding the project. + +When no wheels are found for a source distribution, pip will attempt to build a +wheel using the package's build system. If the build is successful, this wheel +is added to the cache and used in subsequent installs for the same package +version. + +```{versionchanged} 20.0 +pip now caches wheels when building from an immutable Git reference +(i.e. a commit hash). +``` + +## Avoiding caching + +pip tries to use its cache whenever possible, and it is designed do the right +thing by default. + +In some cases, pip's caching behaviour can be undesirable. As an example, if you +have package with optional C extensions, that generates a pure Python wheel +when the C extension can’t be built, pip will use that cached wheel even when +you later invoke it from an environment that could have built those optional C +extensions. This is because pip is seeing a cached wheel for that matches the +package being built, and pip assumes that the result of building a package from +a package index is deterministic. + +The recommended approach for dealing with these situations is to directly +install from a source distribution instead of letting pip auto-discover the +package when it is trying to install. Installing directly from a source +distribution will make pip build a wheel, regardless of whether there is a +matching cached wheel. This usually means doing something like: + +```{pip-cli} +$ pip download sampleproject==1.0.0 --no-binary :all: +$ pip install sampleproject-1.0.0.tar.gz +``` + +It is also a good idea to remove the offending cached wheel using the +{ref}`pip cache` command. + +## Cache management + +The {ref}`pip cache` command can be used to manage pip's cache. + +The exact filesystem structure of pip's cache is considered to be an +implementation detail and may change between any two versions of pip. + +## Disabling caching + +pip's caching behaviour is disabled by passing the ``--no-cache-dir`` option. + +It is, however, recommended to **NOT** disable pip's caching. Doing so can +significantly slow down pip (due to repeated operations and package builds) +and result in significantly more network usage. diff --git a/docs/html/topics/configuration.md b/docs/html/topics/configuration.md new file mode 100644 index 000000000..90799d574 --- /dev/null +++ b/docs/html/topics/configuration.md @@ -0,0 +1,226 @@ +# Configuration + +pip allows a user to change its behaviour via 3 mechanisms: + +- command line options +- environment variables +- configuration files + +This page explains how the configuration files and environment variables work, +and how they are related to pip's various command line options. + +## Configuration Files + +Configuration files can change the default values for command line option. +They are written using a standard INI style configuration files. + +pip has 3 "levels" of configuration files: + +- `global`: system-wide configuration file, shared across users. +- `user`: per-user configuration file. +- `site`: per-environment configuration file; i.e. per-virtualenv. + +### Location + +pip's configuration files are located in fairly standard locations. This +location is different on different operating systems, and has some additional +complexity for backwards compatibility reasons. + +```{tab} Unix + +Global +: {file}`/etc/pip.conf` + + Alternatively, it may be in a "pip" subdirectory of any of the paths set + in the environment variable `XDG_CONFIG_DIRS` (if it exists), for + example {file}`/etc/xdg/pip/pip.conf`. + +User +: {file}`$HOME/.config/pip/pip.conf`, which respects the `XDG_CONFIG_HOME` environment variable. + + The legacy "per-user" configuration file is also loaded, if it exists: {file}`$HOME/.pip/pip.conf`. + +Site +: {file}`$VIRTUAL_ENV/pip.conf` +``` + +```{tab} MacOS + +Global +: {file}`/Library/Application Support/pip/pip.conf` + +User +: {file}`$HOME/Library/Application Support/pip/pip.conf` + if directory `$HOME/Library/Application Support/pip` exists + else {file}`$HOME/.config/pip/pip.conf` + + The legacy "per-user" configuration file is also loaded, if it exists: {file}`$HOME/.pip/pip.conf`. + +Site +: {file}`$VIRTUAL_ENV/pip.conf` +``` + +```{tab} Windows + +Global +: * On Windows 7 and later: {file}`C:\\ProgramData\\pip\\pip.ini` + (hidden but writeable) + * On Windows Vista: Global configuration is not supported. + * On Windows XP: + {file}`C:\\Documents and Settings\\All Users\\Application Data\\pip\\pip.ini` + +User +: {file}`%APPDATA%\\pip\\pip.ini` + + The legacy "per-user" configuration file is also loaded, if it exists: {file}`%HOME%\\pip\\pip.ini` + +Site +: {file}`%VIRTUAL_ENV%\\pip.ini` +``` + +### `PIP_CONFIG_FILE` + +Additionally, the environment variable `PIP_CONFIG_FILE` can be used to specify +a configuration file that's loaded first, and whose values are overridden by +the values set in the aforementioned files. Setting this to {ref}`os.devnull` +disables the loading of _all_ configuration files. + +### Loading order + +When multiple configuration files are found, pip combines them in the following +order: + +- `PIP_CONFIG_FILE`, if given. +- Global +- User +- Site + +Each file read overrides any values read from previous files, so if the +global timeout is specified in both the global file and the per-user file +then the latter value will be used. + +### Naming + +The names of the settings are derived from the long command line option. + +As an example, if you want to use a different package index (`--index-url`) and +set the HTTP timeout (`--default-timeout`) to 60 seconds, your config file would +look like this: + +```ini +[global] +timeout = 60 +index-url = https://download.zope.org/ppix +``` + +### Per-command section + +Each subcommand can be configured optionally in its own section. This overrides +the global setting with the same name. + +As an example, if you want to decrease the `timeout` to `10` seconds when +running the {ref}`pip freeze`, and use `60` seconds for all other commands: + +```ini +[global] +timeout = 60 + +[freeze] +timeout = 10 +``` + +### Boolean options + +Boolean options like `--ignore-installed` or `--no-dependencies` can be set +like this: + +```ini +[install] +ignore-installed = true +no-dependencies = yes +``` + +To enable the boolean options `--no-compile`, `--no-warn-script-location` and +`--no-cache-dir`, falsy values have to be used: + +```ini +[global] +no-cache-dir = false + +[install] +no-compile = no +no-warn-script-location = false +``` + +### Repeatable options + +For options which can be repeated like `--verbose` and `--quiet`, a +non-negative integer can be used to represent the level to be specified: + +```ini +[global] +quiet = 0 +verbose = 2 +``` + +It is possible to append values to a section within a configuration file. This +is applicable to appending options like `--find-links` or `--trusted-host`, +which can be written on multiple lines: + +```ini +[global] +find-links = + http://download.example.com + +[install] +find-links = + http://mirror1.example.com + http://mirror2.example.com + +trusted-host = + mirror1.example.com + mirror2.example.com +``` + +This enables users to add additional values in the order of entry for such +command line arguments. + +## Environment Variables + +pip's command line options can be set with environment variables using the +format `PIP_` . Dashes (`-`) have to be replaced with +underscores (`_`). + +- `PIP_DEFAULT_TIMEOUT=60` is the same as `--default-timeout=60` +- ``` + PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" + ``` + + is the same as + + ``` + --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com + ``` + +Repeatable options that do not take a value (such as `--verbose`) can be +specified using the number of repetitions: + +- `PIP_VERBOSE=3` is the same as `pip install -vvv` + +```{note} +Environment variables set to an empty string (like with `export X=` on Unix) will **not** be treated as false. +Use `no`, `false` or `0` instead. +``` + +## Precedence / Override order + +Command line options have override environment variables, which override the +values in a configuration file. Within the configuration file, values in +command-specific sections over values in the global section. + +Examples: + +- `--host=foo` overrides `PIP_HOST=foo` +- `PIP_HOST=foo` overrides a config file with `[global] host = foo` +- A command specific section in the config file `[] host = bar` + overrides the option with same name in the `[global]` config file section. diff --git a/docs/html/topics/index.md b/docs/html/topics/index.md new file mode 100644 index 000000000..478aacf2a --- /dev/null +++ b/docs/html/topics/index.md @@ -0,0 +1,16 @@ +# Topic Guides + +These pages provide detailed information on individual topics. + +```{note} +This section of the documentation is currently being fleshed out. See +{issue}`9475` for more details. +``` + +```{toctree} +:maxdepth: 1 + +authentication +caching +configuration +``` diff --git a/docs/html/usage.rst b/docs/html/usage.rst deleted file mode 100644 index ab1e9737f..000000000 --- a/docs/html/usage.rst +++ /dev/null @@ -1,7 +0,0 @@ -:orphan: - -===== -Usage -===== - -The "Usage" section is now covered in the :doc:`Reference Guide ` diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 900ad5034..e86fdb48c 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -63,72 +63,17 @@ For more information and examples, see the :ref:`pip install` reference. Basic Authentication Credentials ================================ -pip supports basic authentication credentials. Basically, in the URL there is -a username and password separated by ``:``. - -``https://[username[:password]@]pypi.company.com/simple`` - -Certain special characters are not valid in the authentication part of URLs. -If the user or password part of your login credentials contain any of the -special characters -`here `_ -then they must be percent-encoded. For example, for a -user with username "user" and password "he//o" accessing a repository at -pypi.company.com, the index URL with credentials would look like: - -``https://user:he%2F%2Fo@pypi.company.com`` - -Support for percent-encoded authentication in index URLs was added in pip 10.0.0 -(in `#3236 `_). Users that must use authentication -for their Python repository on systems with older pip versions should make the latest -get-pip.py available in their environment to bootstrap pip to a recent-enough version. - -For indexes that only require single-part authentication tokens, provide the token -as the "username" and do not provide a password, for example - - -``https://0123456789abcdef@pypi.company.com`` - +This is now covered in :doc:`topics/authentication`. netrc Support ------------- -If no credentials are part of the URL, pip will attempt to get authentication credentials -for the URL’s hostname from the user’s .netrc file. This behaviour comes from the underlying -use of `requests`_ which in turn delegates it to the `Python standard library`_. - -The .netrc file contains login and initialization information used by the auto-login process. -It resides in the user's home directory. The .netrc file format is simple. You specify lines -with a machine name and follow that with lines for the login and password that are -associated with that machine. Machine name is the hostname in your URL. - -An example .netrc for the host example.com with a user named 'daniel', using the password -'qwerty' would look like: - -.. code-block:: shell - - machine example.com - login daniel - password qwerty - -As mentioned in the `standard library docs `_, -only ASCII characters are allowed. Whitespace and non-printable characters are not allowed in passwords. - +This is now covered in :doc:`topics/authentication`. Keyring Support --------------- -pip also supports credentials stored in your keyring using the `keyring`_ -library. Note that ``keyring`` will need to be installed separately, as pip -does not come with it included. - -.. code-block:: shell - - pip install keyring - echo your-password | keyring set pypi.company.com your-username - pip install your-package --index-url https://pypi.company.com/ - -.. _keyring: https://pypi.org/project/keyring/ - +This is now covered in :doc:`topics/authentication`. Using a Proxy Server ==================== @@ -492,242 +437,26 @@ For more information and examples, see the :ref:`pip search` reference. Configuration ============= +This is now covered in :doc:`topics/configuration`. + .. _config-file: Config file ----------- -pip allows you to set all command line option defaults in a standard ini -style config file. - -The names and locations of the configuration files vary slightly across -platforms. You may have per-user, per-virtualenv or global (shared amongst -all users) configuration: - -**Per-user**: - -* On Unix the default configuration file is: :file:`$HOME/.config/pip/pip.conf` - which respects the ``XDG_CONFIG_HOME`` environment variable. -* On macOS the configuration file is - :file:`$HOME/Library/Application Support/pip/pip.conf` - if directory ``$HOME/Library/Application Support/pip`` exists - else :file:`$HOME/.config/pip/pip.conf`. -* On Windows the configuration file is :file:`%APPDATA%\\pip\\pip.ini`. - -There is also a legacy per-user configuration file which is also respected. -To find its location: - -* On Unix and macOS the configuration file is: :file:`$HOME/.pip/pip.conf` -* On Windows the configuration file is: :file:`%HOME%\\pip\\pip.ini` - -You can set a custom path location for this config file using the environment -variable ``PIP_CONFIG_FILE``. - -**Inside a virtualenv**: - -* On Unix and macOS the file is :file:`$VIRTUAL_ENV/pip.conf` -* On Windows the file is: :file:`%VIRTUAL_ENV%\\pip.ini` - -**Global**: - -* On Unix the file may be located in :file:`/etc/pip.conf`. Alternatively - it may be in a "pip" subdirectory of any of the paths set in the - environment variable ``XDG_CONFIG_DIRS`` (if it exists), for example - :file:`/etc/xdg/pip/pip.conf`. -* On macOS the file is: :file:`/Library/Application Support/pip/pip.conf` -* On Windows XP the file is: - :file:`C:\\Documents and Settings\\All Users\\Application Data\\pip\\pip.ini` -* On Windows 7 and later the file is hidden, but writeable at - :file:`C:\\ProgramData\\pip\\pip.ini` -* Global configuration is not supported on Windows Vista. - -The global configuration file is shared by all Python installations. - -If multiple configuration files are found by pip then they are combined in -the following order: - -1. The global file is read -2. The per-user file is read -3. The virtualenv-specific file is read - -Each file read overrides any values read from previous files, so if the -global timeout is specified in both the global file and the per-user file -then the latter value will be used. - -The names of the settings are derived from the long command line option, e.g. -if you want to use a different package index (``--index-url``) and set the -HTTP timeout (``--default-timeout``) to 60 seconds your config file would -look like this: - -.. code-block:: ini - - [global] - timeout = 60 - index-url = https://download.zope.org/ppix - -Each subcommand can be configured optionally in its own section so that every -global setting with the same name will be overridden; e.g. decreasing the -``timeout`` to ``10`` seconds when running the ``freeze`` -(:ref:`pip freeze`) command and using -``60`` seconds for all other commands is possible with: - -.. code-block:: ini - - [global] - timeout = 60 - - [freeze] - timeout = 10 - - -Boolean options like ``--ignore-installed`` or ``--no-dependencies`` can be -set like this: - -.. code-block:: ini - - [install] - ignore-installed = true - no-dependencies = yes - -To enable the boolean options ``--no-compile``, ``--no-warn-script-location`` -and ``--no-cache-dir``, falsy values have to be used: - -.. code-block:: ini - - [global] - no-cache-dir = false - - [install] - no-compile = no - no-warn-script-location = false - -For options which can be repeated like ``--verbose`` and ``--quiet``, -a non-negative integer can be used to represent the level to be specified: - -.. code-block:: ini - - [global] - quiet = 0 - verbose = 2 - -It is possible to append values to a section within a configuration file such as the pip.ini file. -This is applicable to appending options like ``--find-links`` or ``--trusted-host``, -which can be written on multiple lines: - -.. code-block:: ini - - [global] - find-links = - http://download.example.com - - [install] - find-links = - http://mirror1.example.com - http://mirror2.example.com - - trusted-host = - mirror1.example.com - mirror2.example.com - -This enables users to add additional values in the order of entry for such command line arguments. - +This is now covered in :doc:`topics/configuration`. Environment Variables --------------------- -pip's command line options can be set with environment variables using the -format ``PIP_`` . Dashes (``-``) have to be replaced with -underscores (``_``). - -For example, to set the default timeout: - -.. tab:: Unix/macOS - - .. code-block:: shell - - export PIP_DEFAULT_TIMEOUT=60 - -.. tab:: Windows - - .. code-block:: shell - - set PIP_DEFAULT_TIMEOUT=60 - -This is the same as passing the option to pip directly: - -.. tab:: Unix/macOS - - .. code-block:: shell - - python -m pip --default-timeout=60 [...] - -.. tab:: Windows - - .. code-block:: shell - - py -m pip --default-timeout=60 [...] - -For command line options which can be repeated, use a space to separate -multiple values. For example: - -.. tab:: Unix/macOS - - .. code-block:: shell - - export PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" - -.. tab:: Windows - - .. code-block:: shell - - set PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" - -is the same as calling: - -.. tab:: Unix/macOS - - .. code-block:: shell - - python -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com - -.. tab:: Windows - - .. code-block:: shell - - py -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com - -Options that do not take a value, but can be repeated (such as ``--verbose``) -can be specified using the number of repetitions, so:: - - export PIP_VERBOSE=3 - -is the same as calling:: - - pip install -vvv - -.. note:: - - Environment variables set to be empty string will not be treated as false. - Please use ``no``, ``false`` or ``0`` instead. - +This is now covered in :doc:`topics/configuration`. .. _config-precedence: Config Precedence ----------------- -Command line options have precedence over environment variables, which have -precedence over the config file. - -Within the config file, command specific sections have precedence over the -global section. - -Examples: - -- ``--host=foo`` overrides ``PIP_HOST=foo`` -- ``PIP_HOST=foo`` overrides a config file with ``[global] host = foo`` -- A command specific section in the config file ``[] host = bar`` - overrides the option with same name in the ``[global]`` config file section +This is now covered in :doc:`topics/configuration`. Command Completion @@ -833,7 +562,7 @@ those specified on the command-line or via a requirements file) while requirements). As an example, say ``SomePackage`` has a dependency, ``SomeDependency``, and -both of them are already installed but are not the latest avaialable versions: +both of them are already installed but are not the latest available versions: - ``pip install SomePackage``: will not upgrade the existing ``SomePackage`` or ``SomeDependency``. @@ -1904,6 +1633,4 @@ announcements on the `low-traffic packaging announcements list`_ and .. _low-traffic packaging announcements list: https://mail.python.org/mailman3/lists/pypi-announce.python.org/ .. _our survey on upgrades that create conflicts: https://docs.google.com/forms/d/e/1FAIpQLSeBkbhuIlSofXqCyhi3kGkLmtrpPOEBwr6iJA6SzHdxWKfqdA/viewform .. _the official Python blog: https://blog.python.org/ -.. _requests: https://requests.readthedocs.io/en/latest/user/authentication/#netrc-authentication -.. _Python standard library: https://docs.python.org/3/library/netrc.html .. _Python Windows launcher: https://docs.python.org/3/using/windows.html#launcher diff --git a/news/0d757310-0e1d-4887-9076-a1eb3c55d9fa.trivial.rst b/news/0d757310-0e1d-4887-9076-a1eb3c55d9fa.trivial.rst new file mode 100644 index 000000000..c5abc9762 --- /dev/null +++ b/news/0d757310-0e1d-4887-9076-a1eb3c55d9fa.trivial.rst @@ -0,0 +1 @@ +Fix typos in several files. diff --git a/news/10004.trivial.rst b/news/10004.trivial.rst new file mode 100644 index 000000000..128cf866d --- /dev/null +++ b/news/10004.trivial.rst @@ -0,0 +1 @@ +Annotate ``typing.List`` into ``tools.tox_pip.pip()`` diff --git a/news/10018.trivial.rst b/news/10018.trivial.rst new file mode 100644 index 000000000..c6950c59a --- /dev/null +++ b/news/10018.trivial.rst @@ -0,0 +1 @@ +Use annotations from the ``typing`` module on some functions. diff --git a/news/10020.bugfix.rst b/news/10020.bugfix.rst new file mode 100644 index 000000000..9425626fb --- /dev/null +++ b/news/10020.bugfix.rst @@ -0,0 +1 @@ +Remove unused optional ``tornado`` import in vendored ``tenacity`` to prevent old versions of Tornado from breaking pip. diff --git a/news/10031.bugfix.rst b/news/10031.bugfix.rst new file mode 100644 index 000000000..8b5332bb0 --- /dev/null +++ b/news/10031.bugfix.rst @@ -0,0 +1 @@ +Require ``setup.cfg``-only projects to be built via PEP 517, by requiring an explicit dependency on setuptools declared in pyproject.toml. diff --git a/news/10043.bugfix.rst b/news/10043.bugfix.rst new file mode 100644 index 000000000..29d78f7bc --- /dev/null +++ b/news/10043.bugfix.rst @@ -0,0 +1 @@ +Update vendored six to 1.16.0 and urllib3 to 1.26.5 diff --git a/news/10045.feature.rst b/news/10045.feature.rst new file mode 100644 index 000000000..7c7b53725 --- /dev/null +++ b/news/10045.feature.rst @@ -0,0 +1 @@ +Added a warning message for errors caused due to Long Paths being disabled on Windows. diff --git a/news/10047.trivial.rst b/news/10047.trivial.rst new file mode 100644 index 000000000..edd324342 --- /dev/null +++ b/news/10047.trivial.rst @@ -0,0 +1 @@ +Convert type annotations into proper annotations in ``noxfile.py``. diff --git a/news/10064.trivial.rst b/news/10064.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/10080.bugfix.rst b/news/10080.bugfix.rst new file mode 100644 index 000000000..f1aa2d6a8 --- /dev/null +++ b/news/10080.bugfix.rst @@ -0,0 +1 @@ +Correctly allow PEP 517 projects to be detected without warnings in ``pip freeze``. diff --git a/news/3931.bugfix.rst b/news/3931.bugfix.rst new file mode 100644 index 000000000..0ebb9e495 --- /dev/null +++ b/news/3931.bugfix.rst @@ -0,0 +1 @@ +Prefer credentials from the URL over the previously-obtained credentials from URLs of the same domain, so it is possible to use different credentials on the same index server for different ``--extra-index-url`` options. diff --git a/news/7975.feature.rst b/news/7975.feature.rst new file mode 100644 index 000000000..b0638939b --- /dev/null +++ b/news/7975.feature.rst @@ -0,0 +1,2 @@ +Add new subcommand ``pip index`` used to interact with indexes, and implement +``pip index version`` to list available versions of a package. diff --git a/news/8954.feature.rst b/news/8954.feature.rst new file mode 100644 index 000000000..05ec68d04 --- /dev/null +++ b/news/8954.feature.rst @@ -0,0 +1,9 @@ +When pip is asked to uninstall a project without the dist-info/RECORD file +it will no longer traceback with FileNotFoundError, +but it will provide a better error message instead, such as:: + + ERROR: Cannot uninstall foobar 0.1, RECORD file not found. You might be able to recover from this via: 'pip install --force-reinstall --no-deps foobar==0.1'. + +When dist-info/INSTALLER is present and contains some useful information, the info is included in the error message instead:: + + ERROR: Cannot uninstall foobar 0.1, RECORD file not found. Hint: The package was installed by rpm. diff --git a/news/8EB21BAE-9CD0-424E-AF3B-651960B50C93.trivial.rst b/news/8EB21BAE-9CD0-424E-AF3B-651960B50C93.trivial.rst new file mode 100644 index 000000000..42719640c --- /dev/null +++ b/news/8EB21BAE-9CD0-424E-AF3B-651960B50C93.trivial.rst @@ -0,0 +1 @@ +mailmap: Clean up Git entries diff --git a/news/9450.feature.rst b/news/9450.feature.rst new file mode 100644 index 000000000..5e54ad301 --- /dev/null +++ b/news/9450.feature.rst @@ -0,0 +1,3 @@ +Add an additional level of verbosity. ``--verbose`` (and the shorthand ``-v``) now +contains significantly less output, and users that need complete full debug-level output +should pass it twice (``--verbose --verbose`` or ``-vv``). diff --git a/news/9455.feature.rst b/news/9455.feature.rst new file mode 100644 index 000000000..f33f33174 --- /dev/null +++ b/news/9455.feature.rst @@ -0,0 +1,2 @@ +New resolver: The order of dependencies resolution has been tweaked to traverse +the dependency graph in a more breadth-first approach. diff --git a/news/9822.bugfix.rst b/news/9822.bugfix.rst new file mode 100644 index 000000000..8a692c3ff --- /dev/null +++ b/news/9822.bugfix.rst @@ -0,0 +1,3 @@ +Fix :ref:`pip freeze` to output packages :ref:`installed from git ` +in the correct ``git+protocol://git.example.com/MyProject#egg=MyProject`` format +rather than the old and no longer supported ``git+git@`` format. diff --git a/news/9925.feature.rst b/news/9925.feature.rst new file mode 100644 index 000000000..8c2401f60 --- /dev/null +++ b/news/9925.feature.rst @@ -0,0 +1,3 @@ +New resolver: A distribution's ``Requires-Python`` metadata is now checked +before its Python dependencies. This makes the resolver fail quicker when +there's an interpreter version conflict. diff --git a/news/9987.feature.rst b/news/9987.feature.rst new file mode 100644 index 000000000..0cf451638 --- /dev/null +++ b/news/9987.feature.rst @@ -0,0 +1 @@ +Include ``rustc`` version in pip's ``User-Agent``, when the system has ``rustc``. diff --git a/noxfile.py b/noxfile.py index d9e344543..becd0c304 100644 --- a/noxfile.py +++ b/noxfile.py @@ -33,8 +33,7 @@ AUTHORS_FILE = "AUTHORS.txt" VERSION_FILE = "src/pip/__init__.py" -def run_with_protected_pip(session, *arguments): - # type: (nox.Session, *str) -> None +def run_with_protected_pip(session: nox.Session, *arguments: str) -> None: """Do a session.run("pip", *arguments), using a "protected" pip. This invokes a wrapper script, that forwards calls to original virtualenv @@ -48,8 +47,7 @@ def run_with_protected_pip(session, *arguments): session.run(*command, env=env, silent=True) -def should_update_common_wheels(): - # type: () -> bool +def should_update_common_wheels() -> bool: # If the cache hasn't been created, create it. if not os.path.exists(LOCATIONS["common-wheels"]): return True @@ -73,8 +71,7 @@ def should_update_common_wheels(): # `tox -e ...` until this note is removed. # ----------------------------------------------------------------------------- @nox.session(python=["3.6", "3.7", "3.8", "3.9", "pypy3"]) -def test(session): - # type: (nox.Session) -> None +def test(session: nox.Session) -> None: # Get the common wheels. if should_update_common_wheels(): # fmt: off @@ -122,8 +119,7 @@ def test(session): @nox.session -def docs(session): - # type: (nox.Session) -> None +def docs(session: nox.Session) -> None: session.install("-e", ".") session.install("-r", REQUIREMENTS["docs"]) @@ -150,8 +146,7 @@ def docs(session): @nox.session(name="docs-live") -def docs_live(session): - # type: (nox.Session) -> None +def docs_live(session: nox.Session) -> None: session.install("-e", ".") session.install("-r", REQUIREMENTS["docs"], "sphinx-autobuild") @@ -166,8 +161,7 @@ def docs_live(session): @nox.session -def lint(session): - # type: (nox.Session) -> None +def lint(session: nox.Session) -> None: session.install("pre-commit") if session.posargs: @@ -179,8 +173,7 @@ def lint(session): @nox.session -def vendoring(session): - # type: (nox.Session) -> None +def vendoring(session: nox.Session) -> None: session.install("vendoring>=0.3.0") if "--upgrade" not in session.posargs: @@ -238,8 +231,7 @@ def vendoring(session): # Release Commands # ----------------------------------------------------------------------------- @nox.session(name="prepare-release") -def prepare_release(session): - # type: (nox.Session) -> None +def prepare_release(session: nox.Session) -> None: version = release.get_version_from_arguments(session) if not version: session.error("Usage: nox -s prepare-release -- ") @@ -272,8 +264,7 @@ def prepare_release(session): @nox.session(name="build-release") -def build_release(session): - # type: (nox.Session) -> None +def build_release(session: nox.Session) -> None: version = release.get_version_from_arguments(session) if not version: session.error("Usage: nox -s build-release -- YY.N[.P]") @@ -304,8 +295,7 @@ def build_release(session): shutil.copy(dist, final) -def build_dists(session): - # type: (nox.Session) -> List[str] +def build_dists(session: nox.Session) -> List[str]: """Return dists with valid metadata.""" session.log( "# Check if there's any Git-untracked files before building the wheel", @@ -333,8 +323,7 @@ def build_dists(session): @nox.session(name="upload-release") -def upload_release(session): - # type: (nox.Session) -> None +def upload_release(session: nox.Session) -> None: version = release.get_version_from_arguments(session) if not version: session.error("Usage: nox -s upload-release -- YY.N[.P]") diff --git a/src/pip/__init__.py b/src/pip/__init__.py index c35872ca9..67722d05e 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -3,8 +3,7 @@ from typing import List, Optional __version__ = "21.2.dev0" -def main(args=None): - # type: (Optional[List[str]]) -> int +def main(args: Optional[List[str]] = None) -> int: """This is an internal API only meant for use by pip's own console scripts. For additional details, see https://github.com/pypa/pip/issues/7498. diff --git a/src/pip/_internal/__init__.py b/src/pip/_internal/__init__.py index 41071cd86..6afb5c627 100755 --- a/src/pip/_internal/__init__.py +++ b/src/pip/_internal/__init__.py @@ -1,10 +1,14 @@ from typing import List, Optional import pip._internal.utils.inject_securetransport # noqa +from pip._internal.utils import _log + +# init_logging() must be called before any call to logging.getLogger() +# which happens at import of most modules. +_log.init_logging() -def main(args=None): - # type: (Optional[List[str]]) -> int +def main(args: (Optional[List[str]]) = None) -> int: """This is preserved for old console scripts that may still be referencing it. diff --git a/src/pip/_internal/cli/autocompletion.py b/src/pip/_internal/cli/autocompletion.py index 3b1d2ac9b..2018ba2d4 100644 --- a/src/pip/_internal/cli/autocompletion.py +++ b/src/pip/_internal/cli/autocompletion.py @@ -12,8 +12,7 @@ from pip._internal.commands import commands_dict, create_command from pip._internal.utils.misc import get_installed_distributions -def autocomplete(): - # type: () -> None +def autocomplete() -> None: """Entry Point for completion of main and subcommand options.""" # Don't complete if user hasn't sourced bash_completion file. if "PIP_AUTO_COMPLETE" not in os.environ: @@ -107,8 +106,9 @@ def autocomplete(): sys.exit(1) -def get_path_completion_type(cwords, cword, opts): - # type: (List[str], int, Iterable[Any]) -> Optional[str] +def get_path_completion_type( + cwords: List[str], cword: int, opts: Iterable[Any] +) -> Optional[str]: """Get the type of path completion (``file``, ``dir``, ``path`` or None) :param cwords: same as the environmental variable ``COMP_WORDS`` @@ -130,8 +130,7 @@ def get_path_completion_type(cwords, cword, opts): return None -def auto_complete_paths(current, completion_type): - # type: (str, str) -> Iterable[str] +def auto_complete_paths(current: str, completion_type: str) -> Iterable[str]: """If ``completion_type`` is ``file`` or ``path``, list all regular files and directories starting with ``current``; otherwise only list directories starting with ``current``. diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py index b59420dda..37f9e65fa 100644 --- a/src/pip/_internal/cli/base_command.py +++ b/src/pip/_internal/cli/base_command.py @@ -43,8 +43,7 @@ class Command(CommandContextMixIn): usage = None # type: str ignore_require_venv = False # type: bool - def __init__(self, name, summary, isolated=False): - # type: (str, str, bool) -> None + def __init__(self, name: str, summary: str, isolated: bool = False) -> None: super().__init__() self.name = name @@ -74,12 +73,10 @@ class Command(CommandContextMixIn): self.add_options() - def add_options(self): - # type: () -> None + def add_options(self) -> None: pass - def handle_pip_version_check(self, options): - # type: (Values) -> None + def handle_pip_version_check(self, options: Values) -> None: """ This is a no-op so that commands by default do not do the pip version check. @@ -88,25 +85,21 @@ class Command(CommandContextMixIn): # are present. assert not hasattr(options, "no_index") - def run(self, options, args): - # type: (Values, List[Any]) -> int + def run(self, options: Values, args: List[Any]) -> int: raise NotImplementedError - def parse_args(self, args): - # type: (List[str]) -> Tuple[Any, Any] + def parse_args(self, args: List[str]) -> Tuple[Any, Any]: # factored out for testability return self.parser.parse_args(args) - def main(self, args): - # type: (List[str]) -> int + def main(self, args: List[str]) -> int: try: with self.main_context(): return self._main(args) finally: logging.shutdown() - def _main(self, args): - # type: (List[str]) -> int + def _main(self, args: List[str]) -> int: # We must initialize this before the tempdir manager, otherwise the # configuration would not be accessible by the time we clean up the # tempdir manager. diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index f71c0b020..e0a672951 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -31,8 +31,7 @@ from pip._internal.utils.hashes import STRONG_HASHES from pip._internal.utils.misc import strtobool -def raise_option_error(parser, option, msg): - # type: (OptionParser, Option, str) -> None +def raise_option_error(parser: OptionParser, option: Option, msg: str) -> None: """ Raise an option parsing error using parser.error(). @@ -46,8 +45,7 @@ def raise_option_error(parser, option, msg): parser.error(msg) -def make_option_group(group, parser): - # type: (Dict[str, Any], ConfigOptionParser) -> OptionGroup +def make_option_group(group: Dict[str, Any], parser: ConfigOptionParser) -> OptionGroup: """ Return an OptionGroup object group -- assumed to be dict with 'name' and 'options' keys @@ -59,8 +57,9 @@ def make_option_group(group, parser): return option_group -def check_install_build_global(options, check_options=None): - # type: (Values, Optional[Values]) -> None +def check_install_build_global( + options: Values, check_options: Optional[Values] = None +) -> None: """Disable wheels if per-setup.py call options are set. :param options: The OptionParser options to update. @@ -70,8 +69,7 @@ def check_install_build_global(options, check_options=None): if check_options is None: check_options = options - def getname(n): - # type: (str) -> Optional[Any] + def getname(n: str) -> Optional[Any]: return getattr(check_options, n, None) names = ["build_options", "global_options", "install_options"] @@ -85,8 +83,7 @@ def check_install_build_global(options, check_options=None): ) -def check_dist_restriction(options, check_target=False): - # type: (Values, bool) -> None +def check_dist_restriction(options: Values, check_target: bool = False) -> None: """Function for determining if custom platform options are allowed. :param options: The OptionParser options. @@ -126,13 +123,11 @@ def check_dist_restriction(options, check_target=False): ) -def _path_option_check(option, opt, value): - # type: (Option, str, str) -> str +def _path_option_check(option: Option, opt: str, value: str) -> str: return os.path.expanduser(value) -def _package_name_option_check(option, opt, value): - # type: (Option, str, str) -> str +def _package_name_option_check(option: Option, opt: str, value: str) -> str: return canonicalize_name(value) @@ -287,8 +282,7 @@ timeout = partial( ) # type: Callable[..., Option] -def exists_action(): - # type: () -> Option +def exists_action() -> Option: return Option( # Option when path already exist "--exists-action", @@ -343,8 +337,7 @@ index_url = partial( ) # type: Callable[..., Option] -def extra_index_url(): - # type: () -> Option +def extra_index_url() -> Option: return Option( "--extra-index-url", dest="extra_index_urls", @@ -367,8 +360,7 @@ no_index = partial( ) # type: Callable[..., Option] -def find_links(): - # type: () -> Option +def find_links() -> Option: return Option( "-f", "--find-links", @@ -378,14 +370,13 @@ def find_links(): metavar="url", help="If a URL or path to an html file, then parse for links to " "archives such as sdist (.tar.gz) or wheel (.whl) files. " - "If a local path or file:// URL that's a directory, " + "If a local path or file:// URL that's a directory, " "then look for archives in the directory listing. " "Links to VCS project URLs are not supported.", ) -def trusted_host(): - # type: () -> Option +def trusted_host() -> Option: return Option( "--trusted-host", dest="trusted_hosts", @@ -397,8 +388,7 @@ def trusted_host(): ) -def constraints(): - # type: () -> Option +def constraints() -> Option: return Option( "-c", "--constraint", @@ -411,8 +401,7 @@ def constraints(): ) -def requirements(): - # type: () -> Option +def requirements() -> Option: return Option( "-r", "--requirement", @@ -425,8 +414,7 @@ def requirements(): ) -def editable(): - # type: () -> Option +def editable() -> Option: return Option( "-e", "--editable", @@ -441,8 +429,7 @@ def editable(): ) -def _handle_src(option, opt_str, value, parser): - # type: (Option, str, str, OptionParser) -> None +def _handle_src(option: Option, opt_str: str, value: str, parser: OptionParser) -> None: value = os.path.abspath(value) setattr(parser.values, option.dest, value) @@ -465,14 +452,14 @@ src = partial( ) # type: Callable[..., Option] -def _get_format_control(values, option): - # type: (Values, Option) -> Any +def _get_format_control(values: Values, option: Option) -> Any: """Get a format_control object.""" return getattr(values, option.dest) -def _handle_no_binary(option, opt_str, value, parser): - # type: (Option, str, str, OptionParser) -> None +def _handle_no_binary( + option: Option, opt_str: str, value: str, parser: OptionParser +) -> None: existing = _get_format_control(parser.values, option) FormatControl.handle_mutual_excludes( value, @@ -481,8 +468,9 @@ def _handle_no_binary(option, opt_str, value, parser): ) -def _handle_only_binary(option, opt_str, value, parser): - # type: (Option, str, str, OptionParser) -> None +def _handle_only_binary( + option: Option, opt_str: str, value: str, parser: OptionParser +) -> None: existing = _get_format_control(parser.values, option) FormatControl.handle_mutual_excludes( value, @@ -491,8 +479,7 @@ def _handle_only_binary(option, opt_str, value, parser): ) -def no_binary(): - # type: () -> Option +def no_binary() -> Option: format_control = FormatControl(set(), set()) return Option( "--no-binary", @@ -510,8 +497,7 @@ def no_binary(): ) -def only_binary(): - # type: () -> Option +def only_binary() -> Option: format_control = FormatControl(set(), set()) return Option( "--only-binary", @@ -545,8 +531,7 @@ platforms = partial( # This was made a separate function for unit-testing purposes. -def _convert_python_version(value): - # type: (str) -> Tuple[Tuple[int, ...], Optional[str]] +def _convert_python_version(value: str) -> Tuple[Tuple[int, ...], Optional[str]]: """ Convert a version string like "3", "37", or "3.7.3" into a tuple of ints. @@ -575,8 +560,9 @@ def _convert_python_version(value): return (version_info, None) -def _handle_python_version(option, opt_str, value, parser): - # type: (Option, str, str, OptionParser) -> None +def _handle_python_version( + option: Option, opt_str: str, value: str, parser: OptionParser +) -> None: """ Handle a provided --python-version value. """ @@ -646,16 +632,14 @@ abis = partial( ) # type: Callable[..., Option] -def add_target_python_options(cmd_opts): - # type: (OptionGroup) -> None +def add_target_python_options(cmd_opts: OptionGroup) -> None: cmd_opts.add_option(platforms()) cmd_opts.add_option(python_version()) cmd_opts.add_option(implementation()) cmd_opts.add_option(abis()) -def make_target_python(options): - # type: (Values) -> TargetPython +def make_target_python(options: Values) -> TargetPython: target_python = TargetPython( platforms=options.platforms, py_version_info=options.python_version, @@ -666,8 +650,7 @@ def make_target_python(options): return target_python -def prefer_binary(): - # type: () -> Option +def prefer_binary() -> Option: return Option( "--prefer-binary", dest="prefer_binary", @@ -688,8 +671,9 @@ cache_dir = partial( ) # type: Callable[..., Option] -def _handle_no_cache_dir(option, opt, value, parser): - # type: (Option, str, str, OptionParser) -> None +def _handle_no_cache_dir( + option: Option, opt: str, value: str, parser: OptionParser +) -> None: """ Process a value provided for the --no-cache-dir option. @@ -767,8 +751,9 @@ no_build_isolation = partial( ) # type: Callable[..., Option] -def _handle_no_use_pep517(option, opt, value, parser): - # type: (Option, str, str, OptionParser) -> None +def _handle_no_use_pep517( + option: Option, opt: str, value: str, parser: OptionParser +) -> None: """ Process a value provided for the --no-use-pep517 option. @@ -871,8 +856,9 @@ disable_pip_version_check = partial( ) # type: Callable[..., Option] -def _handle_merge_hash(option, opt_str, value, parser): - # type: (Option, str, str, OptionParser) -> None +def _handle_merge_hash( + option: Option, opt_str: str, value: str, parser: OptionParser +) -> None: """Given a value spelled "algo:digest", append the digest to a list pointed to in a dict by the algo name.""" if not parser.values.hashes: @@ -931,8 +917,7 @@ list_path = partial( ) # type: Callable[..., Option] -def check_list_path_option(options): - # type: (Values) -> None +def check_list_path_option(options: Values) -> None: if options.path and (options.user or options.local): raise CommandError("Cannot combine '--path' with '--user' or '--local'") diff --git a/src/pip/_internal/cli/command_context.py b/src/pip/_internal/cli/command_context.py index 375a2e366..ed6832237 100644 --- a/src/pip/_internal/cli/command_context.py +++ b/src/pip/_internal/cli/command_context.py @@ -5,15 +5,13 @@ _T = TypeVar("_T", covariant=True) class CommandContextMixIn: - def __init__(self): - # type: () -> None + def __init__(self) -> None: super().__init__() self._in_main_context = False self._main_context = ExitStack() @contextmanager - def main_context(self): - # type: () -> Iterator[None] + def main_context(self) -> Iterator[None]: assert not self._in_main_context self._in_main_context = True @@ -23,8 +21,7 @@ class CommandContextMixIn: finally: self._in_main_context = False - def enter_context(self, context_provider): - # type: (ContextManager[_T]) -> _T + def enter_context(self, context_provider: ContextManager[_T]) -> _T: assert self._in_main_context return self._main_context.enter_context(context_provider) diff --git a/src/pip/_internal/cli/main.py b/src/pip/_internal/cli/main.py index 7ae074b59..0e3122154 100644 --- a/src/pip/_internal/cli/main.py +++ b/src/pip/_internal/cli/main.py @@ -42,8 +42,7 @@ logger = logging.getLogger(__name__) # main, this should not be an issue in practice. -def main(args=None): - # type: (Optional[List[str]]) -> int +def main(args: Optional[List[str]] = None) -> int: if args is None: args = sys.argv[1:] diff --git a/src/pip/_internal/cli/main_parser.py b/src/pip/_internal/cli/main_parser.py index d0f58fe42..3666ab04c 100644 --- a/src/pip/_internal/cli/main_parser.py +++ b/src/pip/_internal/cli/main_parser.py @@ -14,8 +14,7 @@ from pip._internal.utils.misc import get_pip_version, get_prog __all__ = ["create_main_parser", "parse_command"] -def create_main_parser(): - # type: () -> ConfigOptionParser +def create_main_parser() -> ConfigOptionParser: """Creates and returns the main parser for pip's CLI""" parser = ConfigOptionParser( @@ -46,8 +45,7 @@ def create_main_parser(): return parser -def parse_command(args): - # type: (List[str]) -> Tuple[str, List[str]] +def parse_command(args: List[str]) -> Tuple[str, List[str]]: parser = create_main_parser() # Note: parser calls disable_interspersed_args(), so the result of this diff --git a/src/pip/_internal/cli/parser.py b/src/pip/_internal/cli/parser.py index 16523c5a1..efdf57e0b 100644 --- a/src/pip/_internal/cli/parser.py +++ b/src/pip/_internal/cli/parser.py @@ -18,20 +18,19 @@ logger = logging.getLogger(__name__) class PrettyHelpFormatter(optparse.IndentedHelpFormatter): """A prettier/less verbose help formatter for optparse.""" - def __init__(self, *args, **kwargs): - # type: (*Any, **Any) -> None + def __init__(self, *args: Any, **kwargs: Any) -> None: # help position must be aligned with __init__.parseopts.description kwargs["max_help_position"] = 30 kwargs["indent_increment"] = 1 kwargs["width"] = shutil.get_terminal_size()[0] - 2 super().__init__(*args, **kwargs) - def format_option_strings(self, option): - # type: (optparse.Option) -> str + def format_option_strings(self, option: optparse.Option) -> str: return self._format_option_strings(option) - def _format_option_strings(self, option, mvarfmt=" <{}>", optsep=", "): - # type: (optparse.Option, str, str) -> str + def _format_option_strings( + self, option: optparse.Option, mvarfmt: str = " <{}>", optsep: str = ", " + ) -> str: """ Return a comma-separated list of option strings and metavars. @@ -55,14 +54,12 @@ class PrettyHelpFormatter(optparse.IndentedHelpFormatter): return "".join(opts) - def format_heading(self, heading): - # type: (str) -> str + def format_heading(self, heading: str) -> str: if heading == "Options": return "" return heading + ":\n" - def format_usage(self, usage): - # type: (str) -> str + def format_usage(self, usage: str) -> str: """ Ensure there is only one newline between usage and the first heading if there is no description. @@ -70,8 +67,7 @@ class PrettyHelpFormatter(optparse.IndentedHelpFormatter): msg = "\nUsage: {}\n".format(self.indent_lines(textwrap.dedent(usage), " ")) return msg - def format_description(self, description): - # type: (str) -> str + def format_description(self, description: str) -> str: # leave full control over description to us if description: if hasattr(self.parser, "main"): @@ -89,16 +85,14 @@ class PrettyHelpFormatter(optparse.IndentedHelpFormatter): else: return "" - def format_epilog(self, epilog): - # type: (str) -> str + def format_epilog(self, epilog: str) -> str: # leave full control over epilog to us if epilog: return epilog else: return "" - def indent_lines(self, text, indent): - # type: (str, str) -> str + def indent_lines(self, text: str, indent: str) -> str: new_lines = [indent + line for line in text.split("\n")] return "\n".join(new_lines) @@ -112,8 +106,7 @@ class UpdatingDefaultsHelpFormatter(PrettyHelpFormatter): Also redact auth from url type options """ - def expand_default(self, option): - # type: (optparse.Option) -> str + def expand_default(self, option: optparse.Option) -> str: default_values = None if self.parser is not None: assert isinstance(self.parser, ConfigOptionParser) @@ -137,8 +130,9 @@ class UpdatingDefaultsHelpFormatter(PrettyHelpFormatter): class CustomOptionParser(optparse.OptionParser): - def insert_option_group(self, idx, *args, **kwargs): - # type: (int, Any, Any) -> optparse.OptionGroup + def insert_option_group( + self, idx: int, *args: Any, **kwargs: Any + ) -> optparse.OptionGroup: """Insert an OptionGroup at a given position.""" group = self.add_option_group(*args, **kwargs) @@ -148,8 +142,7 @@ class CustomOptionParser(optparse.OptionParser): return group @property - def option_list_all(self): - # type: () -> List[optparse.Option] + def option_list_all(self) -> List[optparse.Option]: """Get a list of all options, including those in option groups.""" res = self.option_list[:] for i in self.option_groups: @@ -164,28 +157,25 @@ class ConfigOptionParser(CustomOptionParser): def __init__( self, - *args, # type: Any - name, # type: str - isolated=False, # type: bool - **kwargs, # type: Any - ): - # type: (...) -> None + *args: Any, + name: str, + isolated: bool = False, + **kwargs: Any, + ) -> None: self.name = name self.config = Configuration(isolated) assert self.name super().__init__(*args, **kwargs) - def check_default(self, option, key, val): - # type: (optparse.Option, str, Any) -> Any + def check_default(self, option: optparse.Option, key: str, val: Any) -> Any: try: return option.check_value(key, val) except optparse.OptionValueError as exc: print(f"An error occurred during configuration: {exc}") sys.exit(3) - def _get_ordered_configuration_items(self): - # type: () -> Iterator[Tuple[str, Any]] + def _get_ordered_configuration_items(self) -> Iterator[Tuple[str, Any]]: # Configuration gives keys in an unordered manner. Order them. override_order = ["global", self.name, ":env:"] @@ -211,8 +201,7 @@ class ConfigOptionParser(CustomOptionParser): for key, val in section_items[section]: yield key, val - def _update_defaults(self, defaults): - # type: (Dict[str, Any]) -> Dict[str, Any] + def _update_defaults(self, defaults: Dict[str, Any]) -> Dict[str, Any]: """Updates the given defaults with values from the config files and the environ. Does a little special handling for certain types of options (lists).""" @@ -276,8 +265,7 @@ class ConfigOptionParser(CustomOptionParser): self.values = None return defaults - def get_default_values(self): - # type: () -> optparse.Values + def get_default_values(self) -> optparse.Values: """Overriding to make updating the defaults after instantiation of the option parser possible, _update_defaults() does the dirty work.""" if not self.process_default_values: @@ -299,7 +287,6 @@ class ConfigOptionParser(CustomOptionParser): defaults[option.dest] = option.check_value(opt_str, default) return optparse.Values(defaults) - def error(self, msg): - # type: (str) -> None + def error(self, msg: str) -> None: self.print_usage(sys.stderr) self.exit(UNKNOWN_ERROR, f"{msg}\n") diff --git a/src/pip/_internal/commands/__init__.py b/src/pip/_internal/commands/__init__.py index 31c985fdc..e1fb87884 100644 --- a/src/pip/_internal/commands/__init__.py +++ b/src/pip/_internal/commands/__init__.py @@ -59,6 +59,10 @@ commands_dict = OrderedDict([ 'pip._internal.commands.cache', 'CacheCommand', "Inspect and manage pip's wheel cache.", )), + ('index', CommandInfo( + 'pip._internal.commands.index', 'IndexCommand', + "Inspect information available from package indexes.", + )), ('wheel', CommandInfo( 'pip._internal.commands.wheel', 'WheelCommand', 'Build wheels from your requirements.', diff --git a/src/pip/_internal/commands/cache.py b/src/pip/_internal/commands/cache.py index 5155a5053..fac9823c1 100644 --- a/src/pip/_internal/commands/cache.py +++ b/src/pip/_internal/commands/cache.py @@ -1,4 +1,3 @@ -import logging import os import textwrap from optparse import Values @@ -8,8 +7,9 @@ import pip._internal.utils.filesystem as filesystem from pip._internal.cli.base_command import Command from pip._internal.cli.status_codes import ERROR, SUCCESS from pip._internal.exceptions import CommandError, PipError +from pip._internal.utils.logging import getLogger -logger = logging.getLogger(__name__) +logger = getLogger(__name__) class CacheCommand(Command): @@ -184,8 +184,8 @@ class CacheCommand(Command): for filename in files: os.unlink(filename) - logger.debug('Removed %s', filename) - logger.info('Files removed: %s', len(files)) + logger.verbose("Removed %s", filename) + logger.info("Files removed: %s", len(files)) def purge_cache(self, options, args): # type: (Values, List[Any]) -> None diff --git a/src/pip/_internal/commands/index.py b/src/pip/_internal/commands/index.py new file mode 100644 index 000000000..4bfc4e9e3 --- /dev/null +++ b/src/pip/_internal/commands/index.py @@ -0,0 +1,143 @@ +import logging +from optparse import Values +from typing import Any, Iterable, List, Optional, Union + +from pip._vendor.packaging.version import LegacyVersion, Version + +from pip._internal.cli import cmdoptions +from pip._internal.cli.req_command import IndexGroupCommand +from pip._internal.cli.status_codes import ERROR, SUCCESS +from pip._internal.commands.search import print_dist_installation_info +from pip._internal.exceptions import CommandError, DistributionNotFound, PipError +from pip._internal.index.collector import LinkCollector +from pip._internal.index.package_finder import PackageFinder +from pip._internal.models.selection_prefs import SelectionPreferences +from pip._internal.models.target_python import TargetPython +from pip._internal.network.session import PipSession +from pip._internal.utils.misc import write_output + +logger = logging.getLogger(__name__) + + +class IndexCommand(IndexGroupCommand): + """ + Inspect information available from package indexes. + """ + + usage = """ + %prog versions + """ + + def add_options(self): + # type: () -> None + cmdoptions.add_target_python_options(self.cmd_opts) + + self.cmd_opts.add_option(cmdoptions.ignore_requires_python()) + self.cmd_opts.add_option(cmdoptions.pre()) + self.cmd_opts.add_option(cmdoptions.no_binary()) + self.cmd_opts.add_option(cmdoptions.only_binary()) + + index_opts = cmdoptions.make_option_group( + cmdoptions.index_group, + self.parser, + ) + + self.parser.insert_option_group(0, index_opts) + self.parser.insert_option_group(0, self.cmd_opts) + + def run(self, options, args): + # type: (Values, List[Any]) -> int + handlers = { + "versions": self.get_available_package_versions, + } + + logger.warning( + "pip index is currently an experimental command. " + "It may be removed/changed in a future release " + "without prior warning." + ) + + # Determine action + if not args or args[0] not in handlers: + logger.error( + "Need an action (%s) to perform.", + ", ".join(sorted(handlers)), + ) + return ERROR + + action = args[0] + + # Error handling happens here, not in the action-handlers. + try: + handlers[action](options, args[1:]) + except PipError as e: + logger.error(e.args[0]) + return ERROR + + return SUCCESS + + def _build_package_finder( + self, + options, # type: Values + session, # type: PipSession + target_python=None, # type: Optional[TargetPython] + ignore_requires_python=None, # type: Optional[bool] + ): + # type: (...) -> PackageFinder + """ + Create a package finder appropriate to the index command. + """ + link_collector = LinkCollector.create(session, options=options) + + # Pass allow_yanked=False to ignore yanked versions. + selection_prefs = SelectionPreferences( + allow_yanked=False, + allow_all_prereleases=options.pre, + ignore_requires_python=ignore_requires_python, + ) + + return PackageFinder.create( + link_collector=link_collector, + selection_prefs=selection_prefs, + target_python=target_python, + ) + + def get_available_package_versions(self, options, args): + # type: (Values, List[Any]) -> None + if len(args) != 1: + raise CommandError('You need to specify exactly one argument') + + target_python = cmdoptions.make_target_python(options) + query = args[0] + + with self._build_session(options) as session: + finder = self._build_package_finder( + options=options, + session=session, + target_python=target_python, + ignore_requires_python=options.ignore_requires_python, + ) + + versions: Iterable[Union[LegacyVersion, Version]] = ( + candidate.version + for candidate in finder.find_all_candidates(query) + ) + + if not options.pre: + # Remove prereleases + versions = (version for version in versions + if not version.is_prerelease) + versions = set(versions) + + if not versions: + raise DistributionNotFound( + 'No matching distribution found for {}'.format(query)) + + formatted_versions = [str(ver) for ver in sorted( + versions, reverse=True)] + latest = formatted_versions[0] + + write_output('{} ({})'.format(query, latest)) + write_output('Available versions: {}'.format( + ', '.join(formatted_versions))) + print_dist_installation_info(query, latest) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 6932f5a6d..f2969630e 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -1,5 +1,4 @@ import errno -import logging import operator import os import shutil @@ -26,8 +25,10 @@ from pip._internal.operations.check import ConflictDetails, check_install_confli from pip._internal.req import install_given_reqs from pip._internal.req.req_install import InstallRequirement from pip._internal.req.req_tracker import get_requirement_tracker +from pip._internal.utils.compat import WINDOWS from pip._internal.utils.distutils_args import parse_distutils_args from pip._internal.utils.filesystem import test_writable_dir +from pip._internal.utils.logging import getLogger from pip._internal.utils.misc import ( ensure_dir, get_pip_version, @@ -45,7 +46,7 @@ from pip._internal.wheel_builder import ( should_build_for_install_command, ) -logger = logging.getLogger(__name__) +logger = getLogger(__name__) def get_check_binary_allowed(format_control): @@ -238,7 +239,7 @@ class InstallCommand(RequirementCommand): install_options = options.install_options or [] - logger.debug("Using %s", get_pip_version()) + logger.verbose("Using %s", get_pip_version()) options.use_user_site = decide_user_install( options.use_user_site, prefix_path=options.prefix_path, @@ -737,4 +738,16 @@ def create_os_error_message(error, show_traceback, using_user_site): parts.append(permissions_part) parts.append(".\n") + # Suggest the user to enable Long Paths if path length is + # more than 260 + if (WINDOWS and error.errno == errno.ENOENT and error.filename and + len(error.filename) > 260): + parts.append( + "HINT: This error might have occurred since " + "this system does not have Windows Long Path " + "support enabled. You can find information on " + "how to enable this at " + "https://pip.pypa.io/warnings/enable-long-paths\n" + ) + return "".join(parts).strip() + "\n" diff --git a/src/pip/_internal/commands/search.py b/src/pip/_internal/commands/search.py index d66e82347..3bfd29afc 100644 --- a/src/pip/_internal/commands/search.py +++ b/src/pip/_internal/commands/search.py @@ -114,6 +114,23 @@ def transform_hits(hits): return list(packages.values()) +def print_dist_installation_info(name, latest): + # type: (str, str) -> None + env = get_default_environment() + dist = env.get_distribution(name) + if dist is not None: + with indent_log(): + if dist.version == latest: + write_output('INSTALLED: %s (latest)', dist.version) + else: + write_output('INSTALLED: %s', dist.version) + if parse_version(latest).pre: + write_output('LATEST: %s (pre-release; install' + ' with "pip install --pre")', latest) + else: + write_output('LATEST: %s', latest) + + def print_results(hits, name_column_width=None, terminal_width=None): # type: (List[TransformedHit], Optional[int], Optional[int]) -> None if not hits: @@ -124,7 +141,6 @@ def print_results(hits, name_column_width=None, terminal_width=None): for hit in hits ]) + 4 - env = get_default_environment() for hit in hits: name = hit['name'] summary = hit['summary'] or '' @@ -141,18 +157,7 @@ def print_results(hits, name_column_width=None, terminal_width=None): line = f'{name_latest:{name_column_width}} - {summary}' try: write_output(line) - dist = env.get_distribution(name) - if dist is not None: - with indent_log(): - if dist.version == latest: - write_output('INSTALLED: %s (latest)', dist.version) - else: - write_output('INSTALLED: %s', dist.version) - if parse_version(latest).pre: - write_output('LATEST: %s (pre-release; install' - ' with "pip install --pre")', latest) - else: - write_output('LATEST: %s', latest) + print_dist_installation_info(name, latest) except UnicodeEncodeError: pass diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index 86d0be407..ebee38395 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -150,7 +150,7 @@ class Link(KeyBasedCompareMixin): def url_without_fragment(self): # type: () -> str scheme, netloc, path, query, fragment = self._parsed_url - return urllib.parse.urlunsplit((scheme, netloc, path, query, None)) + return urllib.parse.urlunsplit((scheme, netloc, path, query, '')) _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)') diff --git a/src/pip/_internal/models/wheel.py b/src/pip/_internal/models/wheel.py index 0a582b305..827ebca91 100644 --- a/src/pip/_internal/models/wheel.py +++ b/src/pip/_internal/models/wheel.py @@ -69,7 +69,7 @@ class Wheel: def find_most_preferred_tag(self, tags, tag_to_priority): # type: (List[Tag], Dict[Tag, int]) -> int """Return the priority of the most preferred tag that one of the wheel's file - tag combinations acheives in the given list of supported tags using the given + tag combinations achieves in the given list of supported tags using the given tag_to_priority mapping, where lower priorities are more-preferred. This is used in place of support_index_min in some cases in order to avoid diff --git a/src/pip/_internal/network/auth.py b/src/pip/_internal/network/auth.py index bd54a5cba..1457c92fe 100644 --- a/src/pip/_internal/network/auth.py +++ b/src/pip/_internal/network/auth.py @@ -4,7 +4,6 @@ Contains interface (MultiDomainBasicAuth) and associated glue code for providing credentials in the context of network requests. """ -import logging import urllib.parse from typing import Any, Dict, List, Optional, Tuple @@ -12,6 +11,7 @@ from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth from pip._vendor.requests.models import Request, Response from pip._vendor.requests.utils import get_netrc_auth +from pip._internal.utils.logging import getLogger from pip._internal.utils.misc import ( ask, ask_input, @@ -21,7 +21,7 @@ from pip._internal.utils.misc import ( ) from pip._internal.vcs.versioncontrol import AuthInfo -logger = logging.getLogger(__name__) +logger = getLogger(__name__) Credentials = Tuple[str, str, str] @@ -170,13 +170,12 @@ class MultiDomainBasicAuth(AuthBase): """ url, netloc, _ = split_auth_netloc_from_url(original_url) - # Use any stored credentials that we have for this netloc - username, password = self.passwords.get(netloc, (None, None)) + # Try to get credentials from original url + username, password = self._get_new_credentials(original_url) + # If credentials not found, use any stored credentials for this netloc if username is None and password is None: - # No stored credentials. Acquire new credentials without prompting - # the user. (e.g. from netrc, keyring, or the URL itself) - username, password = self._get_new_credentials(original_url) + username, password = self.passwords.get(netloc, (None, None)) if username is not None or password is not None: # Convert the username and password if they're None, so that diff --git a/src/pip/_internal/network/lazy_wheel.py b/src/pip/_internal/network/lazy_wheel.py index b877d3b7a..781cb0154 100644 --- a/src/pip/_internal/network/lazy_wheel.py +++ b/src/pip/_internal/network/lazy_wheel.py @@ -123,7 +123,7 @@ class LazyZipOverHTTP: def tell(self): # type: () -> int - """Return the current possition.""" + """Return the current position.""" return self._file.tell() def truncate(self, size=None): diff --git a/src/pip/_internal/network/session.py b/src/pip/_internal/network/session.py index 4af800f12..dca263744 100644 --- a/src/pip/_internal/network/session.py +++ b/src/pip/_internal/network/session.py @@ -19,6 +19,8 @@ import logging import mimetypes import os import platform +import shutil +import subprocess import sys import urllib.parse import warnings @@ -163,6 +165,21 @@ def user_agent(): if setuptools_dist is not None: data["setuptools_version"] = str(setuptools_dist.version) + if shutil.which("rustc") is not None: + # If for any reason `rustc --version` fails, silently ignore it + try: + rustc_output = subprocess.check_output( + ["rustc", "--version"], stderr=subprocess.STDOUT, timeout=.5 + ) + except Exception: + pass + else: + if rustc_output.startswith(b"rustc "): + # The format of `rustc --version` is: + # `b'rustc 1.52.1 (9bc8c42bb 2021-05-09)\n'` + # We extract just the middle (1.52.1) part + data["rustc_version"] = rustc_output.split(b" ")[1].decode() + # Use None rather than False so as not to give the impression that # pip knows it is not being run under CI. Rather, it is a null or # inconclusive result. Also, we include some value rather than no diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py index f34a9d4be..3cda5c8c9 100644 --- a/src/pip/_internal/operations/freeze.py +++ b/src/pip/_internal/operations/freeze.py @@ -176,7 +176,7 @@ def get_requirement_info(dist): location = os.path.normcase(os.path.abspath(dist.location)) - from pip._internal.vcs import RemoteNotFoundError, vcs + from pip._internal.vcs import RemoteNotFoundError, RemoteNotValidError, vcs vcs_backend = vcs.get_backend_for_dir(location) if vcs_backend is None: @@ -200,6 +200,14 @@ def get_requirement_info(dist): ) ] return (location, True, comments) + except RemoteNotValidError as ex: + req = dist.as_requirement() + comments = [ + f"# Editable {type(vcs_backend).__name__} install ({req}) with " + f"either a deleted local remote or invalid URI:", + f"# '{ex.url}'", + ] + return (location, True, comments) except BadCommand: logger.warning( diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 3d074f9f6..247e63fc8 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -39,7 +39,7 @@ from pip._internal.utils.deprecation import deprecated from pip._internal.utils.filesystem import copy2_fixed from pip._internal.utils.hashes import Hashes, MissingHashes from pip._internal.utils.logging import indent_log -from pip._internal.utils.misc import display_path, hide_url, rmtree +from pip._internal.utils.misc import display_path, hide_url, is_installable_dir, rmtree from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.unpacking import unpack_file from pip._internal.vcs import vcs @@ -376,7 +376,7 @@ class RequirementPreparer: # installation. # FIXME: this won't upgrade when there's an existing # package unpacked in `req.source_dir` - if os.path.exists(os.path.join(req.source_dir, 'setup.py')): + if is_installable_dir(req.source_dir): raise PreviousBuildDirError( "pip can't proceed with requirements '{}' due to a" "pre-existing build directory ({}). This is likely " diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index 3f9e7dd77..0887102ea 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -248,8 +248,8 @@ def _looks_like_path(name): def _get_url_from_path(path, name): # type: (str, str) -> Optional[str] """ - First, it checks whether a provided path is an installable directory - (e.g. it has a setup.py). If it is, returns the path. + First, it checks whether a provided path is an installable directory. If it + is, returns the path. If false, check if the path is an archive file (such as a .whl). The function checks if the path is a file. If false, if the path has diff --git a/src/pip/_internal/req/req_uninstall.py b/src/pip/_internal/req/req_uninstall.py index b72234175..83decc61d 100644 --- a/src/pip/_internal/req/req_uninstall.py +++ b/src/pip/_internal/req/req_uninstall.py @@ -1,6 +1,5 @@ import csv import functools -import logging import os import sys import sysconfig @@ -13,7 +12,7 @@ from pip._vendor.pkg_resources import Distribution from pip._internal.exceptions import UninstallationError from pip._internal.locations import get_bin_prefix, get_bin_user from pip._internal.utils.compat import WINDOWS -from pip._internal.utils.logging import indent_log +from pip._internal.utils.logging import getLogger, indent_log from pip._internal.utils.misc import ( ask, dist_in_usersite, @@ -26,7 +25,7 @@ from pip._internal.utils.misc import ( ) from pip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory -logger = logging.getLogger(__name__) +logger = getLogger(__name__) def _script_names(dist, script_name, is_gui): @@ -74,8 +73,27 @@ def uninstallation_paths(dist): the .pyc and .pyo in the same directory. UninstallPathSet.add() takes care of the __pycache__ .py[co]. + + If RECORD is not found, raises UninstallationError, + with possible information from the INSTALLER file. + + https://packaging.python.org/specifications/recording-installed-packages/ """ - r = csv.reader(dist.get_metadata_lines('RECORD')) + try: + r = csv.reader(dist.get_metadata_lines('RECORD')) + except FileNotFoundError as missing_record_exception: + msg = 'Cannot uninstall {dist}, RECORD file not found.'.format(dist=dist) + try: + installer = next(dist.get_metadata_lines('INSTALLER')) + if not installer or installer == 'pip': + raise ValueError() + except (OSError, StopIteration, ValueError): + dep = '{}=={}'.format(dist.project_name, dist.version) + msg += (" You might be able to recover from this via: " + "'pip install --force-reinstall --no-deps {}'.".format(dep)) + else: + msg += ' Hint: The package was installed by {}.'.format(installer) + raise UninstallationError(msg) from missing_record_exception for row in r: path = os.path.join(dist.location, row[0]) yield path @@ -384,7 +402,7 @@ class UninstallPathSet: for path in sorted(compact(for_rename)): moved.stash(path) - logger.debug('Removing file or directory %s', path) + logger.verbose('Removing file or directory %s', path) for pth in self.pth.values(): pth.remove() @@ -599,7 +617,7 @@ class UninstallPthEntries: def remove(self): # type: () -> None - logger.debug('Removing pth entries from %s:', self.file) + logger.verbose('Removing pth entries from %s:', self.file) # If the file doesn't exist, log a warning and return if not os.path.isfile(self.file): @@ -620,7 +638,7 @@ class UninstallPthEntries: lines[-1] = lines[-1] + endline.encode("utf-8") for entry in self.entries: try: - logger.debug('Removing entry: %s', entry) + logger.verbose('Removing entry: %s', entry) lines.remove((entry + endline).encode("utf-8")) except ValueError: pass diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index da516ad3c..e496e10dd 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -32,6 +32,9 @@ BaseCandidate = Union[ "LinkCandidate", ] +# Avoid conflicting with the PyPI package "Python". +REQUIRES_PYTHON_IDENTIFIER = cast(NormalizedName, "") + def as_base_candidate(candidate: Candidate) -> Optional[BaseCandidate]: """The runtime version of BaseCandidate.""" @@ -578,13 +581,12 @@ class RequiresPythonCandidate(Candidate): @property def project_name(self): # type: () -> NormalizedName - # Avoid conflicting with the PyPI package "Python". - return cast(NormalizedName, "") + return REQUIRES_PYTHON_IDENTIFIER @property def name(self): # type: () -> str - return self.project_name + return REQUIRES_PYTHON_IDENTIFIER @property def version(self): diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index 0be58fd3b..e6b5bd544 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -1,8 +1,11 @@ +import collections +import math from typing import TYPE_CHECKING, Dict, Iterable, Iterator, Mapping, Sequence, Union from pip._vendor.resolvelib.providers import AbstractProvider from .base import Candidate, Constraint, Requirement +from .candidates import REQUIRES_PYTHON_IDENTIFIER from .factory import Factory if TYPE_CHECKING: @@ -59,6 +62,7 @@ class PipProvider(_ProviderBase): self._ignore_dependencies = ignore_dependencies self._upgrade_strategy = upgrade_strategy self._user_requested = user_requested + self._known_depths: Dict[str, float] = collections.defaultdict(lambda: math.inf) def identify(self, requirement_or_candidate): # type: (Union[Requirement, Candidate]) -> str @@ -78,48 +82,47 @@ class PipProvider(_ProviderBase): Currently pip considers the followings in order: - * Prefer if any of the known requirements points to an explicit URL. - * If equal, prefer if any requirements contain ``===`` and ``==``. - * If equal, prefer if requirements include version constraints, e.g. - ``>=`` and ``<``. - * If equal, prefer user-specified (non-transitive) requirements, and - order user-specified requirements by the order they are specified. + * Prefer if any of the known requirements is "direct", e.g. points to an + explicit URL. + * If equal, prefer if any requirement is "pinned", i.e. contains + operator ``===`` or ``==``. + * If equal, calculate an approximate "depth" and resolve requirements + closer to the user-specified requirements first. + * Order user-specified requirements by the order they are specified. + * If equal, prefers "non-free" requirements, i.e. contains at least one + operator, such as ``>=`` or ``<``. * If equal, order alphabetically for consistency (helps debuggability). """ + lookups = (r.get_candidate_lookup() for r, _ in information[identifier]) + candidate, ireqs = zip(*lookups) + operators = [ + specifier.operator + for specifier_set in (ireq.specifier for ireq in ireqs if ireq) + for specifier in specifier_set + ] - def _get_restrictive_rating(requirements): - # type: (Iterable[Requirement]) -> int - """Rate how restrictive a set of requirements are. + direct = candidate is not None + pinned = any(op[:2] == "==" for op in operators) + unfree = bool(operators) - ``Requirement.get_candidate_lookup()`` returns a 2-tuple for - lookup. The first element is ``Optional[Candidate]`` and the - second ``Optional[InstallRequirement]``. + try: + requested_order: Union[int, float] = self._user_requested[identifier] + except KeyError: + requested_order = math.inf + parent_depths = ( + self._known_depths[parent.name] if parent is not None else 0.0 + for _, parent in information[identifier] + ) + inferred_depth = min(d for d in parent_depths) + 1.0 + self._known_depths[identifier] = inferred_depth + else: + inferred_depth = 1.0 - * If the requirement is an explicit one, the explicitly-required - candidate is returned as the first element. - * If the requirement is based on a PEP 508 specifier, the backing - ``InstallRequirement`` is returned as the second element. + requested_order = self._user_requested.get(identifier, math.inf) - We use the first element to check whether there is an explicit - requirement, and the second for equality operator. - """ - lookups = (r.get_candidate_lookup() for r in requirements) - cands, ireqs = zip(*lookups) - if any(cand is not None for cand in cands): - return 0 - spec_sets = (ireq.specifier for ireq in ireqs if ireq) - operators = [ - specifier.operator for spec_set in spec_sets for specifier in spec_set - ] - if any(op in ("==", "===") for op in operators): - return 1 - if operators: - return 2 - # A "bare" requirement without any version requirements. - return 3 - - rating = _get_restrictive_rating(r for r, _ in information[identifier]) - order = self._user_requested.get(identifier, float("inf")) + # Requires-Python has only one candidate and the check is basically + # free, so we always do it first to avoid needless work if it fails. + requires_python = identifier == REQUIRES_PYTHON_IDENTIFIER # HACK: Setuptools have a very long and solid backward compatibility # track record, and extremely few projects would request a narrow, @@ -131,7 +134,16 @@ class PipProvider(_ProviderBase): # while we work on "proper" branch pruning techniques. delay_this = identifier == "setuptools" - return (delay_this, rating, order, identifier) + return ( + not requires_python, + delay_this, + not direct, + not pinned, + inferred_depth, + requested_order, + not unfree, + identifier, + ) def find_matches( self, diff --git a/src/pip/_internal/utils/_log.py b/src/pip/_internal/utils/_log.py new file mode 100644 index 000000000..92c4c6a19 --- /dev/null +++ b/src/pip/_internal/utils/_log.py @@ -0,0 +1,38 @@ +"""Customize logging + +Defines custom logger class for the `logger.verbose(...)` method. + +init_logging() must be called before any other modules that call logging.getLogger. +""" + +import logging +from typing import Any, cast + +# custom log level for `--verbose` output +# between DEBUG and INFO +VERBOSE = 15 + + +class VerboseLogger(logging.Logger): + """Custom Logger, defining a verbose log-level + + VERBOSE is between INFO and DEBUG. + """ + + def verbose(self, msg: str, *args: Any, **kwargs: Any) -> None: + return self.log(VERBOSE, msg, *args, **kwargs) + + +def getLogger(name: str) -> VerboseLogger: + """logging.getLogger, but ensures our VerboseLogger class is returned""" + return cast(VerboseLogger, logging.getLogger(name)) + + +def init_logging() -> None: + """Register our VerboseLogger and VERBOSE log level. + + Should be called before any calls to getLogger(), + i.e. in pip._internal.__init__ + """ + logging.setLoggerClass(VerboseLogger) + logging.addLevelName(VERBOSE, "VERBOSE") diff --git a/src/pip/_internal/utils/filesystem.py b/src/pip/_internal/utils/filesystem.py index 3db97dc41..177a6b4fb 100644 --- a/src/pip/_internal/utils/filesystem.py +++ b/src/pip/_internal/utils/filesystem.py @@ -98,7 +98,7 @@ def adjacent_tmp_file(path, **kwargs): os.fsync(result.fileno()) -# Tenacity raises RetryError by default, explictly raise the original exception +# Tenacity raises RetryError by default, explicitly raise the original exception _replace_retry = retry(reraise=True, stop=stop_after_delay(1), wait=wait_fixed(0.25)) replace = _replace_retry(os.replace) diff --git a/src/pip/_internal/utils/logging.py b/src/pip/_internal/utils/logging.py index 45798d54f..0569b9248 100644 --- a/src/pip/_internal/utils/logging.py +++ b/src/pip/_internal/utils/logging.py @@ -4,9 +4,10 @@ import logging import logging.handlers import os import sys -from logging import Filter, getLogger +from logging import Filter from typing import IO, Any, Callable, Iterator, Optional, TextIO, Type, cast +from pip._internal.utils._log import VERBOSE, getLogger from pip._internal.utils.compat import WINDOWS from pip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX from pip._internal.utils.misc import ensure_dir @@ -272,18 +273,20 @@ def setup_logging(verbosity, no_color, user_log_file): """ # Determine the level to be logging at. - if verbosity >= 1: - level = "DEBUG" + if verbosity >= 2: + level_number = logging.DEBUG + elif verbosity == 1: + level_number = VERBOSE elif verbosity == -1: - level = "WARNING" + level_number = logging.WARNING elif verbosity == -2: - level = "ERROR" + level_number = logging.ERROR elif verbosity <= -3: - level = "CRITICAL" + level_number = logging.CRITICAL else: - level = "INFO" + level_number = logging.INFO - level_number = getattr(logging, level) + level = logging.getLevelName(level_number) # The "root" logger should match the "console" level *unless* we also need # to log to a user log file. diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index a4ad35be6..99ebea30c 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -128,7 +128,7 @@ def get_prog(): # Retry every half second for up to 3 seconds -# Tenacity raises RetryError by default, explictly raise the original exception +# Tenacity raises RetryError by default, explicitly raise the original exception @retry(reraise=True, stop=stop_after_delay(3), wait=wait_fixed(0.5)) def rmtree(dir, ignore_errors=False): # type: (AnyStr, bool) -> None @@ -270,13 +270,20 @@ def tabulate(rows): def is_installable_dir(path: str) -> bool: - """Is path is a directory containing pyproject.toml, setup.cfg or setup.py?""" + """Is path is a directory containing pyproject.toml or setup.py? + + If pyproject.toml exists, this is a PEP 517 project. Otherwise we look for + a legacy setuptools layout by identifying setup.py. We don't check for the + setup.cfg because using it without setup.py is only available for PEP 517 + projects, which are already covered by the pyproject.toml check. + """ if not os.path.isdir(path): return False - return any( - os.path.isfile(os.path.join(path, signifier)) - for signifier in ("pyproject.toml", "setup.cfg", "setup.py") - ) + if os.path.isfile(os.path.join(path, "pyproject.toml")): + return True + if os.path.isfile(os.path.join(path, "setup.py")): + return True + return False def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): diff --git a/src/pip/_internal/utils/subprocess.py b/src/pip/_internal/utils/subprocess.py index 2c8cf2123..da052ee69 100644 --- a/src/pip/_internal/utils/subprocess.py +++ b/src/pip/_internal/utils/subprocess.py @@ -6,7 +6,7 @@ from typing import Any, Callable, Iterable, List, Mapping, Optional, Union from pip._internal.cli.spinners import SpinnerInterface, open_spinner from pip._internal.exceptions import InstallationSubprocessError -from pip._internal.utils.logging import subprocess_logger +from pip._internal.utils.logging import VERBOSE, subprocess_logger from pip._internal.utils.misc import HiddenText CommandArgs = List[Union[str, HiddenText]] @@ -144,10 +144,10 @@ def call_subprocess( log_subprocess = subprocess_logger.info used_level = logging.INFO else: - # Then log the subprocess output using DEBUG. This also ensures + # Then log the subprocess output using VERBOSE. This also ensures # it will be logged to the log file (aka user_log), if enabled. - log_subprocess = subprocess_logger.debug - used_level = logging.DEBUG + log_subprocess = subprocess_logger.verbose + used_level = VERBOSE # Whether the subprocess will be visible in the console. showing_subprocess = subprocess_logger.getEffectiveLevel() <= used_level diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index 30025d632..b6beddbe6 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -8,6 +8,7 @@ import pip._internal.vcs.mercurial import pip._internal.vcs.subversion # noqa: F401 from pip._internal.vcs.versioncontrol import ( # noqa: F401 RemoteNotFoundError, + RemoteNotValidError, is_url, make_vcs_requirement_url, vcs, diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index b7c1b9fe7..b860e350a 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -1,5 +1,6 @@ import logging import os.path +import pathlib import re import urllib.parse import urllib.request @@ -14,9 +15,10 @@ from pip._internal.utils.subprocess import make_command from pip._internal.vcs.versioncontrol import ( AuthInfo, RemoteNotFoundError, + RemoteNotValidError, RevOptions, VersionControl, - find_path_to_setup_from_repo_root, + find_path_to_project_root_from_repo_root, vcs, ) @@ -29,6 +31,18 @@ logger = logging.getLogger(__name__) HASH_REGEX = re.compile('^[a-fA-F0-9]{40}$') +# SCP (Secure copy protocol) shorthand. e.g. 'git@example.com:foo/bar.git' +SCP_REGEX = re.compile(r"""^ + # Optional user, e.g. 'git@' + (\w+@)? + # Server, e.g. 'github.com'. + ([^/:]+): + # The server-side path. e.g. 'user/project.git'. Must start with an + # alphanumeric character so as not to be confusable with a Windows paths + # like 'C:/foo/bar' or 'C:\foo\bar'. + (\w[^:]*) +$""", re.VERBOSE) + def looks_like_hash(sha): # type: (str) -> bool @@ -328,7 +342,39 @@ class Git(VersionControl): found_remote = remote break url = found_remote.split(' ')[1] - return url.strip() + return cls._git_remote_to_pip_url(url.strip()) + + @staticmethod + def _git_remote_to_pip_url(url): + # type: (str) -> str + """ + Convert a remote url from what git uses to what pip accepts. + + There are 3 legal forms **url** may take: + + 1. A fully qualified url: ssh://git@example.com/foo/bar.git + 2. A local project.git folder: /path/to/bare/repository.git + 3. SCP shorthand for form 1: git@example.com:foo/bar.git + + Form 1 is output as-is. Form 2 must be converted to URI and form 3 must + be converted to form 1. + + See the corresponding test test_git_remote_url_to_pip() for examples of + sample inputs/outputs. + """ + if re.match(r"\w+://", url): + # This is already valid. Pass it though as-is. + return url + if os.path.exists(url): + # A local bare remote (git clone --mirror). + # Needs a file:// prefix. + return pathlib.PurePath(url).as_uri() + scp_match = SCP_REGEX.match(url) + if scp_match: + # Add an ssh:// prefix and replace the ':' with a '/'. + return scp_match.expand(r"ssh://\1\2/\3") + # Otherwise, bail out. + raise RemoteNotValidError(url) @classmethod def has_commit(cls, location, rev): @@ -364,8 +410,8 @@ class Git(VersionControl): def get_subdirectory(cls, location): # type: (str) -> Optional[str] """ - Return the path to setup.py, relative to the repo root. - Return None if setup.py is in the repo root. + Return the path to Python project root, relative to the repo root. + Return None if the project root is in the repo root. """ # find the repo root git_dir = cls.run_command( @@ -377,7 +423,7 @@ class Git(VersionControl): if not os.path.isabs(git_dir): git_dir = os.path.join(location, git_dir) repo_root = os.path.abspath(os.path.join(git_dir, '..')) - return find_path_to_setup_from_repo_root(location, repo_root) + return find_path_to_project_root_from_repo_root(location, repo_root) @classmethod def get_url_rev_and_auth(cls, url): @@ -446,5 +492,12 @@ class Git(VersionControl): return None return os.path.normpath(r.rstrip('\r\n')) + @staticmethod + def should_add_vcs_url_prefix(repo_url): + # type: (str) -> bool + """In either https or ssh form, requirements must be prefixed with git+. + """ + return True + vcs.register(Git) diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index b4f887d32..8f8b09bd2 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -10,7 +10,7 @@ from pip._internal.utils.urls import path_to_url from pip._internal.vcs.versioncontrol import ( RevOptions, VersionControl, - find_path_to_setup_from_repo_root, + find_path_to_project_root_from_repo_root, vcs, ) @@ -120,8 +120,8 @@ class Mercurial(VersionControl): def get_subdirectory(cls, location): # type: (str) -> Optional[str] """ - Return the path to setup.py, relative to the repo root. - Return None if setup.py is in the repo root. + Return the path to Python project root, relative to the repo root. + Return None if the project root is in the repo root. """ # find the repo root repo_root = cls.run_command( @@ -129,7 +129,7 @@ class Mercurial(VersionControl): ).strip() if not os.path.isabs(repo_root): repo_root = os.path.abspath(os.path.join(location, repo_root)) - return find_path_to_setup_from_repo_root(location, repo_root) + return find_path_to_project_root_from_repo_root(location, repo_root) @classmethod def get_repository_root(cls, location): diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py index 4d1237ca0..965e0b425 100644 --- a/src/pip/_internal/vcs/subversion.py +++ b/src/pip/_internal/vcs/subversion.py @@ -7,6 +7,7 @@ from pip._internal.utils.misc import ( HiddenText, display_path, is_console_interactive, + is_installable_dir, split_auth_from_netloc, ) from pip._internal.utils.subprocess import CommandArgs, make_command @@ -111,18 +112,17 @@ class Subversion(VersionControl): @classmethod def get_remote_url(cls, location): # type: (str) -> str - # In cases where the source is in a subdirectory, not alongside - # setup.py we have to look up in the location until we find a real - # setup.py + # In cases where the source is in a subdirectory, we have to look up in + # the location until we find a valid project root. orig_location = location - while not os.path.exists(os.path.join(location, 'setup.py')): + while not is_installable_dir(location): last_location = location location = os.path.dirname(location) if location == last_location: # We've traversed up to the root of the filesystem without - # finding setup.py + # finding a Python project. logger.warning( - "Could not find setup.py for directory %s (tried all " + "Could not find Python project for directory %s (tried all " "parent directories)", orig_location, ) diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py index 97977b579..cddd78c5e 100644 --- a/src/pip/_internal/vcs/versioncontrol.py +++ b/src/pip/_internal/vcs/versioncontrol.py @@ -27,6 +27,7 @@ from pip._internal.utils.misc import ( display_path, hide_url, hide_value, + is_installable_dir, rmtree, ) from pip._internal.utils.subprocess import CommandArgs, call_subprocess, make_command @@ -68,23 +69,23 @@ def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None): return req -def find_path_to_setup_from_repo_root(location, repo_root): +def find_path_to_project_root_from_repo_root(location, repo_root): # type: (str, str) -> Optional[str] """ - Find the path to `setup.py` by searching up the filesystem from `location`. - Return the path to `setup.py` relative to `repo_root`. - Return None if `setup.py` is in `repo_root` or cannot be found. + Find the the Python project's root by searching up the filesystem from + `location`. Return the path to project root relative to `repo_root`. + Return None if the project root is `repo_root`, or cannot be found. """ - # find setup.py + # find project root. orig_location = location - while not os.path.exists(os.path.join(location, 'setup.py')): + while not is_installable_dir(location): last_location = location location = os.path.dirname(location) if location == last_location: # We've traversed up to the root of the filesystem without - # finding setup.py + # finding a Python project. logger.warning( - "Could not find setup.py for directory %s (tried all " + "Could not find a Python project for directory %s (tried all " "parent directories)", orig_location, ) @@ -100,6 +101,12 @@ class RemoteNotFoundError(Exception): pass +class RemoteNotValidError(Exception): + def __init__(self, url: str): + super().__init__(url) + self.url = url + + class RevOptions: """ @@ -290,8 +297,8 @@ class VersionControl: def get_subdirectory(cls, location): # type: (str) -> Optional[str] """ - Return the path to setup.py, relative to the repo root. - Return None if setup.py is in the repo root. + Return the path to Python project root, relative to the repo root. + Return None if the project root is in the repo root. """ return None diff --git a/src/pip/_vendor/six.py b/src/pip/_vendor/six.py index 83f69783d..4e15675d8 100644 --- a/src/pip/_vendor/six.py +++ b/src/pip/_vendor/six.py @@ -29,7 +29,7 @@ import sys import types __author__ = "Benjamin Peterson " -__version__ = "1.15.0" +__version__ = "1.16.0" # Useful for very coarse version differentiation. @@ -71,6 +71,11 @@ else: MAXSIZE = int((1 << 63) - 1) del X +if PY34: + from importlib.util import spec_from_loader +else: + spec_from_loader = None + def _add_doc(func, doc): """Add documentation to a function.""" @@ -186,6 +191,11 @@ class _SixMetaPathImporter(object): return self return None + def find_spec(self, fullname, path, target=None): + if fullname in self.known_modules: + return spec_from_loader(fullname, self) + return None + def __get_module(self, fullname): try: return self.known_modules[fullname] @@ -223,6 +233,12 @@ class _SixMetaPathImporter(object): return None get_source = get_code # same as get_code + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + pass + _importer = _SixMetaPathImporter(__name__) diff --git a/src/pip/_vendor/urllib3/_version.py b/src/pip/_vendor/urllib3/_version.py index 97c983300..e95fd5228 100644 --- a/src/pip/_vendor/urllib3/_version.py +++ b/src/pip/_vendor/urllib3/_version.py @@ -1,2 +1,2 @@ # This file is protected via CODEOWNERS -__version__ = "1.26.4" +__version__ = "1.26.5" diff --git a/src/pip/_vendor/urllib3/connection.py b/src/pip/_vendor/urllib3/connection.py index 45580b7e1..efa19af5b 100644 --- a/src/pip/_vendor/urllib3/connection.py +++ b/src/pip/_vendor/urllib3/connection.py @@ -201,7 +201,7 @@ class HTTPConnection(_HTTPConnection, object): self._prepare_conn(conn) def putrequest(self, method, url, *args, **kwargs): - """""" + """ """ # Empty docstring because the indentation of CPython's implementation # is broken but we don't want this method in our documentation. match = _CONTAINS_CONTROL_CHAR_RE.search(method) @@ -214,7 +214,7 @@ class HTTPConnection(_HTTPConnection, object): return _HTTPConnection.putrequest(self, method, url, *args, **kwargs) def putheader(self, header, *values): - """""" + """ """ if not any(isinstance(v, str) and v == SKIP_HEADER for v in values): _HTTPConnection.putheader(self, header, *values) elif six.ensure_str(header.lower()) not in SKIPPABLE_HEADERS: diff --git a/src/pip/_vendor/urllib3/connectionpool.py b/src/pip/_vendor/urllib3/connectionpool.py index 4708c5bfc..40183214e 100644 --- a/src/pip/_vendor/urllib3/connectionpool.py +++ b/src/pip/_vendor/urllib3/connectionpool.py @@ -318,7 +318,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): pass def _get_timeout(self, timeout): - """ Helper that always returns a :class:`urllib3.util.Timeout` """ + """Helper that always returns a :class:`urllib3.util.Timeout`""" if timeout is _Default: return self.timeout.clone() diff --git a/src/pip/_vendor/urllib3/contrib/pyopenssl.py b/src/pip/_vendor/urllib3/contrib/pyopenssl.py index bc5c114fa..c43146279 100644 --- a/src/pip/_vendor/urllib3/contrib/pyopenssl.py +++ b/src/pip/_vendor/urllib3/contrib/pyopenssl.py @@ -76,6 +76,7 @@ import sys from .. import util from ..packages import six +from ..util.ssl_ import PROTOCOL_TLS_CLIENT __all__ = ["inject_into_urllib3", "extract_from_urllib3"] @@ -85,6 +86,7 @@ HAS_SNI = True # Map from urllib3 to PyOpenSSL compatible parameter-values. _openssl_versions = { util.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD, + PROTOCOL_TLS_CLIENT: OpenSSL.SSL.SSLv23_METHOD, ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD, } diff --git a/src/pip/_vendor/urllib3/contrib/securetransport.py b/src/pip/_vendor/urllib3/contrib/securetransport.py index 8f058f507..b97555454 100644 --- a/src/pip/_vendor/urllib3/contrib/securetransport.py +++ b/src/pip/_vendor/urllib3/contrib/securetransport.py @@ -67,6 +67,7 @@ import weakref from pip._vendor import six from .. import util +from ..util.ssl_ import PROTOCOL_TLS_CLIENT from ._securetransport.bindings import CoreFoundation, Security, SecurityConst from ._securetransport.low_level import ( _assert_no_error, @@ -154,7 +155,8 @@ CIPHER_SUITES = [ # TLSv1 and a high of TLSv1.2. For everything else, we pin to that version. # TLSv1 to 1.2 are supported on macOS 10.8+ _protocol_to_min_max = { - util.PROTOCOL_TLS: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12) + util.PROTOCOL_TLS: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12), + PROTOCOL_TLS_CLIENT: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12), } if hasattr(ssl, "PROTOCOL_SSLv2"): diff --git a/src/pip/_vendor/urllib3/packages/six.py b/src/pip/_vendor/urllib3/packages/six.py index 314424099..d7ab761fc 100644 --- a/src/pip/_vendor/urllib3/packages/six.py +++ b/src/pip/_vendor/urllib3/packages/six.py @@ -1,4 +1,4 @@ -# Copyright (c) 2010-2019 Benjamin Peterson +# Copyright (c) 2010-2020 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -29,7 +29,7 @@ import sys import types __author__ = "Benjamin Peterson " -__version__ = "1.12.0" +__version__ = "1.16.0" # Useful for very coarse version differentiation. @@ -71,6 +71,11 @@ else: MAXSIZE = int((1 << 63) - 1) del X +if PY34: + from importlib.util import spec_from_loader +else: + spec_from_loader = None + def _add_doc(func, doc): """Add documentation to a function.""" @@ -182,6 +187,11 @@ class _SixMetaPathImporter(object): return self return None + def find_spec(self, fullname, path, target=None): + if fullname in self.known_modules: + return spec_from_loader(fullname, self) + return None + def __get_module(self, fullname): try: return self.known_modules[fullname] @@ -220,6 +230,12 @@ class _SixMetaPathImporter(object): get_source = get_code # same as get_code + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + pass + _importer = _SixMetaPathImporter(__name__) @@ -260,9 +276,19 @@ _moved_attributes = [ ), MovedModule("builtins", "__builtin__"), MovedModule("configparser", "ConfigParser"), + MovedModule( + "collections_abc", + "collections", + "collections.abc" if sys.version_info >= (3, 3) else "collections", + ), MovedModule("copyreg", "copy_reg"), MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), - MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), + MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"), + MovedModule( + "_dummy_thread", + "dummy_thread", + "_dummy_thread" if sys.version_info < (3, 9) else "_thread", + ), MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), MovedModule("http_cookies", "Cookie", "http.cookies"), MovedModule("html_entities", "htmlentitydefs", "html.entities"), @@ -307,7 +333,9 @@ _moved_attributes = [ ] # Add windows specific modules. if sys.platform == "win32": - _moved_attributes += [MovedModule("winreg", "_winreg")] + _moved_attributes += [ + MovedModule("winreg", "_winreg"), + ] for attr in _moved_attributes: setattr(_MovedItems, attr.name, attr) @@ -476,7 +504,7 @@ class Module_six_moves_urllib_robotparser(_LazyModule): _urllib_robotparser_moved_attributes = [ - MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser") + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), ] for attr in _urllib_robotparser_moved_attributes: setattr(Module_six_moves_urllib_robotparser, attr.name, attr) @@ -678,9 +706,11 @@ if PY3: if sys.version_info[1] <= 1: _assertRaisesRegex = "assertRaisesRegexp" _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" else: _assertRaisesRegex = "assertRaisesRegex" _assertRegex = "assertRegex" + _assertNotRegex = "assertNotRegex" else: def b(s): @@ -707,6 +737,7 @@ else: _assertCountEqual = "assertItemsEqual" _assertRaisesRegex = "assertRaisesRegexp" _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" _add_doc(b, """Byte literal""") _add_doc(u, """Text literal""") @@ -723,6 +754,10 @@ def assertRegex(self, *args, **kwargs): return getattr(self, _assertRegex)(*args, **kwargs) +def assertNotRegex(self, *args, **kwargs): + return getattr(self, _assertNotRegex)(*args, **kwargs) + + if PY3: exec_ = getattr(moves.builtins, "exec") @@ -762,18 +797,7 @@ else: ) -if sys.version_info[:2] == (3, 2): - exec_( - """def raise_from(value, from_value): - try: - if from_value is None: - raise value - raise value from from_value - finally: - value = None -""" - ) -elif sys.version_info[:2] > (3, 2): +if sys.version_info[:2] > (3,): exec_( """def raise_from(value, from_value): try: @@ -863,19 +887,41 @@ if sys.version_info[:2] < (3, 3): _add_doc(reraise, """Reraise an exception.""") if sys.version_info[0:2] < (3, 4): + # This does exactly the same what the :func:`py3:functools.update_wrapper` + # function does on Python versions after 3.2. It sets the ``__wrapped__`` + # attribute on ``wrapper`` object and it doesn't raise an error if any of + # the attributes mentioned in ``assigned`` and ``updated`` are missing on + # ``wrapped`` object. + def _update_wrapper( + wrapper, + wrapped, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES, + ): + for attr in assigned: + try: + value = getattr(wrapped, attr) + except AttributeError: + continue + else: + setattr(wrapper, attr, value) + for attr in updated: + getattr(wrapper, attr).update(getattr(wrapped, attr, {})) + wrapper.__wrapped__ = wrapped + return wrapper + + _update_wrapper.__doc__ = functools.update_wrapper.__doc__ def wraps( wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, updated=functools.WRAPPER_UPDATES, ): - def wrapper(f): - f = functools.wraps(wrapped, assigned, updated)(f) - f.__wrapped__ = wrapped - return f - - return wrapper + return functools.partial( + _update_wrapper, wrapped=wrapped, assigned=assigned, updated=updated + ) + wraps.__doc__ = functools.wraps.__doc__ else: wraps = functools.wraps @@ -888,7 +934,15 @@ def with_metaclass(meta, *bases): # the actual metaclass. class metaclass(type): def __new__(cls, name, this_bases, d): - return meta(name, bases, d) + if sys.version_info[:2] >= (3, 7): + # This version introduced PEP 560 that requires a bit + # of extra care (we mimic what is done by __build_class__). + resolved_bases = types.resolve_bases(bases) + if resolved_bases is not bases: + d["__orig_bases__"] = bases + else: + resolved_bases = bases + return meta(name, resolved_bases, d) @classmethod def __prepare__(cls, name, this_bases): @@ -928,12 +982,11 @@ def ensure_binary(s, encoding="utf-8", errors="strict"): - `str` -> encoded to `bytes` - `bytes` -> `bytes` """ + if isinstance(s, binary_type): + return s if isinstance(s, text_type): return s.encode(encoding, errors) - elif isinstance(s, binary_type): - return s - else: - raise TypeError("not expecting type '%s'" % type(s)) + raise TypeError("not expecting type '%s'" % type(s)) def ensure_str(s, encoding="utf-8", errors="strict"): @@ -947,12 +1000,15 @@ def ensure_str(s, encoding="utf-8", errors="strict"): - `str` -> `str` - `bytes` -> decoded to `str` """ - if not isinstance(s, (text_type, binary_type)): - raise TypeError("not expecting type '%s'" % type(s)) + # Optimization: Fast return for the common case. + if type(s) is str: + return s if PY2 and isinstance(s, text_type): - s = s.encode(encoding, errors) + return s.encode(encoding, errors) elif PY3 and isinstance(s, binary_type): - s = s.decode(encoding, errors) + return s.decode(encoding, errors) + elif not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) return s @@ -977,7 +1033,7 @@ def ensure_text(s, encoding="utf-8", errors="strict"): def python_2_unicode_compatible(klass): """ - A decorator that defines __unicode__ and __str__ methods under Python 2. + A class decorator that defines __unicode__ and __str__ methods under Python 2. Under Python 3 it does nothing. To support Python 2 and 3 with a single code base, define a __str__ method diff --git a/src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py b/src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py index 6b12fd90a..ef3fde520 100644 --- a/src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py +++ b/src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py @@ -1,9 +1,11 @@ import sys try: - # Our match_hostname function is the same as 3.5's, so we only want to + # Our match_hostname function is the same as 3.10's, so we only want to # import the match_hostname function if it's at least that good. - if sys.version_info < (3, 5): + # We also fallback on Python 3.10+ because our code doesn't emit + # deprecation warnings and is the same as Python 3.10 otherwise. + if sys.version_info < (3, 5) or sys.version_info >= (3, 10): raise ImportError("Fallback to vendored code") from ssl import CertificateError, match_hostname diff --git a/src/pip/_vendor/urllib3/util/connection.py b/src/pip/_vendor/urllib3/util/connection.py index f1e5d37f8..facfa0dd2 100644 --- a/src/pip/_vendor/urllib3/util/connection.py +++ b/src/pip/_vendor/urllib3/util/connection.py @@ -118,7 +118,7 @@ def allowed_gai_family(): def _has_ipv6(host): - """ Returns True if the system can bind an IPv6 address. """ + """Returns True if the system can bind an IPv6 address.""" sock = None has_ipv6 = False diff --git a/src/pip/_vendor/urllib3/util/retry.py b/src/pip/_vendor/urllib3/util/retry.py index d25a41b42..180e82b8c 100644 --- a/src/pip/_vendor/urllib3/util/retry.py +++ b/src/pip/_vendor/urllib3/util/retry.py @@ -321,7 +321,7 @@ class Retry(object): @classmethod def from_int(cls, retries, redirect=True, default=None): - """ Backwards-compatibility for the old retries format.""" + """Backwards-compatibility for the old retries format.""" if retries is None: retries = default if default is not None else cls.DEFAULT @@ -374,7 +374,7 @@ class Retry(object): return seconds def get_retry_after(self, response): - """ Get the value of Retry-After in seconds. """ + """Get the value of Retry-After in seconds.""" retry_after = response.getheader("Retry-After") @@ -468,7 +468,7 @@ class Retry(object): ) def is_exhausted(self): - """ Are we out of retries? """ + """Are we out of retries?""" retry_counts = ( self.total, self.connect, diff --git a/src/pip/_vendor/urllib3/util/ssl_.py b/src/pip/_vendor/urllib3/util/ssl_.py index 763da82bb..a012e5e13 100644 --- a/src/pip/_vendor/urllib3/util/ssl_.py +++ b/src/pip/_vendor/urllib3/util/ssl_.py @@ -71,6 +71,11 @@ except ImportError: except ImportError: PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 +try: + from ssl import PROTOCOL_TLS_CLIENT +except ImportError: + PROTOCOL_TLS_CLIENT = PROTOCOL_TLS + try: from ssl import OP_NO_COMPRESSION, OP_NO_SSLv2, OP_NO_SSLv3 @@ -278,7 +283,11 @@ def create_urllib3_context( Constructed SSLContext object with specified options :rtype: SSLContext """ - context = SSLContext(ssl_version or PROTOCOL_TLS) + # PROTOCOL_TLS is deprecated in Python 3.10 + if not ssl_version or ssl_version == PROTOCOL_TLS: + ssl_version = PROTOCOL_TLS_CLIENT + + context = SSLContext(ssl_version) context.set_ciphers(ciphers or DEFAULT_CIPHERS) @@ -313,13 +322,25 @@ def create_urllib3_context( ) is not None: context.post_handshake_auth = True - context.verify_mode = cert_reqs - if ( - getattr(context, "check_hostname", None) is not None - ): # Platform-specific: Python 3.2 - # We do our own verification, including fingerprints and alternative - # hostnames. So disable it here - context.check_hostname = False + def disable_check_hostname(): + if ( + getattr(context, "check_hostname", None) is not None + ): # Platform-specific: Python 3.2 + # We do our own verification, including fingerprints and alternative + # hostnames. So disable it here + context.check_hostname = False + + # The order of the below lines setting verify_mode and check_hostname + # matter due to safe-guards SSLContext has to prevent an SSLContext with + # check_hostname=True, verify_mode=NONE/OPTIONAL. This is made even more + # complex because we don't know whether PROTOCOL_TLS_CLIENT will be used + # or not so we don't know the initial state of the freshly created SSLContext. + if cert_reqs == ssl.CERT_REQUIRED: + context.verify_mode = cert_reqs + disable_check_hostname() + else: + disable_check_hostname() + context.verify_mode = cert_reqs # Enable logging of TLS session keys via defacto standard environment variable # 'SSLKEYLOGFILE', if the feature is available (Python 3.8+). Skip empty values. diff --git a/src/pip/_vendor/urllib3/util/ssltransport.py b/src/pip/_vendor/urllib3/util/ssltransport.py index ca00233c9..0ed97b644 100644 --- a/src/pip/_vendor/urllib3/util/ssltransport.py +++ b/src/pip/_vendor/urllib3/util/ssltransport.py @@ -193,7 +193,7 @@ class SSLTransport: raise def _ssl_io_loop(self, func, *args): - """ Performs an I/O loop between incoming/outgoing and the socket.""" + """Performs an I/O loop between incoming/outgoing and the socket.""" should_loop = True ret = None diff --git a/src/pip/_vendor/urllib3/util/url.py b/src/pip/_vendor/urllib3/util/url.py index 66c8795b1..3651c4318 100644 --- a/src/pip/_vendor/urllib3/util/url.py +++ b/src/pip/_vendor/urllib3/util/url.py @@ -63,12 +63,12 @@ IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT + "$") BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT[2:-2] + "$") ZONE_ID_RE = re.compile("(" + ZONE_ID_PAT + r")\]$") -SUBAUTHORITY_PAT = (u"^(?:(.*)@)?(%s|%s|%s)(?::([0-9]{0,5}))?$") % ( +_HOST_PORT_PAT = ("^(%s|%s|%s)(?::([0-9]{0,5}))?$") % ( REG_NAME_PAT, IPV4_PAT, IPV6_ADDRZ_PAT, ) -SUBAUTHORITY_RE = re.compile(SUBAUTHORITY_PAT, re.UNICODE | re.DOTALL) +_HOST_PORT_RE = re.compile(_HOST_PORT_PAT, re.UNICODE | re.DOTALL) UNRESERVED_CHARS = set( "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-~" @@ -365,7 +365,9 @@ def parse_url(url): scheme = scheme.lower() if authority: - auth, host, port = SUBAUTHORITY_RE.match(authority).groups() + auth, _, host_port = authority.rpartition("@") + auth = auth or None + host, port = _HOST_PORT_RE.match(host_port).groups() if auth and normalize_uri: auth = _encode_invalid_chars(auth, USERINFO_CHARS) if port == "": diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 6c9732e97..8eb8a5d20 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -13,10 +13,10 @@ requests==2.25.1 certifi==2020.12.05 chardet==4.0.0 idna==3.1 - urllib3==1.26.4 + urllib3==1.26.5 resolvelib==0.7.0 setuptools==44.0.0 -six==1.15.0 +six==1.16.0 tenacity==7.0.0 toml==0.10.2 webencodings==0.5.1 diff --git a/src/pip/py.typed b/src/pip/py.typed index 0b44fd9b5..493b53e4e 100644 --- a/src/pip/py.typed +++ b/src/pip/py.typed @@ -1,4 +1,4 @@ pip is a command line program. While it is implemented in Python, and so is available for import, you must not use pip's internal APIs in this way. Typing -information is provided as a convenience only and is not a gaurantee. Expect +information is provided as a convenience only and is not a guarantee. Expect unannounced changes to the API and types in releases. diff --git a/tests/functional/test_broken_stdout.py b/tests/functional/test_broken_stdout.py index cb98e31f0..4baa4348b 100644 --- a/tests/functional/test_broken_stdout.py +++ b/tests/functional/test_broken_stdout.py @@ -65,7 +65,7 @@ def test_broken_stdout_pipe__verbose(deprecated_python): Test a broken pipe to stdout with verbose logging enabled. """ stderr, returncode = setup_broken_stdout_test( - ['pip', '-v', 'list'], deprecated_python=deprecated_python, + ['pip', '-vv', 'list'], deprecated_python=deprecated_python, ) # Check that a traceback occurs and that it occurs at most once. diff --git a/tests/functional/test_freeze.py b/tests/functional/test_freeze.py index 858e43931..0af29dd0c 100644 --- a/tests/functional/test_freeze.py +++ b/tests/functional/test_freeze.py @@ -409,7 +409,6 @@ def test_freeze_git_remote(script, tmpdir): expect_stderr=True, ) origin_remote = pkg_version - other_remote = pkg_version + '-other' # check frozen remote after clone result = script.pip('freeze', expect_stderr=True) expected = textwrap.dedent( @@ -417,19 +416,31 @@ def test_freeze_git_remote(script, tmpdir): ...-e git+{remote}@...#egg=version_pkg ... """ - ).format(remote=origin_remote).strip() + ).format(remote=path_to_url(origin_remote)).strip() _check_output(result.stdout, expected) # check frozen remote when there is no remote named origin - script.run('git', 'remote', 'remove', 'origin', cwd=repo_dir) - script.run('git', 'remote', 'add', 'other', other_remote, cwd=repo_dir) + script.run('git', 'remote', 'rename', 'origin', 'other', cwd=repo_dir) result = script.pip('freeze', expect_stderr=True) expected = textwrap.dedent( """ ...-e git+{remote}@...#egg=version_pkg ... """ - ).format(remote=other_remote).strip() + ).format(remote=path_to_url(origin_remote)).strip() _check_output(result.stdout, expected) + # When the remote is a local path, it must exist. + # If it doesn't, it gets flagged as invalid. + other_remote = pkg_version + '-other' + script.run('git', 'remote', 'set-url', 'other', other_remote, cwd=repo_dir) + result = script.pip('freeze', expect_stderr=True) + expected = os.path.normcase(textwrap.dedent( + f""" + ...# Editable Git...(version-pkg...)... + # '{other_remote}' + -e {repo_dir}... + """ + ).strip()) + _check_output(os.path.normcase(result.stdout), expected) # when there are more than one origin, priority is given to the # remote named origin script.run('git', 'remote', 'add', 'origin', origin_remote, cwd=repo_dir) @@ -439,7 +450,7 @@ def test_freeze_git_remote(script, tmpdir): ...-e git+{remote}@...#egg=version_pkg ... """ - ).format(remote=origin_remote).strip() + ).format(remote=path_to_url(origin_remote)).strip() _check_output(result.stdout, expected) diff --git a/tests/functional/test_index.py b/tests/functional/test_index.py new file mode 100644 index 000000000..004e672a5 --- /dev/null +++ b/tests/functional/test_index.py @@ -0,0 +1,75 @@ +import pytest + +from pip._internal.cli.status_codes import ERROR, SUCCESS +from pip._internal.commands import create_command + + +@pytest.mark.network +def test_list_all_versions_basic_search(script): + """ + End to end test of index versions command. + """ + output = script.pip('index', 'versions', 'pip', allow_stderr_warning=True) + assert 'Available versions:' in output.stdout + assert ( + '20.2.3, 20.2.2, 20.2.1, 20.2, 20.1.1, 20.1, 20.0.2' + ', 20.0.1, 19.3.1, 19.3, 19.2.3, 19.2.2, 19.2.1, 19.2, 19.1.1' + ', 19.1, 19.0.3, 19.0.2, 19.0.1, 19.0, 18.1, 18.0, 10.0.1, 10.0.0, ' + '9.0.3, 9.0.2, 9.0.1, 9.0.0, 8.1.2, 8.1.1, ' + '8.1.0, 8.0.3, 8.0.2, 8.0.1, 8.0.0, 7.1.2, 7.1.1, 7.1.0, 7.0.3, ' + '7.0.2, 7.0.1, 7.0.0, 6.1.1, 6.1.0, 6.0.8, 6.0.7, 6.0.6, 6.0.5, ' + '6.0.4, 6.0.3, 6.0.2, 6.0.1, 6.0, 1.5.6, 1.5.5, 1.5.4, 1.5.3, ' + '1.5.2, 1.5.1, 1.5, 1.4.1, 1.4, 1.3.1, 1.3, 1.2.1, 1.2, 1.1, 1.0.2,' + ' 1.0.1, 1.0, 0.8.3, 0.8.2, 0.8.1, 0.8, 0.7.2, 0.7.1, 0.7, 0.6.3, ' + '0.6.2, 0.6.1, 0.6, 0.5.1, 0.5, 0.4, 0.3.1, ' + '0.3, 0.2.1, 0.2' in output.stdout + ) + + +@pytest.mark.network +def test_list_all_versions_search_with_pre(script): + """ + See that adding the --pre flag adds pre-releases + """ + output = script.pip( + 'index', 'versions', 'pip', '--pre', allow_stderr_warning=True) + assert 'Available versions:' in output.stdout + assert ( + '20.2.3, 20.2.2, 20.2.1, 20.2, 20.2b1, 20.1.1, 20.1, 20.1b1, 20.0.2' + ', 20.0.1, 19.3.1, 19.3, 19.2.3, 19.2.2, 19.2.1, 19.2, 19.1.1' + ', 19.1, 19.0.3, 19.0.2, 19.0.1, 19.0, 18.1, 18.0, 10.0.1, 10.0.0, ' + '10.0.0b2, 10.0.0b1, 9.0.3, 9.0.2, 9.0.1, 9.0.0, 8.1.2, 8.1.1, ' + '8.1.0, 8.0.3, 8.0.2, 8.0.1, 8.0.0, 7.1.2, 7.1.1, 7.1.0, 7.0.3, ' + '7.0.2, 7.0.1, 7.0.0, 6.1.1, 6.1.0, 6.0.8, 6.0.7, 6.0.6, 6.0.5, ' + '6.0.4, 6.0.3, 6.0.2, 6.0.1, 6.0, 1.5.6, 1.5.5, 1.5.4, 1.5.3, ' + '1.5.2, 1.5.1, 1.5, 1.4.1, 1.4, 1.3.1, 1.3, 1.2.1, 1.2, 1.1, 1.0.2,' + ' 1.0.1, 1.0, 0.8.3, 0.8.2, 0.8.1, 0.8, 0.7.2, 0.7.1, 0.7, 0.6.3, ' + '0.6.2, 0.6.1, 0.6, 0.5.1, 0.5, 0.4, 0.3.1, ' + '0.3, 0.2.1, 0.2' in output.stdout + ) + + +@pytest.mark.network +def test_list_all_versions_returns_no_matches_found_when_name_not_exact(): + """ + Test that non exact name do not match + """ + command = create_command('index') + cmdline = "versions pand" + with command.main_context(): + options, args = command.parse_args(cmdline.split()) + status = command.run(options, args) + assert status == ERROR + + +@pytest.mark.network +def test_list_all_versions_returns_matches_found_when_name_is_exact(): + """ + Test that exact name matches + """ + command = create_command('index') + cmdline = "versions pandas" + with command.main_context(): + options, args = command.parse_args(cmdline.split()) + status = command.run(options, args) + assert status == SUCCESS diff --git a/tests/functional/test_new_resolver_errors.py b/tests/functional/test_new_resolver_errors.py index b4d63a996..b2e7af7c6 100644 --- a/tests/functional/test_new_resolver_errors.py +++ b/tests/functional/test_new_resolver_errors.py @@ -1,3 +1,4 @@ +import pathlib import sys from tests.lib import create_basic_wheel_for_package, create_test_package_with_setup @@ -73,3 +74,34 @@ def test_new_resolver_requires_python_error(script): # conflict, not the compatible one. assert incompatible_python in result.stderr, str(result) assert compatible_python not in result.stderr, str(result) + + +def test_new_resolver_checks_requires_python_before_dependencies(script): + incompatible_python = "<{0.major}.{0.minor}".format(sys.version_info) + + pkg_dep = create_basic_wheel_for_package( + script, + name="pkg-dep", + version="1", + ) + create_basic_wheel_for_package( + script, + name="pkg-root", + version="1", + # Refer the dependency by URL to prioritise it as much as possible, + # to test that Requires-Python is *still* inspected first. + depends=[f"pkg-dep@{pathlib.Path(pkg_dep).as_uri()}"], + requires_python=incompatible_python, + ) + + result = script.pip( + "install", "--no-cache-dir", + "--no-index", "--find-links", script.scratch_path, + "pkg-root", + expect_error=True, + ) + + # Resolution should fail because of pkg-a's Requires-Python. + # This check should be done before pkg-b, so pkg-b should never be pulled. + assert incompatible_python in result.stderr, str(result) + assert "pkg-b" not in result.stderr, str(result) diff --git a/tests/functional/test_new_resolver_hashes.py b/tests/functional/test_new_resolver_hashes.py index 02397616a..fed1be5e8 100644 --- a/tests/functional/test_new_resolver_hashes.py +++ b/tests/functional/test_new_resolver_hashes.py @@ -102,7 +102,7 @@ def test_new_resolver_hash_intersect(script, requirements_template, message): "--no-deps", "--no-index", "--find-links", find_links.index_html, - "--verbose", + "-vv", "--requirement", requirements_txt, ) @@ -134,7 +134,7 @@ def test_new_resolver_hash_intersect_from_constraint(script): "--no-deps", "--no-index", "--find-links", find_links.index_html, - "--verbose", + "-vv", "--constraint", constraints_txt, "--requirement", requirements_txt, ) diff --git a/tests/functional/test_uninstall.py b/tests/functional/test_uninstall.py index 878e713ed..cbce8746a 100644 --- a/tests/functional/test_uninstall.py +++ b/tests/functional/test_uninstall.py @@ -476,6 +476,51 @@ def test_uninstall_wheel(script, data): assert_all_changes(result, result2, []) +@pytest.mark.parametrize('installer', [FileNotFoundError, IsADirectoryError, + '', os.linesep, b'\xc0\xff\xee', 'pip', + 'MegaCorp Cloud Install-O-Matic']) +def test_uninstall_without_record_fails(script, data, installer): + """ + Test uninstalling a package installed without RECORD + """ + package = data.packages.joinpath("simple.dist-0.1-py2.py3-none-any.whl") + result = script.pip('install', package, '--no-index') + dist_info_folder = script.site_packages / 'simple.dist-0.1.dist-info' + result.did_create(dist_info_folder) + + # Remove RECORD + record_path = dist_info_folder / 'RECORD' + (script.base_path / record_path).unlink() + ignore_changes = [record_path] + + # Populate, remove or otherwise break INSTALLER + installer_path = dist_info_folder / 'INSTALLER' + ignore_changes += [installer_path] + installer_path = script.base_path / installer_path + if installer in (FileNotFoundError, IsADirectoryError): + installer_path.unlink() + if installer is IsADirectoryError: + installer_path.mkdir() + else: + if isinstance(installer, bytes): + installer_path.write_bytes(installer) + else: + installer_path.write_text(installer + os.linesep) + + result2 = script.pip('uninstall', 'simple.dist', '-y', expect_error=True) + expected_error_message = ('ERROR: Cannot uninstall simple.dist 0.1, ' + 'RECORD file not found.') + if not isinstance(installer, str) or not installer.strip() or installer == 'pip': + expected_error_message += (" You might be able to recover from this via: " + "'pip install --force-reinstall --no-deps " + "simple.dist==0.1'.") + elif installer: + expected_error_message += (' Hint: The package was installed by ' + '{}.'.format(installer)) + assert result2.stderr.rstrip() == expected_error_message + assert_all_changes(result.files_after, result2, ignore_changes) + + @pytest.mark.skipif("sys.platform == 'win32'") def test_uninstall_with_symlink(script, data, tmpdir): """ diff --git a/tests/unit/test_base_command.py b/tests/unit/test_base_command.py index 9b57339c0..fa16d2fd7 100644 --- a/tests/unit/test_base_command.py +++ b/tests/unit/test_base_command.py @@ -80,7 +80,7 @@ class TestCommand: """ Test raising BrokenStdoutLoggingError with debug logging enabled. """ - stderr = self.call_main(capsys, ['-v']) + stderr = self.call_main(capsys, ['-vv']) assert 'ERROR: Pipe to stdout was broken' in stderr assert 'Traceback (most recent call last):' in stderr diff --git a/tests/unit/test_commands.py b/tests/unit/test_commands.py index f34f7e538..cb144c5f6 100644 --- a/tests/unit/test_commands.py +++ b/tests/unit/test_commands.py @@ -11,7 +11,8 @@ from pip._internal.commands import commands_dict, create_command # These are the expected names of the commands whose classes inherit from # IndexGroupCommand. -EXPECTED_INDEX_GROUP_COMMANDS = ['download', 'install', 'list', 'wheel'] +EXPECTED_INDEX_GROUP_COMMANDS = [ + 'download', 'index', 'install', 'list', 'wheel'] def check_commands(pred, expected): @@ -49,7 +50,9 @@ def test_session_commands(): def is_session_command(command): return isinstance(command, SessionCommandMixin) - expected = ['download', 'install', 'list', 'search', 'uninstall', 'wheel'] + expected = [ + 'download', 'index', 'install', 'list', 'search', 'uninstall', 'wheel' + ] check_commands(is_session_command, expected) diff --git a/tests/unit/test_network_auth.py b/tests/unit/test_network_auth.py index 44c739d86..ce528d0ac 100644 --- a/tests/unit/test_network_auth.py +++ b/tests/unit/test_network_auth.py @@ -47,11 +47,29 @@ def test_get_credentials_parses_correctly(input_url, url, username, password): ) -def test_get_credentials_uses_cached_credentials(): +def test_get_credentials_not_to_uses_cached_credentials(): auth = MultiDomainBasicAuth() auth.passwords['example.com'] = ('user', 'pass') got = auth._get_url_and_credentials("http://foo:bar@example.com/path") + expected = ('http://example.com/path', 'foo', 'bar') + assert got == expected + + +def test_get_credentials_not_to_uses_cached_credentials_only_username(): + auth = MultiDomainBasicAuth() + auth.passwords['example.com'] = ('user', 'pass') + + got = auth._get_url_and_credentials("http://foo@example.com/path") + expected = ('http://example.com/path', 'foo', '') + assert got == expected + + +def test_get_credentials_uses_cached_credentials(): + auth = MultiDomainBasicAuth() + auth.passwords['example.com'] = ('user', 'pass') + + got = auth._get_url_and_credentials("http://example.com/path") expected = ('http://example.com/path', 'user', 'pass') assert got == expected diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index 3c534c9ee..be1e5815f 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -598,13 +598,16 @@ class TestParseRequirements: with open(tmpdir.joinpath('req1.txt'), 'w') as fp: fp.write(template.format(*map(make_var, env_vars))) + # Construct the session outside the monkey-patch, since it access the + # env + session = PipSession() with patch('pip._internal.req.req_file.os.getenv') as getenv: getenv.side_effect = lambda n: env_vars[n] reqs = list(parse_reqfile( tmpdir.joinpath('req1.txt'), finder=finder, - session=PipSession() + session=session )) assert len(reqs) == 1, \ @@ -623,13 +626,16 @@ class TestParseRequirements: with open(tmpdir.joinpath('req1.txt'), 'w') as fp: fp.write(req_url) + # Construct the session outside the monkey-patch, since it access the + # env + session = PipSession() with patch('pip._internal.req.req_file.os.getenv') as getenv: getenv.return_value = '' reqs = list(parse_reqfile( tmpdir.joinpath('req1.txt'), finder=finder, - session=PipSession() + session=session )) assert len(reqs) == 1, \ diff --git a/tests/unit/test_utils_subprocess.py b/tests/unit/test_utils_subprocess.py index 7a31eeb74..2c21603c7 100644 --- a/tests/unit/test_utils_subprocess.py +++ b/tests/unit/test_utils_subprocess.py @@ -7,6 +7,7 @@ import pytest from pip._internal.cli.spinners import SpinnerInterface from pip._internal.exceptions import InstallationSubprocessError +from pip._internal.utils.logging import VERBOSE from pip._internal.utils.misc import hide_value from pip._internal.utils.subprocess import ( call_subprocess, @@ -127,7 +128,11 @@ def test_make_subprocess_output_error__non_ascii_line(): ) def test_call_subprocess_stdout_only(capfd, monkeypatch, stdout_only, expected): log = [] - monkeypatch.setattr(subprocess_logger, "debug", lambda *args: log.append(args[0])) + monkeypatch.setattr( + subprocess_logger, + "log", + lambda level, *args: log.append(args[0]), + ) out = call_subprocess( [ sys.executable, @@ -233,9 +238,9 @@ class TestCallSubprocess: result = call_subprocess(args, spinner=spinner) expected = (['Hello', 'world'], [ - ('pip.subprocessor', DEBUG, 'Running command '), - ('pip.subprocessor', DEBUG, 'Hello'), - ('pip.subprocessor', DEBUG, 'world'), + ('pip.subprocessor', VERBOSE, 'Running command '), + ('pip.subprocessor', VERBOSE, 'Hello'), + ('pip.subprocessor', VERBOSE, 'world'), ]) # The spinner shouldn't spin in this case since the subprocess # output is already being logged to the console. diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index f86d04d74..305c45fd8 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -1,4 +1,5 @@ import os +import pathlib from unittest import TestCase from unittest.mock import patch @@ -9,7 +10,7 @@ from pip._internal.exceptions import BadCommand, InstallationError from pip._internal.utils.misc import hide_url, hide_value from pip._internal.vcs import make_vcs_requirement_url from pip._internal.vcs.bazaar import Bazaar -from pip._internal.vcs.git import Git, looks_like_hash +from pip._internal.vcs.git import Git, RemoteNotValidError, looks_like_hash from pip._internal.vcs.mercurial import Mercurial from pip._internal.vcs.subversion import Subversion from pip._internal.vcs.versioncontrol import RevOptions, VersionControl @@ -108,10 +109,14 @@ def test_looks_like_hash(sha, expected): @pytest.mark.parametrize('vcs_cls, remote_url, expected', [ - # Git is one of the subclasses using the base class implementation. - (Git, 'git://example.com/MyProject', False), + # Mercurial is one of the subclasses using the base class implementation. + # `hg://` isn't a real prefix but it tests the default behaviour. + (Mercurial, 'hg://user@example.com/MyProject', False), + (Mercurial, 'http://example.com/MyProject', True), + # The Git subclasses should return true in all cases. + (Git, 'git://example.com/MyProject', True), (Git, 'http://example.com/MyProject', True), - # Subversion is the only subclass overriding the base class implementation. + # Subversion also overrides the base class implementation. (Subversion, 'svn://example.com/MyProject', True), ]) def test_should_add_vcs_url_prefix(vcs_cls, remote_url, expected): @@ -119,26 +124,83 @@ def test_should_add_vcs_url_prefix(vcs_cls, remote_url, expected): assert actual == expected +@pytest.mark.parametrize("url, target", [ + # A fully qualified remote url. No changes needed. + ("ssh://bob@server/foo/bar.git", "ssh://bob@server/foo/bar.git"), + ("git://bob@server/foo/bar.git", "git://bob@server/foo/bar.git"), + # User is optional and does not need a default. + ("ssh://server/foo/bar.git", "ssh://server/foo/bar.git"), + # The common scp shorthand for ssh remotes. Pip won't recognise these as + # git remotes until they have a 'ssh://' prefix and the ':' in the middle + # is gone. + ("git@example.com:foo/bar.git", "ssh://git@example.com/foo/bar.git"), + ("example.com:foo.git", "ssh://example.com/foo.git"), + # Http(s) remote names are already complete and should remain unchanged. + ("https://example.com/foo", "https://example.com/foo"), + ("http://example.com/foo/bar.git", "http://example.com/foo/bar.git"), + ("https://bob@example.com/foo", "https://bob@example.com/foo"), + ]) +def test_git_remote_url_to_pip(url, target): + assert Git._git_remote_to_pip_url(url) == target + + +@pytest.mark.parametrize("url, platform", [ + # Windows paths with the ':' drive prefix look dangerously close to SCP. + ("c:/piffle/wiffle/waffle/poffle.git", "nt"), + (r"c:\faffle\waffle\woffle\piffle.git", "nt"), + # Unix paths less so but test them anyway. + ("/muffle/fuffle/pufffle/fluffle.git", "posix"), +]) +def test_paths_are_not_mistaken_for_scp_shorthand(url, platform): + # File paths should not be mistaken for SCP shorthand. If they do then + # 'c:/piffle/wiffle' would end up as 'ssh://c/piffle/wiffle'. + from pip._internal.vcs.git import SCP_REGEX + assert not SCP_REGEX.match(url) + + if platform == os.name: + with pytest.raises(RemoteNotValidError): + Git._git_remote_to_pip_url(url) + + +def test_git_remote_local_path(tmpdir): + path = pathlib.Path(tmpdir, "project.git") + path.mkdir() + # Path must exist to be recognised as a local git remote. + assert Git._git_remote_to_pip_url(str(path)) == path.as_uri() + + @patch('pip._internal.vcs.git.Git.get_remote_url') @patch('pip._internal.vcs.git.Git.get_revision') @patch('pip._internal.vcs.git.Git.get_subdirectory') +@pytest.mark.parametrize( + "git_url, target_url_prefix", + [ + ( + "https://github.com/pypa/pip-test-package", + "git+https://github.com/pypa/pip-test-package", + ), + ( + "git@github.com:pypa/pip-test-package", + "git+ssh://git@github.com/pypa/pip-test-package", + ), + ], + ids=["https", "ssh"], +) @pytest.mark.network def test_git_get_src_requirements( - mock_get_subdirectory, mock_get_revision, mock_get_remote_url + mock_get_subdirectory, mock_get_revision, mock_get_remote_url, + git_url, target_url_prefix, ): - git_url = 'https://github.com/pypa/pip-test-package' sha = '5547fa909e83df8bd743d3978d6667497983a4b7' - mock_get_remote_url.return_value = git_url + mock_get_remote_url.return_value = Git._git_remote_to_pip_url(git_url) mock_get_revision.return_value = sha mock_get_subdirectory.return_value = None ret = Git.get_src_requirement('.', 'pip-test-package') - assert ret == ( - 'git+https://github.com/pypa/pip-test-package' - '@5547fa909e83df8bd743d3978d6667497983a4b7#egg=pip_test_package' - ) + target = f"{target_url_prefix}@{sha}#egg=pip_test_package" + assert ret == target @patch('pip._internal.vcs.git.Git.get_revision_sha') diff --git a/tools/tox_pip.py b/tools/tox_pip.py index fe7621342..6a0e2dae9 100644 --- a/tools/tox_pip.py +++ b/tools/tox_pip.py @@ -9,8 +9,7 @@ VIRTUAL_ENV = os.environ['VIRTUAL_ENV'] TOX_PIP_DIR = os.path.join(VIRTUAL_ENV, 'pip') -def pip(args): - # type: (List[str]) -> None +def pip(args: List[str]) -> None: # First things first, get a recent (stable) version of pip. if not os.path.exists(TOX_PIP_DIR): subprocess.check_call([sys.executable, '-m', 'pip',