From 49fd5db80ef9acab8b88df9033f52ba293d8634e Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 14 Dec 2020 18:50:19 +0200 Subject: [PATCH 01/85] Clean up my Git entries --- .mailmap | 2 ++ news/8EB21BAE-9CD0-424E-AF3B-651960B50C93.trivial.rst | 0 2 files changed, 2 insertions(+) create mode 100644 news/8EB21BAE-9CD0-424E-AF3B-651960B50C93.trivial.rst diff --git a/.mailmap b/.mailmap index 29f9ec039..c8f94a9d8 100644 --- a/.mailmap +++ b/.mailmap @@ -21,6 +21,8 @@ Endoh Takanao Erik M. Bray Gabriel de Perthuis Hsiaoming Yang +Hugo van Kemenade Hugo +Hugo van Kemenade hugovk Igor Kuzmitshov Ilya Baryshev Jakub Stasiak diff --git a/news/8EB21BAE-9CD0-424E-AF3B-651960B50C93.trivial.rst b/news/8EB21BAE-9CD0-424E-AF3B-651960B50C93.trivial.rst new file mode 100644 index 000000000..e69de29bb From 9b3e78474a0da61012403643498ade7a413c4cf8 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 13 Jan 2021 10:15:56 +0100 Subject: [PATCH 02/85] -v shows subprocess output adds VERBOSE custom log level between DEBUG and INFO used when `-v` is given. Now require -vv to enable full debug output. Messages can be logged with VERBOSE level to promote them to `-v` output instead of `-vv` --- news/9450.feature.rst | 2 ++ src/pip/_internal/commands/cache.py | 5 +++-- src/pip/_internal/commands/install.py | 3 ++- src/pip/_internal/req/req_uninstall.py | 8 +++---- src/pip/_internal/utils/logging.py | 22 +++++++++++++------- src/pip/_internal/utils/subprocess.py | 7 ++++--- tests/functional/test_broken_stdout.py | 2 +- tests/functional/test_new_resolver_hashes.py | 4 ++-- tests/unit/test_base_command.py | 2 +- tests/unit/test_utils_subprocess.py | 13 ++++++++---- 10 files changed, 43 insertions(+), 25 deletions(-) create mode 100644 news/9450.feature.rst diff --git a/news/9450.feature.rst b/news/9450.feature.rst new file mode 100644 index 000000000..ca3d8081e --- /dev/null +++ b/news/9450.feature.rst @@ -0,0 +1,2 @@ +Require ``-vv`` for full debug-level output, ``-v`` now enables an intermediate level of verbosity, +e.g. showing subprocess output of ``setup.py install`` but not all dependency resolution debug output. diff --git a/src/pip/_internal/commands/cache.py b/src/pip/_internal/commands/cache.py index 5155a5053..37bdfc999 100644 --- a/src/pip/_internal/commands/cache.py +++ b/src/pip/_internal/commands/cache.py @@ -8,6 +8,7 @@ import pip._internal.utils.filesystem as filesystem from pip._internal.cli.base_command import Command from pip._internal.cli.status_codes import ERROR, SUCCESS from pip._internal.exceptions import CommandError, PipError +from pip._internal.utils.logging import VERBOSE logger = logging.getLogger(__name__) @@ -184,8 +185,8 @@ class CacheCommand(Command): for filename in files: os.unlink(filename) - logger.debug('Removed %s', filename) - logger.info('Files removed: %s', len(files)) + logger.log(VERBOSE, "Removed %s", filename) + logger.info("Files removed: %s", len(files)) def purge_cache(self, options, args): # type: (Values, List[Any]) -> None diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index c4273eda9..93fa2f289 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -28,6 +28,7 @@ from pip._internal.req.req_install import InstallRequirement from pip._internal.req.req_tracker import get_requirement_tracker from pip._internal.utils.distutils_args import parse_distutils_args from pip._internal.utils.filesystem import test_writable_dir +from pip._internal.utils.logging import VERBOSE from pip._internal.utils.misc import ( ensure_dir, get_pip_version, @@ -235,7 +236,7 @@ class InstallCommand(RequirementCommand): install_options = options.install_options or [] - logger.debug("Using %s", get_pip_version()) + logger.log(VERBOSE, "Using %s", get_pip_version()) options.use_user_site = decide_user_install( options.use_user_site, prefix_path=options.prefix_path, diff --git a/src/pip/_internal/req/req_uninstall.py b/src/pip/_internal/req/req_uninstall.py index b72234175..fb04b10ae 100644 --- a/src/pip/_internal/req/req_uninstall.py +++ b/src/pip/_internal/req/req_uninstall.py @@ -13,7 +13,7 @@ from pip._vendor.pkg_resources import Distribution from pip._internal.exceptions import UninstallationError from pip._internal.locations import get_bin_prefix, get_bin_user from pip._internal.utils.compat import WINDOWS -from pip._internal.utils.logging import indent_log +from pip._internal.utils.logging import VERBOSE, indent_log from pip._internal.utils.misc import ( ask, dist_in_usersite, @@ -384,7 +384,7 @@ class UninstallPathSet: for path in sorted(compact(for_rename)): moved.stash(path) - logger.debug('Removing file or directory %s', path) + logger.log(VERBOSE, 'Removing file or directory %s', path) for pth in self.pth.values(): pth.remove() @@ -599,7 +599,7 @@ class UninstallPthEntries: def remove(self): # type: () -> None - logger.debug('Removing pth entries from %s:', self.file) + logger.log(VERBOSE, 'Removing pth entries from %s:', self.file) # If the file doesn't exist, log a warning and return if not os.path.isfile(self.file): @@ -620,7 +620,7 @@ class UninstallPthEntries: lines[-1] = lines[-1] + endline.encode("utf-8") for entry in self.entries: try: - logger.debug('Removing entry: %s', entry) + logger.log(VERBOSE, 'Removing entry: %s', entry) lines.remove((entry + endline).encode("utf-8")) except ValueError: pass diff --git a/src/pip/_internal/utils/logging.py b/src/pip/_internal/utils/logging.py index 45798d54f..028967124 100644 --- a/src/pip/_internal/utils/logging.py +++ b/src/pip/_internal/utils/logging.py @@ -29,6 +29,11 @@ _log_state = threading.local() subprocess_logger = getLogger("pip.subprocessor") +# custom log level for `--verbose` output +# between DEBUG and INFO +VERBOSE = 15 + + class BrokenStdoutLoggingError(Exception): """ Raised if BrokenPipeError occurs for the stdout stream while logging. @@ -271,19 +276,22 @@ def setup_logging(verbosity, no_color, user_log_file): Returns the requested logging level, as its integer value. """ + logging.addLevelName(VERBOSE, "VERBOSE") # Determine the level to be logging at. - if verbosity >= 1: - level = "DEBUG" + if verbosity >= 2: + level_number = logging.DEBUG + elif verbosity == 1: + level_number = VERBOSE elif verbosity == -1: - level = "WARNING" + level_number = logging.WARNING elif verbosity == -2: - level = "ERROR" + level_number = logging.ERROR elif verbosity <= -3: - level = "CRITICAL" + level_number = logging.CRITICAL else: - level = "INFO" + level_number = logging.INFO - level_number = getattr(logging, level) + level = logging.getLevelName(level_number) # The "root" logger should match the "console" level *unless* we also need # to log to a user log file. diff --git a/src/pip/_internal/utils/subprocess.py b/src/pip/_internal/utils/subprocess.py index cfde18700..0e73a5e70 100644 --- a/src/pip/_internal/utils/subprocess.py +++ b/src/pip/_internal/utils/subprocess.py @@ -2,11 +2,12 @@ import logging import os import shlex import subprocess +from functools import partial from typing import Any, Callable, Iterable, List, Mapping, Optional, Union from pip._internal.cli.spinners import SpinnerInterface, open_spinner from pip._internal.exceptions import InstallationSubprocessError -from pip._internal.utils.logging import subprocess_logger +from pip._internal.utils.logging import VERBOSE, subprocess_logger from pip._internal.utils.misc import HiddenText CommandArgs = List[Union[str, HiddenText]] @@ -146,8 +147,8 @@ def call_subprocess( else: # Then log the subprocess output using DEBUG. This also ensures # it will be logged to the log file (aka user_log), if enabled. - log_subprocess = subprocess_logger.debug - used_level = logging.DEBUG + log_subprocess = partial(subprocess_logger.log, VERBOSE) + used_level = VERBOSE # Whether the subprocess will be visible in the console. showing_subprocess = subprocess_logger.getEffectiveLevel() <= used_level diff --git a/tests/functional/test_broken_stdout.py b/tests/functional/test_broken_stdout.py index cb98e31f0..4baa4348b 100644 --- a/tests/functional/test_broken_stdout.py +++ b/tests/functional/test_broken_stdout.py @@ -65,7 +65,7 @@ def test_broken_stdout_pipe__verbose(deprecated_python): Test a broken pipe to stdout with verbose logging enabled. """ stderr, returncode = setup_broken_stdout_test( - ['pip', '-v', 'list'], deprecated_python=deprecated_python, + ['pip', '-vv', 'list'], deprecated_python=deprecated_python, ) # Check that a traceback occurs and that it occurs at most once. diff --git a/tests/functional/test_new_resolver_hashes.py b/tests/functional/test_new_resolver_hashes.py index 854b66418..2968760b2 100644 --- a/tests/functional/test_new_resolver_hashes.py +++ b/tests/functional/test_new_resolver_hashes.py @@ -76,7 +76,7 @@ def test_new_resolver_hash_intersect(script, requirements_template, message): "--no-deps", "--no-index", "--find-links", find_links.index_html, - "--verbose", + "-vv", "--requirement", requirements_txt, ) @@ -108,7 +108,7 @@ def test_new_resolver_hash_intersect_from_constraint(script): "--no-deps", "--no-index", "--find-links", find_links.index_html, - "--verbose", + "-vv", "--constraint", constraints_txt, "--requirement", requirements_txt, ) diff --git a/tests/unit/test_base_command.py b/tests/unit/test_base_command.py index 9b57339c0..fa16d2fd7 100644 --- a/tests/unit/test_base_command.py +++ b/tests/unit/test_base_command.py @@ -80,7 +80,7 @@ class TestCommand: """ Test raising BrokenStdoutLoggingError with debug logging enabled. """ - stderr = self.call_main(capsys, ['-v']) + stderr = self.call_main(capsys, ['-vv']) assert 'ERROR: Pipe to stdout was broken' in stderr assert 'Traceback (most recent call last):' in stderr diff --git a/tests/unit/test_utils_subprocess.py b/tests/unit/test_utils_subprocess.py index 7a31eeb74..2c21603c7 100644 --- a/tests/unit/test_utils_subprocess.py +++ b/tests/unit/test_utils_subprocess.py @@ -7,6 +7,7 @@ import pytest from pip._internal.cli.spinners import SpinnerInterface from pip._internal.exceptions import InstallationSubprocessError +from pip._internal.utils.logging import VERBOSE from pip._internal.utils.misc import hide_value from pip._internal.utils.subprocess import ( call_subprocess, @@ -127,7 +128,11 @@ def test_make_subprocess_output_error__non_ascii_line(): ) def test_call_subprocess_stdout_only(capfd, monkeypatch, stdout_only, expected): log = [] - monkeypatch.setattr(subprocess_logger, "debug", lambda *args: log.append(args[0])) + monkeypatch.setattr( + subprocess_logger, + "log", + lambda level, *args: log.append(args[0]), + ) out = call_subprocess( [ sys.executable, @@ -233,9 +238,9 @@ class TestCallSubprocess: result = call_subprocess(args, spinner=spinner) expected = (['Hello', 'world'], [ - ('pip.subprocessor', DEBUG, 'Running command '), - ('pip.subprocessor', DEBUG, 'Hello'), - ('pip.subprocessor', DEBUG, 'world'), + ('pip.subprocessor', VERBOSE, 'Running command '), + ('pip.subprocessor', VERBOSE, 'Hello'), + ('pip.subprocessor', VERBOSE, 'world'), ]) # The spinner shouldn't spin in this case since the subprocess # output is already being logged to the console. From 5822e39d24f120842da4c0e3a255740636707e78 Mon Sep 17 00:00:00 2001 From: Nicholas Serra Date: Thu, 29 Apr 2021 13:11:58 -0400 Subject: [PATCH 03/85] Set strict_timestamps=False when zip is called for isolated environment --- src/pip/_internal/build_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index cdf043241..d93f940ed 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -58,7 +58,7 @@ def _create_standalone_pip() -> Iterator[str]: with TempDirectory(kind="standalone-pip") as tmp_dir: pip_zip = os.path.join(tmp_dir.path, "__env_pip__.zip") - with zipfile.ZipFile(pip_zip, "w") as zf: + with zipfile.ZipFile(pip_zip, "w", strict_timestamps=False) as zf: for child in source.rglob("*"): zf.write(child, child.relative_to(source.parent).as_posix()) yield os.path.join(pip_zip, "pip") From 1ee1aca7de1bf99238c8a38d0e21fded068a2216 Mon Sep 17 00:00:00 2001 From: Nicholas Serra Date: Thu, 29 Apr 2021 13:15:46 -0400 Subject: [PATCH 04/85] 9910 news --- news/9910.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/9910.bugfix.rst diff --git a/news/9910.bugfix.rst b/news/9910.bugfix.rst new file mode 100644 index 000000000..911181d18 --- /dev/null +++ b/news/9910.bugfix.rst @@ -0,0 +1 @@ +Fix for ValueError('ZIP does not support timestamps before 1980') on GitHub package install. From beeb89c29440958bcc9049a604f234814e06098d Mon Sep 17 00:00:00 2001 From: Nicholas Serra Date: Fri, 30 Apr 2021 13:21:46 -0400 Subject: [PATCH 05/85] Update news/9910.bugfix.rst Co-authored-by: Tzu-ping Chung --- news/9910.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/9910.bugfix.rst b/news/9910.bugfix.rst index 911181d18..ece2f3fdf 100644 --- a/news/9910.bugfix.rst +++ b/news/9910.bugfix.rst @@ -1 +1 @@ -Fix for ValueError('ZIP does not support timestamps before 1980') on GitHub package install. +Allow ZIP to archive files with timestamps earlier than 1980. From 29f1963233bd5c0c47735ca6faa9a6a226ed93f3 Mon Sep 17 00:00:00 2001 From: Dimitri Merejkowsky Date: Sat, 1 May 2021 13:17:29 +0200 Subject: [PATCH 06/85] Fix typo in NEWS.rst --- NEWS.rst | 2 +- news/e0021c1f-5eeb-4f32-bfe8-87752f034acd.trivial.rst | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 news/e0021c1f-5eeb-4f32-bfe8-87752f034acd.trivial.rst diff --git a/NEWS.rst b/NEWS.rst index db5562a02..6ea55c8e1 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -18,7 +18,7 @@ Deprecations and Removals - Temporarily set the new "Value for ... does not match" location warnings level to *DEBUG*, to hide them from casual users. This prepares pip 21.1 for CPython inclusion, while pip maintainers digest the first intake of location mismatch - issues for the ``distutils``-``sysconfig`` trasition. (`#9912 `_) + issues for the ``distutils``-``sysconfig`` transition. (`#9912 `_) Bug Fixes --------- diff --git a/news/e0021c1f-5eeb-4f32-bfe8-87752f034acd.trivial.rst b/news/e0021c1f-5eeb-4f32-bfe8-87752f034acd.trivial.rst new file mode 100644 index 000000000..e69de29bb From 1983ef0edc61fb71960d6b2caf2d0a6baa83ae0e Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 4 May 2021 16:50:52 +0800 Subject: [PATCH 07/85] Better diagnose when setup.py/cfg cannot be found This adds a check before invoking 'egg_info' to make sure either setup.py or setup.cfg actually exists, and emit a clearer error message when neither can be found and the egg_info command can never succeed. --- news/9944.bugfix.rst | 2 ++ src/pip/_internal/req/req_install.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 news/9944.bugfix.rst diff --git a/news/9944.bugfix.rst b/news/9944.bugfix.rst new file mode 100644 index 000000000..58bb70b78 --- /dev/null +++ b/news/9944.bugfix.rst @@ -0,0 +1,2 @@ +Emit clearer error message when a project root does not contain either +``pyproject.toml``, ``setup.py`` or ``setup.cfg``. diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 55c17ac8b..c2eea3712 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -509,6 +509,19 @@ class InstallRequirement: self.unpacked_source_directory, backend, backend_path=backend_path, ) + def _check_setup_py_or_cfg_exists(self) -> bool: + """Check if the requirement actually has a setuptools build file. + + If setup.py does not exist, we also check setup.cfg in the same + directory and allow the directory if that exists. + """ + if os.path.exists(self.setup_py_path): + return True + stem, ext = os.path.splitext(self.setup_py_path) + if ext == ".py" and os.path.exists(f"{stem}.cfg"): + return True + return False + def _generate_metadata(self): # type: () -> str """Invokes metadata generator functions, with the required arguments. @@ -516,6 +529,12 @@ class InstallRequirement: if not self.use_pep517: assert self.unpacked_source_directory + if not self._check_setup_py_or_cfg_exists(): + raise InstallationError( + f'File "setup.py" or "setup.cfg" not found for legacy ' + f'project {self}.' + ) + return generate_metadata_legacy( build_env=self.build_env, setup_py_path=self.setup_py_path, From 8ef383bb23ed0790efcf477e2bfbde2640e3b869 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Thu, 6 May 2021 22:14:28 +0200 Subject: [PATCH 08/85] Handle standalone pip creation from pip wheel This change ensures that when pip is executed from a wheel/zip, standalone pip creation for build environment reuses the source. Resolves: #9953 Co-authored-by: Tzu-ping Chung --- news/9953.bugfix.rst | 1 + src/pip/_internal/build_env.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 news/9953.bugfix.rst diff --git a/news/9953.bugfix.rst b/news/9953.bugfix.rst new file mode 100644 index 000000000..227f012b8 --- /dev/null +++ b/news/9953.bugfix.rst @@ -0,0 +1 @@ +Fix detection of existing standalone pip instance for PEP 517 builds. diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index cdf043241..680e49f5b 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -50,9 +50,9 @@ def _create_standalone_pip() -> Iterator[str]: """ source = pathlib.Path(pip_location).resolve().parent - # Return the current instance if it is already a zip file. This can happen - # if a PEP 517 requirement is an sdist itself. - if not source.is_dir() and source.parent.name == "__env_pip__.zip": + # Return the current instance if `source` is not a directory. We can't build + # a zip from this, and it likely means the instance is already standalone. + if not source.is_dir(): yield str(source) return From 346edd64bc373aac3af278c454d7210663b83dc8 Mon Sep 17 00:00:00 2001 From: Nicholas Serra Date: Sun, 9 May 2021 11:46:47 -0400 Subject: [PATCH 09/85] Update src/pip/_internal/build_env.py Co-authored-by: Tzu-ping Chung --- src/pip/_internal/build_env.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index d93f940ed..7dd1f42e5 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -58,7 +58,10 @@ def _create_standalone_pip() -> Iterator[str]: with TempDirectory(kind="standalone-pip") as tmp_dir: pip_zip = os.path.join(tmp_dir.path, "__env_pip__.zip") - with zipfile.ZipFile(pip_zip, "w", strict_timestamps=False) as zf: + kwargs = {} + if sys.version_info >= (3, 8): + kwargs["strict_timestamps"] = False + with zipfile.ZipFile(pip_zip, "w", **kwargs) as zf: for child in source.rglob("*"): zf.write(child, child.relative_to(source.parent).as_posix()) yield os.path.join(pip_zip, "pip") From f77649e841d7bdff1ce5a29534762fb5be030414 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 5 May 2021 16:35:50 +0200 Subject: [PATCH 10/85] Provide a better error message when uninstalling packages without dist-info/RECORD Fixes https://github.com/pypa/pip/issues/8954 --- news/8954.feature.rst | 9 ++++++ src/pip/_internal/req/req_uninstall.py | 21 +++++++++++- tests/functional/test_uninstall.py | 45 ++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 news/8954.feature.rst diff --git a/news/8954.feature.rst b/news/8954.feature.rst new file mode 100644 index 000000000..05ec68d04 --- /dev/null +++ b/news/8954.feature.rst @@ -0,0 +1,9 @@ +When pip is asked to uninstall a project without the dist-info/RECORD file +it will no longer traceback with FileNotFoundError, +but it will provide a better error message instead, such as:: + + ERROR: Cannot uninstall foobar 0.1, RECORD file not found. You might be able to recover from this via: 'pip install --force-reinstall --no-deps foobar==0.1'. + +When dist-info/INSTALLER is present and contains some useful information, the info is included in the error message instead:: + + ERROR: Cannot uninstall foobar 0.1, RECORD file not found. Hint: The package was installed by rpm. diff --git a/src/pip/_internal/req/req_uninstall.py b/src/pip/_internal/req/req_uninstall.py index b72234175..a11b225ac 100644 --- a/src/pip/_internal/req/req_uninstall.py +++ b/src/pip/_internal/req/req_uninstall.py @@ -74,8 +74,27 @@ def uninstallation_paths(dist): the .pyc and .pyo in the same directory. UninstallPathSet.add() takes care of the __pycache__ .py[co]. + + If RECORD is not found, raises UninstallationError, + with possible information from the INSTALLER file. + + https://packaging.python.org/specifications/recording-installed-packages/ """ - r = csv.reader(dist.get_metadata_lines('RECORD')) + try: + r = csv.reader(dist.get_metadata_lines('RECORD')) + except FileNotFoundError as missing_record_exception: + msg = 'Cannot uninstall {dist}, RECORD file not found.'.format(dist=dist) + try: + installer = next(dist.get_metadata_lines('INSTALLER')) + if not installer or installer == 'pip': + raise ValueError() + except (OSError, StopIteration, ValueError): + dep = '{}=={}'.format(dist.project_name, dist.version) + msg += (" You might be able to recover from this via: " + "'pip install --force-reinstall --no-deps {}'.".format(dep)) + else: + msg += ' Hint: The package was installed by {}.'.format(installer) + raise UninstallationError(msg) from missing_record_exception for row in r: path = os.path.join(dist.location, row[0]) yield path diff --git a/tests/functional/test_uninstall.py b/tests/functional/test_uninstall.py index 878e713ed..cbce8746a 100644 --- a/tests/functional/test_uninstall.py +++ b/tests/functional/test_uninstall.py @@ -476,6 +476,51 @@ def test_uninstall_wheel(script, data): assert_all_changes(result, result2, []) +@pytest.mark.parametrize('installer', [FileNotFoundError, IsADirectoryError, + '', os.linesep, b'\xc0\xff\xee', 'pip', + 'MegaCorp Cloud Install-O-Matic']) +def test_uninstall_without_record_fails(script, data, installer): + """ + Test uninstalling a package installed without RECORD + """ + package = data.packages.joinpath("simple.dist-0.1-py2.py3-none-any.whl") + result = script.pip('install', package, '--no-index') + dist_info_folder = script.site_packages / 'simple.dist-0.1.dist-info' + result.did_create(dist_info_folder) + + # Remove RECORD + record_path = dist_info_folder / 'RECORD' + (script.base_path / record_path).unlink() + ignore_changes = [record_path] + + # Populate, remove or otherwise break INSTALLER + installer_path = dist_info_folder / 'INSTALLER' + ignore_changes += [installer_path] + installer_path = script.base_path / installer_path + if installer in (FileNotFoundError, IsADirectoryError): + installer_path.unlink() + if installer is IsADirectoryError: + installer_path.mkdir() + else: + if isinstance(installer, bytes): + installer_path.write_bytes(installer) + else: + installer_path.write_text(installer + os.linesep) + + result2 = script.pip('uninstall', 'simple.dist', '-y', expect_error=True) + expected_error_message = ('ERROR: Cannot uninstall simple.dist 0.1, ' + 'RECORD file not found.') + if not isinstance(installer, str) or not installer.strip() or installer == 'pip': + expected_error_message += (" You might be able to recover from this via: " + "'pip install --force-reinstall --no-deps " + "simple.dist==0.1'.") + elif installer: + expected_error_message += (' Hint: The package was installed by ' + '{}.'.format(installer)) + assert result2.stderr.rstrip() == expected_error_message + assert_all_changes(result.files_after, result2, ignore_changes) + + @pytest.mark.skipif("sys.platform == 'win32'") def test_uninstall_with_symlink(script, data, tmpdir): """ From f787788a65cf7a8af8b7ef9dc13c0681d94fff5f Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 16 May 2021 21:19:15 -0400 Subject: [PATCH 11/85] Include rustc version in the user agent, if rustc is available Rust is becoming more popular for writing Python extension modules in, this information would be valuable for package maintainers to assess the ecosystem, in the same way glibc or openssl version is. --- src/pip/_internal/network/session.py | 17 +++++++++++++++++ tests/unit/test_req_file.py | 6 ++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/network/session.py b/src/pip/_internal/network/session.py index 4af800f12..78536f1c4 100644 --- a/src/pip/_internal/network/session.py +++ b/src/pip/_internal/network/session.py @@ -19,6 +19,8 @@ import logging import mimetypes import os import platform +import shutil +import subprocess import sys import urllib.parse import warnings @@ -163,6 +165,21 @@ def user_agent(): if setuptools_dist is not None: data["setuptools_version"] = str(setuptools_dist.version) + if shutil.which("rustc") is not None: + # If for any reason `rustc --version` fails, silently ignore it + try: + rustc_output = subprocess.check_output( + ["rustc", "--version"], stderr=subprocess.STDOUT + ) + except Exception: + pass + else: + if rustc_output.startswith(b"rustc "): + # The format of `rustc --version` is: + # `b'rustc 1.52.1 (9bc8c42bb 2021-05-09)\n'` + # We extract just the middle (1.52.1) part + data["rustc_version"] = rustc_output.split(b" ")[1].decode() + # Use None rather than False so as not to give the impression that # pip knows it is not being run under CI. Rather, it is a null or # inconclusive result. Also, we include some value rather than no diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index 3c534c9ee..cecab63ba 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -598,13 +598,14 @@ class TestParseRequirements: with open(tmpdir.joinpath('req1.txt'), 'w') as fp: fp.write(template.format(*map(make_var, env_vars))) + session = PipSession() with patch('pip._internal.req.req_file.os.getenv') as getenv: getenv.side_effect = lambda n: env_vars[n] reqs = list(parse_reqfile( tmpdir.joinpath('req1.txt'), finder=finder, - session=PipSession() + session=session )) assert len(reqs) == 1, \ @@ -623,13 +624,14 @@ class TestParseRequirements: with open(tmpdir.joinpath('req1.txt'), 'w') as fp: fp.write(req_url) + session = PipSession() with patch('pip._internal.req.req_file.os.getenv') as getenv: getenv.return_value = '' reqs = list(parse_reqfile( tmpdir.joinpath('req1.txt'), finder=finder, - session=PipSession() + session=session )) assert len(reqs) == 1, \ From 722ba2dd1634b72ec72c03cad507a12455f3e74c Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Mon, 17 May 2021 08:01:04 -0400 Subject: [PATCH 12/85] Added a new fragment --- news/9987.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/9987.feature.rst diff --git a/news/9987.feature.rst b/news/9987.feature.rst new file mode 100644 index 000000000..0cf451638 --- /dev/null +++ b/news/9987.feature.rst @@ -0,0 +1 @@ +Include ``rustc`` version in pip's ``User-Agent``, when the system has ``rustc``. From 1904e4d66da3cdc0f853dc9c9c99efffe3ca243b Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 11 May 2021 21:48:22 +0800 Subject: [PATCH 13/85] Relax installable dir check to allow cfg-only --- src/pip/_internal/utils/misc.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index 26037dbdc..a4ad35be6 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -269,18 +269,14 @@ def tabulate(rows): return table, sizes -def is_installable_dir(path): - # type: (str) -> bool - """Is path is a directory containing setup.py or pyproject.toml?""" +def is_installable_dir(path: str) -> bool: + """Is path is a directory containing pyproject.toml, setup.cfg or setup.py?""" if not os.path.isdir(path): return False - setup_py = os.path.join(path, "setup.py") - if os.path.isfile(setup_py): - return True - pyproject_toml = os.path.join(path, "pyproject.toml") - if os.path.isfile(pyproject_toml): - return True - return False + return any( + os.path.isfile(os.path.join(path, signifier)) + for signifier in ("pyproject.toml", "setup.cfg", "setup.py") + ) def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): From 248b6785a45a584ca66be2f2fc38721fe5fb8c72 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Mon, 17 May 2021 17:15:27 -0400 Subject: [PATCH 14/85] Added comments --- tests/unit/test_req_file.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index cecab63ba..be1e5815f 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -598,6 +598,8 @@ class TestParseRequirements: with open(tmpdir.joinpath('req1.txt'), 'w') as fp: fp.write(template.format(*map(make_var, env_vars))) + # Construct the session outside the monkey-patch, since it access the + # env session = PipSession() with patch('pip._internal.req.req_file.os.getenv') as getenv: getenv.side_effect = lambda n: env_vars[n] @@ -624,6 +626,8 @@ class TestParseRequirements: with open(tmpdir.joinpath('req1.txt'), 'w') as fp: fp.write(req_url) + # Construct the session outside the monkey-patch, since it access the + # env session = PipSession() with patch('pip._internal.req.req_file.os.getenv') as getenv: getenv.return_value = '' From d18ac6250e458c518d5c70231c65986ffdd32662 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 18 May 2021 22:47:31 +0800 Subject: [PATCH 15/85] Exclude a known incompatible installed candidate The resolver collects previously known incompatibilites and sends them to the provider. But previously the provider does not correctly exclude the currently-installed candidate if it is present in that incompatibility list, causing the resolver to enter a loop trying that same candidate. This patch correctly applies incompat_ids when producing an AlreadyInstalledCandidate and exclude it if its id() is in the set. --- news/9841.bugfix.rst | 3 ++ .../resolution/resolvelib/factory.py | 37 ++++++++++++------- 2 files changed, 27 insertions(+), 13 deletions(-) create mode 100644 news/9841.bugfix.rst diff --git a/news/9841.bugfix.rst b/news/9841.bugfix.rst new file mode 100644 index 000000000..46e00dc22 --- /dev/null +++ b/news/9841.bugfix.rst @@ -0,0 +1,3 @@ +New resolver: Correctly exclude an already installed package if its version is +known to be incompatible to stop the dependency resolution process with a clear +error message. diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 6e3f19518..5816a0ede 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -240,18 +240,29 @@ class Factory: hashes &= ireq.hashes(trust_internet=False) extras |= frozenset(ireq.extras) - # Get the installed version, if it matches, unless the user - # specified `--force-reinstall`, when we want the version from - # the index instead. - installed_candidate = None - if not self._force_reinstall and name in self._installed_dists: - installed_dist = self._installed_dists[name] - if specifier.contains(installed_dist.version, prereleases=True): - installed_candidate = self._make_candidate_from_dist( - dist=installed_dist, - extras=extras, - template=template, - ) + def _get_installed_candidate() -> Optional[Candidate]: + """Get the candidate for the currently-installed version.""" + # If --force-reinstall is set, we want the version from the index + # instead, so we "pretend" there is nothing installed. + if self._force_reinstall: + return None + try: + installed_dist = self._installed_dists[name] + except KeyError: + return None + # Don't use the installed distribution if its version does not fit + # the current dependency graph. + if not specifier.contains(installed_dist.version, prereleases=True): + return None + candidate = self._make_candidate_from_dist( + dist=installed_dist, + extras=extras, + template=template, + ) + # The candidate is a known incompatiblity. Don't use it. + if id(candidate) in incompatible_ids: + return None + return candidate def iter_index_candidate_infos(): # type: () -> Iterator[IndexCandidateInfo] @@ -283,7 +294,7 @@ class Factory: return FoundCandidates( iter_index_candidate_infos, - installed_candidate, + _get_installed_candidate(), prefers_installed, incompatible_ids, ) From 9ab448de634aca8e2bf4ef3f64bc2dbb4641d1cc Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Thu, 20 May 2021 19:24:54 -0400 Subject: [PATCH 16/85] Added a timeout to invoking rustc --- src/pip/_internal/network/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/network/session.py b/src/pip/_internal/network/session.py index 78536f1c4..dca263744 100644 --- a/src/pip/_internal/network/session.py +++ b/src/pip/_internal/network/session.py @@ -169,7 +169,7 @@ def user_agent(): # If for any reason `rustc --version` fails, silently ignore it try: rustc_output = subprocess.check_output( - ["rustc", "--version"], stderr=subprocess.STDOUT + ["rustc", "--version"], stderr=subprocess.STDOUT, timeout=.5 ) except Exception: pass From 8360cab1ba5d83366b3d3227d3c0d957cc5539f4 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 22 May 2021 05:36:01 +0800 Subject: [PATCH 17/85] Test case for backtracking an installed candidate --- tests/functional/test_new_resolver.py | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/functional/test_new_resolver.py b/tests/functional/test_new_resolver.py index 0938768a2..68eb3b1c0 100644 --- a/tests/functional/test_new_resolver.py +++ b/tests/functional/test_new_resolver.py @@ -1827,3 +1827,33 @@ def test_new_resolver_direct_url_with_extras(tmp_path, script): assert not get_created_direct_url(result, "pkg1") assert get_created_direct_url(result, "pkg2") assert not get_created_direct_url(result, "pkg3") + + +def test_new_resolver_modifies_installed_incompatible(script): + create_basic_wheel_for_package(script, name="a", version="1") + create_basic_wheel_for_package(script, name="a", version="2") + create_basic_wheel_for_package(script, name="a", version="3") + create_basic_wheel_for_package(script, name="b", version="1", depends=["a==1"]) + create_basic_wheel_for_package(script, name="b", version="2", depends=["a==2"]) + create_basic_wheel_for_package(script, name="c", version="1", depends=["a!=1"]) + create_basic_wheel_for_package(script, name="c", version="2", depends=["a!=1"]) + create_basic_wheel_for_package(script, name="d", version="1", depends=["b", "c"]) + + script.pip( + "install", + "--no-cache-dir", "--no-index", + "--find-links", script.scratch_path, + "b==1", + ) + + # d-1 depends on b and c. b-1 is already installed and therefore first + # pinned, but later found to be incompatible since the "a==1" dependency + # makes all c versions impossible to satisfy. The resolver should be able to + # discard b-1 and backtrack, so b-2 is selected instead. + script.pip( + "install", + "--no-cache-dir", "--no-index", + "--find-links", script.scratch_path, + "d==1", + ) + assert_installed(script, d="1", c="2", b="2", a="2") From 9fb3b7c795421c39819d8b836a9020a449e6db3e Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 21 May 2021 16:13:19 +0100 Subject: [PATCH 18/85] Add new installation document --- docs/html/installation.md | 73 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 docs/html/installation.md diff --git a/docs/html/installation.md b/docs/html/installation.md new file mode 100644 index 000000000..1e483ce78 --- /dev/null +++ b/docs/html/installation.md @@ -0,0 +1,73 @@ +# Installation + +pip is already installed if you are: + +- working in a + [virtual environment](pypug:Creating\ and\ using\ Virtual\ Environments) + created by [venv](pypug:venv) or [virtualenv](pypug:virtualenv). +- using Python, downloaded from [python.org](https://www.python.org). +- using Python, that has not been patched by a redistributor to remove + `ensurepip`. + +## Compatibility + +The current version of pip works on: + +- Windows, Linux/Unix and MacOS. +- CPython 3.6, 3.7, 3.8, 3.9 and latest PyPy3. + +pip is tested to work on the latest patch version of the Python interpreter, +for each of the minor versions listed above. Previous patch versions are +supported on a best effort approach. + +```{note} +If you're using an older version of pip or Python, it is possible that +the instructions on this page will not work for you. + +pip's maintainers would not be able to provide support for users on such +older versions, and these users are advised to reach out to the +providers of those older versions of pip/Python (eg: your vendor or +Linux distribution) for support. +``` + +### Using {mod}`ensurepip` + +Python >= 3.4 can self-bootstrap pip with the built-in {mod}`ensurepip` module. +To install pip using {mod}`ensurepip`, run: + +```{pip-cli} +$ python -m ensurepip --upgrade +``` + +```{note} +It is strongly recommended to upgrade to the current version of pip using +`--upgrade` when calling ensurepip. It is possible to skip this flag, which +also means that the process would not access the internet. +``` + +More details about how {mod}`ensurepip` works and can be used, is available in +the standard library documentation. + +### Using get-pip.py + +`get-pip.py` is a bootstrapper script, that is capable of installing pip in an +environment. To install pip using `get-pip.py`, you'll want to: + +- Download the `get-pip.py` script, from . + + On most Linux/MacOS machines, this can be done using the command: + + ``` + $ curl -sSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py + ``` + +- Open the terminal/command prompt on your OS, in the folder containing the + `get-pip.py` file and run: + + ```{pip-cli} + $ python get-pip.py + ``` + +More details about this script can be found in [pypa/get-pip]'s README. + +[pypa/get-pip]: https://github.com/pypa/get-pip From 86c79441e9204c37991a69f7a324cb3b880ffda7 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 21 May 2021 16:13:40 +0100 Subject: [PATCH 19/85] Add initial getting-started scaffold --- docs/html/getting-started.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 docs/html/getting-started.md diff --git a/docs/html/getting-started.md b/docs/html/getting-started.md new file mode 100644 index 000000000..a2bd81050 --- /dev/null +++ b/docs/html/getting-started.md @@ -0,0 +1,25 @@ +# Getting Started + +To get started with using pip, you should install Python on your system. + +## Checking if you have a working pip + +The best way to check if you have a working pip installation is to run: + +```{pip-cli} +$ pip --version +pip X.Y.Z from ... (python X.Y) +``` + +If that worked, congratulations! You have a working pip in your environment. + +If you got output that does not look like the sample above, read on -- the rest +of this page has information about how to install pip within a Python +environment that doesn't have it. + +## Next Steps + +As a next step, you'll want to read the +["Installing Packages"](pypug:tutorials/installing-packages) tutorial on +packaging.python.org. That tutorial will guide you through the basics of using +pip for installing packages. From afcefa908861c082c14869dcbfca8c538af0f9a8 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 21 May 2021 18:11:47 +0100 Subject: [PATCH 20/85] Rewrite substantial portions of installation docs --- docs/html/installation.md | 80 +++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 29 deletions(-) diff --git a/docs/html/installation.md b/docs/html/installation.md index 1e483ce78..9bb0cfd33 100644 --- a/docs/html/installation.md +++ b/docs/html/installation.md @@ -3,37 +3,25 @@ pip is already installed if you are: - working in a - [virtual environment](pypug:Creating\ and\ using\ Virtual\ Environments) - created by [venv](pypug:venv) or [virtualenv](pypug:virtualenv). -- using Python, downloaded from [python.org](https://www.python.org). -- using Python, that has not been patched by a redistributor to remove - `ensurepip`. + {ref}`virtual environment ` +- using Python downloaded from [python.org](https://www.python.org) +- using Python that has not been modified by a redistributor to remove + {mod}`ensurepip` -## Compatibility +## Supported Methods -The current version of pip works on: +If your Python environment does not have pip installed, there are 2 mechanisms +to install pip supported directly by pip's maintainers: -- Windows, Linux/Unix and MacOS. -- CPython 3.6, 3.7, 3.8, 3.9 and latest PyPy3. - -pip is tested to work on the latest patch version of the Python interpreter, -for each of the minor versions listed above. Previous patch versions are -supported on a best effort approach. - -```{note} -If you're using an older version of pip or Python, it is possible that -the instructions on this page will not work for you. - -pip's maintainers would not be able to provide support for users on such -older versions, and these users are advised to reach out to the -providers of those older versions of pip/Python (eg: your vendor or -Linux distribution) for support. -``` +- [Using ensurepip](#using-ensurepip) +- [Using get-pip.py](#using-get-pip-py) ### Using {mod}`ensurepip` -Python >= 3.4 can self-bootstrap pip with the built-in {mod}`ensurepip` module. -To install pip using {mod}`ensurepip`, run: +Python comes with an {mod}`ensurepip` module, which can install pip in a +Python environment. + +To install pip using `ensurepip`, run: ```{pip-cli} $ python -m ensurepip --upgrade @@ -45,13 +33,15 @@ It is strongly recommended to upgrade to the current version of pip using also means that the process would not access the internet. ``` -More details about how {mod}`ensurepip` works and can be used, is available in -the standard library documentation. +More details about how {mod}`ensurepip` works and how it can be used, is +available in the standard library documentation. ### Using get-pip.py -`get-pip.py` is a bootstrapper script, that is capable of installing pip in an -environment. To install pip using `get-pip.py`, you'll want to: +`get-pip.py` is a Python script for installing pip in an environment. It uses +a bundled copy of pip to install pip. + +To use `get-pip.py`, you'll want to: - Download the `get-pip.py` script, from . @@ -71,3 +61,35 @@ environment. To install pip using `get-pip.py`, you'll want to: More details about this script can be found in [pypa/get-pip]'s README. [pypa/get-pip]: https://github.com/pypa/get-pip + +## Alternative Methods + +Depending on how you installed Python, there might be other mechanisms +available to you for installing pip such as +{ref}`using Linux package managers `. + +These mechanisms are provided by redistributors of pip, who may have modified +pip to change its behaviour. This has been a frequent source of user confusion, +since it causes a mismatch between documented behaviour in this documentation +and how pip works after those modifications. + +If you face issues when using Python installed using these mechanisms, it is +recommended to request for support from the relevant provider (eg: linux distro +community, cloud provider's support channels, etc). + +## Compatibility + +The current version of pip works on: + +- Windows, Linux and MacOS. +- CPython 3.6, 3.7, 3.8, 3.9 and latest PyPy3. + +pip is tested to work on the latest patch version of the Python interpreter, +for each of the minor versions listed above. Previous patch versions are +supported on a best effort approach. + +pip's maintainers do not provide support for users on older versions of Python, +and these users should request for support from the relevant provider +(eg: linux distro community, cloud provider's support channels, etc). + +[^python]: The `ensurepip` module was added to the Python standard library in Python 3.4. From cbade4e657b59cb2e43ab53f3d1f99933a502c4f Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 21 May 2021 18:45:07 +0100 Subject: [PATCH 21/85] Further tweak the installation page --- docs/html/installation.md | 43 ++++++++++++--------------------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/docs/html/installation.md b/docs/html/installation.md index 9bb0cfd33..5f78572d0 100644 --- a/docs/html/installation.md +++ b/docs/html/installation.md @@ -1,6 +1,6 @@ # Installation -pip is already installed if you are: +Usually, pip is automatically installed if you are: - working in a {ref}`virtual environment ` @@ -13,45 +13,28 @@ pip is already installed if you are: If your Python environment does not have pip installed, there are 2 mechanisms to install pip supported directly by pip's maintainers: -- [Using ensurepip](#using-ensurepip) -- [Using get-pip.py](#using-get-pip-py) +- [`ensurepip`](#using-ensurepip) +- [`get-pip.py`](#using-get-pip-py) -### Using {mod}`ensurepip` +### `ensurepip` Python comes with an {mod}`ensurepip` module, which can install pip in a Python environment. -To install pip using `ensurepip`, run: - ```{pip-cli} $ python -m ensurepip --upgrade ``` -```{note} -It is strongly recommended to upgrade to the current version of pip using -`--upgrade` when calling ensurepip. It is possible to skip this flag, which -also means that the process would not access the internet. -``` - More details about how {mod}`ensurepip` works and how it can be used, is available in the standard library documentation. -### Using get-pip.py +### `get-pip.py` -`get-pip.py` is a Python script for installing pip in an environment. It uses -a bundled copy of pip to install pip. +This is a Python script that uses some bootstrapping logic to install +pip. -To use `get-pip.py`, you'll want to: - -- Download the `get-pip.py` script, from . - - On most Linux/MacOS machines, this can be done using the command: - - ``` - $ curl -sSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py - ``` - -- Open the terminal/command prompt on your OS, in the folder containing the +- Download the script, from . +- Open a terminal/command prompt, `cd` to the folder containing the `get-pip.py` file and run: ```{pip-cli} @@ -73,9 +56,9 @@ pip to change its behaviour. This has been a frequent source of user confusion, since it causes a mismatch between documented behaviour in this documentation and how pip works after those modifications. -If you face issues when using Python installed using these mechanisms, it is -recommended to request for support from the relevant provider (eg: linux distro -community, cloud provider's support channels, etc). +If you face issues when using Python and pip installed using these mechanisms, +it is recommended to request for support from the relevant provider (eg: Linux +distro community, cloud provider support channels, etc). ## Compatibility @@ -90,6 +73,6 @@ supported on a best effort approach. pip's maintainers do not provide support for users on older versions of Python, and these users should request for support from the relevant provider -(eg: linux distro community, cloud provider's support channels, etc). +(eg: Linux distro community, cloud provider support channels, etc). [^python]: The `ensurepip` module was added to the Python standard library in Python 3.4. From 0139009eb0454fb448ab25dafcffcdcdfcb8b71b Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 21 May 2021 18:45:27 +0100 Subject: [PATCH 22/85] Flesh out the Getting Started page --- docs/html/getting-started.md | 99 ++++++++++++++++++++++++++++++++---- 1 file changed, 89 insertions(+), 10 deletions(-) diff --git a/docs/html/getting-started.md b/docs/html/getting-started.md index a2bd81050..3434a30ad 100644 --- a/docs/html/getting-started.md +++ b/docs/html/getting-started.md @@ -1,25 +1,104 @@ # Getting Started -To get started with using pip, you should install Python on your system. +To get started with using pip, you should [install Python] on your system. + +[install Python]: https://realpython.com/installing-python/ ## Checking if you have a working pip -The best way to check if you have a working pip installation is to run: +As a first step, you should check that you have a working Python with pip +installed. This can be done by running the following commands and making +sure that the output looks similar. ```{pip-cli} +$ python --version +Python 3.N.N $ pip --version -pip X.Y.Z from ... (python X.Y) +pip X.Y.Z from ... (python 3.N.N) ``` If that worked, congratulations! You have a working pip in your environment. -If you got output that does not look like the sample above, read on -- the rest -of this page has information about how to install pip within a Python -environment that doesn't have it. +If you got output that does not look like the sample above, please read +the {doc}`installation` page. It provides guidance on how to install pip +within a Python environment that doesn't have it. + +## Common tasks + +### Install a package + +```{pip-cli} +$ pip install sampleproject +[...] +Successfully installed sampleproject +``` + +By default, pip will fetch packages from [Python Package Index][PyPI], a +repository of software for the Python programming language where anyone can +upload packages. + +[PyPI]: https://pypi.org/ + +### Install a package from GitHub + +```{pip-cli} +$ pip install git+https://github.com/pypa/sampleproject.git@main +[...] +Successfully installed sampleproject +``` + +See {ref}`VCS Support` for more information about this syntax. + +### Install a package from a distribution file + +pip can install directly from distribution files as well. They come in 2 forms: + +- {term}`source distribution ` (usually shortened to "sdist") +- {term}`wheel distribution ` (usually shortened to "wheel") + +```{pip-cli} +$ pip install sampleproject-1.0.tar.gz +[...] +Successfully installed sampleproject +$ pip install sampleproject-1.0-py3-none-any.whl +[...] +Successfully installed sampleproject +``` + +### Install multiple packages using a requirements file + +Many Python projects use {file}`requirements.txt` files, to specify the +list of packages that need to be installed for the project to run. To install +the packages listed in that file, you can run: + +```{pip-cli} +$ pip install -r requirements.txt +[...] +Successfully installed sampleproject +``` + +### Upgrade a package + +```{pip-cli} +$ pip install --upgrade sampleproject +Uninstalling sampleproject: + [...] +Proceed (y/n)? y +Successfully uninstalled sampleproject +``` + +### Uninstall a package + +```{pip-cli} +$ pip uninstall sampleproject +Uninstalling sampleproject: + [...] +Proceed (y/n)? y +Successfully uninstalled sampleproject +``` ## Next Steps -As a next step, you'll want to read the -["Installing Packages"](pypug:tutorials/installing-packages) tutorial on -packaging.python.org. That tutorial will guide you through the basics of using -pip for installing packages. +It is recommended to learn about what virtual environments are and how to use +them. This is covered in the ["Installing Packages"](pypug:tutorials/installing-packages) +tutorial on packaging.python.org. From f99ccb15c36b476034ef27749920ea7e2ce76dc4 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 21 May 2021 18:45:47 +0100 Subject: [PATCH 23/85] Update copyright in conf.py, to match copyright.rst --- docs/html/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/html/conf.py b/docs/html/conf.py index 2a4387a35..9e210539e 100644 --- a/docs/html/conf.py +++ b/docs/html/conf.py @@ -30,7 +30,7 @@ extensions = [ # General information about the project. project = "pip" -copyright = "2008-2020, PyPA" +copyright = "The pip developers" # Find the version and release information. # We have a single source of truth for our version number: pip's __init__.py file. From 5734d32546a9b8faa07d11ba0ddc9307e9824897 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 21 May 2021 18:48:36 +0100 Subject: [PATCH 24/85] Change first title in getting-started --- docs/html/getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/html/getting-started.md b/docs/html/getting-started.md index 3434a30ad..42ac2c934 100644 --- a/docs/html/getting-started.md +++ b/docs/html/getting-started.md @@ -4,7 +4,7 @@ To get started with using pip, you should [install Python] on your system. [install Python]: https://realpython.com/installing-python/ -## Checking if you have a working pip +## Ensure you have a working pip As a first step, you should check that you have a working Python with pip installed. This can be done by running the following commands and making From 34d181a5d6213d70f36a385a567b65daf288a21e Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 21 May 2021 18:49:08 +0100 Subject: [PATCH 25/85] quickstart.rst -> getting-started.md --- docs/html/index.md | 4 +- docs/html/quickstart.rst | 139 ++------------------------------------- 2 files changed, 9 insertions(+), 134 deletions(-) diff --git a/docs/html/index.md b/docs/html/index.md index a84c2665d..26b8db4a4 100644 --- a/docs/html/index.md +++ b/docs/html/index.md @@ -10,7 +10,7 @@ install packages from the [Python Package Index][pypi] and other indexes. ```{toctree} :hidden: -quickstart +getting-started installing user_guide cli/index @@ -29,7 +29,7 @@ GitHub If you want to learn about how to use pip, check out the following resources: -- [Quickstart](quickstart) +- [Getting Started](getting-started) - [Python Packaging User Guide](https://packaging.python.org) If you find bugs, need help, or want to talk to the developers, use our mailing diff --git a/docs/html/quickstart.rst b/docs/html/quickstart.rst index 96602a7b3..4385f4a73 100644 --- a/docs/html/quickstart.rst +++ b/docs/html/quickstart.rst @@ -1,136 +1,11 @@ -========== -Quickstart -========== +:orphan: -First, :doc:`install pip `. +.. meta:: -Install a package from `PyPI`_: + :http-equiv=refresh: 3; url=../getting-started/ -.. tab:: Unix/macOS +This page has moved +=================== - .. code-block:: console - - $ python -m pip install SomePackage - [...] - Successfully installed SomePackage - -.. tab:: Windows - - .. code-block:: console - - C:\> py -m pip install SomePackage - [...] - Successfully installed SomePackage - - -Install a package that's already been downloaded from `PyPI`_ or -obtained from elsewhere. This is useful if the target machine does not have a -network connection: - -.. tab:: Unix/macOS - - .. code-block:: console - - $ python -m pip install SomePackage-1.0-py2.py3-none-any.whl - [...] - Successfully installed SomePackage - -.. tab:: Windows - - .. code-block:: console - - C:\> py -m pip install SomePackage-1.0-py2.py3-none-any.whl - [...] - Successfully installed SomePackage - -Show what files were installed: - -.. tab:: Unix/macOS - - .. code-block:: console - - $ python -m pip show --files SomePackage - Name: SomePackage - Version: 1.0 - Location: /my/env/lib/pythonx.x/site-packages - Files: - ../somepackage/__init__.py - [...] - -.. tab:: Windows - - .. code-block:: console - - C:\> py -m pip show --files SomePackage - Name: SomePackage - Version: 1.0 - Location: /my/env/lib/pythonx.x/site-packages - Files: - ../somepackage/__init__.py - [...] - -List what packages are outdated: - -.. tab:: Unix/macOS - - .. code-block:: console - - $ python -m pip list --outdated - SomePackage (Current: 1.0 Latest: 2.0) - -.. tab:: Windows - - .. code-block:: console - - C:\> py -m pip list --outdated - SomePackage (Current: 1.0 Latest: 2.0) - -Upgrade a package: - -.. tab:: Unix/macOS - - .. code-block:: console - - $ python -m pip install --upgrade SomePackage - [...] - Found existing installation: SomePackage 1.0 - Uninstalling SomePackage: - Successfully uninstalled SomePackage - Running setup.py install for SomePackage - Successfully installed SomePackage - -.. tab:: Windows - - .. code-block:: console - - C:\> py -m pip install --upgrade SomePackage - [...] - Found existing installation: SomePackage 1.0 - Uninstalling SomePackage: - Successfully uninstalled SomePackage - Running setup.py install for SomePackage - Successfully installed SomePackage - -Uninstall a package: - -.. tab:: Unix/macOS - - .. code-block:: console - - $ python -m pip uninstall SomePackage - Uninstalling SomePackage: - /my/env/lib/pythonx.x/site-packages/somepackage - Proceed (y/n)? y - Successfully uninstalled SomePackage - -.. tab:: Windows - - .. code-block:: console - - C:\> py -m pip uninstall SomePackage - Uninstalling SomePackage: - /my/env/lib/pythonx.x/site-packages/somepackage - Proceed (y/n)? y - Successfully uninstalled SomePackage - -.. _PyPI: https://pypi.org/ +You should be redirected automatically in 3 seconds. If that didn't +work, here's a link: :doc:`getting-started` From 1b386366a37bf7cbc83fa5b9dc638385c39c3cde Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 21 May 2021 18:51:38 +0100 Subject: [PATCH 26/85] installing.rst -> installation.md --- docs/html/index.md | 2 +- docs/html/installing.rst | 233 ++------------------------------------- 2 files changed, 8 insertions(+), 227 deletions(-) diff --git a/docs/html/index.md b/docs/html/index.md index 26b8db4a4..7cf130c4b 100644 --- a/docs/html/index.md +++ b/docs/html/index.md @@ -11,7 +11,7 @@ install packages from the [Python Package Index][pypi] and other indexes. :hidden: getting-started -installing +installation user_guide cli/index ``` diff --git a/docs/html/installing.rst b/docs/html/installing.rst index 95b21899d..e8d86f344 100644 --- a/docs/html/installing.rst +++ b/docs/html/installing.rst @@ -1,230 +1,11 @@ -.. _`Installation`: +:orphan: -============ -Installation -============ +.. meta:: -Do I need to install pip? -========================= + :http-equiv=refresh: 3; url=../installation/ -pip is already installed if you are using Python 2 >=2.7.9 or Python 3 >=3.4 -downloaded from `python.org `_ or if you are working -in a :ref:`Virtual Environment ` -created by :ref:`pypug:virtualenv` or :ref:`venv `. Just make sure -to :ref:`upgrade pip `. +This page has moved +=================== -Use the following command to check whether pip is installed: - -.. tab:: Unix/macOS - - .. code-block:: console - - $ python -m pip --version - pip X.Y.Z from .../site-packages/pip (python X.Y) - -.. tab:: Windows - - .. code-block:: console - - C:\> py -m pip --version - pip X.Y.Z from ...\site-packages\pip (python X.Y) - -Using Linux Package Managers -============================ - -.. warning:: - - If you installed Python from a package manager on Linux, you should always - install pip for that Python installation using the same source. - -See `pypug:Installing pip/setuptools/wheel with Linux Package Managers `_ -in the Python Packaging User Guide. - -Here are ways to contact a few Linux package maintainers if you run into -problems: - -* `Deadsnakes PPA `_ -* `Debian Python Team `_ (for general - issues related to ``apt``) -* `Red Hat Bugzilla `_ - -pip developers do not have control over how Linux distributions handle pip -installations, and are unable to provide solutions to related issues in -general. - -Using ensurepip -=============== - -Python >=3.4 can self-bootstrap pip with the built-in -:ref:`ensurepip ` module. Refer to the standard library -documentation for more details. Make sure to :ref:`upgrade pip ` -after ``ensurepip`` installs pip. - -See the `Using Linux Package Managers`_ section if your Python reports -``No module named ensurepip`` on Debian and derived systems (e.g. Ubuntu). - - -.. _`get-pip`: - -Installing with get-pip.py -========================== - -.. warning:: - - Be cautious if you are using a Python install that is managed by your operating - system or another package manager. ``get-pip.py`` does not coordinate with - those tools, and may leave your system in an inconsistent state. - -To manually install pip, securely [1]_ download ``get-pip.py`` by following -this link: `get-pip.py -`_. Alternatively, use ``curl``:: - - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py - -Then run the following command in the folder where you -have downloaded ``get-pip.py``: - -.. tab:: Unix/macOS - - .. code-block:: shell - - python get-pip.py - -.. tab:: Windows - - .. code-block:: shell - - py get-pip.py - -``get-pip.py`` also installs :ref:`pypug:setuptools` [2]_ and :ref:`pypug:wheel` -if they are not already. :ref:`pypug:setuptools` is required to install -:term:`source distributions `. Both are -required in order to build a :ref:`Wheel cache` (which improves installation -speed), although neither are required to install pre-built :term:`wheels -`. - -.. note:: - - The get-pip.py script is supported on the same python version as pip. - For the now unsupported Python 2.6, alternate script is available - `here `__. - - -get-pip.py options ------------------- - -.. option:: --no-setuptools - - If set, do not attempt to install :ref:`pypug:setuptools` - -.. option:: --no-wheel - - If set, do not attempt to install :ref:`pypug:wheel` - - -``get-pip.py`` allows :ref:`pip install options ` and the :ref:`general options `. Below are -some examples: - -Install from local copies of pip and setuptools: - -.. tab:: Unix/macOS - - .. code-block:: shell - - python get-pip.py --no-index --find-links=/local/copies - -.. tab:: Windows - - .. code-block:: shell - - py get-pip.py --no-index --find-links=/local/copies - -Install to the user site [3]_: - -.. tab:: Unix/macOS - - .. code-block:: shell - - python get-pip.py --user - -.. tab:: Windows - - .. code-block:: shell - - py get-pip.py --user - -Install behind a proxy: - -.. tab:: Unix/macOS - - .. code-block:: shell - - python get-pip.py --proxy="http://[user:passwd@]proxy.server:port" - -.. tab:: Windows - - .. code-block:: shell - - py get-pip.py --proxy="http://[user:passwd@]proxy.server:port" - -``get-pip.py`` can also be used to install a specified combination of ``pip``, -``setuptools``, and ``wheel`` using the same requirements syntax as pip: - -.. tab:: Unix/macOS - - .. code-block:: shell - - python get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 - -.. tab:: Windows - - .. code-block:: shell - - py get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 - -.. _`Upgrading pip`: - -Upgrading pip -============= - -.. tab:: Unix/macOS - - .. code-block:: shell - - python -m pip install -U pip - -.. tab:: Windows - - .. code-block:: shell - - py -m pip install -U pip - - -.. _compatibility-requirements: - -Python and OS Compatibility -=========================== - -pip works with CPython versions 3.6, 3.7, 3.8, 3.9 and also PyPy. - -This means pip works on the latest patch version of each of these minor -versions. Previous patch versions are supported on a best effort approach. - -pip works on Unix/Linux, macOS, and Windows. - - ----- - -.. [1] "Secure" in this context means using a modern browser or a - tool like ``curl`` that verifies SSL certificates when downloading from - https URLs. - -.. [2] Beginning with pip v1.5.1, ``get-pip.py`` stopped requiring setuptools to - be installed first. - -.. [3] The pip developers are considering making ``--user`` the default for all - installs, including ``get-pip.py`` installs of pip, but at this time, - ``--user`` installs for pip itself, should not be considered to be fully - tested or endorsed. For discussion, see `Issue 1668 - `_. +You should be redirected automatically in 3 seconds. If that didn't +work, here's a link: :doc:`installation` From f9dc946e68e7ed099c410cd7ae317ff45da9e280 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 22 May 2021 12:13:41 +0100 Subject: [PATCH 27/85] Use the footnote in the installation page --- docs/html/installation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/html/installation.md b/docs/html/installation.md index 5f78572d0..da9757271 100644 --- a/docs/html/installation.md +++ b/docs/html/installation.md @@ -18,8 +18,8 @@ to install pip supported directly by pip's maintainers: ### `ensurepip` -Python comes with an {mod}`ensurepip` module, which can install pip in a -Python environment. +Python comes with an {mod}`ensurepip` module[^python], which can install pip in +a Python environment. ```{pip-cli} $ python -m ensurepip --upgrade From 4188fc5438c97c37ee4b2f5781dbaece3afb8ddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 23 May 2021 11:20:12 +0200 Subject: [PATCH 28/85] Bump for development --- src/pip/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/__init__.py b/src/pip/__init__.py index 82f53c38c..c35872ca9 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -1,6 +1,6 @@ from typing import List, Optional -__version__ = "21.1.2" +__version__ = "21.2.dev0" def main(args=None): From 72da651285824d306ce60a11d086abf4c969de43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 23 May 2021 12:30:49 +0200 Subject: [PATCH 29/85] Drop released news fragments --- news/9841.bugfix.rst | 3 --- news/9910.bugfix.rst | 1 - news/9944.bugfix.rst | 2 -- news/9953.bugfix.rst | 1 - news/e0021c1f-5eeb-4f32-bfe8-87752f034acd.trivial.rst | 0 5 files changed, 7 deletions(-) delete mode 100644 news/9841.bugfix.rst delete mode 100644 news/9910.bugfix.rst delete mode 100644 news/9944.bugfix.rst delete mode 100644 news/9953.bugfix.rst delete mode 100644 news/e0021c1f-5eeb-4f32-bfe8-87752f034acd.trivial.rst diff --git a/news/9841.bugfix.rst b/news/9841.bugfix.rst deleted file mode 100644 index 46e00dc22..000000000 --- a/news/9841.bugfix.rst +++ /dev/null @@ -1,3 +0,0 @@ -New resolver: Correctly exclude an already installed package if its version is -known to be incompatible to stop the dependency resolution process with a clear -error message. diff --git a/news/9910.bugfix.rst b/news/9910.bugfix.rst deleted file mode 100644 index ece2f3fdf..000000000 --- a/news/9910.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Allow ZIP to archive files with timestamps earlier than 1980. diff --git a/news/9944.bugfix.rst b/news/9944.bugfix.rst deleted file mode 100644 index 58bb70b78..000000000 --- a/news/9944.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -Emit clearer error message when a project root does not contain either -``pyproject.toml``, ``setup.py`` or ``setup.cfg``. diff --git a/news/9953.bugfix.rst b/news/9953.bugfix.rst deleted file mode 100644 index 227f012b8..000000000 --- a/news/9953.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix detection of existing standalone pip instance for PEP 517 builds. diff --git a/news/e0021c1f-5eeb-4f32-bfe8-87752f034acd.trivial.rst b/news/e0021c1f-5eeb-4f32-bfe8-87752f034acd.trivial.rst deleted file mode 100644 index e69de29bb..000000000 From 5ee933aab81273da3691c97f2a6e7016ecbe0ef9 Mon Sep 17 00:00:00 2001 From: Diego Ramirez Date: Tue, 25 May 2021 09:09:27 -0500 Subject: [PATCH 30/85] Use "typing.List" as an annotation Co-authored-by: Tzu-ping Chung --- news/10004.trivial.rst | 1 + tools/tox_pip.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 news/10004.trivial.rst diff --git a/news/10004.trivial.rst b/news/10004.trivial.rst new file mode 100644 index 000000000..128cf866d --- /dev/null +++ b/news/10004.trivial.rst @@ -0,0 +1 @@ +Annotate ``typing.List`` into ``tools.tox_pip.pip()`` diff --git a/tools/tox_pip.py b/tools/tox_pip.py index fe7621342..6a0e2dae9 100644 --- a/tools/tox_pip.py +++ b/tools/tox_pip.py @@ -9,8 +9,7 @@ VIRTUAL_ENV = os.environ['VIRTUAL_ENV'] TOX_PIP_DIR = os.path.join(VIRTUAL_ENV, 'pip') -def pip(args): - # type: (List[str]) -> None +def pip(args: List[str]) -> None: # First things first, get a recent (stable) version of pip. if not os.path.exists(TOX_PIP_DIR): subprocess.check_call([sys.executable, '-m', 'pip', From 8b521e2cdda294bc66e71b6dd5d3200209932b61 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 28 May 2021 11:22:39 +0100 Subject: [PATCH 31/85] Rephrase the warning printed when run as root on Unix The earlier warning phrasing has some awkwardness and doesn't clearly explain why this action is potentially harmful. The change from "you should" to "it is recommended" is also intentional, to take a different tone. --- src/pip/_internal/cli/req_command.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 3fc00d4f4..7ed623880 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -177,8 +177,9 @@ def warn_if_run_as_root(): if os.getuid() != 0: return logger.warning( - "Running pip as root will break packages and permissions. " - "You should install packages reliably by using venv: " + "Running pip as the 'root' user can result in broken permissions and " + "conflicting behaviour with the system package manager. " + "It is recommended to use a virtual environment instead: " "https://pip.pypa.io/warnings/venv" ) From b3a848af13e5dbc557e380cc9df56837418c3886 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 28 May 2021 12:16:56 +0100 Subject: [PATCH 32/85] Move where show is shown in commands It is for introspecting installed packages, not on the index. --- docs/html/cli/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/html/cli/index.md b/docs/html/cli/index.md index f608da521..a3497c308 100644 --- a/docs/html/cli/index.md +++ b/docs/html/cli/index.md @@ -17,6 +17,7 @@ pip pip_install pip_uninstall pip_list +pip_show pip_freeze pip_check ``` @@ -34,7 +35,6 @@ pip_hash :maxdepth: 1 :caption: Package Index information -pip_show pip_search ``` From 70a2c723e8596e6a8cc9535ffb1dfd54f90b5385 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 28 May 2021 12:43:31 +0100 Subject: [PATCH 33/85] Remove old documentation pages These pages were redirecting users from really old versions of pip. Breaking those links now should be fine, since these pages do not get much traffic. --- docs/html/cookbook.rst | 7 ------- docs/html/logic.rst | 7 ------- docs/html/usage.rst | 7 ------- 3 files changed, 21 deletions(-) delete mode 100644 docs/html/cookbook.rst delete mode 100644 docs/html/logic.rst delete mode 100644 docs/html/usage.rst diff --git a/docs/html/cookbook.rst b/docs/html/cookbook.rst deleted file mode 100644 index efd76af15..000000000 --- a/docs/html/cookbook.rst +++ /dev/null @@ -1,7 +0,0 @@ -:orphan: - -======== -Cookbook -======== - -This content is now covered in the :doc:`User Guide ` diff --git a/docs/html/logic.rst b/docs/html/logic.rst deleted file mode 100644 index 189169a8c..000000000 --- a/docs/html/logic.rst +++ /dev/null @@ -1,7 +0,0 @@ -:orphan: - -================ -Internal Details -================ - -This content is now covered in the :doc:`Architecture section `. diff --git a/docs/html/usage.rst b/docs/html/usage.rst deleted file mode 100644 index ab1e9737f..000000000 --- a/docs/html/usage.rst +++ /dev/null @@ -1,7 +0,0 @@ -:orphan: - -===== -Usage -===== - -The "Usage" section is now covered in the :doc:`Reference Guide ` From 1a0ece17f1f0aad2e66ce459850ca3d3c441d77b Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 28 May 2021 13:01:53 +0100 Subject: [PATCH 34/85] Add a missing reference This was being used in another part of our documentation, to refer to this heading. --- docs/html/installation.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/html/installation.md b/docs/html/installation.md index da9757271..e389a8fa4 100644 --- a/docs/html/installation.md +++ b/docs/html/installation.md @@ -60,6 +60,8 @@ If you face issues when using Python and pip installed using these mechanisms, it is recommended to request for support from the relevant provider (eg: Linux distro community, cloud provider support channels, etc). +(compatibility-requirements)= + ## Compatibility The current version of pip works on: From 7d42fb61ae49f0d53e1b11936cedffdd6897e586 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 28 May 2021 13:03:57 +0100 Subject: [PATCH 35/85] Add a "Topic Guide" section to the documentation This section will contain pages that describe a single topic (like "Authentication", "Caching", etc). --- docs/html/index.md | 1 + docs/html/topics/index.md | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 docs/html/topics/index.md diff --git a/docs/html/index.md b/docs/html/index.md index 7cf130c4b..9ab5df298 100644 --- a/docs/html/index.md +++ b/docs/html/index.md @@ -13,6 +13,7 @@ install packages from the [Python Package Index][pypi] and other indexes. getting-started installation user_guide +topics/index cli/index ``` diff --git a/docs/html/topics/index.md b/docs/html/topics/index.md new file mode 100644 index 000000000..b4f984bc7 --- /dev/null +++ b/docs/html/topics/index.md @@ -0,0 +1,12 @@ +# Topic Guides + +These pages provide detailed information on individual topics. + +```{note} +This section of the documentation is currently being fleshed out. See +{issue}`9475` for more details. +``` + +```{toctree} +:maxdepth: 1 +``` From 3930f5545f1e5f3e3706eedb4b023195f74b655b Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 28 May 2021 13:15:25 +0100 Subject: [PATCH 36/85] Add a topic guide for Authentication --- docs/html/topics/authentication.md | 83 ++++++++++++++++++++++++++++++ docs/html/topics/index.md | 2 + docs/html/user_guide.rst | 63 ++--------------------- 3 files changed, 88 insertions(+), 60 deletions(-) create mode 100644 docs/html/topics/authentication.md diff --git a/docs/html/topics/authentication.md b/docs/html/topics/authentication.md new file mode 100644 index 000000000..e78aff472 --- /dev/null +++ b/docs/html/topics/authentication.md @@ -0,0 +1,83 @@ +# Authentication + +## Basic HTTP authentication + +pip supports basic HTTP-based authentication credentials. This is done by +providing the username (and optionally password) in the URL: + +``` +https://username:password@pypi.company.com/simple +``` + +For indexes that only require single-part authentication tokens, provide the +token as the "username" and do not provide a password: + +``` +https://0123456789abcdef@pypi.company.com/simple +``` + +### Percent-encoding special characters + +```{versionaddded} 10.0 +``` + +Certain special characters are not valid in the credential part of a URL. +If the user or password part of your login credentials contain any of these +[special characters][reserved-chars], then they must be percent-encoded. As an +example, for a user with username `user` and password `he//o` accessing a +repository at `pypi.company.com/simple`, the URL with credentials would look +like: + +``` +https://user:he%2F%2Fo@pypi.company.com/simple +``` + +[reserved-chars]: https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_reserved_characters + +## netrc support + +pip supports loading credentials from a user's `.netrc` file. If no credentials +are part of the URL, pip will attempt to get authentication credentials for the +URL's hostname from the user's `.netrc` file. This behaviour comes from the +underlying use of {pypi}`requests`, which in turn delegates it to the +[Python standard library's `netrc` module][netrc-std-lib]. + +```{note} +As mentioned in the [standard library documentation for netrc][netrc-std-lib], +only ASCII characters are allowed in `.netrc` files. Whitespace and +non-printable characters are not allowed in passwords. +``` + +Below is an example `.netrc`, for the host `example.com`, with a user named +`daniel`, using the password `qwerty`: + +``` +machine example.com +login daniel +password qwerty +``` + +More information about the `.netrc` file format can be found in the GNU [`ftp` +man pages][netrc-docs]. + +[netrc-docs]: https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html +[netrc-std-lib]: https://docs.python.org/3/library/netrc.html + +## Keyring Support + +pip supports loading credentials stored in your keyring using the +{pypi}`keyring` library. + +```bash +$ pip install keyring # install keyring from PyPI +$ echo "your-password" | keyring set pypi.company.com your-username +$ pip install your-package --index-url https://pypi.company.com/ +``` + +Note that `keyring` (the Python package) needs to be installed separately from +pip. This can create a bootstrapping issue if you need the credentials stored in +the keyring to download and install keyring. + +It is, thus, expected that users that wish to use pip's keyring support have +some mechanism for downloading and installing {pypi}`keyring` in their Python +environment. diff --git a/docs/html/topics/index.md b/docs/html/topics/index.md index b4f984bc7..6e815ebc6 100644 --- a/docs/html/topics/index.md +++ b/docs/html/topics/index.md @@ -9,4 +9,6 @@ This section of the documentation is currently being fleshed out. See ```{toctree} :maxdepth: 1 + +authentication ``` diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 900ad5034..a36bc9268 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -63,72 +63,17 @@ For more information and examples, see the :ref:`pip install` reference. Basic Authentication Credentials ================================ -pip supports basic authentication credentials. Basically, in the URL there is -a username and password separated by ``:``. - -``https://[username[:password]@]pypi.company.com/simple`` - -Certain special characters are not valid in the authentication part of URLs. -If the user or password part of your login credentials contain any of the -special characters -`here `_ -then they must be percent-encoded. For example, for a -user with username "user" and password "he//o" accessing a repository at -pypi.company.com, the index URL with credentials would look like: - -``https://user:he%2F%2Fo@pypi.company.com`` - -Support for percent-encoded authentication in index URLs was added in pip 10.0.0 -(in `#3236 `_). Users that must use authentication -for their Python repository on systems with older pip versions should make the latest -get-pip.py available in their environment to bootstrap pip to a recent-enough version. - -For indexes that only require single-part authentication tokens, provide the token -as the "username" and do not provide a password, for example - - -``https://0123456789abcdef@pypi.company.com`` - +This is now covered in {doc}`topics/authentication`. netrc Support ------------- -If no credentials are part of the URL, pip will attempt to get authentication credentials -for the URL’s hostname from the user’s .netrc file. This behaviour comes from the underlying -use of `requests`_ which in turn delegates it to the `Python standard library`_. - -The .netrc file contains login and initialization information used by the auto-login process. -It resides in the user's home directory. The .netrc file format is simple. You specify lines -with a machine name and follow that with lines for the login and password that are -associated with that machine. Machine name is the hostname in your URL. - -An example .netrc for the host example.com with a user named 'daniel', using the password -'qwerty' would look like: - -.. code-block:: shell - - machine example.com - login daniel - password qwerty - -As mentioned in the `standard library docs `_, -only ASCII characters are allowed. Whitespace and non-printable characters are not allowed in passwords. - +This is now covered in {doc}`topics/authentication`. Keyring Support --------------- -pip also supports credentials stored in your keyring using the `keyring`_ -library. Note that ``keyring`` will need to be installed separately, as pip -does not come with it included. - -.. code-block:: shell - - pip install keyring - echo your-password | keyring set pypi.company.com your-username - pip install your-package --index-url https://pypi.company.com/ - -.. _keyring: https://pypi.org/project/keyring/ - +This is now covered in {doc}`topics/authentication`. Using a Proxy Server ==================== @@ -1904,6 +1849,4 @@ announcements on the `low-traffic packaging announcements list`_ and .. _low-traffic packaging announcements list: https://mail.python.org/mailman3/lists/pypi-announce.python.org/ .. _our survey on upgrades that create conflicts: https://docs.google.com/forms/d/e/1FAIpQLSeBkbhuIlSofXqCyhi3kGkLmtrpPOEBwr6iJA6SzHdxWKfqdA/viewform .. _the official Python blog: https://blog.python.org/ -.. _requests: https://requests.readthedocs.io/en/latest/user/authentication/#netrc-authentication -.. _Python standard library: https://docs.python.org/3/library/netrc.html .. _Python Windows launcher: https://docs.python.org/3/using/windows.html#launcher From b8f1fcf863081fde0b9d558759c0e3c46ce09a12 Mon Sep 17 00:00:00 2001 From: Ben Darnell <> Date: Fri, 28 May 2021 16:01:41 +0000 Subject: [PATCH 37/85] Avoid importing a non-vendored version of Tornado Code depending on this conditional import could break if an old version of Tornado is present in the environment, rendering pip unusable. --- news/10020.bugfix.rst | 1 + src/pip/_vendor/tenacity/__init__.py | 10 ++++++---- tools/vendoring/patches/tenacity.patch | 21 +++++++++++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 news/10020.bugfix.rst create mode 100644 tools/vendoring/patches/tenacity.patch diff --git a/news/10020.bugfix.rst b/news/10020.bugfix.rst new file mode 100644 index 000000000..9425626fb --- /dev/null +++ b/news/10020.bugfix.rst @@ -0,0 +1 @@ +Remove unused optional ``tornado`` import in vendored ``tenacity`` to prevent old versions of Tornado from breaking pip. diff --git a/src/pip/_vendor/tenacity/__init__.py b/src/pip/_vendor/tenacity/__init__.py index 5f8cb5058..42e9d8940 100644 --- a/src/pip/_vendor/tenacity/__init__.py +++ b/src/pip/_vendor/tenacity/__init__.py @@ -22,10 +22,12 @@ try: except ImportError: iscoroutinefunction = None -try: - import tornado -except ImportError: - tornado = None +# Replace a conditional import with a hard-coded None so that pip does +# not attempt to use tornado even if it is present in the environment. +# If tornado is non-None, tenacity will attempt to execute some code +# that is sensitive to the version of tornado, which could break pip +# if an old version is found. +tornado = None import sys import threading diff --git a/tools/vendoring/patches/tenacity.patch b/tools/vendoring/patches/tenacity.patch new file mode 100644 index 000000000..006588b36 --- /dev/null +++ b/tools/vendoring/patches/tenacity.patch @@ -0,0 +1,21 @@ +diff --git a/src/pip/_vendor/tenacity/__init__.py b/src/pip/_vendor/tenacity/__init__.py +index 5f8cb5058..42e9d8940 100644 +--- a/src/pip/_vendor/tenacity/__init__.py ++++ b/src/pip/_vendor/tenacity/__init__.py +@@ -22,10 +22,12 @@ try: + except ImportError: + iscoroutinefunction = None + +-try: +- import tornado +-except ImportError: +- tornado = None ++# Replace a conditional import with a hard-coded None so that pip does ++# not attempt to use tornado even if it is present in the environment. ++# If tornado is non-None, tenacity will attempt to execute some code ++# that is sensitive to the version of tornado, which could break pip ++# if an old version is found. ++tornado = None + + import sys + import threading From 47f7c63ed7bad74205e767f0d7d83237a2eaf7f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Sat, 29 May 2021 16:11:31 +0100 Subject: [PATCH 38/85] remove support for setup.cfg only projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per the discussion in #9945. Signed-off-by: Filipe Laíns --- news/10031.bugfix.rst | 1 + src/pip/_internal/req/req_install.py | 18 ++---------------- 2 files changed, 3 insertions(+), 16 deletions(-) create mode 100644 news/10031.bugfix.rst diff --git a/news/10031.bugfix.rst b/news/10031.bugfix.rst new file mode 100644 index 000000000..8b5332bb0 --- /dev/null +++ b/news/10031.bugfix.rst @@ -0,0 +1 @@ +Require ``setup.cfg``-only projects to be built via PEP 517, by requiring an explicit dependency on setuptools declared in pyproject.toml. diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index c2eea3712..f4d625141 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -509,19 +509,6 @@ class InstallRequirement: self.unpacked_source_directory, backend, backend_path=backend_path, ) - def _check_setup_py_or_cfg_exists(self) -> bool: - """Check if the requirement actually has a setuptools build file. - - If setup.py does not exist, we also check setup.cfg in the same - directory and allow the directory if that exists. - """ - if os.path.exists(self.setup_py_path): - return True - stem, ext = os.path.splitext(self.setup_py_path) - if ext == ".py" and os.path.exists(f"{stem}.cfg"): - return True - return False - def _generate_metadata(self): # type: () -> str """Invokes metadata generator functions, with the required arguments. @@ -529,10 +516,9 @@ class InstallRequirement: if not self.use_pep517: assert self.unpacked_source_directory - if not self._check_setup_py_or_cfg_exists(): + if not os.path.exists(self.setup_py_path): raise InstallationError( - f'File "setup.py" or "setup.cfg" not found for legacy ' - f'project {self}.' + f'File "setup.py" not found for legacy project {self}.' ) return generate_metadata_legacy( From b97e2e0c8dcb19bfc60e7d4bc988b9ac9fb42efc Mon Sep 17 00:00:00 2001 From: snook92 Date: Mon, 31 May 2021 22:23:38 +0200 Subject: [PATCH 39/85] fix index_urls if multiple creds for same domain --- src/pip/_internal/network/auth.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/network/auth.py b/src/pip/_internal/network/auth.py index bd54a5cba..3097da151 100644 --- a/src/pip/_internal/network/auth.py +++ b/src/pip/_internal/network/auth.py @@ -173,10 +173,11 @@ class MultiDomainBasicAuth(AuthBase): # Use any stored credentials that we have for this netloc username, password = self.passwords.get(netloc, (None, None)) - if username is None and password is None: - # No stored credentials. Acquire new credentials without prompting - # the user. (e.g. from netrc, keyring, or the URL itself) - username, password = self._get_new_credentials(original_url) + # still grab if different creds for same domain + username_candidate, password_candidate = self._get_new_credentials(original_url) + if username_candidate is not None and password_candidate is not None: + username = username_candidate + password = password_candidate if username is not None or password is not None: # Convert the username and password if they're None, so that From 9eee1a4755f35e59b7491d5e86b82ddc8848f4da Mon Sep 17 00:00:00 2001 From: snook92 Date: Mon, 31 May 2021 22:34:23 +0200 Subject: [PATCH 40/85] Issue 3931 news file --- news/3931.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/3931.bugfix.rst diff --git a/news/3931.bugfix.rst b/news/3931.bugfix.rst new file mode 100644 index 000000000..21509ee9f --- /dev/null +++ b/news/3931.bugfix.rst @@ -0,0 +1 @@ +fix basic auth credentials in --extra-index-urls when using different creds for same domain From ff03baee676bdb17abbf4477a0d173725e135d02 Mon Sep 17 00:00:00 2001 From: Bruno S Date: Tue, 1 Jun 2021 09:55:35 +0200 Subject: [PATCH 41/85] network auth / allow username alone and fix tests --- src/pip/_internal/network/auth.py | 6 ++++-- tests/unit/test_network_auth.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/network/auth.py b/src/pip/_internal/network/auth.py index 3097da151..2759a497c 100644 --- a/src/pip/_internal/network/auth.py +++ b/src/pip/_internal/network/auth.py @@ -175,9 +175,11 @@ class MultiDomainBasicAuth(AuthBase): # still grab if different creds for same domain username_candidate, password_candidate = self._get_new_credentials(original_url) - if username_candidate is not None and password_candidate is not None: + if username_candidate is not None: username = username_candidate - password = password_candidate + # Accept password only if username has been setted + if password_candidate is not None: + password = password_candidate if username is not None or password is not None: # Convert the username and password if they're None, so that diff --git a/tests/unit/test_network_auth.py b/tests/unit/test_network_auth.py index 44c739d86..31ce47ef8 100644 --- a/tests/unit/test_network_auth.py +++ b/tests/unit/test_network_auth.py @@ -47,12 +47,12 @@ def test_get_credentials_parses_correctly(input_url, url, username, password): ) -def test_get_credentials_uses_cached_credentials(): +def test_get_credentials_not_to_uses_cached_credentials(): auth = MultiDomainBasicAuth() auth.passwords['example.com'] = ('user', 'pass') got = auth._get_url_and_credentials("http://foo:bar@example.com/path") - expected = ('http://example.com/path', 'user', 'pass') + expected = ('http://example.com/path', 'foo', 'bar') assert got == expected From e620e8ee79b8ff0d4d17c76e9912add56e69c81b Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Tue, 1 Jun 2021 11:12:58 +0300 Subject: [PATCH 42/85] Add content to .trivial file --- news/8EB21BAE-9CD0-424E-AF3B-651960B50C93.trivial.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/news/8EB21BAE-9CD0-424E-AF3B-651960B50C93.trivial.rst b/news/8EB21BAE-9CD0-424E-AF3B-651960B50C93.trivial.rst index e69de29bb..42719640c 100644 --- a/news/8EB21BAE-9CD0-424E-AF3B-651960B50C93.trivial.rst +++ b/news/8EB21BAE-9CD0-424E-AF3B-651960B50C93.trivial.rst @@ -0,0 +1 @@ +mailmap: Clean up Git entries From c8638adc5a5561b887b6a209436c2275cf94cd3d Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 18 May 2021 23:50:44 +0800 Subject: [PATCH 43/85] Check Requires-Python before other dependencies This makes the resolver fail quicker when there's a interpreter version conflict, and avoid additional (useless) downloads. --- news/9925.feature.rst | 3 ++ .../resolution/resolvelib/candidates.py | 8 +++-- .../resolution/resolvelib/provider.py | 7 +++- tests/functional/test_new_resolver_errors.py | 32 +++++++++++++++++++ 4 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 news/9925.feature.rst diff --git a/news/9925.feature.rst b/news/9925.feature.rst new file mode 100644 index 000000000..8c2401f60 --- /dev/null +++ b/news/9925.feature.rst @@ -0,0 +1,3 @@ +New resolver: A distribution's ``Requires-Python`` metadata is now checked +before its Python dependencies. This makes the resolver fail quicker when +there's an interpreter version conflict. diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index da516ad3c..e496e10dd 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -32,6 +32,9 @@ BaseCandidate = Union[ "LinkCandidate", ] +# Avoid conflicting with the PyPI package "Python". +REQUIRES_PYTHON_IDENTIFIER = cast(NormalizedName, "") + def as_base_candidate(candidate: Candidate) -> Optional[BaseCandidate]: """The runtime version of BaseCandidate.""" @@ -578,13 +581,12 @@ class RequiresPythonCandidate(Candidate): @property def project_name(self): # type: () -> NormalizedName - # Avoid conflicting with the PyPI package "Python". - return cast(NormalizedName, "") + return REQUIRES_PYTHON_IDENTIFIER @property def name(self): # type: () -> str - return self.project_name + return REQUIRES_PYTHON_IDENTIFIER @property def version(self): diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index 0be58fd3b..9a8c29980 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Dict, Iterable, Iterator, Mapping, Sequence, U from pip._vendor.resolvelib.providers import AbstractProvider from .base import Candidate, Constraint, Requirement +from .candidates import REQUIRES_PYTHON_IDENTIFIER from .factory import Factory if TYPE_CHECKING: @@ -121,6 +122,10 @@ class PipProvider(_ProviderBase): rating = _get_restrictive_rating(r for r, _ in information[identifier]) order = self._user_requested.get(identifier, float("inf")) + # Requires-Python has only one candidate and the check is basically + # free, so we always do it first to avoid needless work if it fails. + requires_python = identifier == REQUIRES_PYTHON_IDENTIFIER + # HACK: Setuptools have a very long and solid backward compatibility # track record, and extremely few projects would request a narrow, # non-recent version range of it since that would break a lot things. @@ -131,7 +136,7 @@ class PipProvider(_ProviderBase): # while we work on "proper" branch pruning techniques. delay_this = identifier == "setuptools" - return (delay_this, rating, order, identifier) + return (not requires_python, delay_this, rating, order, identifier) def find_matches( self, diff --git a/tests/functional/test_new_resolver_errors.py b/tests/functional/test_new_resolver_errors.py index b4d63a996..b2e7af7c6 100644 --- a/tests/functional/test_new_resolver_errors.py +++ b/tests/functional/test_new_resolver_errors.py @@ -1,3 +1,4 @@ +import pathlib import sys from tests.lib import create_basic_wheel_for_package, create_test_package_with_setup @@ -73,3 +74,34 @@ def test_new_resolver_requires_python_error(script): # conflict, not the compatible one. assert incompatible_python in result.stderr, str(result) assert compatible_python not in result.stderr, str(result) + + +def test_new_resolver_checks_requires_python_before_dependencies(script): + incompatible_python = "<{0.major}.{0.minor}".format(sys.version_info) + + pkg_dep = create_basic_wheel_for_package( + script, + name="pkg-dep", + version="1", + ) + create_basic_wheel_for_package( + script, + name="pkg-root", + version="1", + # Refer the dependency by URL to prioritise it as much as possible, + # to test that Requires-Python is *still* inspected first. + depends=[f"pkg-dep@{pathlib.Path(pkg_dep).as_uri()}"], + requires_python=incompatible_python, + ) + + result = script.pip( + "install", "--no-cache-dir", + "--no-index", "--find-links", script.scratch_path, + "pkg-root", + expect_error=True, + ) + + # Resolution should fail because of pkg-a's Requires-Python. + # This check should be done before pkg-b, so pkg-b should never be pulled. + assert incompatible_python in result.stderr, str(result) + assert "pkg-b" not in result.stderr, str(result) From dca1299b1802f5254f0ed202980cf872d28ecd86 Mon Sep 17 00:00:00 2001 From: Bruno S Date: Tue, 1 Jun 2021 21:12:34 +0200 Subject: [PATCH 44/85] network auth / minor changes and more tests --- src/pip/_internal/network/auth.py | 14 +++++--------- tests/unit/test_network_auth.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/pip/_internal/network/auth.py b/src/pip/_internal/network/auth.py index 2759a497c..819687bae 100644 --- a/src/pip/_internal/network/auth.py +++ b/src/pip/_internal/network/auth.py @@ -170,16 +170,12 @@ class MultiDomainBasicAuth(AuthBase): """ url, netloc, _ = split_auth_netloc_from_url(original_url) - # Use any stored credentials that we have for this netloc - username, password = self.passwords.get(netloc, (None, None)) + # Try to get credentials from original url + username, password = self._get_new_credentials(original_url) - # still grab if different creds for same domain - username_candidate, password_candidate = self._get_new_credentials(original_url) - if username_candidate is not None: - username = username_candidate - # Accept password only if username has been setted - if password_candidate is not None: - password = password_candidate + # If credentials not found, use any stored credentials for this netloc + if username is None and password is None: + username, password = self.passwords.get(netloc, (None, None)) if username is not None or password is not None: # Convert the username and password if they're None, so that diff --git a/tests/unit/test_network_auth.py b/tests/unit/test_network_auth.py index 31ce47ef8..ce528d0ac 100644 --- a/tests/unit/test_network_auth.py +++ b/tests/unit/test_network_auth.py @@ -56,6 +56,24 @@ def test_get_credentials_not_to_uses_cached_credentials(): assert got == expected +def test_get_credentials_not_to_uses_cached_credentials_only_username(): + auth = MultiDomainBasicAuth() + auth.passwords['example.com'] = ('user', 'pass') + + got = auth._get_url_and_credentials("http://foo@example.com/path") + expected = ('http://example.com/path', 'foo', '') + assert got == expected + + +def test_get_credentials_uses_cached_credentials(): + auth = MultiDomainBasicAuth() + auth.passwords['example.com'] = ('user', 'pass') + + got = auth._get_url_and_credentials("http://example.com/path") + expected = ('http://example.com/path', 'user', 'pass') + assert got == expected + + def test_get_index_url_credentials(): auth = MultiDomainBasicAuth(index_urls=[ "http://foo:bar@example.com/path" From 11259a4d4ca3b3350f15a8fdea96a290685e563c Mon Sep 17 00:00:00 2001 From: Bruno S Date: Tue, 1 Jun 2021 23:16:13 +0200 Subject: [PATCH 45/85] update bugfix description --- news/3931.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/3931.bugfix.rst b/news/3931.bugfix.rst index 21509ee9f..f97ab955f 100644 --- a/news/3931.bugfix.rst +++ b/news/3931.bugfix.rst @@ -1 +1 @@ -fix basic auth credentials in --extra-index-urls when using different creds for same domain +Fix incorrect credentials caching when using multiple package indexes sharing same domain but different credentials \ No newline at end of file From 6b421bf8751c2d54370de26d54e28dc1b2c407ad Mon Sep 17 00:00:00 2001 From: Bruno S Date: Tue, 1 Jun 2021 23:26:22 +0200 Subject: [PATCH 46/85] update bugfix description --- news/3931.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/3931.bugfix.rst b/news/3931.bugfix.rst index f97ab955f..fe5e85341 100644 --- a/news/3931.bugfix.rst +++ b/news/3931.bugfix.rst @@ -1 +1 @@ -Fix incorrect credentials caching when using multiple package indexes sharing same domain but different credentials \ No newline at end of file +Prefer credentials from the URL over the previously-obtained credentials from URLs of the same domain, so it is possible to use different credentials on the same index server for different `--extra-index-url` options. \ No newline at end of file From 0468eb01a54a324fbb866c7ae78bdc32b0387bf5 Mon Sep 17 00:00:00 2001 From: Bruno S Date: Wed, 2 Jun 2021 16:32:02 +0200 Subject: [PATCH 47/85] 3931.bugfix.rst / add double backticks and trailing new line --- news/3931.bugfix.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/news/3931.bugfix.rst b/news/3931.bugfix.rst index fe5e85341..ded51d626 100644 --- a/news/3931.bugfix.rst +++ b/news/3931.bugfix.rst @@ -1 +1,2 @@ -Prefer credentials from the URL over the previously-obtained credentials from URLs of the same domain, so it is possible to use different credentials on the same index server for different `--extra-index-url` options. \ No newline at end of file +Prefer credentials from the URL over the previously-obtained credentials from URLs of the same domain, so it is possible to use different credentials on the same index server for different ``--extra-index-url`` options. + From c22e15edc87b9a18c465ad4cedd802730af5ea91 Mon Sep 17 00:00:00 2001 From: Bruno S Date: Wed, 2 Jun 2021 17:39:44 +0200 Subject: [PATCH 48/85] news/3931.bugfix.rst / fix pre-commit errors --- news/3931.bugfix.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/news/3931.bugfix.rst b/news/3931.bugfix.rst index ded51d626..0ebb9e495 100644 --- a/news/3931.bugfix.rst +++ b/news/3931.bugfix.rst @@ -1,2 +1 @@ Prefer credentials from the URL over the previously-obtained credentials from URLs of the same domain, so it is possible to use different credentials on the same index server for different ``--extra-index-url`` options. - From b8e7a70fd53b3131cc8d1b043f1b115dfb54bbf7 Mon Sep 17 00:00:00 2001 From: Dirk Stolle Date: Thu, 3 Jun 2021 16:44:42 +0200 Subject: [PATCH 49/85] Fix typos (#10001) --- NEWS.rst | 4 ++-- docs/html/user_guide.rst | 2 +- news/0d757310-0e1d-4887-9076-a1eb3c55d9fa.trivial.rst | 1 + src/pip/_internal/models/wheel.py | 2 +- src/pip/_internal/network/lazy_wheel.py | 2 +- src/pip/_internal/utils/filesystem.py | 2 +- src/pip/_internal/utils/misc.py | 2 +- src/pip/py.typed | 2 +- 8 files changed, 9 insertions(+), 8 deletions(-) create mode 100644 news/0d757310-0e1d-4887-9076-a1eb3c55d9fa.trivial.rst diff --git a/NEWS.rst b/NEWS.rst index 69655db57..5688da3ba 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -350,7 +350,7 @@ Features - When installing a git URL that refers to a commit that is not available locally after git clone, attempt to fetch it from the remote. (`#8815 `_) - Include http subdirectory in ``pip cache info`` and ``pip cache purge`` commands. (`#8892 `_) -- Cache package listings on index packages so they are guarenteed to stay stable +- Cache package listings on index packages so they are guaranteed to stay stable during a pip command session. This also improves performance when a index page is accessed multiple times during the command session. (`#8905 `_) - New resolver: Tweak resolution logic to improve user experience when @@ -422,7 +422,7 @@ Features and considered good enough. (`#8023 `_) - Improve error message friendliness when an environment has packages with corrupted metadata. (`#8676 `_) -- Cache package listings on index packages so they are guarenteed to stay stable +- Cache package listings on index packages so they are guaranteed to stay stable during a pip command session. This also improves performance when a index page is accessed multiple times during the command session. (`#8905 `_) - New resolver: Tweak resolution logic to improve user experience when diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index a36bc9268..cfcdfc182 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -778,7 +778,7 @@ those specified on the command-line or via a requirements file) while requirements). As an example, say ``SomePackage`` has a dependency, ``SomeDependency``, and -both of them are already installed but are not the latest avaialable versions: +both of them are already installed but are not the latest available versions: - ``pip install SomePackage``: will not upgrade the existing ``SomePackage`` or ``SomeDependency``. diff --git a/news/0d757310-0e1d-4887-9076-a1eb3c55d9fa.trivial.rst b/news/0d757310-0e1d-4887-9076-a1eb3c55d9fa.trivial.rst new file mode 100644 index 000000000..c5abc9762 --- /dev/null +++ b/news/0d757310-0e1d-4887-9076-a1eb3c55d9fa.trivial.rst @@ -0,0 +1 @@ +Fix typos in several files. diff --git a/src/pip/_internal/models/wheel.py b/src/pip/_internal/models/wheel.py index 0a582b305..827ebca91 100644 --- a/src/pip/_internal/models/wheel.py +++ b/src/pip/_internal/models/wheel.py @@ -69,7 +69,7 @@ class Wheel: def find_most_preferred_tag(self, tags, tag_to_priority): # type: (List[Tag], Dict[Tag, int]) -> int """Return the priority of the most preferred tag that one of the wheel's file - tag combinations acheives in the given list of supported tags using the given + tag combinations achieves in the given list of supported tags using the given tag_to_priority mapping, where lower priorities are more-preferred. This is used in place of support_index_min in some cases in order to avoid diff --git a/src/pip/_internal/network/lazy_wheel.py b/src/pip/_internal/network/lazy_wheel.py index b877d3b7a..781cb0154 100644 --- a/src/pip/_internal/network/lazy_wheel.py +++ b/src/pip/_internal/network/lazy_wheel.py @@ -123,7 +123,7 @@ class LazyZipOverHTTP: def tell(self): # type: () -> int - """Return the current possition.""" + """Return the current position.""" return self._file.tell() def truncate(self, size=None): diff --git a/src/pip/_internal/utils/filesystem.py b/src/pip/_internal/utils/filesystem.py index 3db97dc41..177a6b4fb 100644 --- a/src/pip/_internal/utils/filesystem.py +++ b/src/pip/_internal/utils/filesystem.py @@ -98,7 +98,7 @@ def adjacent_tmp_file(path, **kwargs): os.fsync(result.fileno()) -# Tenacity raises RetryError by default, explictly raise the original exception +# Tenacity raises RetryError by default, explicitly raise the original exception _replace_retry = retry(reraise=True, stop=stop_after_delay(1), wait=wait_fixed(0.25)) replace = _replace_retry(os.replace) diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index a4ad35be6..d88f3f46a 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -128,7 +128,7 @@ def get_prog(): # Retry every half second for up to 3 seconds -# Tenacity raises RetryError by default, explictly raise the original exception +# Tenacity raises RetryError by default, explicitly raise the original exception @retry(reraise=True, stop=stop_after_delay(3), wait=wait_fixed(0.5)) def rmtree(dir, ignore_errors=False): # type: (AnyStr, bool) -> None diff --git a/src/pip/py.typed b/src/pip/py.typed index 0b44fd9b5..493b53e4e 100644 --- a/src/pip/py.typed +++ b/src/pip/py.typed @@ -1,4 +1,4 @@ pip is a command line program. While it is implemented in Python, and so is available for import, you must not use pip's internal APIs in this way. Typing -information is provided as a convenience only and is not a gaurantee. Expect +information is provided as a convenience only and is not a guarantee. Expect unannounced changes to the API and types in releases. From f533671b0ca9689855b7bdda67f44108387fe2a9 Mon Sep 17 00:00:00 2001 From: bwoodsend Date: Wed, 21 Apr 2021 10:06:41 +0100 Subject: [PATCH 50/85] Fix pip freeze to use modern format for git repos (#9822) Pip dropped support for `git+ssh@` style requirements (see #7554) in favour of `git+ssh://` but didn't propagate the change to `pip freeze` which resultantly returns invalid requirements. Fix this behaviour. Fixes #9625. --- news/9822.bugfix.rst | 3 ++ src/pip/_internal/vcs/git.py | 39 +++++++++++++++++++- tests/functional/test_freeze.py | 22 ++++++++---- tests/unit/test_vcs.py | 64 +++++++++++++++++++++++++++------ 4 files changed, 111 insertions(+), 17 deletions(-) create mode 100644 news/9822.bugfix.rst diff --git a/news/9822.bugfix.rst b/news/9822.bugfix.rst new file mode 100644 index 000000000..8a692c3ff --- /dev/null +++ b/news/9822.bugfix.rst @@ -0,0 +1,3 @@ +Fix :ref:`pip freeze` to output packages :ref:`installed from git ` +in the correct ``git+protocol://git.example.com/MyProject#egg=MyProject`` format +rather than the old and no longer supported ``git+git@`` format. diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index 9f24ccdf5..bc5a5ac2b 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -1,5 +1,6 @@ import logging import os.path +import pathlib import re import urllib.parse import urllib.request @@ -322,7 +323,36 @@ class Git(VersionControl): found_remote = remote break url = found_remote.split(' ')[1] - return url.strip() + return cls._git_remote_to_pip_url(url.strip()) + + @staticmethod + def _git_remote_to_pip_url(url): + # type: (str) -> str + """ + Convert a remote url from what git uses to what pip accepts. + + There are 3 legal forms **url** may take: + + 1. A fully qualified url: ssh://git@example.com/foo/bar.git + 2. A local project.git folder: /path/to/bare/repository.git + 3. SCP shorthand for form 1: git@example.com:foo/bar.git + + Form 1 is output as-is. Form 2 must be converted to URI and form 3 must + be converted to form 1. + + See the corresponding test test_git_remote_url_to_pip() for examples of + sample inputs/outputs. + """ + if re.match(r"\w+://", url): + # This is already valid. Pass it though as-is. + return url + if os.path.exists(url): + # A local bare remote (git clone --mirror). + # Needs a file:// prefix. + return pathlib.PurePath(url).as_uri() + # SCP shorthand. e.g. git@example.com:foo/bar.git + # Should add an ssh:// prefix and replace the ':' with a '/'. + return "ssh://" + url.replace(":", "/") @classmethod def has_commit(cls, location, rev): @@ -440,5 +470,12 @@ class Git(VersionControl): return None return os.path.normpath(r.rstrip('\r\n')) + @staticmethod + def should_add_vcs_url_prefix(repo_url): + # type: (str) -> bool + """In either https or ssh form, requirements must be prefixed with git+. + """ + return True + vcs.register(Git) diff --git a/tests/functional/test_freeze.py b/tests/functional/test_freeze.py index 858e43931..9b51b0a53 100644 --- a/tests/functional/test_freeze.py +++ b/tests/functional/test_freeze.py @@ -409,7 +409,6 @@ def test_freeze_git_remote(script, tmpdir): expect_stderr=True, ) origin_remote = pkg_version - other_remote = pkg_version + '-other' # check frozen remote after clone result = script.pip('freeze', expect_stderr=True) expected = textwrap.dedent( @@ -417,18 +416,29 @@ def test_freeze_git_remote(script, tmpdir): ...-e git+{remote}@...#egg=version_pkg ... """ - ).format(remote=origin_remote).strip() + ).format(remote=path_to_url(origin_remote)).strip() _check_output(result.stdout, expected) # check frozen remote when there is no remote named origin - script.run('git', 'remote', 'remove', 'origin', cwd=repo_dir) - script.run('git', 'remote', 'add', 'other', other_remote, cwd=repo_dir) + script.run('git', 'remote', 'rename', 'origin', 'other', cwd=repo_dir) result = script.pip('freeze', expect_stderr=True) expected = textwrap.dedent( """ ...-e git+{remote}@...#egg=version_pkg ... """ - ).format(remote=other_remote).strip() + ).format(remote=path_to_url(origin_remote)).strip() + _check_output(result.stdout, expected) + # When the remote is a local path, it must exist. Otherwise it is assumed to + # be an ssh:// remote. This is a side effect and not intentional behaviour. + other_remote = pkg_version + '-other' + script.run('git', 'remote', 'set-url', 'other', other_remote, cwd=repo_dir) + result = script.pip('freeze', expect_stderr=True) + expected = textwrap.dedent( + """ + ...-e git+ssh://{remote}@...#egg=version_pkg + ... + """ + ).format(remote=other_remote.replace(":", "/")).strip() _check_output(result.stdout, expected) # when there are more than one origin, priority is given to the # remote named origin @@ -439,7 +449,7 @@ def test_freeze_git_remote(script, tmpdir): ...-e git+{remote}@...#egg=version_pkg ... """ - ).format(remote=origin_remote).strip() + ).format(remote=path_to_url(origin_remote)).strip() _check_output(result.stdout, expected) diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index f86d04d74..3dc19b6d8 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -1,4 +1,5 @@ import os +import pathlib from unittest import TestCase from unittest.mock import patch @@ -108,10 +109,14 @@ def test_looks_like_hash(sha, expected): @pytest.mark.parametrize('vcs_cls, remote_url, expected', [ - # Git is one of the subclasses using the base class implementation. - (Git, 'git://example.com/MyProject', False), + # Mercurial is one of the subclasses using the base class implementation. + # `hg://` isn't a real prefix but it tests the default behaviour. + (Mercurial, 'hg://user@example.com/MyProject', False), + (Mercurial, 'http://example.com/MyProject', True), + # The Git subclasses should return true in all cases. + (Git, 'git://example.com/MyProject', True), (Git, 'http://example.com/MyProject', True), - # Subversion is the only subclass overriding the base class implementation. + # Subversion also overrides the base class implementation. (Subversion, 'svn://example.com/MyProject', True), ]) def test_should_add_vcs_url_prefix(vcs_cls, remote_url, expected): @@ -119,26 +124,65 @@ def test_should_add_vcs_url_prefix(vcs_cls, remote_url, expected): assert actual == expected +@pytest.mark.parametrize("url, target", [ + # A fully qualified remote url. No changes needed. + ("ssh://bob@server/foo/bar.git", "ssh://bob@server/foo/bar.git"), + ("git://bob@server/foo/bar.git", "git://bob@server/foo/bar.git"), + # User is optional and does not need a default. + ("ssh://server/foo/bar.git", "ssh://server/foo/bar.git"), + # The common scp shorthand for ssh remotes. Pip won't recognise these as + # git remotes until they have a 'ssh://' prefix and the ':' in the middle + # is gone. + ("git@example.com:foo/bar.git", "ssh://git@example.com/foo/bar.git"), + ("example.com:foo.git", "ssh://example.com/foo.git"), + # Http(s) remote names are already complete and should remain unchanged. + ("https://example.com/foo", "https://example.com/foo"), + ("http://example.com/foo/bar.git", "http://example.com/foo/bar.git"), + ("https://bob@example.com/foo", "https://bob@example.com/foo"), + ]) +def test_git_remote_url_to_pip(url, target): + assert Git._git_remote_to_pip_url(url) == target + + +def test_git_remote_local_path(tmpdir): + path = pathlib.Path(tmpdir, "project.git") + path.mkdir() + # Path must exist to be recognised as a local git remote. + assert Git._git_remote_to_pip_url(str(path)) == path.as_uri() + + @patch('pip._internal.vcs.git.Git.get_remote_url') @patch('pip._internal.vcs.git.Git.get_revision') @patch('pip._internal.vcs.git.Git.get_subdirectory') +@pytest.mark.parametrize( + "git_url, target_url_prefix", + [ + ( + "https://github.com/pypa/pip-test-package", + "git+https://github.com/pypa/pip-test-package", + ), + ( + "git@github.com:pypa/pip-test-package", + "git+ssh://git@github.com/pypa/pip-test-package", + ), + ], + ids=["https", "ssh"], +) @pytest.mark.network def test_git_get_src_requirements( - mock_get_subdirectory, mock_get_revision, mock_get_remote_url + mock_get_subdirectory, mock_get_revision, mock_get_remote_url, + git_url, target_url_prefix, ): - git_url = 'https://github.com/pypa/pip-test-package' sha = '5547fa909e83df8bd743d3978d6667497983a4b7' - mock_get_remote_url.return_value = git_url + mock_get_remote_url.return_value = Git._git_remote_to_pip_url(git_url) mock_get_revision.return_value = sha mock_get_subdirectory.return_value = None ret = Git.get_src_requirement('.', 'pip-test-package') - assert ret == ( - 'git+https://github.com/pypa/pip-test-package' - '@5547fa909e83df8bd743d3978d6667497983a4b7#egg=pip_test_package' - ) + target = f"{target_url_prefix}@{sha}#egg=pip_test_package" + assert ret == target @patch('pip._internal.vcs.git.Git.get_revision_sha') From 8b8fa2bd5f67d03f35f615092603969eb9563c70 Mon Sep 17 00:00:00 2001 From: bwoodsend Date: Sat, 1 May 2021 15:27:35 +0100 Subject: [PATCH 51/85] Explicitly recognise SCP-shorthand git remotes. --- src/pip/_internal/operations/freeze.py | 10 +++++++++- src/pip/_internal/vcs/__init__.py | 1 + src/pip/_internal/vcs/git.py | 22 +++++++++++++++++++--- src/pip/_internal/vcs/versioncontrol.py | 6 ++++++ tests/functional/test_freeze.py | 17 +++++++++-------- tests/unit/test_vcs.py | 20 +++++++++++++++++++- 6 files changed, 63 insertions(+), 13 deletions(-) diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py index f34a9d4be..3cda5c8c9 100644 --- a/src/pip/_internal/operations/freeze.py +++ b/src/pip/_internal/operations/freeze.py @@ -176,7 +176,7 @@ def get_requirement_info(dist): location = os.path.normcase(os.path.abspath(dist.location)) - from pip._internal.vcs import RemoteNotFoundError, vcs + from pip._internal.vcs import RemoteNotFoundError, RemoteNotValidError, vcs vcs_backend = vcs.get_backend_for_dir(location) if vcs_backend is None: @@ -200,6 +200,14 @@ def get_requirement_info(dist): ) ] return (location, True, comments) + except RemoteNotValidError as ex: + req = dist.as_requirement() + comments = [ + f"# Editable {type(vcs_backend).__name__} install ({req}) with " + f"either a deleted local remote or invalid URI:", + f"# '{ex.url}'", + ] + return (location, True, comments) except BadCommand: logger.warning( diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index 30025d632..b6beddbe6 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -8,6 +8,7 @@ import pip._internal.vcs.mercurial import pip._internal.vcs.subversion # noqa: F401 from pip._internal.vcs.versioncontrol import ( # noqa: F401 RemoteNotFoundError, + RemoteNotValidError, is_url, make_vcs_requirement_url, vcs, diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index bc5a5ac2b..3c46c250e 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -15,6 +15,7 @@ from pip._internal.utils.subprocess import make_command from pip._internal.vcs.versioncontrol import ( AuthInfo, RemoteNotFoundError, + RemoteNotValidError, RevOptions, VersionControl, find_path_to_setup_from_repo_root, @@ -30,6 +31,18 @@ logger = logging.getLogger(__name__) HASH_REGEX = re.compile('^[a-fA-F0-9]{40}$') +# SCP (Secure copy protocol) shorthand. e.g. 'git@example.com:foo/bar.git' +SCP_REGEX = re.compile(r"""^ + # Optional user, e.g. 'git@' + (\w+@)? + # Server, e.g. 'github.com'. + ([^/:]+): + # The server-side path. e.g. 'user/project.git'. Must start with an + # alphanumeric character so as not to be confusable with a Windows paths + # like 'C:/foo/bar' or 'C:\foo\bar'. + (\w[^:]*) +$""", re.VERBOSE) + def looks_like_hash(sha): # type: (str) -> bool @@ -350,9 +363,12 @@ class Git(VersionControl): # A local bare remote (git clone --mirror). # Needs a file:// prefix. return pathlib.PurePath(url).as_uri() - # SCP shorthand. e.g. git@example.com:foo/bar.git - # Should add an ssh:// prefix and replace the ':' with a '/'. - return "ssh://" + url.replace(":", "/") + scp_match = SCP_REGEX.match(url) + if scp_match: + # Add an ssh:// prefix and replace the ':' with a '/'. + return scp_match.expand(r"ssh://\1\2/\3") + # Otherwise, bail out. + raise RemoteNotValidError(url) @classmethod def has_commit(cls, location, rev): diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py index 97977b579..d06c81032 100644 --- a/src/pip/_internal/vcs/versioncontrol.py +++ b/src/pip/_internal/vcs/versioncontrol.py @@ -100,6 +100,12 @@ class RemoteNotFoundError(Exception): pass +class RemoteNotValidError(Exception): + def __init__(self, url: str): + super().__init__(url) + self.url = url + + class RevOptions: """ diff --git a/tests/functional/test_freeze.py b/tests/functional/test_freeze.py index 9b51b0a53..0af29dd0c 100644 --- a/tests/functional/test_freeze.py +++ b/tests/functional/test_freeze.py @@ -428,18 +428,19 @@ def test_freeze_git_remote(script, tmpdir): """ ).format(remote=path_to_url(origin_remote)).strip() _check_output(result.stdout, expected) - # When the remote is a local path, it must exist. Otherwise it is assumed to - # be an ssh:// remote. This is a side effect and not intentional behaviour. + # When the remote is a local path, it must exist. + # If it doesn't, it gets flagged as invalid. other_remote = pkg_version + '-other' script.run('git', 'remote', 'set-url', 'other', other_remote, cwd=repo_dir) result = script.pip('freeze', expect_stderr=True) - expected = textwrap.dedent( + expected = os.path.normcase(textwrap.dedent( + f""" + ...# Editable Git...(version-pkg...)... + # '{other_remote}' + -e {repo_dir}... """ - ...-e git+ssh://{remote}@...#egg=version_pkg - ... - """ - ).format(remote=other_remote.replace(":", "/")).strip() - _check_output(result.stdout, expected) + ).strip()) + _check_output(os.path.normcase(result.stdout), expected) # when there are more than one origin, priority is given to the # remote named origin script.run('git', 'remote', 'add', 'origin', origin_remote, cwd=repo_dir) diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 3dc19b6d8..305c45fd8 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -10,7 +10,7 @@ from pip._internal.exceptions import BadCommand, InstallationError from pip._internal.utils.misc import hide_url, hide_value from pip._internal.vcs import make_vcs_requirement_url from pip._internal.vcs.bazaar import Bazaar -from pip._internal.vcs.git import Git, looks_like_hash +from pip._internal.vcs.git import Git, RemoteNotValidError, looks_like_hash from pip._internal.vcs.mercurial import Mercurial from pip._internal.vcs.subversion import Subversion from pip._internal.vcs.versioncontrol import RevOptions, VersionControl @@ -144,6 +144,24 @@ def test_git_remote_url_to_pip(url, target): assert Git._git_remote_to_pip_url(url) == target +@pytest.mark.parametrize("url, platform", [ + # Windows paths with the ':' drive prefix look dangerously close to SCP. + ("c:/piffle/wiffle/waffle/poffle.git", "nt"), + (r"c:\faffle\waffle\woffle\piffle.git", "nt"), + # Unix paths less so but test them anyway. + ("/muffle/fuffle/pufffle/fluffle.git", "posix"), +]) +def test_paths_are_not_mistaken_for_scp_shorthand(url, platform): + # File paths should not be mistaken for SCP shorthand. If they do then + # 'c:/piffle/wiffle' would end up as 'ssh://c/piffle/wiffle'. + from pip._internal.vcs.git import SCP_REGEX + assert not SCP_REGEX.match(url) + + if platform == os.name: + with pytest.raises(RemoteNotValidError): + Git._git_remote_to_pip_url(url) + + def test_git_remote_local_path(tmpdir): path = pathlib.Path(tmpdir, "project.git") path.mkdir() From 0170e37046ba1759444738344e276810233896cd Mon Sep 17 00:00:00 2001 From: Deepyaman Datta Date: Mon, 7 Jun 2021 05:42:32 -0400 Subject: [PATCH 52/85] Remove second space after comma in find_links help --- src/pip/_internal/cli/cmdoptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index f71c0b020..f49173fa5 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -378,7 +378,7 @@ def find_links(): metavar="url", help="If a URL or path to an html file, then parse for links to " "archives such as sdist (.tar.gz) or wheel (.whl) files. " - "If a local path or file:// URL that's a directory, " + "If a local path or file:// URL that's a directory, " "then look for archives in the directory listing. " "Links to VCS project URLs are not supported.", ) From 1e016d299478bc001b579266b013c3dce5878817 Mon Sep 17 00:00:00 2001 From: Diego Ramirez Date: Mon, 7 Jun 2021 08:19:23 -0500 Subject: [PATCH 53/85] Convert type hint comments into annotations Co-authored-by: Pradyun Gedam Co-authored-by: Tzu-ping Chung --- news/10018.trivial.rst | 1 + src/pip/__init__.py | 3 +- src/pip/_internal/__init__.py | 3 +- src/pip/_internal/cli/autocompletion.py | 11 ++- src/pip/_internal/cli/base_command.py | 21 ++--- src/pip/_internal/cli/cmdoptions.py | 101 ++++++++++------------- src/pip/_internal/cli/command_context.py | 9 +- src/pip/_internal/cli/main.py | 3 +- src/pip/_internal/cli/main_parser.py | 6 +- src/pip/_internal/cli/parser.py | 63 ++++++-------- 10 files changed, 89 insertions(+), 132 deletions(-) create mode 100644 news/10018.trivial.rst diff --git a/news/10018.trivial.rst b/news/10018.trivial.rst new file mode 100644 index 000000000..c6950c59a --- /dev/null +++ b/news/10018.trivial.rst @@ -0,0 +1 @@ +Use annotations from the ``typing`` module on some functions. diff --git a/src/pip/__init__.py b/src/pip/__init__.py index c35872ca9..67722d05e 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -3,8 +3,7 @@ from typing import List, Optional __version__ = "21.2.dev0" -def main(args=None): - # type: (Optional[List[str]]) -> int +def main(args: Optional[List[str]] = None) -> int: """This is an internal API only meant for use by pip's own console scripts. For additional details, see https://github.com/pypa/pip/issues/7498. diff --git a/src/pip/_internal/__init__.py b/src/pip/_internal/__init__.py index 41071cd86..627d0e1ec 100755 --- a/src/pip/_internal/__init__.py +++ b/src/pip/_internal/__init__.py @@ -3,8 +3,7 @@ from typing import List, Optional import pip._internal.utils.inject_securetransport # noqa -def main(args=None): - # type: (Optional[List[str]]) -> int +def main(args: (Optional[List[str]]) = None) -> int: """This is preserved for old console scripts that may still be referencing it. diff --git a/src/pip/_internal/cli/autocompletion.py b/src/pip/_internal/cli/autocompletion.py index 3b1d2ac9b..2018ba2d4 100644 --- a/src/pip/_internal/cli/autocompletion.py +++ b/src/pip/_internal/cli/autocompletion.py @@ -12,8 +12,7 @@ from pip._internal.commands import commands_dict, create_command from pip._internal.utils.misc import get_installed_distributions -def autocomplete(): - # type: () -> None +def autocomplete() -> None: """Entry Point for completion of main and subcommand options.""" # Don't complete if user hasn't sourced bash_completion file. if "PIP_AUTO_COMPLETE" not in os.environ: @@ -107,8 +106,9 @@ def autocomplete(): sys.exit(1) -def get_path_completion_type(cwords, cword, opts): - # type: (List[str], int, Iterable[Any]) -> Optional[str] +def get_path_completion_type( + cwords: List[str], cword: int, opts: Iterable[Any] +) -> Optional[str]: """Get the type of path completion (``file``, ``dir``, ``path`` or None) :param cwords: same as the environmental variable ``COMP_WORDS`` @@ -130,8 +130,7 @@ def get_path_completion_type(cwords, cword, opts): return None -def auto_complete_paths(current, completion_type): - # type: (str, str) -> Iterable[str] +def auto_complete_paths(current: str, completion_type: str) -> Iterable[str]: """If ``completion_type`` is ``file`` or ``path``, list all regular files and directories starting with ``current``; otherwise only list directories starting with ``current``. diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py index b59420dda..37f9e65fa 100644 --- a/src/pip/_internal/cli/base_command.py +++ b/src/pip/_internal/cli/base_command.py @@ -43,8 +43,7 @@ class Command(CommandContextMixIn): usage = None # type: str ignore_require_venv = False # type: bool - def __init__(self, name, summary, isolated=False): - # type: (str, str, bool) -> None + def __init__(self, name: str, summary: str, isolated: bool = False) -> None: super().__init__() self.name = name @@ -74,12 +73,10 @@ class Command(CommandContextMixIn): self.add_options() - def add_options(self): - # type: () -> None + def add_options(self) -> None: pass - def handle_pip_version_check(self, options): - # type: (Values) -> None + def handle_pip_version_check(self, options: Values) -> None: """ This is a no-op so that commands by default do not do the pip version check. @@ -88,25 +85,21 @@ class Command(CommandContextMixIn): # are present. assert not hasattr(options, "no_index") - def run(self, options, args): - # type: (Values, List[Any]) -> int + def run(self, options: Values, args: List[Any]) -> int: raise NotImplementedError - def parse_args(self, args): - # type: (List[str]) -> Tuple[Any, Any] + def parse_args(self, args: List[str]) -> Tuple[Any, Any]: # factored out for testability return self.parser.parse_args(args) - def main(self, args): - # type: (List[str]) -> int + def main(self, args: List[str]) -> int: try: with self.main_context(): return self._main(args) finally: logging.shutdown() - def _main(self, args): - # type: (List[str]) -> int + def _main(self, args: List[str]) -> int: # We must initialize this before the tempdir manager, otherwise the # configuration would not be accessible by the time we clean up the # tempdir manager. diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index f49173fa5..e0a672951 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -31,8 +31,7 @@ from pip._internal.utils.hashes import STRONG_HASHES from pip._internal.utils.misc import strtobool -def raise_option_error(parser, option, msg): - # type: (OptionParser, Option, str) -> None +def raise_option_error(parser: OptionParser, option: Option, msg: str) -> None: """ Raise an option parsing error using parser.error(). @@ -46,8 +45,7 @@ def raise_option_error(parser, option, msg): parser.error(msg) -def make_option_group(group, parser): - # type: (Dict[str, Any], ConfigOptionParser) -> OptionGroup +def make_option_group(group: Dict[str, Any], parser: ConfigOptionParser) -> OptionGroup: """ Return an OptionGroup object group -- assumed to be dict with 'name' and 'options' keys @@ -59,8 +57,9 @@ def make_option_group(group, parser): return option_group -def check_install_build_global(options, check_options=None): - # type: (Values, Optional[Values]) -> None +def check_install_build_global( + options: Values, check_options: Optional[Values] = None +) -> None: """Disable wheels if per-setup.py call options are set. :param options: The OptionParser options to update. @@ -70,8 +69,7 @@ def check_install_build_global(options, check_options=None): if check_options is None: check_options = options - def getname(n): - # type: (str) -> Optional[Any] + def getname(n: str) -> Optional[Any]: return getattr(check_options, n, None) names = ["build_options", "global_options", "install_options"] @@ -85,8 +83,7 @@ def check_install_build_global(options, check_options=None): ) -def check_dist_restriction(options, check_target=False): - # type: (Values, bool) -> None +def check_dist_restriction(options: Values, check_target: bool = False) -> None: """Function for determining if custom platform options are allowed. :param options: The OptionParser options. @@ -126,13 +123,11 @@ def check_dist_restriction(options, check_target=False): ) -def _path_option_check(option, opt, value): - # type: (Option, str, str) -> str +def _path_option_check(option: Option, opt: str, value: str) -> str: return os.path.expanduser(value) -def _package_name_option_check(option, opt, value): - # type: (Option, str, str) -> str +def _package_name_option_check(option: Option, opt: str, value: str) -> str: return canonicalize_name(value) @@ -287,8 +282,7 @@ timeout = partial( ) # type: Callable[..., Option] -def exists_action(): - # type: () -> Option +def exists_action() -> Option: return Option( # Option when path already exist "--exists-action", @@ -343,8 +337,7 @@ index_url = partial( ) # type: Callable[..., Option] -def extra_index_url(): - # type: () -> Option +def extra_index_url() -> Option: return Option( "--extra-index-url", dest="extra_index_urls", @@ -367,8 +360,7 @@ no_index = partial( ) # type: Callable[..., Option] -def find_links(): - # type: () -> Option +def find_links() -> Option: return Option( "-f", "--find-links", @@ -384,8 +376,7 @@ def find_links(): ) -def trusted_host(): - # type: () -> Option +def trusted_host() -> Option: return Option( "--trusted-host", dest="trusted_hosts", @@ -397,8 +388,7 @@ def trusted_host(): ) -def constraints(): - # type: () -> Option +def constraints() -> Option: return Option( "-c", "--constraint", @@ -411,8 +401,7 @@ def constraints(): ) -def requirements(): - # type: () -> Option +def requirements() -> Option: return Option( "-r", "--requirement", @@ -425,8 +414,7 @@ def requirements(): ) -def editable(): - # type: () -> Option +def editable() -> Option: return Option( "-e", "--editable", @@ -441,8 +429,7 @@ def editable(): ) -def _handle_src(option, opt_str, value, parser): - # type: (Option, str, str, OptionParser) -> None +def _handle_src(option: Option, opt_str: str, value: str, parser: OptionParser) -> None: value = os.path.abspath(value) setattr(parser.values, option.dest, value) @@ -465,14 +452,14 @@ src = partial( ) # type: Callable[..., Option] -def _get_format_control(values, option): - # type: (Values, Option) -> Any +def _get_format_control(values: Values, option: Option) -> Any: """Get a format_control object.""" return getattr(values, option.dest) -def _handle_no_binary(option, opt_str, value, parser): - # type: (Option, str, str, OptionParser) -> None +def _handle_no_binary( + option: Option, opt_str: str, value: str, parser: OptionParser +) -> None: existing = _get_format_control(parser.values, option) FormatControl.handle_mutual_excludes( value, @@ -481,8 +468,9 @@ def _handle_no_binary(option, opt_str, value, parser): ) -def _handle_only_binary(option, opt_str, value, parser): - # type: (Option, str, str, OptionParser) -> None +def _handle_only_binary( + option: Option, opt_str: str, value: str, parser: OptionParser +) -> None: existing = _get_format_control(parser.values, option) FormatControl.handle_mutual_excludes( value, @@ -491,8 +479,7 @@ def _handle_only_binary(option, opt_str, value, parser): ) -def no_binary(): - # type: () -> Option +def no_binary() -> Option: format_control = FormatControl(set(), set()) return Option( "--no-binary", @@ -510,8 +497,7 @@ def no_binary(): ) -def only_binary(): - # type: () -> Option +def only_binary() -> Option: format_control = FormatControl(set(), set()) return Option( "--only-binary", @@ -545,8 +531,7 @@ platforms = partial( # This was made a separate function for unit-testing purposes. -def _convert_python_version(value): - # type: (str) -> Tuple[Tuple[int, ...], Optional[str]] +def _convert_python_version(value: str) -> Tuple[Tuple[int, ...], Optional[str]]: """ Convert a version string like "3", "37", or "3.7.3" into a tuple of ints. @@ -575,8 +560,9 @@ def _convert_python_version(value): return (version_info, None) -def _handle_python_version(option, opt_str, value, parser): - # type: (Option, str, str, OptionParser) -> None +def _handle_python_version( + option: Option, opt_str: str, value: str, parser: OptionParser +) -> None: """ Handle a provided --python-version value. """ @@ -646,16 +632,14 @@ abis = partial( ) # type: Callable[..., Option] -def add_target_python_options(cmd_opts): - # type: (OptionGroup) -> None +def add_target_python_options(cmd_opts: OptionGroup) -> None: cmd_opts.add_option(platforms()) cmd_opts.add_option(python_version()) cmd_opts.add_option(implementation()) cmd_opts.add_option(abis()) -def make_target_python(options): - # type: (Values) -> TargetPython +def make_target_python(options: Values) -> TargetPython: target_python = TargetPython( platforms=options.platforms, py_version_info=options.python_version, @@ -666,8 +650,7 @@ def make_target_python(options): return target_python -def prefer_binary(): - # type: () -> Option +def prefer_binary() -> Option: return Option( "--prefer-binary", dest="prefer_binary", @@ -688,8 +671,9 @@ cache_dir = partial( ) # type: Callable[..., Option] -def _handle_no_cache_dir(option, opt, value, parser): - # type: (Option, str, str, OptionParser) -> None +def _handle_no_cache_dir( + option: Option, opt: str, value: str, parser: OptionParser +) -> None: """ Process a value provided for the --no-cache-dir option. @@ -767,8 +751,9 @@ no_build_isolation = partial( ) # type: Callable[..., Option] -def _handle_no_use_pep517(option, opt, value, parser): - # type: (Option, str, str, OptionParser) -> None +def _handle_no_use_pep517( + option: Option, opt: str, value: str, parser: OptionParser +) -> None: """ Process a value provided for the --no-use-pep517 option. @@ -871,8 +856,9 @@ disable_pip_version_check = partial( ) # type: Callable[..., Option] -def _handle_merge_hash(option, opt_str, value, parser): - # type: (Option, str, str, OptionParser) -> None +def _handle_merge_hash( + option: Option, opt_str: str, value: str, parser: OptionParser +) -> None: """Given a value spelled "algo:digest", append the digest to a list pointed to in a dict by the algo name.""" if not parser.values.hashes: @@ -931,8 +917,7 @@ list_path = partial( ) # type: Callable[..., Option] -def check_list_path_option(options): - # type: (Values) -> None +def check_list_path_option(options: Values) -> None: if options.path and (options.user or options.local): raise CommandError("Cannot combine '--path' with '--user' or '--local'") diff --git a/src/pip/_internal/cli/command_context.py b/src/pip/_internal/cli/command_context.py index 375a2e366..ed6832237 100644 --- a/src/pip/_internal/cli/command_context.py +++ b/src/pip/_internal/cli/command_context.py @@ -5,15 +5,13 @@ _T = TypeVar("_T", covariant=True) class CommandContextMixIn: - def __init__(self): - # type: () -> None + def __init__(self) -> None: super().__init__() self._in_main_context = False self._main_context = ExitStack() @contextmanager - def main_context(self): - # type: () -> Iterator[None] + def main_context(self) -> Iterator[None]: assert not self._in_main_context self._in_main_context = True @@ -23,8 +21,7 @@ class CommandContextMixIn: finally: self._in_main_context = False - def enter_context(self, context_provider): - # type: (ContextManager[_T]) -> _T + def enter_context(self, context_provider: ContextManager[_T]) -> _T: assert self._in_main_context return self._main_context.enter_context(context_provider) diff --git a/src/pip/_internal/cli/main.py b/src/pip/_internal/cli/main.py index 7ae074b59..0e3122154 100644 --- a/src/pip/_internal/cli/main.py +++ b/src/pip/_internal/cli/main.py @@ -42,8 +42,7 @@ logger = logging.getLogger(__name__) # main, this should not be an issue in practice. -def main(args=None): - # type: (Optional[List[str]]) -> int +def main(args: Optional[List[str]] = None) -> int: if args is None: args = sys.argv[1:] diff --git a/src/pip/_internal/cli/main_parser.py b/src/pip/_internal/cli/main_parser.py index d0f58fe42..3666ab04c 100644 --- a/src/pip/_internal/cli/main_parser.py +++ b/src/pip/_internal/cli/main_parser.py @@ -14,8 +14,7 @@ from pip._internal.utils.misc import get_pip_version, get_prog __all__ = ["create_main_parser", "parse_command"] -def create_main_parser(): - # type: () -> ConfigOptionParser +def create_main_parser() -> ConfigOptionParser: """Creates and returns the main parser for pip's CLI""" parser = ConfigOptionParser( @@ -46,8 +45,7 @@ def create_main_parser(): return parser -def parse_command(args): - # type: (List[str]) -> Tuple[str, List[str]] +def parse_command(args: List[str]) -> Tuple[str, List[str]]: parser = create_main_parser() # Note: parser calls disable_interspersed_args(), so the result of this diff --git a/src/pip/_internal/cli/parser.py b/src/pip/_internal/cli/parser.py index 16523c5a1..efdf57e0b 100644 --- a/src/pip/_internal/cli/parser.py +++ b/src/pip/_internal/cli/parser.py @@ -18,20 +18,19 @@ logger = logging.getLogger(__name__) class PrettyHelpFormatter(optparse.IndentedHelpFormatter): """A prettier/less verbose help formatter for optparse.""" - def __init__(self, *args, **kwargs): - # type: (*Any, **Any) -> None + def __init__(self, *args: Any, **kwargs: Any) -> None: # help position must be aligned with __init__.parseopts.description kwargs["max_help_position"] = 30 kwargs["indent_increment"] = 1 kwargs["width"] = shutil.get_terminal_size()[0] - 2 super().__init__(*args, **kwargs) - def format_option_strings(self, option): - # type: (optparse.Option) -> str + def format_option_strings(self, option: optparse.Option) -> str: return self._format_option_strings(option) - def _format_option_strings(self, option, mvarfmt=" <{}>", optsep=", "): - # type: (optparse.Option, str, str) -> str + def _format_option_strings( + self, option: optparse.Option, mvarfmt: str = " <{}>", optsep: str = ", " + ) -> str: """ Return a comma-separated list of option strings and metavars. @@ -55,14 +54,12 @@ class PrettyHelpFormatter(optparse.IndentedHelpFormatter): return "".join(opts) - def format_heading(self, heading): - # type: (str) -> str + def format_heading(self, heading: str) -> str: if heading == "Options": return "" return heading + ":\n" - def format_usage(self, usage): - # type: (str) -> str + def format_usage(self, usage: str) -> str: """ Ensure there is only one newline between usage and the first heading if there is no description. @@ -70,8 +67,7 @@ class PrettyHelpFormatter(optparse.IndentedHelpFormatter): msg = "\nUsage: {}\n".format(self.indent_lines(textwrap.dedent(usage), " ")) return msg - def format_description(self, description): - # type: (str) -> str + def format_description(self, description: str) -> str: # leave full control over description to us if description: if hasattr(self.parser, "main"): @@ -89,16 +85,14 @@ class PrettyHelpFormatter(optparse.IndentedHelpFormatter): else: return "" - def format_epilog(self, epilog): - # type: (str) -> str + def format_epilog(self, epilog: str) -> str: # leave full control over epilog to us if epilog: return epilog else: return "" - def indent_lines(self, text, indent): - # type: (str, str) -> str + def indent_lines(self, text: str, indent: str) -> str: new_lines = [indent + line for line in text.split("\n")] return "\n".join(new_lines) @@ -112,8 +106,7 @@ class UpdatingDefaultsHelpFormatter(PrettyHelpFormatter): Also redact auth from url type options """ - def expand_default(self, option): - # type: (optparse.Option) -> str + def expand_default(self, option: optparse.Option) -> str: default_values = None if self.parser is not None: assert isinstance(self.parser, ConfigOptionParser) @@ -137,8 +130,9 @@ class UpdatingDefaultsHelpFormatter(PrettyHelpFormatter): class CustomOptionParser(optparse.OptionParser): - def insert_option_group(self, idx, *args, **kwargs): - # type: (int, Any, Any) -> optparse.OptionGroup + def insert_option_group( + self, idx: int, *args: Any, **kwargs: Any + ) -> optparse.OptionGroup: """Insert an OptionGroup at a given position.""" group = self.add_option_group(*args, **kwargs) @@ -148,8 +142,7 @@ class CustomOptionParser(optparse.OptionParser): return group @property - def option_list_all(self): - # type: () -> List[optparse.Option] + def option_list_all(self) -> List[optparse.Option]: """Get a list of all options, including those in option groups.""" res = self.option_list[:] for i in self.option_groups: @@ -164,28 +157,25 @@ class ConfigOptionParser(CustomOptionParser): def __init__( self, - *args, # type: Any - name, # type: str - isolated=False, # type: bool - **kwargs, # type: Any - ): - # type: (...) -> None + *args: Any, + name: str, + isolated: bool = False, + **kwargs: Any, + ) -> None: self.name = name self.config = Configuration(isolated) assert self.name super().__init__(*args, **kwargs) - def check_default(self, option, key, val): - # type: (optparse.Option, str, Any) -> Any + def check_default(self, option: optparse.Option, key: str, val: Any) -> Any: try: return option.check_value(key, val) except optparse.OptionValueError as exc: print(f"An error occurred during configuration: {exc}") sys.exit(3) - def _get_ordered_configuration_items(self): - # type: () -> Iterator[Tuple[str, Any]] + def _get_ordered_configuration_items(self) -> Iterator[Tuple[str, Any]]: # Configuration gives keys in an unordered manner. Order them. override_order = ["global", self.name, ":env:"] @@ -211,8 +201,7 @@ class ConfigOptionParser(CustomOptionParser): for key, val in section_items[section]: yield key, val - def _update_defaults(self, defaults): - # type: (Dict[str, Any]) -> Dict[str, Any] + def _update_defaults(self, defaults: Dict[str, Any]) -> Dict[str, Any]: """Updates the given defaults with values from the config files and the environ. Does a little special handling for certain types of options (lists).""" @@ -276,8 +265,7 @@ class ConfigOptionParser(CustomOptionParser): self.values = None return defaults - def get_default_values(self): - # type: () -> optparse.Values + def get_default_values(self) -> optparse.Values: """Overriding to make updating the defaults after instantiation of the option parser possible, _update_defaults() does the dirty work.""" if not self.process_default_values: @@ -299,7 +287,6 @@ class ConfigOptionParser(CustomOptionParser): defaults[option.dest] = option.check_value(opt_str, default) return optparse.Values(defaults) - def error(self, msg): - # type: (str) -> None + def error(self, msg: str) -> None: self.print_usage(sys.stderr) self.exit(UNKNOWN_ERROR, f"{msg}\n") From 5394d340fb3a0b31a8e1909dd6872ecc36f75fbe Mon Sep 17 00:00:00 2001 From: Dominic Davis-Foster Date: Mon, 7 Jun 2021 14:34:07 +0100 Subject: [PATCH 54/85] Update vendored six to 1.16.0 --- src/pip/_vendor/six.py | 18 ++- src/pip/_vendor/urllib3/_version.py | 2 +- src/pip/_vendor/urllib3/connection.py | 4 +- src/pip/_vendor/urllib3/connectionpool.py | 2 +- src/pip/_vendor/urllib3/contrib/pyopenssl.py | 2 + .../urllib3/contrib/securetransport.py | 4 +- src/pip/_vendor/urllib3/packages/six.py | 122 +++++++++++++----- .../packages/ssl_match_hostname/__init__.py | 6 +- src/pip/_vendor/urllib3/util/connection.py | 2 +- src/pip/_vendor/urllib3/util/retry.py | 6 +- src/pip/_vendor/urllib3/util/ssl_.py | 37 ++++-- src/pip/_vendor/urllib3/util/ssltransport.py | 2 +- src/pip/_vendor/urllib3/util/url.py | 8 +- src/pip/_vendor/vendor.txt | 4 +- 14 files changed, 160 insertions(+), 59 deletions(-) diff --git a/src/pip/_vendor/six.py b/src/pip/_vendor/six.py index 83f69783d..4e15675d8 100644 --- a/src/pip/_vendor/six.py +++ b/src/pip/_vendor/six.py @@ -29,7 +29,7 @@ import sys import types __author__ = "Benjamin Peterson " -__version__ = "1.15.0" +__version__ = "1.16.0" # Useful for very coarse version differentiation. @@ -71,6 +71,11 @@ else: MAXSIZE = int((1 << 63) - 1) del X +if PY34: + from importlib.util import spec_from_loader +else: + spec_from_loader = None + def _add_doc(func, doc): """Add documentation to a function.""" @@ -186,6 +191,11 @@ class _SixMetaPathImporter(object): return self return None + def find_spec(self, fullname, path, target=None): + if fullname in self.known_modules: + return spec_from_loader(fullname, self) + return None + def __get_module(self, fullname): try: return self.known_modules[fullname] @@ -223,6 +233,12 @@ class _SixMetaPathImporter(object): return None get_source = get_code # same as get_code + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + pass + _importer = _SixMetaPathImporter(__name__) diff --git a/src/pip/_vendor/urllib3/_version.py b/src/pip/_vendor/urllib3/_version.py index 97c983300..e95fd5228 100644 --- a/src/pip/_vendor/urllib3/_version.py +++ b/src/pip/_vendor/urllib3/_version.py @@ -1,2 +1,2 @@ # This file is protected via CODEOWNERS -__version__ = "1.26.4" +__version__ = "1.26.5" diff --git a/src/pip/_vendor/urllib3/connection.py b/src/pip/_vendor/urllib3/connection.py index 45580b7e1..efa19af5b 100644 --- a/src/pip/_vendor/urllib3/connection.py +++ b/src/pip/_vendor/urllib3/connection.py @@ -201,7 +201,7 @@ class HTTPConnection(_HTTPConnection, object): self._prepare_conn(conn) def putrequest(self, method, url, *args, **kwargs): - """""" + """ """ # Empty docstring because the indentation of CPython's implementation # is broken but we don't want this method in our documentation. match = _CONTAINS_CONTROL_CHAR_RE.search(method) @@ -214,7 +214,7 @@ class HTTPConnection(_HTTPConnection, object): return _HTTPConnection.putrequest(self, method, url, *args, **kwargs) def putheader(self, header, *values): - """""" + """ """ if not any(isinstance(v, str) and v == SKIP_HEADER for v in values): _HTTPConnection.putheader(self, header, *values) elif six.ensure_str(header.lower()) not in SKIPPABLE_HEADERS: diff --git a/src/pip/_vendor/urllib3/connectionpool.py b/src/pip/_vendor/urllib3/connectionpool.py index 4708c5bfc..40183214e 100644 --- a/src/pip/_vendor/urllib3/connectionpool.py +++ b/src/pip/_vendor/urllib3/connectionpool.py @@ -318,7 +318,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): pass def _get_timeout(self, timeout): - """ Helper that always returns a :class:`urllib3.util.Timeout` """ + """Helper that always returns a :class:`urllib3.util.Timeout`""" if timeout is _Default: return self.timeout.clone() diff --git a/src/pip/_vendor/urllib3/contrib/pyopenssl.py b/src/pip/_vendor/urllib3/contrib/pyopenssl.py index bc5c114fa..c43146279 100644 --- a/src/pip/_vendor/urllib3/contrib/pyopenssl.py +++ b/src/pip/_vendor/urllib3/contrib/pyopenssl.py @@ -76,6 +76,7 @@ import sys from .. import util from ..packages import six +from ..util.ssl_ import PROTOCOL_TLS_CLIENT __all__ = ["inject_into_urllib3", "extract_from_urllib3"] @@ -85,6 +86,7 @@ HAS_SNI = True # Map from urllib3 to PyOpenSSL compatible parameter-values. _openssl_versions = { util.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD, + PROTOCOL_TLS_CLIENT: OpenSSL.SSL.SSLv23_METHOD, ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD, } diff --git a/src/pip/_vendor/urllib3/contrib/securetransport.py b/src/pip/_vendor/urllib3/contrib/securetransport.py index 8f058f507..b97555454 100644 --- a/src/pip/_vendor/urllib3/contrib/securetransport.py +++ b/src/pip/_vendor/urllib3/contrib/securetransport.py @@ -67,6 +67,7 @@ import weakref from pip._vendor import six from .. import util +from ..util.ssl_ import PROTOCOL_TLS_CLIENT from ._securetransport.bindings import CoreFoundation, Security, SecurityConst from ._securetransport.low_level import ( _assert_no_error, @@ -154,7 +155,8 @@ CIPHER_SUITES = [ # TLSv1 and a high of TLSv1.2. For everything else, we pin to that version. # TLSv1 to 1.2 are supported on macOS 10.8+ _protocol_to_min_max = { - util.PROTOCOL_TLS: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12) + util.PROTOCOL_TLS: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12), + PROTOCOL_TLS_CLIENT: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12), } if hasattr(ssl, "PROTOCOL_SSLv2"): diff --git a/src/pip/_vendor/urllib3/packages/six.py b/src/pip/_vendor/urllib3/packages/six.py index 314424099..d7ab761fc 100644 --- a/src/pip/_vendor/urllib3/packages/six.py +++ b/src/pip/_vendor/urllib3/packages/six.py @@ -1,4 +1,4 @@ -# Copyright (c) 2010-2019 Benjamin Peterson +# Copyright (c) 2010-2020 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -29,7 +29,7 @@ import sys import types __author__ = "Benjamin Peterson " -__version__ = "1.12.0" +__version__ = "1.16.0" # Useful for very coarse version differentiation. @@ -71,6 +71,11 @@ else: MAXSIZE = int((1 << 63) - 1) del X +if PY34: + from importlib.util import spec_from_loader +else: + spec_from_loader = None + def _add_doc(func, doc): """Add documentation to a function.""" @@ -182,6 +187,11 @@ class _SixMetaPathImporter(object): return self return None + def find_spec(self, fullname, path, target=None): + if fullname in self.known_modules: + return spec_from_loader(fullname, self) + return None + def __get_module(self, fullname): try: return self.known_modules[fullname] @@ -220,6 +230,12 @@ class _SixMetaPathImporter(object): get_source = get_code # same as get_code + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + pass + _importer = _SixMetaPathImporter(__name__) @@ -260,9 +276,19 @@ _moved_attributes = [ ), MovedModule("builtins", "__builtin__"), MovedModule("configparser", "ConfigParser"), + MovedModule( + "collections_abc", + "collections", + "collections.abc" if sys.version_info >= (3, 3) else "collections", + ), MovedModule("copyreg", "copy_reg"), MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), - MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), + MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"), + MovedModule( + "_dummy_thread", + "dummy_thread", + "_dummy_thread" if sys.version_info < (3, 9) else "_thread", + ), MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), MovedModule("http_cookies", "Cookie", "http.cookies"), MovedModule("html_entities", "htmlentitydefs", "html.entities"), @@ -307,7 +333,9 @@ _moved_attributes = [ ] # Add windows specific modules. if sys.platform == "win32": - _moved_attributes += [MovedModule("winreg", "_winreg")] + _moved_attributes += [ + MovedModule("winreg", "_winreg"), + ] for attr in _moved_attributes: setattr(_MovedItems, attr.name, attr) @@ -476,7 +504,7 @@ class Module_six_moves_urllib_robotparser(_LazyModule): _urllib_robotparser_moved_attributes = [ - MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser") + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), ] for attr in _urllib_robotparser_moved_attributes: setattr(Module_six_moves_urllib_robotparser, attr.name, attr) @@ -678,9 +706,11 @@ if PY3: if sys.version_info[1] <= 1: _assertRaisesRegex = "assertRaisesRegexp" _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" else: _assertRaisesRegex = "assertRaisesRegex" _assertRegex = "assertRegex" + _assertNotRegex = "assertNotRegex" else: def b(s): @@ -707,6 +737,7 @@ else: _assertCountEqual = "assertItemsEqual" _assertRaisesRegex = "assertRaisesRegexp" _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" _add_doc(b, """Byte literal""") _add_doc(u, """Text literal""") @@ -723,6 +754,10 @@ def assertRegex(self, *args, **kwargs): return getattr(self, _assertRegex)(*args, **kwargs) +def assertNotRegex(self, *args, **kwargs): + return getattr(self, _assertNotRegex)(*args, **kwargs) + + if PY3: exec_ = getattr(moves.builtins, "exec") @@ -762,18 +797,7 @@ else: ) -if sys.version_info[:2] == (3, 2): - exec_( - """def raise_from(value, from_value): - try: - if from_value is None: - raise value - raise value from from_value - finally: - value = None -""" - ) -elif sys.version_info[:2] > (3, 2): +if sys.version_info[:2] > (3,): exec_( """def raise_from(value, from_value): try: @@ -863,19 +887,41 @@ if sys.version_info[:2] < (3, 3): _add_doc(reraise, """Reraise an exception.""") if sys.version_info[0:2] < (3, 4): + # This does exactly the same what the :func:`py3:functools.update_wrapper` + # function does on Python versions after 3.2. It sets the ``__wrapped__`` + # attribute on ``wrapper`` object and it doesn't raise an error if any of + # the attributes mentioned in ``assigned`` and ``updated`` are missing on + # ``wrapped`` object. + def _update_wrapper( + wrapper, + wrapped, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES, + ): + for attr in assigned: + try: + value = getattr(wrapped, attr) + except AttributeError: + continue + else: + setattr(wrapper, attr, value) + for attr in updated: + getattr(wrapper, attr).update(getattr(wrapped, attr, {})) + wrapper.__wrapped__ = wrapped + return wrapper + + _update_wrapper.__doc__ = functools.update_wrapper.__doc__ def wraps( wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, updated=functools.WRAPPER_UPDATES, ): - def wrapper(f): - f = functools.wraps(wrapped, assigned, updated)(f) - f.__wrapped__ = wrapped - return f - - return wrapper + return functools.partial( + _update_wrapper, wrapped=wrapped, assigned=assigned, updated=updated + ) + wraps.__doc__ = functools.wraps.__doc__ else: wraps = functools.wraps @@ -888,7 +934,15 @@ def with_metaclass(meta, *bases): # the actual metaclass. class metaclass(type): def __new__(cls, name, this_bases, d): - return meta(name, bases, d) + if sys.version_info[:2] >= (3, 7): + # This version introduced PEP 560 that requires a bit + # of extra care (we mimic what is done by __build_class__). + resolved_bases = types.resolve_bases(bases) + if resolved_bases is not bases: + d["__orig_bases__"] = bases + else: + resolved_bases = bases + return meta(name, resolved_bases, d) @classmethod def __prepare__(cls, name, this_bases): @@ -928,12 +982,11 @@ def ensure_binary(s, encoding="utf-8", errors="strict"): - `str` -> encoded to `bytes` - `bytes` -> `bytes` """ + if isinstance(s, binary_type): + return s if isinstance(s, text_type): return s.encode(encoding, errors) - elif isinstance(s, binary_type): - return s - else: - raise TypeError("not expecting type '%s'" % type(s)) + raise TypeError("not expecting type '%s'" % type(s)) def ensure_str(s, encoding="utf-8", errors="strict"): @@ -947,12 +1000,15 @@ def ensure_str(s, encoding="utf-8", errors="strict"): - `str` -> `str` - `bytes` -> decoded to `str` """ - if not isinstance(s, (text_type, binary_type)): - raise TypeError("not expecting type '%s'" % type(s)) + # Optimization: Fast return for the common case. + if type(s) is str: + return s if PY2 and isinstance(s, text_type): - s = s.encode(encoding, errors) + return s.encode(encoding, errors) elif PY3 and isinstance(s, binary_type): - s = s.decode(encoding, errors) + return s.decode(encoding, errors) + elif not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) return s @@ -977,7 +1033,7 @@ def ensure_text(s, encoding="utf-8", errors="strict"): def python_2_unicode_compatible(klass): """ - A decorator that defines __unicode__ and __str__ methods under Python 2. + A class decorator that defines __unicode__ and __str__ methods under Python 2. Under Python 3 it does nothing. To support Python 2 and 3 with a single code base, define a __str__ method diff --git a/src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py b/src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py index 6b12fd90a..ef3fde520 100644 --- a/src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py +++ b/src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py @@ -1,9 +1,11 @@ import sys try: - # Our match_hostname function is the same as 3.5's, so we only want to + # Our match_hostname function is the same as 3.10's, so we only want to # import the match_hostname function if it's at least that good. - if sys.version_info < (3, 5): + # We also fallback on Python 3.10+ because our code doesn't emit + # deprecation warnings and is the same as Python 3.10 otherwise. + if sys.version_info < (3, 5) or sys.version_info >= (3, 10): raise ImportError("Fallback to vendored code") from ssl import CertificateError, match_hostname diff --git a/src/pip/_vendor/urllib3/util/connection.py b/src/pip/_vendor/urllib3/util/connection.py index f1e5d37f8..facfa0dd2 100644 --- a/src/pip/_vendor/urllib3/util/connection.py +++ b/src/pip/_vendor/urllib3/util/connection.py @@ -118,7 +118,7 @@ def allowed_gai_family(): def _has_ipv6(host): - """ Returns True if the system can bind an IPv6 address. """ + """Returns True if the system can bind an IPv6 address.""" sock = None has_ipv6 = False diff --git a/src/pip/_vendor/urllib3/util/retry.py b/src/pip/_vendor/urllib3/util/retry.py index d25a41b42..180e82b8c 100644 --- a/src/pip/_vendor/urllib3/util/retry.py +++ b/src/pip/_vendor/urllib3/util/retry.py @@ -321,7 +321,7 @@ class Retry(object): @classmethod def from_int(cls, retries, redirect=True, default=None): - """ Backwards-compatibility for the old retries format.""" + """Backwards-compatibility for the old retries format.""" if retries is None: retries = default if default is not None else cls.DEFAULT @@ -374,7 +374,7 @@ class Retry(object): return seconds def get_retry_after(self, response): - """ Get the value of Retry-After in seconds. """ + """Get the value of Retry-After in seconds.""" retry_after = response.getheader("Retry-After") @@ -468,7 +468,7 @@ class Retry(object): ) def is_exhausted(self): - """ Are we out of retries? """ + """Are we out of retries?""" retry_counts = ( self.total, self.connect, diff --git a/src/pip/_vendor/urllib3/util/ssl_.py b/src/pip/_vendor/urllib3/util/ssl_.py index 763da82bb..a012e5e13 100644 --- a/src/pip/_vendor/urllib3/util/ssl_.py +++ b/src/pip/_vendor/urllib3/util/ssl_.py @@ -71,6 +71,11 @@ except ImportError: except ImportError: PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 +try: + from ssl import PROTOCOL_TLS_CLIENT +except ImportError: + PROTOCOL_TLS_CLIENT = PROTOCOL_TLS + try: from ssl import OP_NO_COMPRESSION, OP_NO_SSLv2, OP_NO_SSLv3 @@ -278,7 +283,11 @@ def create_urllib3_context( Constructed SSLContext object with specified options :rtype: SSLContext """ - context = SSLContext(ssl_version or PROTOCOL_TLS) + # PROTOCOL_TLS is deprecated in Python 3.10 + if not ssl_version or ssl_version == PROTOCOL_TLS: + ssl_version = PROTOCOL_TLS_CLIENT + + context = SSLContext(ssl_version) context.set_ciphers(ciphers or DEFAULT_CIPHERS) @@ -313,13 +322,25 @@ def create_urllib3_context( ) is not None: context.post_handshake_auth = True - context.verify_mode = cert_reqs - if ( - getattr(context, "check_hostname", None) is not None - ): # Platform-specific: Python 3.2 - # We do our own verification, including fingerprints and alternative - # hostnames. So disable it here - context.check_hostname = False + def disable_check_hostname(): + if ( + getattr(context, "check_hostname", None) is not None + ): # Platform-specific: Python 3.2 + # We do our own verification, including fingerprints and alternative + # hostnames. So disable it here + context.check_hostname = False + + # The order of the below lines setting verify_mode and check_hostname + # matter due to safe-guards SSLContext has to prevent an SSLContext with + # check_hostname=True, verify_mode=NONE/OPTIONAL. This is made even more + # complex because we don't know whether PROTOCOL_TLS_CLIENT will be used + # or not so we don't know the initial state of the freshly created SSLContext. + if cert_reqs == ssl.CERT_REQUIRED: + context.verify_mode = cert_reqs + disable_check_hostname() + else: + disable_check_hostname() + context.verify_mode = cert_reqs # Enable logging of TLS session keys via defacto standard environment variable # 'SSLKEYLOGFILE', if the feature is available (Python 3.8+). Skip empty values. diff --git a/src/pip/_vendor/urllib3/util/ssltransport.py b/src/pip/_vendor/urllib3/util/ssltransport.py index ca00233c9..0ed97b644 100644 --- a/src/pip/_vendor/urllib3/util/ssltransport.py +++ b/src/pip/_vendor/urllib3/util/ssltransport.py @@ -193,7 +193,7 @@ class SSLTransport: raise def _ssl_io_loop(self, func, *args): - """ Performs an I/O loop between incoming/outgoing and the socket.""" + """Performs an I/O loop between incoming/outgoing and the socket.""" should_loop = True ret = None diff --git a/src/pip/_vendor/urllib3/util/url.py b/src/pip/_vendor/urllib3/util/url.py index 66c8795b1..3651c4318 100644 --- a/src/pip/_vendor/urllib3/util/url.py +++ b/src/pip/_vendor/urllib3/util/url.py @@ -63,12 +63,12 @@ IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT + "$") BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT[2:-2] + "$") ZONE_ID_RE = re.compile("(" + ZONE_ID_PAT + r")\]$") -SUBAUTHORITY_PAT = (u"^(?:(.*)@)?(%s|%s|%s)(?::([0-9]{0,5}))?$") % ( +_HOST_PORT_PAT = ("^(%s|%s|%s)(?::([0-9]{0,5}))?$") % ( REG_NAME_PAT, IPV4_PAT, IPV6_ADDRZ_PAT, ) -SUBAUTHORITY_RE = re.compile(SUBAUTHORITY_PAT, re.UNICODE | re.DOTALL) +_HOST_PORT_RE = re.compile(_HOST_PORT_PAT, re.UNICODE | re.DOTALL) UNRESERVED_CHARS = set( "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-~" @@ -365,7 +365,9 @@ def parse_url(url): scheme = scheme.lower() if authority: - auth, host, port = SUBAUTHORITY_RE.match(authority).groups() + auth, _, host_port = authority.rpartition("@") + auth = auth or None + host, port = _HOST_PORT_RE.match(host_port).groups() if auth and normalize_uri: auth = _encode_invalid_chars(auth, USERINFO_CHARS) if port == "": diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 6c9732e97..8eb8a5d20 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -13,10 +13,10 @@ requests==2.25.1 certifi==2020.12.05 chardet==4.0.0 idna==3.1 - urllib3==1.26.4 + urllib3==1.26.5 resolvelib==0.7.0 setuptools==44.0.0 -six==1.15.0 +six==1.16.0 tenacity==7.0.0 toml==0.10.2 webencodings==0.5.1 From d532a4cf449215299f902f81500689e9c5334954 Mon Sep 17 00:00:00 2001 From: Dominic Davis-Foster Date: Mon, 7 Jun 2021 14:42:04 +0100 Subject: [PATCH 55/85] Add news entry --- news/10043.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/10043.bugfix.rst diff --git a/news/10043.bugfix.rst b/news/10043.bugfix.rst new file mode 100644 index 000000000..29d78f7bc --- /dev/null +++ b/news/10043.bugfix.rst @@ -0,0 +1 @@ +Update vendored six to 1.16.0 and urllib3 to 1.26.5 From 9cbe7f60adaaaa9d060a6165a57098a526ad7f82 Mon Sep 17 00:00:00 2001 From: OBITORASU Date: Tue, 8 Jun 2021 21:09:09 +0530 Subject: [PATCH 56/85] Warning for user when Windows path limit is exceeded --- src/pip/_internal/commands/install.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 6932f5a6d..a35ac06ae 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -26,6 +26,7 @@ from pip._internal.operations.check import ConflictDetails, check_install_confli from pip._internal.req import install_given_reqs from pip._internal.req.req_install import InstallRequirement from pip._internal.req.req_tracker import get_requirement_tracker +from pip._internal.utils.compat import WINDOWS from pip._internal.utils.distutils_args import parse_distutils_args from pip._internal.utils.filesystem import test_writable_dir from pip._internal.utils.misc import ( @@ -440,6 +441,13 @@ class InstallCommand(RequirementCommand): message = create_os_error_message( error, show_traceback, options.use_user_site, ) + if WINDOWS and len(message) >= 280: + logger.warning( + 'The following error can potentially be caused ' + 'because Long Paths is disabled on your system. ' + 'Please set LongPathsEnabled to 1 in the ' + 'registry and try again.' + ) logger.error(message, exc_info=show_traceback) # noqa return ERROR From 7204b9240db717ec60b67bef946c446b37db59a0 Mon Sep 17 00:00:00 2001 From: OBITORASU Date: Tue, 8 Jun 2021 21:17:21 +0530 Subject: [PATCH 57/85] add news entry for issue 10045 --- news/10045.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/10045.feature.rst diff --git a/news/10045.feature.rst b/news/10045.feature.rst new file mode 100644 index 000000000..201f66bf7 --- /dev/null +++ b/news/10045.feature.rst @@ -0,0 +1 @@ +Added a warning message for errors caused due to Long Paths being disabled on Windows. \ No newline at end of file From 5c735941532a224b9cac4c8d3d36b7f3c367ab7f Mon Sep 17 00:00:00 2001 From: OBITORASU Date: Tue, 8 Jun 2021 21:40:45 +0530 Subject: [PATCH 58/85] change message to error --- src/pip/_internal/commands/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index a35ac06ae..aeed0f3cd 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -441,7 +441,7 @@ class InstallCommand(RequirementCommand): message = create_os_error_message( error, show_traceback, options.use_user_site, ) - if WINDOWS and len(message) >= 280: + if WINDOWS and len(error) >= 280: logger.warning( 'The following error can potentially be caused ' 'because Long Paths is disabled on your system. ' From e22799bd8e409bdab09d62f9baaed2fb0caa6aea Mon Sep 17 00:00:00 2001 From: OBITORASU Date: Tue, 8 Jun 2021 21:57:16 +0530 Subject: [PATCH 59/85] update padding around default limit --- src/pip/_internal/commands/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index aeed0f3cd..58f47f0b8 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -441,7 +441,7 @@ class InstallCommand(RequirementCommand): message = create_os_error_message( error, show_traceback, options.use_user_site, ) - if WINDOWS and len(error) >= 280: + if WINDOWS and len(error) >= 270: logger.warning( 'The following error can potentially be caused ' 'because Long Paths is disabled on your system. ' From fe4ab16d3f56039ba345bf815da6408ea0776911 Mon Sep 17 00:00:00 2001 From: OBITORASU Date: Tue, 8 Jun 2021 22:30:47 +0530 Subject: [PATCH 60/85] add reference link and extra checks --- src/pip/_internal/commands/install.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 58f47f0b8..654120759 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -441,13 +441,18 @@ class InstallCommand(RequirementCommand): message = create_os_error_message( error, show_traceback, options.use_user_site, ) - if WINDOWS and len(error) >= 270: - logger.warning( - 'The following error can potentially be caused ' - 'because Long Paths is disabled on your system. ' - 'Please set LongPathsEnabled to 1 in the ' - 'registry and try again.' - ) + if WINDOWS and len(str(error)) >= 270: + if error.errno == errno.ENOENT: + logger.warning( + 'The following error can potentially be caused ' + 'because Long Paths is disabled on your system. ' + 'Please set LongPathsEnabled to 1 in the ' + 'registry and try again. For further instructions ' + 'please refer to the documentation: https://docs.' + 'microsoft.com/en-us/windows/win32/fileio/maximum' + '-file-path-limitation?tabs=cmd#enable-long-paths' + '-in-windows-10-version-1607-and-later' + ) logger.error(message, exc_info=show_traceback) # noqa return ERROR From 4dbf953ba25a9ea438a4947fc410892a1e16cd46 Mon Sep 17 00:00:00 2001 From: OBITORASU Date: Tue, 8 Jun 2021 22:31:08 +0530 Subject: [PATCH 61/85] fix lint issue in docs --- news/10045.feature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/10045.feature.rst b/news/10045.feature.rst index 201f66bf7..7c7b53725 100644 --- a/news/10045.feature.rst +++ b/news/10045.feature.rst @@ -1 +1 @@ -Added a warning message for errors caused due to Long Paths being disabled on Windows. \ No newline at end of file +Added a warning message for errors caused due to Long Paths being disabled on Windows. From b0626f61da337e97a4462241c7977386075b5a53 Mon Sep 17 00:00:00 2001 From: Diego Ramirez Date: Tue, 8 Jun 2021 13:19:10 -0500 Subject: [PATCH 62/85] Make proper annotations on `noxfile.py` As I said on the pypa PR 10018, it is necessary to convert the commentaries into proper annotations. --- noxfile.py | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/noxfile.py b/noxfile.py index d9e344543..becd0c304 100644 --- a/noxfile.py +++ b/noxfile.py @@ -33,8 +33,7 @@ AUTHORS_FILE = "AUTHORS.txt" VERSION_FILE = "src/pip/__init__.py" -def run_with_protected_pip(session, *arguments): - # type: (nox.Session, *str) -> None +def run_with_protected_pip(session: nox.Session, *arguments: str) -> None: """Do a session.run("pip", *arguments), using a "protected" pip. This invokes a wrapper script, that forwards calls to original virtualenv @@ -48,8 +47,7 @@ def run_with_protected_pip(session, *arguments): session.run(*command, env=env, silent=True) -def should_update_common_wheels(): - # type: () -> bool +def should_update_common_wheels() -> bool: # If the cache hasn't been created, create it. if not os.path.exists(LOCATIONS["common-wheels"]): return True @@ -73,8 +71,7 @@ def should_update_common_wheels(): # `tox -e ...` until this note is removed. # ----------------------------------------------------------------------------- @nox.session(python=["3.6", "3.7", "3.8", "3.9", "pypy3"]) -def test(session): - # type: (nox.Session) -> None +def test(session: nox.Session) -> None: # Get the common wheels. if should_update_common_wheels(): # fmt: off @@ -122,8 +119,7 @@ def test(session): @nox.session -def docs(session): - # type: (nox.Session) -> None +def docs(session: nox.Session) -> None: session.install("-e", ".") session.install("-r", REQUIREMENTS["docs"]) @@ -150,8 +146,7 @@ def docs(session): @nox.session(name="docs-live") -def docs_live(session): - # type: (nox.Session) -> None +def docs_live(session: nox.Session) -> None: session.install("-e", ".") session.install("-r", REQUIREMENTS["docs"], "sphinx-autobuild") @@ -166,8 +161,7 @@ def docs_live(session): @nox.session -def lint(session): - # type: (nox.Session) -> None +def lint(session: nox.Session) -> None: session.install("pre-commit") if session.posargs: @@ -179,8 +173,7 @@ def lint(session): @nox.session -def vendoring(session): - # type: (nox.Session) -> None +def vendoring(session: nox.Session) -> None: session.install("vendoring>=0.3.0") if "--upgrade" not in session.posargs: @@ -238,8 +231,7 @@ def vendoring(session): # Release Commands # ----------------------------------------------------------------------------- @nox.session(name="prepare-release") -def prepare_release(session): - # type: (nox.Session) -> None +def prepare_release(session: nox.Session) -> None: version = release.get_version_from_arguments(session) if not version: session.error("Usage: nox -s prepare-release -- ") @@ -272,8 +264,7 @@ def prepare_release(session): @nox.session(name="build-release") -def build_release(session): - # type: (nox.Session) -> None +def build_release(session: nox.Session) -> None: version = release.get_version_from_arguments(session) if not version: session.error("Usage: nox -s build-release -- YY.N[.P]") @@ -304,8 +295,7 @@ def build_release(session): shutil.copy(dist, final) -def build_dists(session): - # type: (nox.Session) -> List[str] +def build_dists(session: nox.Session) -> List[str]: """Return dists with valid metadata.""" session.log( "# Check if there's any Git-untracked files before building the wheel", @@ -333,8 +323,7 @@ def build_dists(session): @nox.session(name="upload-release") -def upload_release(session): - # type: (nox.Session) -> None +def upload_release(session: nox.Session) -> None: version = release.get_version_from_arguments(session) if not version: session.error("Usage: nox -s upload-release -- YY.N[.P]") From 6069a3d23762b9fefe398292ae0d6bfc4e1c8166 Mon Sep 17 00:00:00 2001 From: Diego Ramirez Date: Tue, 8 Jun 2021 13:25:10 -0500 Subject: [PATCH 63/85] Create 10047.trivial.rst This is the respective news entry for this pull request. --- news/10047.trivial.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/10047.trivial.rst diff --git a/news/10047.trivial.rst b/news/10047.trivial.rst new file mode 100644 index 000000000..edd324342 --- /dev/null +++ b/news/10047.trivial.rst @@ -0,0 +1 @@ +Convert type annotations into proper annotations in ``noxfile.py``. From 5e2c0cc8f77ac5af3d7d4bb0d71b6fe0a66c7d46 Mon Sep 17 00:00:00 2001 From: OBITORASU Date: Wed, 9 Jun 2021 00:30:29 +0530 Subject: [PATCH 64/85] add check to verify error.filename is not none --- src/pip/_internal/commands/install.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 654120759..203a1de27 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -441,18 +441,18 @@ class InstallCommand(RequirementCommand): message = create_os_error_message( error, show_traceback, options.use_user_site, ) - if WINDOWS and len(str(error)) >= 270: - if error.errno == errno.ENOENT: - logger.warning( - 'The following error can potentially be caused ' - 'because Long Paths is disabled on your system. ' - 'Please set LongPathsEnabled to 1 in the ' - 'registry and try again. For further instructions ' - 'please refer to the documentation: https://docs.' - 'microsoft.com/en-us/windows/win32/fileio/maximum' - '-file-path-limitation?tabs=cmd#enable-long-paths' - '-in-windows-10-version-1607-and-later' - ) + if (WINDOWS and error.errno == errno.ENOENT and error.filename and + len(str(error)) >= 270): + logger.warning( + 'The following error can potentially be caused ' + 'because Long Paths is disabled on your system. ' + 'Please set LongPathsEnabled to 1 in the ' + 'registry and try again. For further instructions ' + 'please refer to the documentation: https://docs.' + 'microsoft.com/en-us/windows/win32/fileio/maximum' + '-file-path-limitation?tabs=cmd#enable-long-paths' + '-in-windows-10-version-1607-and-later' + ) logger.error(message, exc_info=show_traceback) # noqa return ERROR From 45930078a65a53f762705aab5bbf92e663a73a8d Mon Sep 17 00:00:00 2001 From: OBITORASU Date: Wed, 9 Jun 2021 00:33:23 +0530 Subject: [PATCH 65/85] use error.filename instead of error --- src/pip/_internal/commands/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 203a1de27..54576dabe 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -442,7 +442,7 @@ class InstallCommand(RequirementCommand): error, show_traceback, options.use_user_site, ) if (WINDOWS and error.errno == errno.ENOENT and error.filename and - len(str(error)) >= 270): + len(error.filename) >= 260): logger.warning( 'The following error can potentially be caused ' 'because Long Paths is disabled on your system. ' From 67dcdf5e110a430e2a544061726f16971d122ba4 Mon Sep 17 00:00:00 2001 From: OBITORASU Date: Wed, 9 Jun 2021 00:53:17 +0530 Subject: [PATCH 66/85] use error.filename, remove padding and shorten url --- src/pip/_internal/commands/install.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 54576dabe..8ccee9f34 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -442,16 +442,14 @@ class InstallCommand(RequirementCommand): error, show_traceback, options.use_user_site, ) if (WINDOWS and error.errno == errno.ENOENT and error.filename and - len(error.filename) >= 260): + len(error.filename) > 260): logger.warning( 'The following error can potentially be caused ' 'because Long Paths is disabled on your system. ' 'Please set LongPathsEnabled to 1 in the ' 'registry and try again. For further instructions ' - 'please refer to the documentation: https://docs.' - 'microsoft.com/en-us/windows/win32/fileio/maximum' - '-file-path-limitation?tabs=cmd#enable-long-paths' - '-in-windows-10-version-1607-and-later' + 'please refer to the documentation: ' + 'https://pip.pypa.io/warnings/enable-long-paths' ) logger.error(message, exc_info=show_traceback) # noqa From 560d2e97ae9bc51a94d8a3c81f6ee3718d7a202f Mon Sep 17 00:00:00 2001 From: OBITORASU Date: Wed, 9 Jun 2021 01:05:54 +0530 Subject: [PATCH 67/85] move the check to create_os_error_message --- src/pip/_internal/commands/install.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 8ccee9f34..808bb5a1c 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -441,16 +441,6 @@ class InstallCommand(RequirementCommand): message = create_os_error_message( error, show_traceback, options.use_user_site, ) - if (WINDOWS and error.errno == errno.ENOENT and error.filename and - len(error.filename) > 260): - logger.warning( - 'The following error can potentially be caused ' - 'because Long Paths is disabled on your system. ' - 'Please set LongPathsEnabled to 1 in the ' - 'registry and try again. For further instructions ' - 'please refer to the documentation: ' - 'https://pip.pypa.io/warnings/enable-long-paths' - ) logger.error(message, exc_info=show_traceback) # noqa return ERROR @@ -748,4 +738,17 @@ def create_os_error_message(error, show_traceback, using_user_site): parts.append(permissions_part) parts.append(".\n") + # Suggest the user to enable Long Paths if path length is + # more than 260 + if (WINDOWS and error.errno == errno.ENOENT and error.filename and + len(error.filename) > 260): + parts.append( + "The following error can potentially be caused " + "because Long Paths is disabled on your system. " + "Please set LongPathsEnabled to 1 in the " + "registry and try again. For further instructions " + "please refer to the documentation: " + "https://pip.pypa.io/warnings/enable-long-paths" + ) + return "".join(parts).strip() + "\n" From 1458166a28a30c77d04bcc8ffc05396afc03f715 Mon Sep 17 00:00:00 2001 From: OBITORASU Date: Wed, 9 Jun 2021 01:10:01 +0530 Subject: [PATCH 68/85] fix the wordings --- src/pip/_internal/commands/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 808bb5a1c..ba157f1f8 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -743,7 +743,7 @@ def create_os_error_message(error, show_traceback, using_user_site): if (WINDOWS and error.errno == errno.ENOENT and error.filename and len(error.filename) > 260): parts.append( - "The following error can potentially be caused " + "This error can potentially be caused " "because Long Paths is disabled on your system. " "Please set LongPathsEnabled to 1 in the " "registry and try again. For further instructions " From 9ba78c964e6aa61fafd530743fdfba08ae88f754 Mon Sep 17 00:00:00 2001 From: OBITORASU Date: Wed, 9 Jun 2021 11:19:46 +0530 Subject: [PATCH 69/85] change wordings for the info message --- src/pip/_internal/commands/install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index ba157f1f8..03286ab26 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -743,8 +743,8 @@ def create_os_error_message(error, show_traceback, using_user_site): if (WINDOWS and error.errno == errno.ENOENT and error.filename and len(error.filename) > 260): parts.append( - "This error can potentially be caused " - "because Long Paths is disabled on your system. " + "A potential cause to this error is " + "Long Paths beign disabled on your system. " "Please set LongPathsEnabled to 1 in the " "registry and try again. For further instructions " "please refer to the documentation: " From 9f0338a416f07d8a8b7d98bbf47c815c29452a9e Mon Sep 17 00:00:00 2001 From: OBITORASU Date: Fri, 11 Jun 2021 10:49:48 +0530 Subject: [PATCH 70/85] fix typo --- src/pip/_internal/commands/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 03286ab26..b4fbcc305 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -744,7 +744,7 @@ def create_os_error_message(error, show_traceback, using_user_site): len(error.filename) > 260): parts.append( "A potential cause to this error is " - "Long Paths beign disabled on your system. " + "Long Paths being disabled on your system. " "Please set LongPathsEnabled to 1 in the " "registry and try again. For further instructions " "please refer to the documentation: " From 7a64c944b8c895db68fa40842694f429c934377f Mon Sep 17 00:00:00 2001 From: OBITORASU Date: Fri, 11 Jun 2021 15:13:53 +0530 Subject: [PATCH 71/85] minor rephrasing of message --- src/pip/_internal/commands/install.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index b4fbcc305..2fa6a6274 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -743,12 +743,11 @@ def create_os_error_message(error, show_traceback, using_user_site): if (WINDOWS and error.errno == errno.ENOENT and error.filename and len(error.filename) > 260): parts.append( - "A potential cause to this error is " - "Long Paths being disabled on your system. " - "Please set LongPathsEnabled to 1 in the " - "registry and try again. For further instructions " - "please refer to the documentation: " - "https://pip.pypa.io/warnings/enable-long-paths" + "HINT: This error might have occurred since " + "this system does not have Windows Long Path " + "support enabled. You can find information on " + "how to enable this at " + "https://pip.pypa.io/warnings/enable-long-paths\n" ) return "".join(parts).strip() + "\n" From c87af084224d560942a6dc7b6e879a12b610c454 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 11 Jun 2021 11:14:59 +0200 Subject: [PATCH 72/85] use setLoggerClass to define log.verbose loaded in pip._internals.__init__ must use utils.logging.getLogger to get the right type annotation instead of logging.getLogger, despite no actual difference in behavior --- src/pip/_internal/__init__.py | 5 ++++ src/pip/_internal/commands/cache.py | 7 ++--- src/pip/_internal/commands/install.py | 7 ++--- src/pip/_internal/network/auth.py | 4 +-- src/pip/_internal/req/req_uninstall.py | 11 ++++---- src/pip/_internal/utils/_log.py | 38 ++++++++++++++++++++++++++ src/pip/_internal/utils/logging.py | 9 ++---- src/pip/_internal/utils/subprocess.py | 5 ++-- 8 files changed, 60 insertions(+), 26 deletions(-) create mode 100644 src/pip/_internal/utils/_log.py diff --git a/src/pip/_internal/__init__.py b/src/pip/_internal/__init__.py index 41071cd86..6db42ae25 100755 --- a/src/pip/_internal/__init__.py +++ b/src/pip/_internal/__init__.py @@ -1,6 +1,11 @@ from typing import List, Optional import pip._internal.utils.inject_securetransport # noqa +from pip._internal.utils import _log + +# init_logging() must be called before any call to logging.getLogger() +# which happens at import of most modules. +_log.init_logging() def main(args=None): diff --git a/src/pip/_internal/commands/cache.py b/src/pip/_internal/commands/cache.py index 37bdfc999..fac9823c1 100644 --- a/src/pip/_internal/commands/cache.py +++ b/src/pip/_internal/commands/cache.py @@ -1,4 +1,3 @@ -import logging import os import textwrap from optparse import Values @@ -8,9 +7,9 @@ import pip._internal.utils.filesystem as filesystem from pip._internal.cli.base_command import Command from pip._internal.cli.status_codes import ERROR, SUCCESS from pip._internal.exceptions import CommandError, PipError -from pip._internal.utils.logging import VERBOSE +from pip._internal.utils.logging import getLogger -logger = logging.getLogger(__name__) +logger = getLogger(__name__) class CacheCommand(Command): @@ -185,7 +184,7 @@ class CacheCommand(Command): for filename in files: os.unlink(filename) - logger.log(VERBOSE, "Removed %s", filename) + logger.verbose("Removed %s", filename) logger.info("Files removed: %s", len(files)) def purge_cache(self, options, args): diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 93fa2f289..ff95d61f5 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -1,5 +1,4 @@ import errno -import logging import operator import os import shutil @@ -28,7 +27,7 @@ from pip._internal.req.req_install import InstallRequirement from pip._internal.req.req_tracker import get_requirement_tracker from pip._internal.utils.distutils_args import parse_distutils_args from pip._internal.utils.filesystem import test_writable_dir -from pip._internal.utils.logging import VERBOSE +from pip._internal.utils.logging import getLogger from pip._internal.utils.misc import ( ensure_dir, get_pip_version, @@ -43,7 +42,7 @@ from pip._internal.wheel_builder import ( should_build_for_install_command, ) -logger = logging.getLogger(__name__) +logger = getLogger(__name__) def get_check_binary_allowed(format_control): @@ -236,7 +235,7 @@ class InstallCommand(RequirementCommand): install_options = options.install_options or [] - logger.log(VERBOSE, "Using %s", get_pip_version()) + logger.verbose("Using %s", get_pip_version()) options.use_user_site = decide_user_install( options.use_user_site, prefix_path=options.prefix_path, diff --git a/src/pip/_internal/network/auth.py b/src/pip/_internal/network/auth.py index bd54a5cba..f92b4af2b 100644 --- a/src/pip/_internal/network/auth.py +++ b/src/pip/_internal/network/auth.py @@ -4,7 +4,6 @@ Contains interface (MultiDomainBasicAuth) and associated glue code for providing credentials in the context of network requests. """ -import logging import urllib.parse from typing import Any, Dict, List, Optional, Tuple @@ -12,6 +11,7 @@ from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth from pip._vendor.requests.models import Request, Response from pip._vendor.requests.utils import get_netrc_auth +from pip._internal.utils.logging import getLogger from pip._internal.utils.misc import ( ask, ask_input, @@ -21,7 +21,7 @@ from pip._internal.utils.misc import ( ) from pip._internal.vcs.versioncontrol import AuthInfo -logger = logging.getLogger(__name__) +logger = getLogger(__name__) Credentials = Tuple[str, str, str] diff --git a/src/pip/_internal/req/req_uninstall.py b/src/pip/_internal/req/req_uninstall.py index fb04b10ae..54c6f3c0d 100644 --- a/src/pip/_internal/req/req_uninstall.py +++ b/src/pip/_internal/req/req_uninstall.py @@ -1,6 +1,5 @@ import csv import functools -import logging import os import sys import sysconfig @@ -13,7 +12,7 @@ from pip._vendor.pkg_resources import Distribution from pip._internal.exceptions import UninstallationError from pip._internal.locations import get_bin_prefix, get_bin_user from pip._internal.utils.compat import WINDOWS -from pip._internal.utils.logging import VERBOSE, indent_log +from pip._internal.utils.logging import getLogger, indent_log from pip._internal.utils.misc import ( ask, dist_in_usersite, @@ -26,7 +25,7 @@ from pip._internal.utils.misc import ( ) from pip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory -logger = logging.getLogger(__name__) +logger = getLogger(__name__) def _script_names(dist, script_name, is_gui): @@ -384,7 +383,7 @@ class UninstallPathSet: for path in sorted(compact(for_rename)): moved.stash(path) - logger.log(VERBOSE, 'Removing file or directory %s', path) + logger.verbose('Removing file or directory %s', path) for pth in self.pth.values(): pth.remove() @@ -599,7 +598,7 @@ class UninstallPthEntries: def remove(self): # type: () -> None - logger.log(VERBOSE, 'Removing pth entries from %s:', self.file) + logger.verbose('Removing pth entries from %s:', self.file) # If the file doesn't exist, log a warning and return if not os.path.isfile(self.file): @@ -620,7 +619,7 @@ class UninstallPthEntries: lines[-1] = lines[-1] + endline.encode("utf-8") for entry in self.entries: try: - logger.log(VERBOSE, 'Removing entry: %s', entry) + logger.verbose('Removing entry: %s', entry) lines.remove((entry + endline).encode("utf-8")) except ValueError: pass diff --git a/src/pip/_internal/utils/_log.py b/src/pip/_internal/utils/_log.py new file mode 100644 index 000000000..92c4c6a19 --- /dev/null +++ b/src/pip/_internal/utils/_log.py @@ -0,0 +1,38 @@ +"""Customize logging + +Defines custom logger class for the `logger.verbose(...)` method. + +init_logging() must be called before any other modules that call logging.getLogger. +""" + +import logging +from typing import Any, cast + +# custom log level for `--verbose` output +# between DEBUG and INFO +VERBOSE = 15 + + +class VerboseLogger(logging.Logger): + """Custom Logger, defining a verbose log-level + + VERBOSE is between INFO and DEBUG. + """ + + def verbose(self, msg: str, *args: Any, **kwargs: Any) -> None: + return self.log(VERBOSE, msg, *args, **kwargs) + + +def getLogger(name: str) -> VerboseLogger: + """logging.getLogger, but ensures our VerboseLogger class is returned""" + return cast(VerboseLogger, logging.getLogger(name)) + + +def init_logging() -> None: + """Register our VerboseLogger and VERBOSE log level. + + Should be called before any calls to getLogger(), + i.e. in pip._internal.__init__ + """ + logging.setLoggerClass(VerboseLogger) + logging.addLevelName(VERBOSE, "VERBOSE") diff --git a/src/pip/_internal/utils/logging.py b/src/pip/_internal/utils/logging.py index 028967124..0569b9248 100644 --- a/src/pip/_internal/utils/logging.py +++ b/src/pip/_internal/utils/logging.py @@ -4,9 +4,10 @@ import logging import logging.handlers import os import sys -from logging import Filter, getLogger +from logging import Filter from typing import IO, Any, Callable, Iterator, Optional, TextIO, Type, cast +from pip._internal.utils._log import VERBOSE, getLogger from pip._internal.utils.compat import WINDOWS from pip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX from pip._internal.utils.misc import ensure_dir @@ -29,11 +30,6 @@ _log_state = threading.local() subprocess_logger = getLogger("pip.subprocessor") -# custom log level for `--verbose` output -# between DEBUG and INFO -VERBOSE = 15 - - class BrokenStdoutLoggingError(Exception): """ Raised if BrokenPipeError occurs for the stdout stream while logging. @@ -276,7 +272,6 @@ def setup_logging(verbosity, no_color, user_log_file): Returns the requested logging level, as its integer value. """ - logging.addLevelName(VERBOSE, "VERBOSE") # Determine the level to be logging at. if verbosity >= 2: level_number = logging.DEBUG diff --git a/src/pip/_internal/utils/subprocess.py b/src/pip/_internal/utils/subprocess.py index 0e73a5e70..19756e1ac 100644 --- a/src/pip/_internal/utils/subprocess.py +++ b/src/pip/_internal/utils/subprocess.py @@ -2,7 +2,6 @@ import logging import os import shlex import subprocess -from functools import partial from typing import Any, Callable, Iterable, List, Mapping, Optional, Union from pip._internal.cli.spinners import SpinnerInterface, open_spinner @@ -145,9 +144,9 @@ def call_subprocess( log_subprocess = subprocess_logger.info used_level = logging.INFO else: - # Then log the subprocess output using DEBUG. This also ensures + # Then log the subprocess output using VERBOSE. This also ensures # it will be logged to the log file (aka user_log), if enabled. - log_subprocess = partial(subprocess_logger.log, VERBOSE) + log_subprocess = subprocess_logger.verbose used_level = VERBOSE # Whether the subprocess will be visible in the console. From 5bc83d44a36da8b959a580b938c424c1190a6e49 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 11 Jun 2021 11:43:47 +0100 Subject: [PATCH 73/85] Fix typo in redirect to topics/authentication --- docs/html/user_guide.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index cfcdfc182..967d78aef 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -63,17 +63,17 @@ For more information and examples, see the :ref:`pip install` reference. Basic Authentication Credentials ================================ -This is now covered in {doc}`topics/authentication`. +This is now covered in :doc:`topics/authentication`. netrc Support ------------- -This is now covered in {doc}`topics/authentication`. +This is now covered in :doc:`topics/authentication`. Keyring Support --------------- -This is now covered in {doc}`topics/authentication`. +This is now covered in :doc:`topics/authentication`. Using a Proxy Server ==================== From 9687f4598f8ae45dfd1401ac36e0fa5b3fce6cd7 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 11 Jun 2021 11:44:26 +0100 Subject: [PATCH 74/85] Add an intersphinx link, to compatibility section This was being used in a different part of our documentation. --- docs/html/installation.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/html/installation.md b/docs/html/installation.md index da9757271..e389a8fa4 100644 --- a/docs/html/installation.md +++ b/docs/html/installation.md @@ -60,6 +60,8 @@ If you face issues when using Python and pip installed using these mechanisms, it is recommended to request for support from the relevant provider (eg: Linux distro community, cloud provider support channels, etc). +(compatibility-requirements)= + ## Compatibility The current version of pip works on: From de8882808bdfe29e3cb1146afd7ec656cf839357 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 11 Jun 2021 11:44:39 +0100 Subject: [PATCH 75/85] Fix typo in version added directive --- docs/html/topics/authentication.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/html/topics/authentication.md b/docs/html/topics/authentication.md index e78aff472..981aab5ab 100644 --- a/docs/html/topics/authentication.md +++ b/docs/html/topics/authentication.md @@ -18,7 +18,7 @@ https://0123456789abcdef@pypi.company.com/simple ### Percent-encoding special characters -```{versionaddded} 10.0 +```{versionadded} 10.0 ``` Certain special characters are not valid in the credential part of a URL. From 3751878b42c9914e1c71aeec62b97ec47fb9accf Mon Sep 17 00:00:00 2001 From: Noah Gorny Date: Fri, 11 Jun 2021 14:01:14 +0300 Subject: [PATCH 76/85] Implement new command 'pip index versions' --- news/7975.feature.rst | 2 + src/pip/_internal/commands/__init__.py | 4 + src/pip/_internal/commands/index.py | 143 +++++++++++++++++++++++++ src/pip/_internal/commands/search.py | 31 +++--- tests/functional/test_index.py | 75 +++++++++++++ tests/unit/test_commands.py | 7 +- 6 files changed, 247 insertions(+), 15 deletions(-) create mode 100644 news/7975.feature.rst create mode 100644 src/pip/_internal/commands/index.py create mode 100644 tests/functional/test_index.py diff --git a/news/7975.feature.rst b/news/7975.feature.rst new file mode 100644 index 000000000..b0638939b --- /dev/null +++ b/news/7975.feature.rst @@ -0,0 +1,2 @@ +Add new subcommand ``pip index`` used to interact with indexes, and implement +``pip index version`` to list available versions of a package. diff --git a/src/pip/_internal/commands/__init__.py b/src/pip/_internal/commands/__init__.py index 31c985fdc..e1fb87884 100644 --- a/src/pip/_internal/commands/__init__.py +++ b/src/pip/_internal/commands/__init__.py @@ -59,6 +59,10 @@ commands_dict = OrderedDict([ 'pip._internal.commands.cache', 'CacheCommand', "Inspect and manage pip's wheel cache.", )), + ('index', CommandInfo( + 'pip._internal.commands.index', 'IndexCommand', + "Inspect information available from package indexes.", + )), ('wheel', CommandInfo( 'pip._internal.commands.wheel', 'WheelCommand', 'Build wheels from your requirements.', diff --git a/src/pip/_internal/commands/index.py b/src/pip/_internal/commands/index.py new file mode 100644 index 000000000..4bfc4e9e3 --- /dev/null +++ b/src/pip/_internal/commands/index.py @@ -0,0 +1,143 @@ +import logging +from optparse import Values +from typing import Any, Iterable, List, Optional, Union + +from pip._vendor.packaging.version import LegacyVersion, Version + +from pip._internal.cli import cmdoptions +from pip._internal.cli.req_command import IndexGroupCommand +from pip._internal.cli.status_codes import ERROR, SUCCESS +from pip._internal.commands.search import print_dist_installation_info +from pip._internal.exceptions import CommandError, DistributionNotFound, PipError +from pip._internal.index.collector import LinkCollector +from pip._internal.index.package_finder import PackageFinder +from pip._internal.models.selection_prefs import SelectionPreferences +from pip._internal.models.target_python import TargetPython +from pip._internal.network.session import PipSession +from pip._internal.utils.misc import write_output + +logger = logging.getLogger(__name__) + + +class IndexCommand(IndexGroupCommand): + """ + Inspect information available from package indexes. + """ + + usage = """ + %prog versions + """ + + def add_options(self): + # type: () -> None + cmdoptions.add_target_python_options(self.cmd_opts) + + self.cmd_opts.add_option(cmdoptions.ignore_requires_python()) + self.cmd_opts.add_option(cmdoptions.pre()) + self.cmd_opts.add_option(cmdoptions.no_binary()) + self.cmd_opts.add_option(cmdoptions.only_binary()) + + index_opts = cmdoptions.make_option_group( + cmdoptions.index_group, + self.parser, + ) + + self.parser.insert_option_group(0, index_opts) + self.parser.insert_option_group(0, self.cmd_opts) + + def run(self, options, args): + # type: (Values, List[Any]) -> int + handlers = { + "versions": self.get_available_package_versions, + } + + logger.warning( + "pip index is currently an experimental command. " + "It may be removed/changed in a future release " + "without prior warning." + ) + + # Determine action + if not args or args[0] not in handlers: + logger.error( + "Need an action (%s) to perform.", + ", ".join(sorted(handlers)), + ) + return ERROR + + action = args[0] + + # Error handling happens here, not in the action-handlers. + try: + handlers[action](options, args[1:]) + except PipError as e: + logger.error(e.args[0]) + return ERROR + + return SUCCESS + + def _build_package_finder( + self, + options, # type: Values + session, # type: PipSession + target_python=None, # type: Optional[TargetPython] + ignore_requires_python=None, # type: Optional[bool] + ): + # type: (...) -> PackageFinder + """ + Create a package finder appropriate to the index command. + """ + link_collector = LinkCollector.create(session, options=options) + + # Pass allow_yanked=False to ignore yanked versions. + selection_prefs = SelectionPreferences( + allow_yanked=False, + allow_all_prereleases=options.pre, + ignore_requires_python=ignore_requires_python, + ) + + return PackageFinder.create( + link_collector=link_collector, + selection_prefs=selection_prefs, + target_python=target_python, + ) + + def get_available_package_versions(self, options, args): + # type: (Values, List[Any]) -> None + if len(args) != 1: + raise CommandError('You need to specify exactly one argument') + + target_python = cmdoptions.make_target_python(options) + query = args[0] + + with self._build_session(options) as session: + finder = self._build_package_finder( + options=options, + session=session, + target_python=target_python, + ignore_requires_python=options.ignore_requires_python, + ) + + versions: Iterable[Union[LegacyVersion, Version]] = ( + candidate.version + for candidate in finder.find_all_candidates(query) + ) + + if not options.pre: + # Remove prereleases + versions = (version for version in versions + if not version.is_prerelease) + versions = set(versions) + + if not versions: + raise DistributionNotFound( + 'No matching distribution found for {}'.format(query)) + + formatted_versions = [str(ver) for ver in sorted( + versions, reverse=True)] + latest = formatted_versions[0] + + write_output('{} ({})'.format(query, latest)) + write_output('Available versions: {}'.format( + ', '.join(formatted_versions))) + print_dist_installation_info(query, latest) diff --git a/src/pip/_internal/commands/search.py b/src/pip/_internal/commands/search.py index d66e82347..3bfd29afc 100644 --- a/src/pip/_internal/commands/search.py +++ b/src/pip/_internal/commands/search.py @@ -114,6 +114,23 @@ def transform_hits(hits): return list(packages.values()) +def print_dist_installation_info(name, latest): + # type: (str, str) -> None + env = get_default_environment() + dist = env.get_distribution(name) + if dist is not None: + with indent_log(): + if dist.version == latest: + write_output('INSTALLED: %s (latest)', dist.version) + else: + write_output('INSTALLED: %s', dist.version) + if parse_version(latest).pre: + write_output('LATEST: %s (pre-release; install' + ' with "pip install --pre")', latest) + else: + write_output('LATEST: %s', latest) + + def print_results(hits, name_column_width=None, terminal_width=None): # type: (List[TransformedHit], Optional[int], Optional[int]) -> None if not hits: @@ -124,7 +141,6 @@ def print_results(hits, name_column_width=None, terminal_width=None): for hit in hits ]) + 4 - env = get_default_environment() for hit in hits: name = hit['name'] summary = hit['summary'] or '' @@ -141,18 +157,7 @@ def print_results(hits, name_column_width=None, terminal_width=None): line = f'{name_latest:{name_column_width}} - {summary}' try: write_output(line) - dist = env.get_distribution(name) - if dist is not None: - with indent_log(): - if dist.version == latest: - write_output('INSTALLED: %s (latest)', dist.version) - else: - write_output('INSTALLED: %s', dist.version) - if parse_version(latest).pre: - write_output('LATEST: %s (pre-release; install' - ' with "pip install --pre")', latest) - else: - write_output('LATEST: %s', latest) + print_dist_installation_info(name, latest) except UnicodeEncodeError: pass diff --git a/tests/functional/test_index.py b/tests/functional/test_index.py new file mode 100644 index 000000000..004e672a5 --- /dev/null +++ b/tests/functional/test_index.py @@ -0,0 +1,75 @@ +import pytest + +from pip._internal.cli.status_codes import ERROR, SUCCESS +from pip._internal.commands import create_command + + +@pytest.mark.network +def test_list_all_versions_basic_search(script): + """ + End to end test of index versions command. + """ + output = script.pip('index', 'versions', 'pip', allow_stderr_warning=True) + assert 'Available versions:' in output.stdout + assert ( + '20.2.3, 20.2.2, 20.2.1, 20.2, 20.1.1, 20.1, 20.0.2' + ', 20.0.1, 19.3.1, 19.3, 19.2.3, 19.2.2, 19.2.1, 19.2, 19.1.1' + ', 19.1, 19.0.3, 19.0.2, 19.0.1, 19.0, 18.1, 18.0, 10.0.1, 10.0.0, ' + '9.0.3, 9.0.2, 9.0.1, 9.0.0, 8.1.2, 8.1.1, ' + '8.1.0, 8.0.3, 8.0.2, 8.0.1, 8.0.0, 7.1.2, 7.1.1, 7.1.0, 7.0.3, ' + '7.0.2, 7.0.1, 7.0.0, 6.1.1, 6.1.0, 6.0.8, 6.0.7, 6.0.6, 6.0.5, ' + '6.0.4, 6.0.3, 6.0.2, 6.0.1, 6.0, 1.5.6, 1.5.5, 1.5.4, 1.5.3, ' + '1.5.2, 1.5.1, 1.5, 1.4.1, 1.4, 1.3.1, 1.3, 1.2.1, 1.2, 1.1, 1.0.2,' + ' 1.0.1, 1.0, 0.8.3, 0.8.2, 0.8.1, 0.8, 0.7.2, 0.7.1, 0.7, 0.6.3, ' + '0.6.2, 0.6.1, 0.6, 0.5.1, 0.5, 0.4, 0.3.1, ' + '0.3, 0.2.1, 0.2' in output.stdout + ) + + +@pytest.mark.network +def test_list_all_versions_search_with_pre(script): + """ + See that adding the --pre flag adds pre-releases + """ + output = script.pip( + 'index', 'versions', 'pip', '--pre', allow_stderr_warning=True) + assert 'Available versions:' in output.stdout + assert ( + '20.2.3, 20.2.2, 20.2.1, 20.2, 20.2b1, 20.1.1, 20.1, 20.1b1, 20.0.2' + ', 20.0.1, 19.3.1, 19.3, 19.2.3, 19.2.2, 19.2.1, 19.2, 19.1.1' + ', 19.1, 19.0.3, 19.0.2, 19.0.1, 19.0, 18.1, 18.0, 10.0.1, 10.0.0, ' + '10.0.0b2, 10.0.0b1, 9.0.3, 9.0.2, 9.0.1, 9.0.0, 8.1.2, 8.1.1, ' + '8.1.0, 8.0.3, 8.0.2, 8.0.1, 8.0.0, 7.1.2, 7.1.1, 7.1.0, 7.0.3, ' + '7.0.2, 7.0.1, 7.0.0, 6.1.1, 6.1.0, 6.0.8, 6.0.7, 6.0.6, 6.0.5, ' + '6.0.4, 6.0.3, 6.0.2, 6.0.1, 6.0, 1.5.6, 1.5.5, 1.5.4, 1.5.3, ' + '1.5.2, 1.5.1, 1.5, 1.4.1, 1.4, 1.3.1, 1.3, 1.2.1, 1.2, 1.1, 1.0.2,' + ' 1.0.1, 1.0, 0.8.3, 0.8.2, 0.8.1, 0.8, 0.7.2, 0.7.1, 0.7, 0.6.3, ' + '0.6.2, 0.6.1, 0.6, 0.5.1, 0.5, 0.4, 0.3.1, ' + '0.3, 0.2.1, 0.2' in output.stdout + ) + + +@pytest.mark.network +def test_list_all_versions_returns_no_matches_found_when_name_not_exact(): + """ + Test that non exact name do not match + """ + command = create_command('index') + cmdline = "versions pand" + with command.main_context(): + options, args = command.parse_args(cmdline.split()) + status = command.run(options, args) + assert status == ERROR + + +@pytest.mark.network +def test_list_all_versions_returns_matches_found_when_name_is_exact(): + """ + Test that exact name matches + """ + command = create_command('index') + cmdline = "versions pandas" + with command.main_context(): + options, args = command.parse_args(cmdline.split()) + status = command.run(options, args) + assert status == SUCCESS diff --git a/tests/unit/test_commands.py b/tests/unit/test_commands.py index f34f7e538..cb144c5f6 100644 --- a/tests/unit/test_commands.py +++ b/tests/unit/test_commands.py @@ -11,7 +11,8 @@ from pip._internal.commands import commands_dict, create_command # These are the expected names of the commands whose classes inherit from # IndexGroupCommand. -EXPECTED_INDEX_GROUP_COMMANDS = ['download', 'install', 'list', 'wheel'] +EXPECTED_INDEX_GROUP_COMMANDS = [ + 'download', 'index', 'install', 'list', 'wheel'] def check_commands(pred, expected): @@ -49,7 +50,9 @@ def test_session_commands(): def is_session_command(command): return isinstance(command, SessionCommandMixin) - expected = ['download', 'install', 'list', 'search', 'uninstall', 'wheel'] + expected = [ + 'download', 'index', 'install', 'list', 'search', 'uninstall', 'wheel' + ] check_commands(is_session_command, expected) From 119671fc1adf4fb6b78cb0989d83a7cc301cc082 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 11 Jun 2021 12:07:45 +0100 Subject: [PATCH 77/85] Reword the changelog entry for verbosity changes --- news/9450.feature.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/news/9450.feature.rst b/news/9450.feature.rst index ca3d8081e..f7b21c735 100644 --- a/news/9450.feature.rst +++ b/news/9450.feature.rst @@ -1,2 +1 @@ -Require ``-vv`` for full debug-level output, ``-v`` now enables an intermediate level of verbosity, -e.g. showing subprocess output of ``setup.py install`` but not all dependency resolution debug output. +Add an additional level of verbosity. ``--verbose`` now contains significantly less output, and users that need complete full debug-level output should pass it twice. From c495eab2acb0c7adde09740a46a138b33bc6ca00 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 11 Jun 2021 12:11:50 +0100 Subject: [PATCH 78/85] Incorporate suggested rephrasing Co-authored-by: Tzu-ping Chung --- news/9450.feature.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/news/9450.feature.rst b/news/9450.feature.rst index f7b21c735..5e54ad301 100644 --- a/news/9450.feature.rst +++ b/news/9450.feature.rst @@ -1 +1,3 @@ -Add an additional level of verbosity. ``--verbose`` now contains significantly less output, and users that need complete full debug-level output should pass it twice. +Add an additional level of verbosity. ``--verbose`` (and the shorthand ``-v``) now +contains significantly less output, and users that need complete full debug-level output +should pass it twice (``--verbose --verbose`` or ``-vv``). From d63f06a881a54823fcc4798989d532797e772789 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 28 May 2021 16:11:58 +0100 Subject: [PATCH 79/85] Add a topic guide: Caching --- docs/html/cli/pip_install.rst | 52 +-------------------- docs/html/topics/caching.md | 86 +++++++++++++++++++++++++++++++++++ docs/html/topics/index.md | 1 + 3 files changed, 89 insertions(+), 50 deletions(-) create mode 100644 docs/html/topics/caching.md diff --git a/docs/html/cli/pip_install.rst b/docs/html/cli/pip_install.rst index 9ebb6e3f7..6bc15349d 100644 --- a/docs/html/cli/pip_install.rst +++ b/docs/html/cli/pip_install.rst @@ -574,62 +574,14 @@ overridden by using ``--cert`` option or by using ``PIP_CERT``, Caching ------- -Starting with v6.0, pip provides an on-by-default cache which functions -similarly to that of a web browser. While the cache is on by default and is -designed do the right thing by default you can disable the cache and always -access PyPI by utilizing the ``--no-cache-dir`` option. - -When making any HTTP request pip will first check its local cache to determine -if it has a suitable response stored for that request which has not expired. If -it does then it simply returns that response and doesn't make the request. - -If it has a response stored, but it has expired, then it will attempt to make a -conditional request to refresh the cache which will either return an empty -response telling pip to simply use the cached item (and refresh the expiration -timer) or it will return a whole new response which pip can then store in the -cache. - -While this cache attempts to minimize network activity, it does not prevent -network access altogether. If you want a local install solution that -circumvents accessing PyPI, see :ref:`Installing from local packages`. - -The default location for the cache directory depends on the operating system: - -Unix - :file:`~/.cache/pip` and it respects the ``XDG_CACHE_HOME`` directory. -macOS - :file:`~/Library/Caches/pip`. -Windows - :file:`\\pip\\Cache` - -Run ``pip cache dir`` to show the cache directory and see :ref:`pip cache` to -inspect and manage pip’s cache. - +This is now covered in :doc:`../topics/caching` .. _`Wheel cache`: Wheel Cache ^^^^^^^^^^^ -pip will read from the subdirectory ``wheels`` within the pip cache directory -and use any packages found there. This is disabled via the same -``--no-cache-dir`` option that disables the HTTP cache. The internal structure -of that is not part of the pip API. As of 7.0, pip makes a subdirectory for -each sdist that wheels are built from and places the resulting wheels inside. - -As of version 20.0, pip also caches wheels when building from an immutable Git -reference (i.e. a commit hash). - -pip attempts to choose the best wheels from those built in preference to -building a new wheel. Note that this means when a package has both optional -C extensions and builds ``py`` tagged wheels when the C extension can't be built -that pip will not attempt to build a better wheel for Pythons that would have -supported it, once any generic wheel is built. To correct this, make sure that -the wheels are built with Python specific tags - e.g. pp on PyPy. - -When no wheels are found for an sdist, pip will attempt to build a wheel -automatically and insert it into the wheel cache. - +This is now covered in :doc:`../topics/caching` .. _`hash-checking mode`: diff --git a/docs/html/topics/caching.md b/docs/html/topics/caching.md new file mode 100644 index 000000000..0f4dfe9b9 --- /dev/null +++ b/docs/html/topics/caching.md @@ -0,0 +1,86 @@ +# Caching + +```{versionadded} 6.0 +``` + +pip provides an on-by-default caching, designed to reduce the amount of time +spent on duplicate downloads and builds. + +## What is cached + +### HTTP responses + +This cache functions like a web browser cache. + +When making any HTTP request, pip will first check its local cache to determine +if it has a suitable response stored for that request which has not expired. If +it does then it returns that response and doesn't re-download the content. + +If it has a response stored but it has expired, then it will attempt to make a +conditional request to refresh the cache which will either return an empty +response telling pip to simply use the cached item (and refresh the expiration +timer) or it will return a whole new response which pip can then store in the +cache. + +While this cache attempts to minimize network activity, it does not prevent +network access altogether. If you want a local install solution that +circumvents accessing PyPI, see {ref}`Installing from local packages`. + +### Locally built wheels + +pip attempts to use wheels from its local wheel cache whenever possible. + +This means that if there is a cached wheel for the same version of a specific +package name, pip will use that wheel instead of rebuilding the project. + +When no wheels are found for a source distribution, pip will attempt to build a +wheel using the package's build system. If the build is successful, this wheel +is added to the cache and used in subsequent installs for the same package +version. + +```{versionchanged} 20.0 +pip now caches wheels when building from an immutable Git reference +(i.e. a commit hash). +``` + +## Avoiding caching + +pip tries to use its cache whenever possible, and it is designed do the right +thing by default. + +In some cases, pip's caching behaviour can be undesirable. As an example, if you +have package with optional C extensions, that generates a pure Python wheel +when the C extension can’t be built, pip will use that cached wheel even when +you later invoke it from an environment that could have built those optional C +extensions. This is because pip is seeing a cached wheel for that matches the +package being built, and pip assumes that the result of building a package from +a package index is deterministic. + +The recommended approach for dealing with these situations is to directly +install from a source distribution instead of letting pip auto-discover the +package when it is trying to install. Installing directly from a source +distribution will make pip build a wheel, regardless of whether there is a +matching cached wheel. This usually means doing something like: + +```{pip-cli} +$ pip download sampleproject==1.0.0 --no-binary :all: +$ pip install sampleproject-1.0.0.tar.gz +``` + +It is also a good idea to remove the offending cached wheel using the +{ref}`pip cache` command. + +## Cache management + +The {ref}`pip cache` command can be used to manage pip's cache. + +The exact filesystem structure of pip's cache is considered to be an +implementation detail and may change between any two versions of pip. + +## Disabling caching + +pip's caching behaviour is disabled by passing the ``--no-cache-dir`` option. + +It is, however, recommended to **NOT** disable pip's caching. Doing so can +significantly slow down pip (due to repeated operations and package builds) +and result in significantly more network usage. diff --git a/docs/html/topics/index.md b/docs/html/topics/index.md index 6e815ebc6..d82e7afdb 100644 --- a/docs/html/topics/index.md +++ b/docs/html/topics/index.md @@ -11,4 +11,5 @@ This section of the documentation is currently being fleshed out. See :maxdepth: 1 authentication +caching ``` From 330cfa38e291844aed984032a3e2b572accdd59a Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 11 Jun 2021 16:12:00 +0100 Subject: [PATCH 80/85] Enable definition lists in MyST This is useful for presenting content. --- docs/html/conf.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/html/conf.py b/docs/html/conf.py index 9e210539e..64fddeffc 100644 --- a/docs/html/conf.py +++ b/docs/html/conf.py @@ -52,6 +52,10 @@ with open(file_with_version) as f: print("pip version:", version) print("pip release:", release) +# -- Options for myst-parser ---------------------------------------------------------- + +myst_enable_extensions = ["deflist"] + # -- Options for smartquotes ---------------------------------------------------------- # Disable the conversion of dashes so that long options like "--find-links" won't From 0ab159a3f2d6ba54515fc99e077c0d22702c4a30 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 11 Jun 2021 16:09:29 +0100 Subject: [PATCH 81/85] Add topic guide: Configuration --- docs/html/topics/configuration.md | 226 ++++++++++++++++++++++++++++++ docs/html/topics/index.md | 1 + docs/html/user_guide.rst | 226 +----------------------------- 3 files changed, 232 insertions(+), 221 deletions(-) create mode 100644 docs/html/topics/configuration.md diff --git a/docs/html/topics/configuration.md b/docs/html/topics/configuration.md new file mode 100644 index 000000000..90799d574 --- /dev/null +++ b/docs/html/topics/configuration.md @@ -0,0 +1,226 @@ +# Configuration + +pip allows a user to change its behaviour via 3 mechanisms: + +- command line options +- environment variables +- configuration files + +This page explains how the configuration files and environment variables work, +and how they are related to pip's various command line options. + +## Configuration Files + +Configuration files can change the default values for command line option. +They are written using a standard INI style configuration files. + +pip has 3 "levels" of configuration files: + +- `global`: system-wide configuration file, shared across users. +- `user`: per-user configuration file. +- `site`: per-environment configuration file; i.e. per-virtualenv. + +### Location + +pip's configuration files are located in fairly standard locations. This +location is different on different operating systems, and has some additional +complexity for backwards compatibility reasons. + +```{tab} Unix + +Global +: {file}`/etc/pip.conf` + + Alternatively, it may be in a "pip" subdirectory of any of the paths set + in the environment variable `XDG_CONFIG_DIRS` (if it exists), for + example {file}`/etc/xdg/pip/pip.conf`. + +User +: {file}`$HOME/.config/pip/pip.conf`, which respects the `XDG_CONFIG_HOME` environment variable. + + The legacy "per-user" configuration file is also loaded, if it exists: {file}`$HOME/.pip/pip.conf`. + +Site +: {file}`$VIRTUAL_ENV/pip.conf` +``` + +```{tab} MacOS + +Global +: {file}`/Library/Application Support/pip/pip.conf` + +User +: {file}`$HOME/Library/Application Support/pip/pip.conf` + if directory `$HOME/Library/Application Support/pip` exists + else {file}`$HOME/.config/pip/pip.conf` + + The legacy "per-user" configuration file is also loaded, if it exists: {file}`$HOME/.pip/pip.conf`. + +Site +: {file}`$VIRTUAL_ENV/pip.conf` +``` + +```{tab} Windows + +Global +: * On Windows 7 and later: {file}`C:\\ProgramData\\pip\\pip.ini` + (hidden but writeable) + * On Windows Vista: Global configuration is not supported. + * On Windows XP: + {file}`C:\\Documents and Settings\\All Users\\Application Data\\pip\\pip.ini` + +User +: {file}`%APPDATA%\\pip\\pip.ini` + + The legacy "per-user" configuration file is also loaded, if it exists: {file}`%HOME%\\pip\\pip.ini` + +Site +: {file}`%VIRTUAL_ENV%\\pip.ini` +``` + +### `PIP_CONFIG_FILE` + +Additionally, the environment variable `PIP_CONFIG_FILE` can be used to specify +a configuration file that's loaded first, and whose values are overridden by +the values set in the aforementioned files. Setting this to {ref}`os.devnull` +disables the loading of _all_ configuration files. + +### Loading order + +When multiple configuration files are found, pip combines them in the following +order: + +- `PIP_CONFIG_FILE`, if given. +- Global +- User +- Site + +Each file read overrides any values read from previous files, so if the +global timeout is specified in both the global file and the per-user file +then the latter value will be used. + +### Naming + +The names of the settings are derived from the long command line option. + +As an example, if you want to use a different package index (`--index-url`) and +set the HTTP timeout (`--default-timeout`) to 60 seconds, your config file would +look like this: + +```ini +[global] +timeout = 60 +index-url = https://download.zope.org/ppix +``` + +### Per-command section + +Each subcommand can be configured optionally in its own section. This overrides +the global setting with the same name. + +As an example, if you want to decrease the `timeout` to `10` seconds when +running the {ref}`pip freeze`, and use `60` seconds for all other commands: + +```ini +[global] +timeout = 60 + +[freeze] +timeout = 10 +``` + +### Boolean options + +Boolean options like `--ignore-installed` or `--no-dependencies` can be set +like this: + +```ini +[install] +ignore-installed = true +no-dependencies = yes +``` + +To enable the boolean options `--no-compile`, `--no-warn-script-location` and +`--no-cache-dir`, falsy values have to be used: + +```ini +[global] +no-cache-dir = false + +[install] +no-compile = no +no-warn-script-location = false +``` + +### Repeatable options + +For options which can be repeated like `--verbose` and `--quiet`, a +non-negative integer can be used to represent the level to be specified: + +```ini +[global] +quiet = 0 +verbose = 2 +``` + +It is possible to append values to a section within a configuration file. This +is applicable to appending options like `--find-links` or `--trusted-host`, +which can be written on multiple lines: + +```ini +[global] +find-links = + http://download.example.com + +[install] +find-links = + http://mirror1.example.com + http://mirror2.example.com + +trusted-host = + mirror1.example.com + mirror2.example.com +``` + +This enables users to add additional values in the order of entry for such +command line arguments. + +## Environment Variables + +pip's command line options can be set with environment variables using the +format `PIP_` . Dashes (`-`) have to be replaced with +underscores (`_`). + +- `PIP_DEFAULT_TIMEOUT=60` is the same as `--default-timeout=60` +- ``` + PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" + ``` + + is the same as + + ``` + --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com + ``` + +Repeatable options that do not take a value (such as `--verbose`) can be +specified using the number of repetitions: + +- `PIP_VERBOSE=3` is the same as `pip install -vvv` + +```{note} +Environment variables set to an empty string (like with `export X=` on Unix) will **not** be treated as false. +Use `no`, `false` or `0` instead. +``` + +## Precedence / Override order + +Command line options have override environment variables, which override the +values in a configuration file. Within the configuration file, values in +command-specific sections over values in the global section. + +Examples: + +- `--host=foo` overrides `PIP_HOST=foo` +- `PIP_HOST=foo` overrides a config file with `[global] host = foo` +- A command specific section in the config file `[] host = bar` + overrides the option with same name in the `[global]` config file section. diff --git a/docs/html/topics/index.md b/docs/html/topics/index.md index d82e7afdb..478aacf2a 100644 --- a/docs/html/topics/index.md +++ b/docs/html/topics/index.md @@ -12,4 +12,5 @@ This section of the documentation is currently being fleshed out. See authentication caching +configuration ``` diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 967d78aef..e86fdb48c 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -437,242 +437,26 @@ For more information and examples, see the :ref:`pip search` reference. Configuration ============= +This is now covered in :doc:`topics/configuration`. + .. _config-file: Config file ----------- -pip allows you to set all command line option defaults in a standard ini -style config file. - -The names and locations of the configuration files vary slightly across -platforms. You may have per-user, per-virtualenv or global (shared amongst -all users) configuration: - -**Per-user**: - -* On Unix the default configuration file is: :file:`$HOME/.config/pip/pip.conf` - which respects the ``XDG_CONFIG_HOME`` environment variable. -* On macOS the configuration file is - :file:`$HOME/Library/Application Support/pip/pip.conf` - if directory ``$HOME/Library/Application Support/pip`` exists - else :file:`$HOME/.config/pip/pip.conf`. -* On Windows the configuration file is :file:`%APPDATA%\\pip\\pip.ini`. - -There is also a legacy per-user configuration file which is also respected. -To find its location: - -* On Unix and macOS the configuration file is: :file:`$HOME/.pip/pip.conf` -* On Windows the configuration file is: :file:`%HOME%\\pip\\pip.ini` - -You can set a custom path location for this config file using the environment -variable ``PIP_CONFIG_FILE``. - -**Inside a virtualenv**: - -* On Unix and macOS the file is :file:`$VIRTUAL_ENV/pip.conf` -* On Windows the file is: :file:`%VIRTUAL_ENV%\\pip.ini` - -**Global**: - -* On Unix the file may be located in :file:`/etc/pip.conf`. Alternatively - it may be in a "pip" subdirectory of any of the paths set in the - environment variable ``XDG_CONFIG_DIRS`` (if it exists), for example - :file:`/etc/xdg/pip/pip.conf`. -* On macOS the file is: :file:`/Library/Application Support/pip/pip.conf` -* On Windows XP the file is: - :file:`C:\\Documents and Settings\\All Users\\Application Data\\pip\\pip.ini` -* On Windows 7 and later the file is hidden, but writeable at - :file:`C:\\ProgramData\\pip\\pip.ini` -* Global configuration is not supported on Windows Vista. - -The global configuration file is shared by all Python installations. - -If multiple configuration files are found by pip then they are combined in -the following order: - -1. The global file is read -2. The per-user file is read -3. The virtualenv-specific file is read - -Each file read overrides any values read from previous files, so if the -global timeout is specified in both the global file and the per-user file -then the latter value will be used. - -The names of the settings are derived from the long command line option, e.g. -if you want to use a different package index (``--index-url``) and set the -HTTP timeout (``--default-timeout``) to 60 seconds your config file would -look like this: - -.. code-block:: ini - - [global] - timeout = 60 - index-url = https://download.zope.org/ppix - -Each subcommand can be configured optionally in its own section so that every -global setting with the same name will be overridden; e.g. decreasing the -``timeout`` to ``10`` seconds when running the ``freeze`` -(:ref:`pip freeze`) command and using -``60`` seconds for all other commands is possible with: - -.. code-block:: ini - - [global] - timeout = 60 - - [freeze] - timeout = 10 - - -Boolean options like ``--ignore-installed`` or ``--no-dependencies`` can be -set like this: - -.. code-block:: ini - - [install] - ignore-installed = true - no-dependencies = yes - -To enable the boolean options ``--no-compile``, ``--no-warn-script-location`` -and ``--no-cache-dir``, falsy values have to be used: - -.. code-block:: ini - - [global] - no-cache-dir = false - - [install] - no-compile = no - no-warn-script-location = false - -For options which can be repeated like ``--verbose`` and ``--quiet``, -a non-negative integer can be used to represent the level to be specified: - -.. code-block:: ini - - [global] - quiet = 0 - verbose = 2 - -It is possible to append values to a section within a configuration file such as the pip.ini file. -This is applicable to appending options like ``--find-links`` or ``--trusted-host``, -which can be written on multiple lines: - -.. code-block:: ini - - [global] - find-links = - http://download.example.com - - [install] - find-links = - http://mirror1.example.com - http://mirror2.example.com - - trusted-host = - mirror1.example.com - mirror2.example.com - -This enables users to add additional values in the order of entry for such command line arguments. - +This is now covered in :doc:`topics/configuration`. Environment Variables --------------------- -pip's command line options can be set with environment variables using the -format ``PIP_`` . Dashes (``-``) have to be replaced with -underscores (``_``). - -For example, to set the default timeout: - -.. tab:: Unix/macOS - - .. code-block:: shell - - export PIP_DEFAULT_TIMEOUT=60 - -.. tab:: Windows - - .. code-block:: shell - - set PIP_DEFAULT_TIMEOUT=60 - -This is the same as passing the option to pip directly: - -.. tab:: Unix/macOS - - .. code-block:: shell - - python -m pip --default-timeout=60 [...] - -.. tab:: Windows - - .. code-block:: shell - - py -m pip --default-timeout=60 [...] - -For command line options which can be repeated, use a space to separate -multiple values. For example: - -.. tab:: Unix/macOS - - .. code-block:: shell - - export PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" - -.. tab:: Windows - - .. code-block:: shell - - set PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" - -is the same as calling: - -.. tab:: Unix/macOS - - .. code-block:: shell - - python -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com - -.. tab:: Windows - - .. code-block:: shell - - py -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com - -Options that do not take a value, but can be repeated (such as ``--verbose``) -can be specified using the number of repetitions, so:: - - export PIP_VERBOSE=3 - -is the same as calling:: - - pip install -vvv - -.. note:: - - Environment variables set to be empty string will not be treated as false. - Please use ``no``, ``false`` or ``0`` instead. - +This is now covered in :doc:`topics/configuration`. .. _config-precedence: Config Precedence ----------------- -Command line options have precedence over environment variables, which have -precedence over the config file. - -Within the config file, command specific sections have precedence over the -global section. - -Examples: - -- ``--host=foo`` overrides ``PIP_HOST=foo`` -- ``PIP_HOST=foo`` overrides a config file with ``[global] host = foo`` -- A command specific section in the config file ``[] host = bar`` - overrides the option with same name in the ``[global]`` config file section +This is now covered in :doc:`topics/configuration`. Command Completion From 185120d725b538ef4bc3813dcab79fad7500d8df Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 13 Jun 2021 11:29:32 -0400 Subject: [PATCH 82/85] Stop relying on undocumented duck typing by `urlunsplit()` There are proposals in CPython to enforce correct types (str, bytes, bytearray) in urllib.parse: bpo-19094 and bpo-22234. --- news/10057.trivial.rst | 0 src/pip/_internal/models/link.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 news/10057.trivial.rst diff --git a/news/10057.trivial.rst b/news/10057.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index 86d0be407..ebee38395 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -150,7 +150,7 @@ class Link(KeyBasedCompareMixin): def url_without_fragment(self): # type: () -> str scheme, netloc, path, query, fragment = self._parsed_url - return urllib.parse.urlunsplit((scheme, netloc, path, query, None)) + return urllib.parse.urlunsplit((scheme, netloc, path, query, '')) _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)') From c4e83dc01d41d03e903087476d3a160d2e1d35b6 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 13 Jun 2021 11:41:32 -0400 Subject: [PATCH 83/85] Rename empty trivial news entry --- news/{10057.trivial.rst => 10064.trivial.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename news/{10057.trivial.rst => 10064.trivial.rst} (100%) diff --git a/news/10057.trivial.rst b/news/10064.trivial.rst similarity index 100% rename from news/10057.trivial.rst rename to news/10064.trivial.rst From f5f91351e7e41991374ef33b8cb0cbe9d852d862 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 1 Jun 2021 03:19:50 +0800 Subject: [PATCH 84/85] Rework resolution ordering to consider "depth" --- news/9455.feature.rst | 2 + .../resolution/resolvelib/provider.py | 83 ++++++++++--------- 2 files changed, 47 insertions(+), 38 deletions(-) create mode 100644 news/9455.feature.rst diff --git a/news/9455.feature.rst b/news/9455.feature.rst new file mode 100644 index 000000000..f33f33174 --- /dev/null +++ b/news/9455.feature.rst @@ -0,0 +1,2 @@ +New resolver: The order of dependencies resolution has been tweaked to traverse +the dependency graph in a more breadth-first approach. diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index 9a8c29980..e6b5bd544 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -1,3 +1,5 @@ +import collections +import math from typing import TYPE_CHECKING, Dict, Iterable, Iterator, Mapping, Sequence, Union from pip._vendor.resolvelib.providers import AbstractProvider @@ -60,6 +62,7 @@ class PipProvider(_ProviderBase): self._ignore_dependencies = ignore_dependencies self._upgrade_strategy = upgrade_strategy self._user_requested = user_requested + self._known_depths: Dict[str, float] = collections.defaultdict(lambda: math.inf) def identify(self, requirement_or_candidate): # type: (Union[Requirement, Candidate]) -> str @@ -79,48 +82,43 @@ class PipProvider(_ProviderBase): Currently pip considers the followings in order: - * Prefer if any of the known requirements points to an explicit URL. - * If equal, prefer if any requirements contain ``===`` and ``==``. - * If equal, prefer if requirements include version constraints, e.g. - ``>=`` and ``<``. - * If equal, prefer user-specified (non-transitive) requirements, and - order user-specified requirements by the order they are specified. + * Prefer if any of the known requirements is "direct", e.g. points to an + explicit URL. + * If equal, prefer if any requirement is "pinned", i.e. contains + operator ``===`` or ``==``. + * If equal, calculate an approximate "depth" and resolve requirements + closer to the user-specified requirements first. + * Order user-specified requirements by the order they are specified. + * If equal, prefers "non-free" requirements, i.e. contains at least one + operator, such as ``>=`` or ``<``. * If equal, order alphabetically for consistency (helps debuggability). """ + lookups = (r.get_candidate_lookup() for r, _ in information[identifier]) + candidate, ireqs = zip(*lookups) + operators = [ + specifier.operator + for specifier_set in (ireq.specifier for ireq in ireqs if ireq) + for specifier in specifier_set + ] - def _get_restrictive_rating(requirements): - # type: (Iterable[Requirement]) -> int - """Rate how restrictive a set of requirements are. + direct = candidate is not None + pinned = any(op[:2] == "==" for op in operators) + unfree = bool(operators) - ``Requirement.get_candidate_lookup()`` returns a 2-tuple for - lookup. The first element is ``Optional[Candidate]`` and the - second ``Optional[InstallRequirement]``. + try: + requested_order: Union[int, float] = self._user_requested[identifier] + except KeyError: + requested_order = math.inf + parent_depths = ( + self._known_depths[parent.name] if parent is not None else 0.0 + for _, parent in information[identifier] + ) + inferred_depth = min(d for d in parent_depths) + 1.0 + self._known_depths[identifier] = inferred_depth + else: + inferred_depth = 1.0 - * If the requirement is an explicit one, the explicitly-required - candidate is returned as the first element. - * If the requirement is based on a PEP 508 specifier, the backing - ``InstallRequirement`` is returned as the second element. - - We use the first element to check whether there is an explicit - requirement, and the second for equality operator. - """ - lookups = (r.get_candidate_lookup() for r in requirements) - cands, ireqs = zip(*lookups) - if any(cand is not None for cand in cands): - return 0 - spec_sets = (ireq.specifier for ireq in ireqs if ireq) - operators = [ - specifier.operator for spec_set in spec_sets for specifier in spec_set - ] - if any(op in ("==", "===") for op in operators): - return 1 - if operators: - return 2 - # A "bare" requirement without any version requirements. - return 3 - - rating = _get_restrictive_rating(r for r, _ in information[identifier]) - order = self._user_requested.get(identifier, float("inf")) + requested_order = self._user_requested.get(identifier, math.inf) # Requires-Python has only one candidate and the check is basically # free, so we always do it first to avoid needless work if it fails. @@ -136,7 +134,16 @@ class PipProvider(_ProviderBase): # while we work on "proper" branch pruning techniques. delay_this = identifier == "setuptools" - return (not requires_python, delay_this, rating, order, identifier) + return ( + not requires_python, + delay_this, + not direct, + not pinned, + inferred_depth, + requested_order, + not unfree, + identifier, + ) def find_matches( self, From 288bffc43c599773608745eea10bfe6b787554aa Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 19 Jun 2021 15:29:04 +0800 Subject: [PATCH 85/85] Unify Python project root detection logic A Python project root is now defined as containing a pyproject.toml, or a setup.py (pre-PEP-517 legacy layout). After this patch, this logic applies to all checks except parse_editable, where we check for setup.py and setup.cfg instead since non-setuptools PEP 517 projects cannot be installed as editable right now. --- news/10080.bugfix.rst | 1 + src/pip/_internal/operations/prepare.py | 4 ++-- src/pip/_internal/req/constructors.py | 4 ++-- src/pip/_internal/utils/misc.py | 17 ++++++++++++----- src/pip/_internal/vcs/git.py | 8 ++++---- src/pip/_internal/vcs/mercurial.py | 8 ++++---- src/pip/_internal/vcs/subversion.py | 12 ++++++------ src/pip/_internal/vcs/versioncontrol.py | 21 +++++++++++---------- 8 files changed, 42 insertions(+), 33 deletions(-) create mode 100644 news/10080.bugfix.rst diff --git a/news/10080.bugfix.rst b/news/10080.bugfix.rst new file mode 100644 index 000000000..f1aa2d6a8 --- /dev/null +++ b/news/10080.bugfix.rst @@ -0,0 +1 @@ +Correctly allow PEP 517 projects to be detected without warnings in ``pip freeze``. diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 3d074f9f6..247e63fc8 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -39,7 +39,7 @@ from pip._internal.utils.deprecation import deprecated from pip._internal.utils.filesystem import copy2_fixed from pip._internal.utils.hashes import Hashes, MissingHashes from pip._internal.utils.logging import indent_log -from pip._internal.utils.misc import display_path, hide_url, rmtree +from pip._internal.utils.misc import display_path, hide_url, is_installable_dir, rmtree from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.unpacking import unpack_file from pip._internal.vcs import vcs @@ -376,7 +376,7 @@ class RequirementPreparer: # installation. # FIXME: this won't upgrade when there's an existing # package unpacked in `req.source_dir` - if os.path.exists(os.path.join(req.source_dir, 'setup.py')): + if is_installable_dir(req.source_dir): raise PreviousBuildDirError( "pip can't proceed with requirements '{}' due to a" "pre-existing build directory ({}). This is likely " diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index 3f9e7dd77..0887102ea 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -248,8 +248,8 @@ def _looks_like_path(name): def _get_url_from_path(path, name): # type: (str, str) -> Optional[str] """ - First, it checks whether a provided path is an installable directory - (e.g. it has a setup.py). If it is, returns the path. + First, it checks whether a provided path is an installable directory. If it + is, returns the path. If false, check if the path is an archive file (such as a .whl). The function checks if the path is a file. If false, if the path has diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index d88f3f46a..99ebea30c 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -270,13 +270,20 @@ def tabulate(rows): def is_installable_dir(path: str) -> bool: - """Is path is a directory containing pyproject.toml, setup.cfg or setup.py?""" + """Is path is a directory containing pyproject.toml or setup.py? + + If pyproject.toml exists, this is a PEP 517 project. Otherwise we look for + a legacy setuptools layout by identifying setup.py. We don't check for the + setup.cfg because using it without setup.py is only available for PEP 517 + projects, which are already covered by the pyproject.toml check. + """ if not os.path.isdir(path): return False - return any( - os.path.isfile(os.path.join(path, signifier)) - for signifier in ("pyproject.toml", "setup.cfg", "setup.py") - ) + if os.path.isfile(os.path.join(path, "pyproject.toml")): + return True + if os.path.isfile(os.path.join(path, "setup.py")): + return True + return False def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index 364ccca6c..b860e350a 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -18,7 +18,7 @@ from pip._internal.vcs.versioncontrol import ( RemoteNotValidError, RevOptions, VersionControl, - find_path_to_setup_from_repo_root, + find_path_to_project_root_from_repo_root, vcs, ) @@ -410,8 +410,8 @@ class Git(VersionControl): def get_subdirectory(cls, location): # type: (str) -> Optional[str] """ - Return the path to setup.py, relative to the repo root. - Return None if setup.py is in the repo root. + Return the path to Python project root, relative to the repo root. + Return None if the project root is in the repo root. """ # find the repo root git_dir = cls.run_command( @@ -423,7 +423,7 @@ class Git(VersionControl): if not os.path.isabs(git_dir): git_dir = os.path.join(location, git_dir) repo_root = os.path.abspath(os.path.join(git_dir, '..')) - return find_path_to_setup_from_repo_root(location, repo_root) + return find_path_to_project_root_from_repo_root(location, repo_root) @classmethod def get_url_rev_and_auth(cls, url): diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index b4f887d32..8f8b09bd2 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -10,7 +10,7 @@ from pip._internal.utils.urls import path_to_url from pip._internal.vcs.versioncontrol import ( RevOptions, VersionControl, - find_path_to_setup_from_repo_root, + find_path_to_project_root_from_repo_root, vcs, ) @@ -120,8 +120,8 @@ class Mercurial(VersionControl): def get_subdirectory(cls, location): # type: (str) -> Optional[str] """ - Return the path to setup.py, relative to the repo root. - Return None if setup.py is in the repo root. + Return the path to Python project root, relative to the repo root. + Return None if the project root is in the repo root. """ # find the repo root repo_root = cls.run_command( @@ -129,7 +129,7 @@ class Mercurial(VersionControl): ).strip() if not os.path.isabs(repo_root): repo_root = os.path.abspath(os.path.join(location, repo_root)) - return find_path_to_setup_from_repo_root(location, repo_root) + return find_path_to_project_root_from_repo_root(location, repo_root) @classmethod def get_repository_root(cls, location): diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py index 4d1237ca0..965e0b425 100644 --- a/src/pip/_internal/vcs/subversion.py +++ b/src/pip/_internal/vcs/subversion.py @@ -7,6 +7,7 @@ from pip._internal.utils.misc import ( HiddenText, display_path, is_console_interactive, + is_installable_dir, split_auth_from_netloc, ) from pip._internal.utils.subprocess import CommandArgs, make_command @@ -111,18 +112,17 @@ class Subversion(VersionControl): @classmethod def get_remote_url(cls, location): # type: (str) -> str - # In cases where the source is in a subdirectory, not alongside - # setup.py we have to look up in the location until we find a real - # setup.py + # In cases where the source is in a subdirectory, we have to look up in + # the location until we find a valid project root. orig_location = location - while not os.path.exists(os.path.join(location, 'setup.py')): + while not is_installable_dir(location): last_location = location location = os.path.dirname(location) if location == last_location: # We've traversed up to the root of the filesystem without - # finding setup.py + # finding a Python project. logger.warning( - "Could not find setup.py for directory %s (tried all " + "Could not find Python project for directory %s (tried all " "parent directories)", orig_location, ) diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py index d06c81032..cddd78c5e 100644 --- a/src/pip/_internal/vcs/versioncontrol.py +++ b/src/pip/_internal/vcs/versioncontrol.py @@ -27,6 +27,7 @@ from pip._internal.utils.misc import ( display_path, hide_url, hide_value, + is_installable_dir, rmtree, ) from pip._internal.utils.subprocess import CommandArgs, call_subprocess, make_command @@ -68,23 +69,23 @@ def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None): return req -def find_path_to_setup_from_repo_root(location, repo_root): +def find_path_to_project_root_from_repo_root(location, repo_root): # type: (str, str) -> Optional[str] """ - Find the path to `setup.py` by searching up the filesystem from `location`. - Return the path to `setup.py` relative to `repo_root`. - Return None if `setup.py` is in `repo_root` or cannot be found. + Find the the Python project's root by searching up the filesystem from + `location`. Return the path to project root relative to `repo_root`. + Return None if the project root is `repo_root`, or cannot be found. """ - # find setup.py + # find project root. orig_location = location - while not os.path.exists(os.path.join(location, 'setup.py')): + while not is_installable_dir(location): last_location = location location = os.path.dirname(location) if location == last_location: # We've traversed up to the root of the filesystem without - # finding setup.py + # finding a Python project. logger.warning( - "Could not find setup.py for directory %s (tried all " + "Could not find a Python project for directory %s (tried all " "parent directories)", orig_location, ) @@ -296,8 +297,8 @@ class VersionControl: def get_subdirectory(cls, location): # type: (str) -> Optional[str] """ - Return the path to setup.py, relative to the repo root. - Return None if setup.py is in the repo root. + Return the path to Python project root, relative to the repo root. + Return None if the project root is in the repo root. """ return None