From 5c9c68c10feca3f4d78cfa44a0fce05337dcdeb2 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 14 Mar 2020 00:28:29 +0530 Subject: [PATCH 001/209] WIP: Add initial draft for CLI deep dive --- .../architecture/command-line-interface.rst | 57 +++++++++++++++++++ docs/html/development/architecture/index.rst | 1 + 2 files changed, 58 insertions(+) create mode 100644 docs/html/development/architecture/command-line-interface.rst diff --git a/docs/html/development/architecture/command-line-interface.rst b/docs/html/development/architecture/command-line-interface.rst new file mode 100644 index 000000000..9bfa91192 --- /dev/null +++ b/docs/html/development/architecture/command-line-interface.rst @@ -0,0 +1,57 @@ +====================== +Command Line Interface +====================== + +The ``pip._internal.cli`` package is responsible for processing and providing +pip's command line interface. This package handles: + +* CLI option definition and parsing +* autocompletion +* dispatching to the various commands +* utilities like progress bars and spinners + +.. note:: + + This section of the documentation is currently being written. pip + developers welcome your help to complete this documentation. If you're + interested in helping out, please let us know in the + `tracking issue `_. + + +.. _cli-overview: + +Overview +======== + +A ``ConfigOptionParser`` instance is used as the "main parser", +for parsing top level args. + +``Command`` then uses another ``ConfigOptionParser`` instance, to parse command-specific args. + +* TODO: How & where options are defined + (cmdoptions, command-specific files). + +* TODO: How & where arguments are processed. + (main_parser, command-specific parser) + +* TODO: How processed arguments are accessed. + (attributes on argument to ``Command.run()``) + +* TODO: How configuration and CLI "blend". + (implemented in ``ConfigOptionParser``) + +* TODO: progress bars and spinners + +* TODO: quirks / standard practices / broad ideas. + (avoiding lists in option def'n, special cased option value types, + ) + + +Future Refactoring Ideas +======================== + +* Change option definition to be a more declarative, consistent, static + data-structure, replacing the current ``partial(Option, ...)`` form +* Move progress bar and spinner to a ``cli.ui`` subpackage +* Move all ``Command`` classes into a ``cli.commands`` subpackage + (including base classes) diff --git a/docs/html/development/architecture/index.rst b/docs/html/development/architecture/index.rst index 094adeede..2053a2d47 100644 --- a/docs/html/development/architecture/index.rst +++ b/docs/html/development/architecture/index.rst @@ -21,6 +21,7 @@ Architecture of pip's internals overview anatomy package-finding + command-line-interface upgrade-options From e0f311b1f42e0a41fe5c1b4d263bbdabf74cdc6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Fri, 10 Jul 2020 00:37:38 +0700 Subject: [PATCH 002/209] Declare constants in configuration.py as such --- ...1e2773-eae6-43a4-b075-55f49e713fb1.trivial | 0 src/pip/_internal/configuration.py | 55 ++++++++----------- 2 files changed, 23 insertions(+), 32 deletions(-) create mode 100644 news/2a1e2773-eae6-43a4-b075-55f49e713fb1.trivial diff --git a/news/2a1e2773-eae6-43a4-b075-55f49e713fb1.trivial b/news/2a1e2773-eae6-43a4-b075-55f49e713fb1.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index e49a5f4f5..13cab9230 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -35,6 +35,20 @@ if MYPY_CHECK_RUNNING: RawConfigParser = configparser.RawConfigParser # Shorthand Kind = NewType("Kind", str) +CONFIG_BASENAME = 'pip.ini' if WINDOWS else 'pip.conf' +ENV_NAMES_IGNORED = "version", "help" + +# The kinds of configurations there are. +kinds = enum( + USER="user", # User Specific + GLOBAL="global", # System Wide + SITE="site", # [Virtual] Environment Specific + ENV="env", # from PIP_CONFIG_FILE + ENV_VAR="env-var", # from Environment Variables +) +OVERRIDE_ORDER = kinds.GLOBAL, kinds.USER, kinds.SITE, kinds.ENV, kinds.ENV_VAR +VALID_LOAD_ONLY = kinds.USER, kinds.GLOBAL, kinds.SITE + logger = logging.getLogger(__name__) @@ -60,19 +74,6 @@ def _disassemble_key(name): return name.split(".", 1) -# The kinds of configurations there are. -kinds = enum( - USER="user", # User Specific - GLOBAL="global", # System Wide - SITE="site", # [Virtual] Environment Specific - ENV="env", # from PIP_CONFIG_FILE - ENV_VAR="env-var", # from Environment Variables -) - - -CONFIG_BASENAME = 'pip.ini' if WINDOWS else 'pip.conf' - - def get_configuration_files(): # type: () -> Dict[Kind, List[str]] global_config_files = [ @@ -114,29 +115,21 @@ class Configuration(object): # type: (bool, Optional[Kind]) -> None super(Configuration, self).__init__() - _valid_load_only = [kinds.USER, kinds.GLOBAL, kinds.SITE, None] - if load_only not in _valid_load_only: + if load_only is not None and load_only not in VALID_LOAD_ONLY: raise ConfigurationError( "Got invalid value for load_only - should be one of {}".format( - ", ".join(map(repr, _valid_load_only[:-1])) + ", ".join(map(repr, VALID_LOAD_ONLY)) ) ) self.isolated = isolated self.load_only = load_only - # The order here determines the override order. - self._override_order = [ - kinds.GLOBAL, kinds.USER, kinds.SITE, kinds.ENV, kinds.ENV_VAR - ] - - self._ignore_env_names = ["version", "help"] - # Because we keep track of where we got the data from self._parsers = { - variant: [] for variant in self._override_order + variant: [] for variant in OVERRIDE_ORDER } # type: Dict[Kind, List[Tuple[str, RawConfigParser]]] self._config = { - variant: {} for variant in self._override_order + variant: {} for variant in OVERRIDE_ORDER } # type: Dict[Kind, Dict[str, Any]] self._modified_parsers = [] # type: List[Tuple[str, RawConfigParser]] @@ -257,7 +250,7 @@ class Configuration(object): # are not needed here. retval = {} - for variant in self._override_order: + for variant in OVERRIDE_ORDER: retval.update(self._config[variant]) return retval @@ -348,12 +341,10 @@ class Configuration(object): # type: () -> Iterable[Tuple[str, str]] """Returns a generator with all environmental vars with prefix PIP_""" for key, val in os.environ.items(): - should_be_yielded = ( - key.startswith("PIP_") and - key[4:].lower() not in self._ignore_env_names - ) - if should_be_yielded: - yield key[4:].lower(), val + if key.startswith("PIP_"): + name = key[4:].lower() + if name not in ENV_NAMES_IGNORED: + yield name, val # XXX: This is patched in the tests. def iter_config_files(self): From 95dfd8b5a7855514c16e5352f5f814c714b62a7e Mon Sep 17 00:00:00 2001 From: Oliver Mannion <125105+tekumara@users.noreply.github.com> Date: Sun, 19 Jul 2020 15:49:41 +1000 Subject: [PATCH 003/209] Ignore require-virtualenv in `pip list` --- news/8603.feature | 1 + src/pip/_internal/commands/list.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 news/8603.feature diff --git a/news/8603.feature b/news/8603.feature new file mode 100644 index 000000000..1f8480baa --- /dev/null +++ b/news/8603.feature @@ -0,0 +1 @@ +Ignore require-virtualenv in ``pip list`` diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index a67d0f8d4..20e9bff2b 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -39,6 +39,7 @@ class ListCommand(IndexGroupCommand): Packages are listed in a case-insensitive sorted order. """ + ignore_require_venv = True usage = """ %prog [options]""" From c564a3d5414690060c7c41197e2cbed6f4ca5233 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Wed, 22 Jul 2020 23:01:30 +0700 Subject: [PATCH 004/209] Use monkeypatch.setenv in config and option tests --- ...3425e4-767e-4d73-bce5-88644b781855.trivial | 0 tests/lib/configuration_helpers.py | 16 --- tests/lib/options_helpers.py | 5 - tests/unit/test_configuration.py | 38 +++---- tests/unit/test_options.py | 106 +++++++----------- 5 files changed, 57 insertions(+), 108 deletions(-) create mode 100644 news/d53425e4-767e-4d73-bce5-88644b781855.trivial diff --git a/news/d53425e4-767e-4d73-bce5-88644b781855.trivial b/news/d53425e4-767e-4d73-bce5-88644b781855.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lib/configuration_helpers.py b/tests/lib/configuration_helpers.py index d33b8ec09..3e3692696 100644 --- a/tests/lib/configuration_helpers.py +++ b/tests/lib/configuration_helpers.py @@ -14,18 +14,6 @@ from pip._internal.utils.misc import ensure_dir kinds = pip._internal.configuration.kinds -def reset_os_environ(old_environ): - """ - Reset os.environ while preserving the same underlying mapping. - """ - # Preserving the same mapping is preferable to assigning a new mapping - # because the latter has interfered with test isolation by, for example, - # preventing time.tzset() from working in subsequent tests after - # changing os.environ['TZ'] in those tests. - os.environ.clear() - os.environ.update(old_environ) - - class ConfigurationMixin(object): def setup(self): @@ -34,14 +22,10 @@ class ConfigurationMixin(object): ) self._files_to_clear = [] - self._old_environ = os.environ.copy() - def teardown(self): for fname in self._files_to_clear: fname.stop() - reset_os_environ(self._old_environ) - def patch_configuration(self, variant, di): old = self.configuration._load_config_files diff --git a/tests/lib/options_helpers.py b/tests/lib/options_helpers.py index 120070abb..2354a818d 100644 --- a/tests/lib/options_helpers.py +++ b/tests/lib/options_helpers.py @@ -1,12 +1,9 @@ """Provides helper classes for testing option handling in pip """ -import os - from pip._internal.cli import cmdoptions from pip._internal.cli.base_command import Command from pip._internal.commands import CommandInfo, commands_dict -from tests.lib.configuration_helpers import reset_os_environ class FakeCommand(Command): @@ -23,11 +20,9 @@ class FakeCommand(Command): class AddFakeCommandMixin(object): def setup(self): - self.environ_before = os.environ.copy() commands_dict['fake'] = CommandInfo( 'tests.lib.options_helpers', 'FakeCommand', 'fake summary', ) def teardown(self): - reset_os_environ(self.environ_before) commands_dict.pop('fake') diff --git a/tests/unit/test_configuration.py b/tests/unit/test_configuration.py index f16252f44..0a45fc136 100644 --- a/tests/unit/test_configuration.py +++ b/tests/unit/test_configuration.py @@ -1,8 +1,6 @@ """Tests for all things related to the configuration """ -import os - import pytest from mock import MagicMock @@ -31,48 +29,48 @@ class TestConfigurationLoading(ConfigurationMixin): self.configuration.load() assert self.configuration.get_value("test.hello") == "3" - def test_environment_config_loading(self): + def test_environment_config_loading(self, monkeypatch): contents = """ [test] hello = 4 """ with self.tmpfile(contents) as config_file: - os.environ["PIP_CONFIG_FILE"] = config_file + monkeypatch.setenv("PIP_CONFIG_FILE", config_file) self.configuration.load() assert self.configuration.get_value("test.hello") == "4", \ self.configuration._config - def test_environment_var_loading(self): - os.environ["PIP_HELLO"] = "5" + def test_environment_var_loading(self, monkeypatch): + monkeypatch.setenv("PIP_HELLO", "5") self.configuration.load() assert self.configuration.get_value(":env:.hello") == "5" @pytest.mark.skipif("sys.platform == 'win32'") - def test_environment_var_does_not_load_lowercase(self): - os.environ["pip_hello"] = "5" + def test_environment_var_does_not_load_lowercase(self, monkeypatch): + monkeypatch.setenv("pip_hello", "5") self.configuration.load() with pytest.raises(ConfigurationError): self.configuration.get_value(":env:.hello") - def test_environment_var_does_not_load_version(self): - os.environ["PIP_VERSION"] = "True" + def test_environment_var_does_not_load_version(self, monkeypatch): + monkeypatch.setenv("PIP_VERSION", "True") self.configuration.load() with pytest.raises(ConfigurationError): self.configuration.get_value(":env:.version") - def test_environment_config_errors_if_malformed(self): + def test_environment_config_errors_if_malformed(self, monkeypatch): contents = """ test] hello = 4 """ with self.tmpfile(contents) as config_file: - os.environ["PIP_CONFIG_FILE"] = config_file + monkeypatch.setenv("PIP_CONFIG_FILE", config_file) with pytest.raises(ConfigurationError) as err: self.configuration.load() @@ -130,36 +128,36 @@ class TestConfigurationPrecedence(ConfigurationMixin): assert self.configuration.get_value("test.hello") == "2" - def test_env_not_overriden_by_environment_var(self): + def test_env_not_overriden_by_environment_var(self, monkeypatch): self.patch_configuration(kinds.ENV, {"test.hello": "1"}) - os.environ["PIP_HELLO"] = "5" + monkeypatch.setenv("PIP_HELLO", "5") self.configuration.load() assert self.configuration.get_value("test.hello") == "1" assert self.configuration.get_value(":env:.hello") == "5" - def test_site_not_overriden_by_environment_var(self): + def test_site_not_overriden_by_environment_var(self, monkeypatch): self.patch_configuration(kinds.SITE, {"test.hello": "2"}) - os.environ["PIP_HELLO"] = "5" + monkeypatch.setenv("PIP_HELLO", "5") self.configuration.load() assert self.configuration.get_value("test.hello") == "2" assert self.configuration.get_value(":env:.hello") == "5" - def test_user_not_overriden_by_environment_var(self): + def test_user_not_overriden_by_environment_var(self, monkeypatch): self.patch_configuration(kinds.USER, {"test.hello": "3"}) - os.environ["PIP_HELLO"] = "5" + monkeypatch.setenv("PIP_HELLO", "5") self.configuration.load() assert self.configuration.get_value("test.hello") == "3" assert self.configuration.get_value(":env:.hello") == "5" - def test_global_not_overriden_by_environment_var(self): + def test_global_not_overriden_by_environment_var(self, monkeypatch): self.patch_configuration(kinds.GLOBAL, {"test.hello": "4"}) - os.environ["PIP_HELLO"] = "5" + monkeypatch.setenv("PIP_HELLO", "5") self.configuration.load() diff --git a/tests/unit/test_options.py b/tests/unit/test_options.py index ce4fc9c25..ff0324ff4 100644 --- a/tests/unit/test_options.py +++ b/tests/unit/test_options.py @@ -10,23 +10,6 @@ from pip._internal.exceptions import PipError from tests.lib.options_helpers import AddFakeCommandMixin -@contextmanager -def temp_environment_variable(name, value): - not_set = object() - original = os.environ[name] if name in os.environ else not_set - os.environ[name] = value - - try: - yield - finally: - # Return the environment variable to its original state. - if original is not_set: - if name in os.environ: - del os.environ[name] - else: - os.environ[name] = original - - @contextmanager def assert_option_error(capsys, expected): """ @@ -70,56 +53,48 @@ class TestOptionPrecedence(AddFakeCommandMixin): } return config[section] - def test_env_override_default_int(self): + def test_env_override_default_int(self, monkeypatch): """ Test that environment variable overrides an int option default. """ - os.environ['PIP_TIMEOUT'] = '-1' + monkeypatch.setenv('PIP_TIMEOUT', '-1') options, args = main(['fake']) assert options.timeout == -1 - def test_env_override_default_append(self): + @pytest.mark.parametrize('values', (['F1'], ['F1', 'F2'])) + def test_env_override_default_append(self, values, monkeypatch): """ Test that environment variable overrides an append option default. """ - os.environ['PIP_FIND_LINKS'] = 'F1' + monkeypatch.setenv('PIP_FIND_LINKS', ' '.join(values)) options, args = main(['fake']) - assert options.find_links == ['F1'] + assert options.find_links == values - os.environ['PIP_FIND_LINKS'] = 'F1 F2' - options, args = main(['fake']) - assert options.find_links == ['F1', 'F2'] - - def test_env_override_default_choice(self): + @pytest.mark.parametrize('choises', (['w'], ['s', 'w'])) + def test_env_override_default_choice(self, choises, monkeypatch): """ Test that environment variable overrides a choice option default. """ - os.environ['PIP_EXISTS_ACTION'] = 'w' + monkeypatch.setenv('PIP_EXISTS_ACTION', ' '.join(choises)) options, args = main(['fake']) - assert options.exists_action == ['w'] + assert options.exists_action == choises - os.environ['PIP_EXISTS_ACTION'] = 's w' - options, args = main(['fake']) - assert options.exists_action == ['s', 'w'] - - def test_env_alias_override_default(self): + @pytest.mark.parametrize('name', ('PIP_LOG_FILE', 'PIP_LOCAL_LOG')) + def test_env_alias_override_default(self, name, monkeypatch): """ When an option has multiple long forms, test that the technique of using the env variable, "PIP_" works for all cases. (e.g. PIP_LOG_FILE and PIP_LOCAL_LOG should all work) """ - os.environ['PIP_LOG_FILE'] = 'override.log' - options, args = main(['fake']) - assert options.log == 'override.log' - os.environ['PIP_LOCAL_LOG'] = 'override.log' + monkeypatch.setenv(name, 'override.log') options, args = main(['fake']) assert options.log == 'override.log' - def test_cli_override_environment(self): + def test_cli_override_environment(self, monkeypatch): """ Test the cli overrides and environment variable """ - os.environ['PIP_TIMEOUT'] = '-1' + monkeypatch.setenv('PIP_TIMEOUT', '-1') options, args = main(['fake', '--timeout', '-2']) assert options.timeout == -2 @@ -136,49 +111,49 @@ class TestOptionPrecedence(AddFakeCommandMixin): 'off', 'no', ]) - def test_cache_dir__PIP_NO_CACHE_DIR(self, pip_no_cache_dir): + def test_cache_dir__PIP_NO_CACHE_DIR(self, pip_no_cache_dir, monkeypatch): """ Test setting the PIP_NO_CACHE_DIR environment variable without passing any command-line flags. """ - os.environ['PIP_NO_CACHE_DIR'] = pip_no_cache_dir + monkeypatch.setenv('PIP_NO_CACHE_DIR', pip_no_cache_dir) options, args = main(['fake']) assert options.cache_dir is False @pytest.mark.parametrize('pip_no_cache_dir', ['yes', 'no']) def test_cache_dir__PIP_NO_CACHE_DIR__with_cache_dir( - self, pip_no_cache_dir + self, pip_no_cache_dir, monkeypatch, ): """ Test setting PIP_NO_CACHE_DIR while also passing an explicit --cache-dir value. """ - os.environ['PIP_NO_CACHE_DIR'] = pip_no_cache_dir + monkeypatch.setenv('PIP_NO_CACHE_DIR', pip_no_cache_dir) options, args = main(['--cache-dir', '/cache/dir', 'fake']) # The command-line flag takes precedence. assert options.cache_dir == '/cache/dir' @pytest.mark.parametrize('pip_no_cache_dir', ['yes', 'no']) def test_cache_dir__PIP_NO_CACHE_DIR__with_no_cache_dir( - self, pip_no_cache_dir + self, pip_no_cache_dir, monkeypatch, ): """ Test setting PIP_NO_CACHE_DIR while also passing --no-cache-dir. """ - os.environ['PIP_NO_CACHE_DIR'] = pip_no_cache_dir + monkeypatch.setenv('PIP_NO_CACHE_DIR', pip_no_cache_dir) options, args = main(['--no-cache-dir', 'fake']) # The command-line flag should take precedence (which has the same # value in this case). assert options.cache_dir is False def test_cache_dir__PIP_NO_CACHE_DIR_invalid__with_no_cache_dir( - self, capsys, + self, monkeypatch, capsys, ): """ Test setting PIP_NO_CACHE_DIR to an invalid value while also passing --no-cache-dir. """ - os.environ['PIP_NO_CACHE_DIR'] = 'maybe' + monkeypatch.setenv('PIP_NO_CACHE_DIR', 'maybe') expected_err = "--no-cache-dir error: invalid truth value 'maybe'" with assert_option_error(capsys, expected=expected_err): main(['--no-cache-dir', 'fake']) @@ -219,52 +194,49 @@ class TestUsePEP517Options(object): options = self.parse_args(['--no-use-pep517']) assert options.use_pep517 is False - def test_PIP_USE_PEP517_true(self): + def test_PIP_USE_PEP517_true(self, monkeypatch): """ Test setting PIP_USE_PEP517 to "true". """ - with temp_environment_variable('PIP_USE_PEP517', 'true'): - options = self.parse_args([]) + monkeypatch.setenv('PIP_USE_PEP517', 'true') + options = self.parse_args([]) # This is an int rather than a boolean because strtobool() in pip's # configuration code returns an int. assert options.use_pep517 == 1 - def test_PIP_USE_PEP517_false(self): + def test_PIP_USE_PEP517_false(self, monkeypatch): """ Test setting PIP_USE_PEP517 to "false". """ - with temp_environment_variable('PIP_USE_PEP517', 'false'): - options = self.parse_args([]) + monkeypatch.setenv('PIP_USE_PEP517', 'false') + options = self.parse_args([]) # This is an int rather than a boolean because strtobool() in pip's # configuration code returns an int. assert options.use_pep517 == 0 - def test_use_pep517_and_PIP_USE_PEP517_false(self): + def test_use_pep517_and_PIP_USE_PEP517_false(self, monkeypatch): """ Test passing --use-pep517 and setting PIP_USE_PEP517 to "false". """ - with temp_environment_variable('PIP_USE_PEP517', 'false'): - options = self.parse_args(['--use-pep517']) + monkeypatch.setenv('PIP_USE_PEP517', 'false') + options = self.parse_args(['--use-pep517']) assert options.use_pep517 is True - def test_no_use_pep517_and_PIP_USE_PEP517_true(self): + def test_no_use_pep517_and_PIP_USE_PEP517_true(self, monkeypatch): """ Test passing --no-use-pep517 and setting PIP_USE_PEP517 to "true". """ - with temp_environment_variable('PIP_USE_PEP517', 'true'): - options = self.parse_args(['--no-use-pep517']) + monkeypatch.setenv('PIP_USE_PEP517', 'true') + options = self.parse_args(['--no-use-pep517']) assert options.use_pep517 is False - def test_PIP_NO_USE_PEP517(self, capsys): + def test_PIP_NO_USE_PEP517(self, monkeypatch, capsys): """ Test setting PIP_NO_USE_PEP517, which isn't allowed. """ - expected_err = ( - '--no-use-pep517 error: A value was passed for --no-use-pep517,\n' - ) - with temp_environment_variable('PIP_NO_USE_PEP517', 'true'): - with assert_option_error(capsys, expected=expected_err): - self.parse_args([]) + monkeypatch.setenv('PIP_NO_USE_PEP517', 'true') + with assert_option_error(capsys, expected='--no-use-pep517 error'): + self.parse_args([]) class TestOptionsInterspersed(AddFakeCommandMixin): From 5b1093fc7521857715938cce13b347108e824b50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Thu, 23 Jul 2020 15:01:37 +0700 Subject: [PATCH 005/209] Use monkeypatch.*env in conftest and tests.lib Session fixtures have to use mock.patch.dict though --- tests/conftest.py | 55 +++++++++++++++++++++++++------------------ tests/lib/__init__.py | 6 +---- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2aab50207..1ca4d45d0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,7 @@ from contextlib import contextmanager import pytest import six +from mock import patch from pip._vendor.contextlib2 import ExitStack, nullcontext from setuptools.wheel import Wheel @@ -104,11 +105,13 @@ def use_new_resolver(request): """Set environment variable to make pip default to the new resolver. """ new_resolver = request.config.getoption("--new-resolver") + features = set(os.environ.get("PIP_USE_FEATURE", "").split()) if new_resolver: - os.environ["PIP_USE_FEATURE"] = "2020-resolver" + features.add("2020-resolver") else: - os.environ.pop("PIP_USE_FEATURE", None) - yield new_resolver + features.discard("2020-resolver") + with patch.dict(os.environ, {"PIP_USE_FEATURE", " ".join(features)}): + yield new_resolver @pytest.fixture(scope='session') @@ -151,7 +154,7 @@ def tmpdir(request, tmpdir): @pytest.fixture(autouse=True) -def isolate(tmpdir): +def isolate(tmpdir, monkeypatch): """ Isolate our tests so that things like global configuration files and the like do not affect our test results. @@ -174,45 +177,51 @@ def isolate(tmpdir): if sys.platform == 'win32': # Note: this will only take effect in subprocesses... home_drive, home_path = os.path.splitdrive(home_dir) - os.environ.update({ - 'USERPROFILE': home_dir, - 'HOMEDRIVE': home_drive, - 'HOMEPATH': home_path, - }) + monkeypatch.setenv('USERPROFILE', home_dir) + monkeypatch.setenv('HOMEDRIVE', home_drive) + monkeypatch.setenv('HOMEPATH', home_path) for env_var, sub_path in ( ('APPDATA', 'AppData/Roaming'), ('LOCALAPPDATA', 'AppData/Local'), ): path = os.path.join(home_dir, *sub_path.split('/')) - os.environ[env_var] = path + monkeypatch.setenv(env_var, path) os.makedirs(path) else: # Set our home directory to our temporary directory, this should force # all of our relative configuration files to be read from here instead # of the user's actual $HOME directory. - os.environ["HOME"] = home_dir + monkeypatch.setenv("HOME", home_dir) # Isolate ourselves from XDG directories - os.environ["XDG_DATA_HOME"] = os.path.join(home_dir, ".local", "share") - os.environ["XDG_CONFIG_HOME"] = os.path.join(home_dir, ".config") - os.environ["XDG_CACHE_HOME"] = os.path.join(home_dir, ".cache") - os.environ["XDG_RUNTIME_DIR"] = os.path.join(home_dir, ".runtime") - os.environ["XDG_DATA_DIRS"] = ":".join([ + monkeypatch.setenv("XDG_DATA_HOME", os.path.join( + home_dir, ".local", "share", + )) + monkeypatch.setenv("XDG_CONFIG_HOME", os.path.join( + home_dir, ".config", + )) + monkeypatch.setenv("XDG_CACHE_HOME", os.path.join(home_dir, ".cache")) + monkeypatch.setenv("XDG_RUNTIME_DIR", os.path.join( + home_dir, ".runtime", + )) + monkeypatch.setenv("XDG_DATA_DIRS", os.pathsep.join([ os.path.join(fake_root, "usr", "local", "share"), os.path.join(fake_root, "usr", "share"), - ]) - os.environ["XDG_CONFIG_DIRS"] = os.path.join(fake_root, "etc", "xdg") + ])) + monkeypatch.setenv("XDG_CONFIG_DIRS", os.path.join( + fake_root, "etc", "xdg", + )) # Configure git, because without an author name/email git will complain # and cause test failures. - os.environ["GIT_CONFIG_NOSYSTEM"] = "1" - os.environ["GIT_AUTHOR_NAME"] = "pip" - os.environ["GIT_AUTHOR_EMAIL"] = "distutils-sig@python.org" + monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") + monkeypatch.setenv("GIT_AUTHOR_NAME", "pip") + monkeypatch.setenv("GIT_AUTHOR_EMAIL", "distutils-sig@python.org") # We want to disable the version check from running in the tests - os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "true" + monkeypatch.setenv("PIP_DISABLE_PIP_VERSION_CHECK", "true") # Make sure tests don't share a requirements tracker. - os.environ.pop('PIP_REQ_TRACKER', None) + monkeypatch.delenv("PIP_REQ_TRACKER", False) # FIXME: Windows... os.makedirs(os.path.join(home_dir, ".config", "git")) diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index a8ea86761..ee243d0e6 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -492,10 +492,7 @@ class PipTestEnvironment(TestFileEnvironment): kwargs.setdefault("cwd", self.scratch_path) # Setup our environment - environ = kwargs.get("environ") - if environ is None: - environ = os.environ.copy() - + environ = kwargs.setdefault("environ", os.environ.copy()) environ["PATH"] = Path.pathsep.join( [self.bin_path] + [environ.get("PATH", [])], ) @@ -504,7 +501,6 @@ class PipTestEnvironment(TestFileEnvironment): environ["PYTHONDONTWRITEBYTECODE"] = "1" # Make sure we get UTF-8 on output, even on Windows... environ["PYTHONIOENCODING"] = "UTF-8" - kwargs["environ"] = environ # Whether all pip invocations should expect stderr # (useful for Python version deprecation) From b795c9a7d6b39eaa5138d51a78d307fbaee96573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Sun, 26 Jul 2020 17:13:04 +0700 Subject: [PATCH 006/209] Abstract away AbstractDistribution in higher-level resolver code --- ...4ae133-3a6d-4a6c-a4c3-fe8d78223498.trivial | 0 src/pip/_internal/operations/prepare.py | 35 +++++++++---------- .../_internal/resolution/legacy/resolver.py | 18 ++++------ .../resolution/resolvelib/candidates.py | 16 ++++----- 4 files changed, 30 insertions(+), 39 deletions(-) create mode 100644 news/094ae133-3a6d-4a6c-a4c3-fe8d78223498.trivial diff --git a/news/094ae133-3a6d-4a6c-a4c3-fe8d78223498.trivial b/news/094ae133-3a6d-4a6c-a4c3-fe8d78223498.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index a5455fcc8..e6a34172d 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -44,8 +44,8 @@ if MYPY_CHECK_RUNNING: ) from mypy_extensions import TypedDict + from pip._vendor.pkg_resources import Distribution - from pip._internal.distributions import AbstractDistribution from pip._internal.index.package_finder import PackageFinder from pip._internal.models.link import Link from pip._internal.network.download import Downloader @@ -78,18 +78,17 @@ logger = logging.getLogger(__name__) def _get_prepared_distribution( - req, # type: InstallRequirement - req_tracker, # type: RequirementTracker - finder, # type: PackageFinder - build_isolation # type: bool + req, # type: InstallRequirement + req_tracker, # type: RequirementTracker + finder, # type: PackageFinder + build_isolation, # type: bool ): - # type: (...) -> AbstractDistribution - """Prepare a distribution for installation. - """ + # type: (...) -> Distribution + """Prepare a distribution for installation.""" abstract_dist = make_distribution_for_install_requirement(req) with req_tracker.track(req): abstract_dist.prepare_distribution_metadata(finder, build_isolation) - return abstract_dist + return abstract_dist.get_pkg_resources_distribution() def unpack_vcs_link(link, location): @@ -450,7 +449,7 @@ class RequirementPreparer(object): return req.hashes(trust_internet=False) or MissingHashes() def prepare_linked_requirement(self, req, parallel_builds=False): - # type: (InstallRequirement, bool) -> AbstractDistribution + # type: (InstallRequirement, bool) -> Distribution """Prepare a requirement to be obtained from req.link.""" assert req.link link = req.link @@ -479,7 +478,7 @@ class RequirementPreparer(object): if local_file: req.local_file_path = local_file.path - abstract_dist = _get_prepared_distribution( + dist = _get_prepared_distribution( req, self.req_tracker, self.finder, self.build_isolation, ) @@ -499,13 +498,13 @@ class RequirementPreparer(object): # Make a .zip of the source_dir we already created. if link.is_vcs: req.archive(self.download_dir) - return abstract_dist + return dist def prepare_editable_requirement( self, req, # type: InstallRequirement ): - # type: (...) -> AbstractDistribution + # type: (...) -> Distribution """Prepare an editable requirement """ assert req.editable, "cannot prepare a non-editable req as editable" @@ -522,7 +521,7 @@ class RequirementPreparer(object): req.ensure_has_source_dir(self.src_dir) req.update_editable(not self._download_should_save) - abstract_dist = _get_prepared_distribution( + dist = _get_prepared_distribution( req, self.req_tracker, self.finder, self.build_isolation, ) @@ -530,14 +529,14 @@ class RequirementPreparer(object): req.archive(self.download_dir) req.check_if_exists(self.use_user_site) - return abstract_dist + return dist def prepare_installed_requirement( self, req, # type: InstallRequirement skip_reason # type: str ): - # type: (...) -> AbstractDistribution + # type: (...) -> Distribution """Prepare an already-installed requirement """ assert req.satisfied_by, "req should have been satisfied but isn't" @@ -557,6 +556,4 @@ class RequirementPreparer(object): 'completely repeatable environment, install into an ' 'empty virtualenv.' ) - abstract_dist = InstalledDistribution(req) - - return abstract_dist + return InstalledDistribution(req).get_pkg_resources_distribution() diff --git a/src/pip/_internal/resolution/legacy/resolver.py b/src/pip/_internal/resolution/legacy/resolver.py index c9b4c6616..a743b5696 100644 --- a/src/pip/_internal/resolution/legacy/resolver.py +++ b/src/pip/_internal/resolution/legacy/resolver.py @@ -42,10 +42,9 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import DefaultDict, List, Optional, Set, Tuple - from pip._vendor import pkg_resources + from pip._vendor.pkg_resources import Distribution from pip._internal.cache import WheelCache - from pip._internal.distributions import AbstractDistribution from pip._internal.index.package_finder import PackageFinder from pip._internal.models.link import Link from pip._internal.operations.prepare import RequirementPreparer @@ -58,7 +57,7 @@ logger = logging.getLogger(__name__) def _check_dist_requires_python( - dist, # type: pkg_resources.Distribution + dist, # type: Distribution version_info, # type: Tuple[int, int, int] ignore_requires_python=False, # type: bool ): @@ -317,8 +316,8 @@ class Resolver(BaseResolver): req.original_link_is_in_wheel_cache = True req.link = cache_entry.link - def _get_abstract_dist_for(self, req): - # type: (InstallRequirement) -> AbstractDistribution + def _get_dist_for(self, req): + # type: (InstallRequirement) -> Distribution """Takes a InstallRequirement and returns a single AbstractDist \ representing a prepared variant of the same. """ @@ -337,7 +336,7 @@ class Resolver(BaseResolver): # We eagerly populate the link, since that's our "legacy" behavior. self._populate_link(req) - abstract_dist = self.preparer.prepare_linked_requirement(req) + dist = self.preparer.prepare_linked_requirement(req) # NOTE # The following portion is for determining if a certain package is @@ -364,8 +363,7 @@ class Resolver(BaseResolver): 'Requirement already satisfied (use --upgrade to upgrade):' ' %s', req, ) - - return abstract_dist + return dist def _resolve_one( self, @@ -385,10 +383,8 @@ class Resolver(BaseResolver): req_to_install.prepared = True - abstract_dist = self._get_abstract_dist_for(req_to_install) - # Parse and return dependencies - dist = abstract_dist.get_pkg_resources_distribution() + dist = self._get_dist_for(req_to_install) # This will raise UnsupportedPythonVersion if the given Python # version isn't compatible with the distribution's Requires-Python. _check_dist_requires_python( diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index c289bb583..3d8e399a7 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -29,7 +29,6 @@ if MYPY_CHECK_RUNNING: from pip._vendor.packaging.version import _BaseVersion from pip._vendor.pkg_resources import Distribution - from pip._internal.distributions import AbstractDistribution from pip._internal.models.link import Link from .base import Requirement @@ -200,8 +199,8 @@ class _InstallRequirementBackedCandidate(Candidate): self._link.file_path if self._link.is_file else self._link ) - def _prepare_abstract_distribution(self): - # type: () -> AbstractDistribution + def _prepare_distribution(self): + # type: () -> Distribution raise NotImplementedError("Override in subclass") def _check_metadata_consistency(self): @@ -222,12 +221,11 @@ class _InstallRequirementBackedCandidate(Candidate): if self._prepared: return try: - abstract_dist = self._prepare_abstract_distribution() + self._dist = self._prepare_distribution() except HashError as e: e.req = self._ireq raise - self._dist = abstract_dist.get_pkg_resources_distribution() assert self._dist is not None, "Distribution already installed" self._check_metadata_consistency() self._prepared = True @@ -324,8 +322,8 @@ class LinkCandidate(_InstallRequirementBackedCandidate): version=version, ) - def _prepare_abstract_distribution(self): - # type: () -> AbstractDistribution + def _prepare_distribution(self): + # type: () -> Distribution return self._factory.preparer.prepare_linked_requirement( self._ireq, parallel_builds=True, ) @@ -352,8 +350,8 @@ class EditableCandidate(_InstallRequirementBackedCandidate): version=version, ) - def _prepare_abstract_distribution(self): - # type: () -> AbstractDistribution + def _prepare_distribution(self): + # type: () -> Distribution return self._factory.preparer.prepare_editable_requirement(self._ireq) From 9eb9319f7e0d5ab564be75ce5fab1f1317228c91 Mon Sep 17 00:00:00 2001 From: Laurie O Date: Mon, 27 Jul 2020 20:59:31 +1000 Subject: [PATCH 007/209] Document keyring support for index basic-auth --- docs/html/user_guide.rst | 15 +++++++++++++++ news/8636.doc | 1 + 2 files changed, 16 insertions(+) create mode 100644 news/8636.doc diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index f2e49fadf..78a57d32a 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -79,6 +79,21 @@ as the "username" and do not provide a password, for example - ``https://0123456789abcdef@pypi.company.com`` +Keyring Support +--------------- + +pip also supports credentials stored in your keyring using the `keyring`_ +library. + +.. code-block:: shell + + pip install keyring + echo your-password | keyring set pypi.company.com your-username + pip install your-package --extra-index-url https://pypi.company.com/ + +.. _keyring: https://pypi.org/project/keyring/ + + Using a Proxy Server ==================== diff --git a/news/8636.doc b/news/8636.doc new file mode 100644 index 000000000..081cf1c7e --- /dev/null +++ b/news/8636.doc @@ -0,0 +1 @@ +Add note and example on keyring support for index basic-auth From bac3c8eb9ac9dc7d2454ee3e2f6695d5812babf7 Mon Sep 17 00:00:00 2001 From: Laurie O Date: Tue, 28 Jul 2020 10:42:01 +1000 Subject: [PATCH 008/209] Explicitly note that pip does not vendor keyring --- docs/html/user_guide.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 78a57d32a..b021cb6dc 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -83,7 +83,8 @@ Keyring Support --------------- pip also supports credentials stored in your keyring using the `keyring`_ -library. +library. Note that ``keyring`` will need to be installed separately, as pip +does not come with it included. .. code-block:: shell From 2439d80a83d9cc23982972b9c9f9c8458774382f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Mon, 13 Jul 2020 11:39:03 +0700 Subject: [PATCH 009/209] Allow specifying verbose/quiet level via config file and env var --- news/8578.bugfix | 4 ++++ src/pip/_internal/cli/parser.py | 36 ++++++++++++++++----------------- 2 files changed, 22 insertions(+), 18 deletions(-) create mode 100644 news/8578.bugfix diff --git a/news/8578.bugfix b/news/8578.bugfix new file mode 100644 index 000000000..3df7ed078 --- /dev/null +++ b/news/8578.bugfix @@ -0,0 +1,4 @@ +Allow specifying verbosity and quiet level via configuration files +and environment variables. Previously these options were treated as +boolean values when read from there while through CLI the level can be +specified. diff --git a/src/pip/_internal/cli/parser.py b/src/pip/_internal/cli/parser.py index 04e00b721..b6b78318a 100644 --- a/src/pip/_internal/cli/parser.py +++ b/src/pip/_internal/cli/parser.py @@ -11,6 +11,7 @@ import sys import textwrap from distutils.util import strtobool +from pip._vendor.contextlib2 import suppress from pip._vendor.six import string_types from pip._internal.cli.status_codes import UNKNOWN_ERROR @@ -197,15 +198,27 @@ class ConfigOptionParser(CustomOptionParser): if option is None: continue - if option.action in ('store_true', 'store_false', 'count'): + if option.action in ('store_true', 'store_false'): try: val = strtobool(val) except ValueError: - error_msg = invalid_config_error_message( - option.action, key, val + self.error( + '{} is not a valid value for {} option, ' # noqa + 'please specify a boolean value like yes/no, ' + 'true/false or 1/0 instead.'.format(val, key) + ) + elif option.action == 'count': + with suppress(ValueError): + val = strtobool(val) + with suppress(ValueError): + val = int(val) + if not isinstance(val, int) or val < 0: + self.error( + '{} is not a valid value for {} option, ' # noqa + 'please instead specify either a non-negative integer ' + 'or a boolean value like yes/no or false/true ' + 'which is equivalent to 1/0.'.format(val, key) ) - self.error(error_msg) - elif option.action == 'append': val = val.split() val = [self.check_default(option, key, v) for v in val] @@ -251,16 +264,3 @@ class ConfigOptionParser(CustomOptionParser): def error(self, msg): self.print_usage(sys.stderr) self.exit(UNKNOWN_ERROR, "{}\n".format(msg)) - - -def invalid_config_error_message(action, key, val): - """Returns a better error message when invalid configuration option - is provided.""" - if action in ('store_true', 'store_false'): - return ("{0} is not a valid value for {1} option, " - "please specify a boolean value like yes/no, " - "true/false or 1/0 instead.").format(val, key) - - return ("{0} is not a valid value for {1} option, " - "please specify a numerical value like 1/0 " - "instead.").format(val, key) From 7a0061d8864d5fab7abc3d0d82ad01a43a0d23cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Wed, 15 Jul 2020 15:26:04 +0700 Subject: [PATCH 010/209] Update docs for setting verbose/quiet in config file or env var Co-authored-by: Paul Moore Co-authored-by: Prashant Sharma Co-authored-by: Xavier Fernandez --- docs/html/user_guide.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index f2e49fadf..ddaaccd57 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -442,6 +442,15 @@ and ``--no-cache-dir``, falsy values have to be used: no-compile = no no-warn-script-location = false +For options which can be repeated like ``--verbose`` and ``--quiet``, +a non-negative integer can be used to represent the level to be specified: + +.. code-block:: ini + + [global] + quiet = 0 + verbose = 2 + It is possible to append values to a section within a configuration file such as the pip.ini file. This is applicable to appending options like ``--find-links`` or ``--trusted-host``, which can be written on multiple lines: @@ -488,6 +497,15 @@ is the same as calling:: pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com +Options that do not take a value, but can be repeated (such as ``--verbose``) +can be specified using the number of repetitions, so:: + + export PIP_VERBOSE=3 + +is the same as calling:: + + pip install -vvv + .. note:: Environment variables set to be empty string will not be treated as false. From a85be3f5557e49f9e35e8abcf9365dd3b6cfabab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Wed, 22 Jul 2020 17:58:27 +0700 Subject: [PATCH 011/209] Test verbose/quiet level specified via env var and config file --- tests/unit/test_options.py | 120 +++++++++++++++++++++++++++++++------ 1 file changed, 102 insertions(+), 18 deletions(-) diff --git a/tests/unit/test_options.py b/tests/unit/test_options.py index ce4fc9c25..df174ea67 100644 --- a/tests/unit/test_options.py +++ b/tests/unit/test_options.py @@ -1,5 +1,6 @@ import os from contextlib import contextmanager +from tempfile import NamedTemporaryFile import pytest @@ -286,6 +287,107 @@ class TestOptionsInterspersed(AddFakeCommandMixin): main(['--find-links', 'F1', 'fake']) +@contextmanager +def tmpconfig(option, value, section='global'): + with NamedTemporaryFile(mode='w', delete=False) as f: + f.write('[{}]\n{}={}\n'.format(section, option, value)) + name = f.name + try: + yield name + finally: + os.unlink(name) + + +class TestCountOptions(AddFakeCommandMixin): + + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', range(4)) + def test_cli_long(self, option, value): + flags = ['--{}'.format(option)] * value + opt1, args1 = main(flags+['fake']) + opt2, args2 = main(['fake']+flags) + assert getattr(opt1, option) == getattr(opt2, option) == value + + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', range(1, 4)) + def test_cli_short(self, option, value): + flag = '-' + option[0]*value + opt1, args1 = main([flag, 'fake']) + opt2, args2 = main(['fake', flag]) + assert getattr(opt1, option) == getattr(opt2, option) == value + + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', range(4)) + def test_env_var(self, option, value, monkeypatch): + monkeypatch.setenv('PIP_'+option.upper(), str(value)) + assert getattr(main(['fake'])[0], option) == value + + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', range(3)) + def test_env_var_integrate_cli(self, option, value, monkeypatch): + monkeypatch.setenv('PIP_'+option.upper(), str(value)) + assert getattr(main(['fake', '--'+option])[0], option) == value + 1 + + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', (-1, 'foobar')) + def test_env_var_invalid(self, option, value, monkeypatch, capsys): + monkeypatch.setenv('PIP_'+option.upper(), str(value)) + with assert_option_error(capsys, expected='a non-negative integer'): + main(['fake']) + + # Undocumented, support for backward compatibility + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', ('no', 'false')) + def test_env_var_false(self, option, value, monkeypatch): + monkeypatch.setenv('PIP_'+option.upper(), str(value)) + assert getattr(main(['fake'])[0], option) == 0 + + # Undocumented, support for backward compatibility + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', ('yes', 'true')) + def test_env_var_true(self, option, value, monkeypatch): + monkeypatch.setenv('PIP_'+option.upper(), str(value)) + assert getattr(main(['fake'])[0], option) == 1 + + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', range(4)) + def test_config_file(self, option, value, monkeypatch): + with tmpconfig(option, value) as name: + monkeypatch.setenv('PIP_CONFIG_FILE', name) + assert getattr(main(['fake'])[0], option) == value + + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', range(3)) + def test_config_file_integrate_cli(self, option, value, monkeypatch): + with tmpconfig(option, value) as name: + monkeypatch.setenv('PIP_CONFIG_FILE', name) + assert getattr(main(['fake', '--'+option])[0], option) == value + 1 + + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', (-1, 'foobar')) + def test_config_file_invalid(self, option, value, monkeypatch, capsys): + with tmpconfig(option, value) as name: + monkeypatch.setenv('PIP_CONFIG_FILE', name) + with assert_option_error(capsys, expected='non-negative integer'): + main(['fake']) + + # Undocumented, support for backward compatibility + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', ('no', 'false')) + def test_config_file_false(self, option, value, monkeypatch): + with tmpconfig(option, value) as name: + monkeypatch.setenv('PIP_CONFIG_FILE', name) + assert getattr(main(['fake'])[0], option) == 0 + + # Undocumented, support for backward compatibility + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', ('yes', 'true')) + def test_config_file_true(self, option, value, monkeypatch): + with tmpconfig(option, value) as name: + monkeypatch.setenv('PIP_CONFIG_FILE', name) + assert getattr(main(['fake'])[0], option) == 1 + + class TestGeneralOptions(AddFakeCommandMixin): # the reason to specifically test general options is due to the @@ -310,24 +412,6 @@ class TestGeneralOptions(AddFakeCommandMixin): assert options1.require_venv assert options2.require_venv - def test_verbose(self): - options1, args1 = main(['--verbose', 'fake']) - options2, args2 = main(['fake', '--verbose']) - assert options1.verbose == options2.verbose == 1 - - def test_quiet(self): - options1, args1 = main(['--quiet', 'fake']) - options2, args2 = main(['fake', '--quiet']) - assert options1.quiet == options2.quiet == 1 - - options3, args3 = main(['--quiet', '--quiet', 'fake']) - options4, args4 = main(['fake', '--quiet', '--quiet']) - assert options3.quiet == options4.quiet == 2 - - options5, args5 = main(['--quiet', '--quiet', '--quiet', 'fake']) - options6, args6 = main(['fake', '--quiet', '--quiet', '--quiet']) - assert options5.quiet == options6.quiet == 3 - def test_log(self): options1, args1 = main(['--log', 'path', 'fake']) options2, args2 = main(['fake', '--log', 'path']) From a28081c28ecf45514d622d31a790cf37405e01d3 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Wed, 29 Jul 2020 07:53:18 +0530 Subject: [PATCH 012/209] Bump for development --- src/pip/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/__init__.py b/src/pip/__init__.py index b67e61d06..5a2f3c317 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -4,7 +4,7 @@ if MYPY_CHECK_RUNNING: from typing import List, Optional -__version__ = "20.2" +__version__ = "20.3.dev0" def main(args=None): From c7af6a420cb069d29fd57928dfbc600a395850f1 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Wed, 29 Jul 2020 23:16:12 +0530 Subject: [PATCH 013/209] Update Code of Conduct references --- README.rst | 4 ++-- docs/html/index.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 4f0f210f0..395b642d6 100644 --- a/README.rst +++ b/README.rst @@ -38,7 +38,7 @@ Code of Conduct --------------- Everyone interacting in the pip project's codebases, issue trackers, chat -rooms, and mailing lists is expected to follow the `PyPA Code of Conduct`_. +rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_. .. _package installer: https://packaging.python.org/guides/tool-recommendations/ .. _Python Package Index: https://pypi.org @@ -54,4 +54,4 @@ rooms, and mailing lists is expected to follow the `PyPA Code of Conduct`_. .. _Development mailing list: https://mail.python.org/mailman3/lists/distutils-sig.python.org/ .. _User IRC: https://webchat.freenode.net/?channels=%23pypa .. _Development IRC: https://webchat.freenode.net/?channels=%23pypa-dev -.. _PyPA Code of Conduct: https://www.pypa.io/en/latest/code-of-conduct/ +.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md diff --git a/docs/html/index.rst b/docs/html/index.rst index ff41fc42b..1ec915585 100644 --- a/docs/html/index.rst +++ b/docs/html/index.rst @@ -35,7 +35,7 @@ Code of Conduct =============== Everyone interacting in the pip project's codebases, issue trackers, chat -rooms, and mailing lists is expected to follow the `PyPA Code of Conduct`_. +rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_. .. _package installer: https://packaging.python.org/guides/tool-recommendations/ .. _Python Package Index: https://pypi.org @@ -49,4 +49,4 @@ rooms, and mailing lists is expected to follow the `PyPA Code of Conduct`_. .. _Development mailing list: https://mail.python.org/mailman3/lists/distutils-sig.python.org/ .. _User IRC: https://webchat.freenode.net/?channels=%23pypa .. _Development IRC: https://webchat.freenode.net/?channels=%23pypa-dev -.. _PyPA Code of Conduct: https://www.pypa.io/en/latest/code-of-conduct/ +.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md From 5dac5dc098df3e6a5b132c041b06ffa739ca6a14 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Wed, 29 Jul 2020 23:16:56 +0530 Subject: [PATCH 014/209] Drop CoC reference from new issue templates --- .github/ISSUE_TEMPLATE/config.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 3babf35bd..8e5c268c1 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,7 +1,7 @@ # Ref: https://help.github.com/en/github/building-a-strong-community/configuring-issue-templates-for-your-repository#configuring-the-template-chooser blank_issues_enabled: true # default contact_links: -- name: 🤷💻🤦 Discourse +- name: 💬 Discourse url: https://discuss.python.org/c/packaging about: | Please ask typical Q&A here: general ideas for Python packaging, @@ -9,6 +9,3 @@ contact_links: - name: '💬 IRC: #pypa @ Freenode' url: https://webchat.freenode.net/#pypa about: Chat with devs -- name: 📝 PyPA Code of Conduct - url: https://www.pypa.io/en/latest/code-of-conduct/ - about: ❤ Be nice to other members of the community. ☮ Behave. From 864f0e0efa2701c9dc1120e6a80485e86da500ce Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Wed, 29 Jul 2020 17:51:58 -0400 Subject: [PATCH 015/209] Explicitly handle incorrect .data paths during wheel install Previously our wheel installation process allowed wheels which contained non-conforming contents in a contained .data directory. After the refactoring to enable direct-from-wheel installation, pip throws an exception when encountering these wheels, but does not include any helpful information to pinpoint the cause. Now if we encounter such a wheel, we trace an error that includes the name of the requirement we're trying to install, the path to the wheel file, the path we didn't understand, and a hint about what we expect. --- src/pip/_internal/operations/install/wheel.py | 10 +++++++++- tests/functional/test_install_wheel.py | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py index 8f73a88b0..0bd7b28be 100644 --- a/src/pip/_internal/operations/install/wheel.py +++ b/src/pip/_internal/operations/install/wheel.py @@ -583,7 +583,15 @@ def _install_wheel( def make_data_scheme_file(record_path): # type: (RecordPath) -> File normed_path = os.path.normpath(record_path) - _, scheme_key, dest_subpath = normed_path.split(os.path.sep, 2) + try: + _, scheme_key, dest_subpath = normed_path.split(os.path.sep, 2) + except ValueError: + message = ( + "Unexpected file in {}: {!r}. .data directory contents" + " should be named like: '/'." + ).format(wheel_path, record_path) + raise InstallationError(message) + scheme_path = scheme_paths[scheme_key] dest_path = os.path.join(scheme_path, dest_subpath) assert_no_path_traversal(scheme_path, dest_path) diff --git a/tests/functional/test_install_wheel.py b/tests/functional/test_install_wheel.py index c53f13ca4..89d24a62f 100644 --- a/tests/functional/test_install_wheel.py +++ b/tests/functional/test_install_wheel.py @@ -681,3 +681,21 @@ def test_correct_package_name_while_creating_wheel_bug(script, package_name): package = create_basic_wheel_for_package(script, package_name, '1.0') wheel_name = os.path.basename(package) assert wheel_name == 'simple_package-1.0-py2.py3-none-any.whl' + + +@pytest.mark.parametrize("name", ["purelib", "abc"]) +def test_wheel_with_file_in_data_dir_has_reasonable_error( + script, tmpdir, name +): + """Normally we expect entities in the .data directory to be in a + subdirectory, but if they are not then we should show a reasonable error + message that includes the path. + """ + wheel_path = make_wheel( + "simple", "0.1.0", extra_data_files={name: "hello world"} + ).save_to_dir(tmpdir) + + result = script.pip( + "install", "--no-index", str(wheel_path), expect_error=True + ) + assert "simple-0.1.0.data/{}".format(name) in result.stderr From 3f9b326c115741a0a60e647e242961d80ccda08a Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Wed, 29 Jul 2020 18:06:25 -0400 Subject: [PATCH 016/209] Provide a reasonable error on invalid scheme keys Originally we would throw an `AttributeError` if a bad scheme key was used. After refactoring we would throw a `KeyError`, which isn't much better. Now we call out the wheel being processed, scheme key we didn't recognize, and provide a list of the valid scheme keys. This would likely be useful for people developing/testing the wheel. --- src/pip/_internal/operations/install/wheel.py | 14 +++++++++++++- tests/functional/test_install_wheel.py | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py index 0bd7b28be..681fc0aa8 100644 --- a/src/pip/_internal/operations/install/wheel.py +++ b/src/pip/_internal/operations/install/wheel.py @@ -592,7 +592,19 @@ def _install_wheel( ).format(wheel_path, record_path) raise InstallationError(message) - scheme_path = scheme_paths[scheme_key] + try: + scheme_path = scheme_paths[scheme_key] + except KeyError: + valid_scheme_keys = ", ".join(sorted(scheme_paths)) + message = ( + "Unknown scheme key used in {}: {} (for file {!r}). .data" + " directory contents should be in subdirectories named" + " with a valid scheme key ({})" + ).format( + wheel_path, scheme_key, record_path, valid_scheme_keys + ) + raise InstallationError(message) + dest_path = os.path.join(scheme_path, dest_subpath) assert_no_path_traversal(scheme_path, dest_path) return ZipBackedFile(record_path, dest_path, zip_file) diff --git a/tests/functional/test_install_wheel.py b/tests/functional/test_install_wheel.py index 89d24a62f..ad4e74967 100644 --- a/tests/functional/test_install_wheel.py +++ b/tests/functional/test_install_wheel.py @@ -699,3 +699,18 @@ def test_wheel_with_file_in_data_dir_has_reasonable_error( "install", "--no-index", str(wheel_path), expect_error=True ) assert "simple-0.1.0.data/{}".format(name) in result.stderr + + +def test_wheel_with_unknown_subdir_in_data_dir_has_reasonable_error( + script, tmpdir +): + wheel_path = make_wheel( + "simple", + "0.1.0", + extra_data_files={"unknown/hello.txt": "hello world"} + ).save_to_dir(tmpdir) + + result = script.pip( + "install", "--no-index", str(wheel_path), expect_error=True + ) + assert "simple-0.1.0.data/unknown/hello.txt" in result.stderr From 127c5b026c9e810a8bfb795662cd1be6567b8b64 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Wed, 29 Jul 2020 18:14:58 -0400 Subject: [PATCH 017/209] Add news --- news/8654.bugfix | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 news/8654.bugfix diff --git a/news/8654.bugfix b/news/8654.bugfix new file mode 100644 index 000000000..ec0df7a90 --- /dev/null +++ b/news/8654.bugfix @@ -0,0 +1,2 @@ +Trace a better error message on installation failure due to invalid ``.data`` +files in wheels. From 1fd5098b2480eee5be4a0033b43cb02605bc0fe1 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 30 Jul 2020 21:58:05 +0800 Subject: [PATCH 018/209] Canonicalize name in check_if_exists The previous implementation uses pkg_resources.get_distribution(), which does not canonicalize the package name correctly, and fails when combined with pip's own get_distribution(), which does canonicalize names. This makes InstallRequirement.check_if_exists() only use pip's own canonicalization logic so different package name forms are matched as expected. --- news/8645.bugfix | 2 ++ src/pip/_internal/req/req_install.py | 28 +++++++------------- tests/functional/test_install_upgrade.py | 33 ++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 19 deletions(-) create mode 100644 news/8645.bugfix diff --git a/news/8645.bugfix b/news/8645.bugfix new file mode 100644 index 000000000..a388d24e4 --- /dev/null +++ b/news/8645.bugfix @@ -0,0 +1,2 @@ +Correctly find already-installed distributions with dot (``.``) in the name +and uninstall them when needed. diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 644930a15..4759f4af6 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -429,25 +429,13 @@ class InstallRequirement(object): """ if self.req is None: return - # get_distribution() will resolve the entire list of requirements - # anyway, and we've already determined that we need the requirement - # in question, so strip the marker so that we don't try to - # evaluate it. - no_marker = Requirement(str(self.req)) - no_marker.marker = None - - # pkg_resources uses the canonical name to look up packages, but - # the name passed passed to get_distribution is not canonicalized - # so we have to explicitly convert it to a canonical name - no_marker.name = canonicalize_name(no_marker.name) - try: - self.satisfied_by = pkg_resources.get_distribution(str(no_marker)) - except pkg_resources.DistributionNotFound: + existing_dist = get_distribution(self.req.name) + if not existing_dist: return - except pkg_resources.VersionConflict: - existing_dist = get_distribution( - self.req.name - ) + + existing_version = existing_dist.parsed_version + if not self.req.specifier.contains(existing_version, prereleases=True): + self.satisfied_by = None if use_user_site: if dist_in_usersite(existing_dist): self.should_reinstall = True @@ -461,11 +449,13 @@ class InstallRequirement(object): else: self.should_reinstall = True else: - if self.editable and self.satisfied_by: + if self.editable: self.should_reinstall = True # when installing editables, nothing pre-existing should ever # satisfy self.satisfied_by = None + else: + self.satisfied_by = existing_dist # Things valid for wheels @property diff --git a/tests/functional/test_install_upgrade.py b/tests/functional/test_install_upgrade.py index e45bf3148..02e221101 100644 --- a/tests/functional/test_install_upgrade.py +++ b/tests/functional/test_install_upgrade.py @@ -1,3 +1,4 @@ +import itertools import os import sys import textwrap @@ -7,6 +8,7 @@ import pytest from tests.lib import pyversion # noqa: F401 from tests.lib import assert_all_changes from tests.lib.local_repos import local_checkout +from tests.lib.wheel import make_wheel @pytest.mark.network @@ -439,3 +441,34 @@ class TestUpgradeDistributeToSetuptools(object): cwd=pip_src, expect_stderr=True, ) + + +@pytest.mark.parametrize("req1, req2", list(itertools.product( + ["foo.bar", "foo_bar", "foo-bar"], ["foo.bar", "foo_bar", "foo-bar"], +))) +def test_install_find_existing_package_canonicalize(script, req1, req2): + """Ensure an already-installed dist is found no matter how the dist name + was normalized on installation. (pypa/pip#8645) + """ + # Create and install a package that's not available in the later stage. + req_container = script.scratch_path.joinpath("foo-bar") + req_container.mkdir() + req_path = make_wheel("foo_bar", "1.0").save_to_dir(req_container) + script.pip("install", "--no-index", req_path) + + # Depend on the previously installed, but now unavailable package. + pkg_container = script.scratch_path.joinpath("pkg") + pkg_container.mkdir() + make_wheel( + "pkg", + "1.0", + metadata_updates={"Requires-Dist": req2}, + ).save_to_dir(pkg_container) + + # Ensure the previously installed package can be correctly used to match + # the dependency. + result = script.pip( + "install", "--no-index", "--find-links", pkg_container, "pkg", + ) + satisfied_message = "Requirement already satisfied: {}".format(req2) + assert satisfied_message in result.stdout, str(result) From 27b4980c6c747b10818fcaf11f0fe1bad72754fb Mon Sep 17 00:00:00 2001 From: Sumana Harihareswara Date: Thu, 30 Jul 2020 13:58:47 -0400 Subject: [PATCH 019/209] Update documentation to reflect updated resolver feature flag Followup to #8371, #8530, #8513. --- .github/ISSUE_TEMPLATE/bug-report.md | 2 +- docs/html/user_guide.rst | 16 ++++++++-------- news/8660.doc | 1 + 3 files changed, 10 insertions(+), 9 deletions(-) create mode 100644 news/8660.doc diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index fdefbe1a4..157be28b6 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -4,7 +4,7 @@ about: Create a report to help us improve --- **Environment** diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 702a97d0c..31887a288 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -783,7 +783,7 @@ specified packages due to conflicting dependencies (a ``ResolutionImpossible`` error). This documentation is specific to the new resolver, which you can use -with the flag ``--unstable-feature=resolver``. +with the flag ``--use-feature=2020-resolver``. Understanding your error message -------------------------------- @@ -1029,13 +1029,13 @@ Changes to the pip dependency resolver in 20.2 (2020) ===================================================== pip 20.1 included an alpha version of the new resolver (hidden behind -an optional ``--unstable-feature=resolver`` flag). pip 20.2 includes a -robust beta of the new resolver (hidden behind an optional -``--use-feature=2020-resolver`` flag) that we encourage you to -test. We will continue to improve the pip dependency resolver in -response to testers' feedback. Please give us feedback through the -`resolver testing survey`_. This will help us prepare to release pip -20.3, with the new resolver on by default, in October. +an optional ``--unstable-feature=resolver`` flag). pip 20.2 removes +that flag, and includes a robust beta of the new resolver (hidden +behind an optional ``--use-feature=2020-resolver`` flag) that we +encourage you to test. We will continue to improve the pip dependency +resolver in response to testers' feedback. Please give us feedback +through the `resolver testing survey`_. This will help us prepare to +release pip 20.3, with the new resolver on by default, in October. Watch out for ------------- diff --git a/news/8660.doc b/news/8660.doc new file mode 100644 index 000000000..45b71cc26 --- /dev/null +++ b/news/8660.doc @@ -0,0 +1 @@ +Fix feature flag name in docs. From c3e1a153fd252a3b279fbd1598a4a9771a120a36 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 31 Jul 2020 06:38:43 +0800 Subject: [PATCH 020/209] Improve SVN version parser SVN has multiple distributions on Windows, e.g. SlikSVN, CollabNet. Some of them suffix the version with a "-{distro}" part, which causes the previous implementation to fail. This patch removes that final part and make the version logic work. --- src/pip/_internal/vcs/subversion.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py index 14825f791..ab134970b 100644 --- a/src/pip/_internal/vcs/subversion.py +++ b/src/pip/_internal/vcs/subversion.py @@ -213,6 +213,8 @@ class Subversion(VersionControl): # compiled Feb 25 2019, 14:20:39 on x86_64-apple-darwin17.0.0 # svn, version 1.7.14 (r1542130) # compiled Mar 28 2018, 08:49:13 on x86_64-pc-linux-gnu + # svn, version 1.12.0-SlikSvn (SlikSvn/1.12.0) + # compiled May 28 2019, 13:44:56 on x86_64-microsoft-windows6.2 version_prefix = 'svn, version ' version = self.run_command(['--version']) @@ -220,7 +222,7 @@ class Subversion(VersionControl): return () version = version[len(version_prefix):].split()[0] - version_list = version.split('.') + version_list = version.partition('-')[0].split('.') try: parsed_version = tuple(map(int, version_list)) except ValueError: From c8596e141009bb8536a43fda43ed335c55721f60 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 31 Jul 2020 06:45:40 +0800 Subject: [PATCH 021/209] News --- news/8665.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/8665.bugfix diff --git a/news/8665.bugfix b/news/8665.bugfix new file mode 100644 index 000000000..0ce458463 --- /dev/null +++ b/news/8665.bugfix @@ -0,0 +1 @@ +Fix SVN version detection for alternative SVN distributions. From f31898e18c5235f43cfd82f170d79ee5df3927bc Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 31 Jul 2020 13:15:23 +0100 Subject: [PATCH 022/209] Fix typos in the docs about conflicting dependencies Previously: - the example wildcard version string was being rendered with a stray space (`== 3.1. *` instead of `== 3.1.*`) due to the markup being split over two lines - the "Dependency Hell" Wikipedia URL 404ed due to the trailing `>` --- docs/html/user_guide.rst | 6 +++--- news/AE707F60-0ABE-4DBA-98AA-59CE8F989386.trivial | 0 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 news/AE707F60-0ABE-4DBA-98AA-59CE8F989386.trivial diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 31887a288..4f276e46a 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -838,8 +838,8 @@ specifying package versions (e.g. ``~=`` or ``*``): semantic versioning.", "``~=3.1``: version ``3.1`` or later, but not version ``4.0`` or later. ``~=3.1.2``: version ``3.1.2`` or later, but not version ``3.2.0`` or later." - ``*``,Can be used at the end of a version number to represent "all", "``== 3. - 1.*``: any version that starts with ``3.1``. Equivalent to ``~=3.1.0``." + ``*``,Can be used at the end of a version number to represent "all", "``== 3.1.*``: + any version that starts with ``3.1``. Equivalent to ``~=3.1.0``." The detailed specification of supported comparison operators can be found in :pep:`440`. @@ -945,7 +945,7 @@ Unfortunately, **the pip team cannot provide support for individual dependency conflict errors**. Please *only* open a ticket on the `pip issue tracker`_ if you believe that your problem has exposed a bug in pip. -.. _dependency hell: https://en.wikipedia.org/wiki/Dependency_hell> +.. _dependency hell: https://en.wikipedia.org/wiki/Dependency_hell .. _Awesome Python: https://python.libhunt.com/ .. _Python user Discourse: https://discuss.python.org/c/users/7 .. _Python user forums: https://www.python.org/community/forums/ diff --git a/news/AE707F60-0ABE-4DBA-98AA-59CE8F989386.trivial b/news/AE707F60-0ABE-4DBA-98AA-59CE8F989386.trivial new file mode 100644 index 000000000..e69de29bb From ea47920767347cc5cfed5e69e9429229e7b9b2ec Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 1 Aug 2020 02:03:45 +0800 Subject: [PATCH 023/209] Add test case for SlikSVN version parsing --- tests/unit/test_vcs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 590cb5c0b..93598c367 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -443,6 +443,9 @@ def test_subversion__call_vcs_version(): ('svn, version 1.10.3 (r1842928)\n' ' compiled Feb 25 2019, 14:20:39 on x86_64-apple-darwin17.0.0', (1, 10, 3)), + ('svn, version 1.12.0-SlikSvn (SlikSvn/1.12.0)\n' + ' compiled May 28 2019, 13:44:56 on x86_64-microsoft-windows6.2', + (1, 12, 0)), ('svn, version 1.9.7 (r1800392)', (1, 9, 7)), ('svn, version 1.9.7a1 (r1800392)', ()), ('svn, version 1.9 (r1800392)', (1, 9)), From 0ef877339a270c760a51c3a61e55c2f4f86f84ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Sat, 1 Aug 2020 16:57:30 +0700 Subject: [PATCH 024/209] Make assertions independent of log prefixes --- ...81396b-ebe4-46a9-b38f-e6b0da97d53a.trivial | 0 tests/functional/test_install_check.py | 20 +++++++++---------- 2 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 news/2581396b-ebe4-46a9-b38f-e6b0da97d53a.trivial diff --git a/news/2581396b-ebe4-46a9-b38f-e6b0da97d53a.trivial b/news/2581396b-ebe4-46a9-b38f-e6b0da97d53a.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/test_install_check.py b/tests/functional/test_install_check.py index 88609422b..96ed6b6e1 100644 --- a/tests/functional/test_install_check.py +++ b/tests/functional/test_install_check.py @@ -1,8 +1,10 @@ from tests.lib import create_test_package_with_setup -def contains_expected_lines(string, expected_lines): - return set(expected_lines) <= set(string.splitlines()) +def assert_contains_expected_lines(string, expected_lines): + lines = string.splitlines() + for expected_line in expected_lines: + assert any(line.endswith(expected_line) for line in lines) def test_check_install_canonicalization(script): @@ -38,7 +40,7 @@ def test_check_install_canonicalization(script): "pkga 1.0 requires SPECIAL.missing, which is not installed.", ] # Deprecated python versions produce an extra warning on stderr - assert contains_expected_lines(result.stderr, expected_lines) + assert_contains_expected_lines(result.stderr, expected_lines) assert result.returncode == 0 # Install the second missing package and expect that there is no warning @@ -55,7 +57,7 @@ def test_check_install_canonicalization(script): expected_lines = [ "No broken requirements found.", ] - assert contains_expected_lines(result.stdout, expected_lines) + assert_contains_expected_lines(result.stdout, expected_lines) assert result.returncode == 0 @@ -85,12 +87,10 @@ def test_check_install_does_not_warn_for_out_of_graph_issues(script): result = script.pip( 'install', '--no-index', pkg_conflict_path, allow_stderr_error=True, ) - assert contains_expected_lines(result.stderr, [ + assert_contains_expected_lines(result.stderr, [ "broken 1.0 requires missing, which is not installed.", - ( - "broken 1.0 requires conflict<1.0, but " - "you'll have conflict 1.0 which is incompatible." - ), + "broken 1.0 requires conflict<1.0, " + "but you'll have conflict 1.0 which is incompatible." ]) # Install unrelated package @@ -105,4 +105,4 @@ def test_check_install_does_not_warn_for_out_of_graph_issues(script): "broken 1.0 requires missing, which is not installed.", "broken 1.0 has requirement conflict<1.0, but you have conflict 1.0.", ] - assert contains_expected_lines(result.stdout, expected_lines) + assert_contains_expected_lines(result.stdout, expected_lines) From e48a0cb7cba70d7184f7a08b49163cf5a6451b61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Sat, 1 Aug 2020 16:59:07 +0700 Subject: [PATCH 025/209] Remove no-longer-used messages --- src/pip/_internal/commands/install.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 8c2c32fd4..77ec210d6 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -21,7 +21,6 @@ from pip._internal.locations import distutils_scheme from pip._internal.operations.check import check_install_conflicts from pip._internal.req import install_given_reqs from pip._internal.req.req_tracker import get_requirement_tracker -from pip._internal.utils.datetime import today_is_later_than from pip._internal.utils.deprecation import deprecated from pip._internal.utils.distutils_args import parse_distutils_args from pip._internal.utils.filesystem import test_writable_dir @@ -558,19 +557,6 @@ class InstallCommand(RequirementCommand): "your packages with the new resolver before it becomes the " "default.\n" ) - elif not today_is_later_than(year=2020, month=7, day=31): - # NOTE: trailing newlines here are intentional - parts.append( - "Pip will install or upgrade your package(s) and its " - "dependencies without taking into account other packages you " - "already have installed. This may cause an uncaught " - "dependency conflict.\n" - ) - form_link = "https://forms.gle/cWKMoDs8sUVE29hz9" - parts.append( - "If you would like pip to take your other packages into " - "account, please tell us here: {}\n".format(form_link) - ) # NOTE: There is some duplication here, with commands/check.py for project_name in missing: From 22aec424d96f48e0054505070eb57b873f0089e0 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com> Date: Sun, 2 Aug 2020 07:46:54 +0530 Subject: [PATCH 026/209] Rewrap lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Nguyễn Gia Phong --- docs/html/user_guide.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 4f276e46a..b0b25677e 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -838,8 +838,9 @@ specifying package versions (e.g. ``~=`` or ``*``): semantic versioning.", "``~=3.1``: version ``3.1`` or later, but not version ``4.0`` or later. ``~=3.1.2``: version ``3.1.2`` or later, but not version ``3.2.0`` or later." - ``*``,Can be used at the end of a version number to represent "all", "``== 3.1.*``: - any version that starts with ``3.1``. Equivalent to ``~=3.1.0``." + ``*``, Can be used at the end of a version number to represent "all", + "``==3.1.*``: any version that starts with ``3.1``. + Equivalent to ``~=3.1.0``." The detailed specification of supported comparison operators can be found in :pep:`440`. From b97c199cf7f2c308a50be8209ef29fb97e630006 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 2 Aug 2020 07:53:02 +0530 Subject: [PATCH 027/209] Point to latest documentation This allows us to update the content users would see as we get feedback. --- src/pip/_internal/resolution/resolvelib/factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index bd7e3efd9..36d6baae9 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -454,6 +454,6 @@ class Factory(object): return DistributionNotFound( "ResolutionImpossible For help visit: " - "https://pip.pypa.io/en/stable/user_guide/" + "https://pip.pypa.io/en/latest/user_guide/" "#fixing-conflicting-dependencies" ) From c412613efe94f3dc7e836352bfe0c86d4f789c47 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 2 Aug 2020 07:55:49 +0530 Subject: [PATCH 028/209] Tweak ResolutionImpossible error line This makes it more consistent with how error "summary" lines look. eg: IndexError: list index out of range ModuleNotFoundError: No module named 'notamodule' --- src/pip/_internal/resolution/resolvelib/factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 36d6baae9..dab23aa09 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -453,7 +453,7 @@ class Factory(object): logger.info(msg) return DistributionNotFound( - "ResolutionImpossible For help visit: " + "ResolutionImpossible: for help visit " "https://pip.pypa.io/en/latest/user_guide/" "#fixing-conflicting-dependencies" ) From b4632d080bdf8124006c03cb829ffb5186a5de52 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sun, 2 Aug 2020 08:20:16 +0800 Subject: [PATCH 029/209] Failing test for new resolver + extras + --no-deps --- tests/yaml/extras.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/yaml/extras.yml b/tests/yaml/extras.yml index 6e2a1b17e..ac68fae49 100644 --- a/tests/yaml/extras.yml +++ b/tests/yaml/extras.yml @@ -40,3 +40,10 @@ cases: - E 1.0.0 - F 1.0.0 skip: old +- + request: + - install: D[extra_1] + options: --no-deps + response: + - state: + - D 1.0.0 From 3ce63a62d76fac211861d42770ca6072c7c97edd Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sun, 2 Aug 2020 08:22:04 +0800 Subject: [PATCH 030/209] Ask candidates for dependencies even on --no-deps ExtrasCandidate need to provide one dependency on the non-extra-ed self. --- .../_internal/resolution/resolvelib/base.py | 4 +-- .../resolution/resolvelib/candidates.py | 30 +++++++++++-------- .../resolution/resolvelib/provider.py | 8 +++-- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/base.py b/src/pip/_internal/resolution/resolvelib/base.py index a155a1101..7f71ef4c4 100644 --- a/src/pip/_internal/resolution/resolvelib/base.py +++ b/src/pip/_internal/resolution/resolvelib/base.py @@ -69,8 +69,8 @@ class Candidate(object): # type: () -> Optional[Link] raise NotImplementedError("Override in subclass") - def iter_dependencies(self): - # type: () -> Iterable[Optional[Requirement]] + def iter_dependencies(self, ignore_dependencies): + # type: (bool) -> Iterable[Optional[Requirement]] raise NotImplementedError("Override in subclass") def get_install_requirement(self): diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index c289bb583..912f72a76 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -275,8 +275,10 @@ class _InstallRequirementBackedCandidate(Candidate): return None return spec - def iter_dependencies(self): - # type: () -> Iterable[Optional[Requirement]] + def iter_dependencies(self, ignore_dependencies): + # type: (bool) -> Iterable[Optional[Requirement]] + if ignore_dependencies: + return for r in self.dist.requires(): yield self._factory.make_requirement_from_spec(str(r), self._ireq) python_dep = self._factory.make_requires_python_requirement( @@ -420,8 +422,10 @@ class AlreadyInstalledCandidate(Candidate): # type: () -> str return "{} {} (Installed)".format(self.name, self.version) - def iter_dependencies(self): - # type: () -> Iterable[Optional[Requirement]] + def iter_dependencies(self, ignore_dependencies): + # type: (bool) -> Iterable[Optional[Requirement]] + if ignore_dependencies: + return for r in self.dist.requires(): yield self._factory.make_requirement_from_spec(str(r), self._ireq) @@ -519,10 +523,16 @@ class ExtrasCandidate(Candidate): # type: () -> Optional[Link] return self.base.source_link - def iter_dependencies(self): - # type: () -> Iterable[Optional[Requirement]] + def iter_dependencies(self, ignore_dependencies): + # type: (bool) -> Iterable[Optional[Requirement]] factory = self.base._factory + # Add a dependency on the exact base + # (See note 2b in the class docstring) + yield factory.make_requirement_from_candidate(self.base) + if ignore_dependencies: + return + # The user may have specified extras that the candidate doesn't # support. We ignore any unsupported extras here. valid_extras = self.extras.intersection(self.base.dist.extras) @@ -535,10 +545,6 @@ class ExtrasCandidate(Candidate): extra ) - # Add a dependency on the exact base - # (See note 2b in the class docstring) - yield factory.make_requirement_from_candidate(self.base) - for r in self.base.dist.requires(valid_extras): requirement = factory.make_requirement_from_spec( str(r), self.base._ireq, valid_extras, @@ -585,8 +591,8 @@ class RequiresPythonCandidate(Candidate): # type: () -> str return "Python {}".format(self.version) - def iter_dependencies(self): - # type: () -> Iterable[Optional[Requirement]] + def iter_dependencies(self, ignore_dependencies): + # type: (bool) -> Iterable[Optional[Requirement]] return () def get_install_requirement(self): diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index 72f162059..50b16a12e 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -145,6 +145,8 @@ class PipProvider(AbstractProvider): def get_dependencies(self, candidate): # type: (Candidate) -> Sequence[Requirement] - if self._ignore_dependencies: - return [] - return [r for r in candidate.iter_dependencies() if r is not None] + return [ + r + for r in candidate.iter_dependencies(self._ignore_dependencies) + if r is not None + ] From 77cedcf52f823039edfcad19af1f933926ba5d62 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sun, 2 Aug 2020 08:24:15 +0800 Subject: [PATCH 031/209] News --- news/8677.bugfix | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 news/8677.bugfix diff --git a/news/8677.bugfix b/news/8677.bugfix new file mode 100644 index 000000000..e9efd8279 --- /dev/null +++ b/news/8677.bugfix @@ -0,0 +1,2 @@ +New resolver: Correctly include the base package when specified with extras +in ``--no-deps`` mode. From 32b5e43c79c3dab421637da5870cf4e6341eab58 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 3 Aug 2020 05:28:10 +0800 Subject: [PATCH 032/209] Flip the flag with another name --- src/pip/_internal/resolution/resolvelib/base.py | 2 +- .../_internal/resolution/resolvelib/candidates.py | 14 +++++++------- .../_internal/resolution/resolvelib/provider.py | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/base.py b/src/pip/_internal/resolution/resolvelib/base.py index 7f71ef4c4..9245747bf 100644 --- a/src/pip/_internal/resolution/resolvelib/base.py +++ b/src/pip/_internal/resolution/resolvelib/base.py @@ -69,7 +69,7 @@ class Candidate(object): # type: () -> Optional[Link] raise NotImplementedError("Override in subclass") - def iter_dependencies(self, ignore_dependencies): + def iter_dependencies(self, with_requires): # type: (bool) -> Iterable[Optional[Requirement]] raise NotImplementedError("Override in subclass") diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 912f72a76..46cc7e7a2 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -275,9 +275,9 @@ class _InstallRequirementBackedCandidate(Candidate): return None return spec - def iter_dependencies(self, ignore_dependencies): + def iter_dependencies(self, with_requires): # type: (bool) -> Iterable[Optional[Requirement]] - if ignore_dependencies: + if not with_requires: return for r in self.dist.requires(): yield self._factory.make_requirement_from_spec(str(r), self._ireq) @@ -422,9 +422,9 @@ class AlreadyInstalledCandidate(Candidate): # type: () -> str return "{} {} (Installed)".format(self.name, self.version) - def iter_dependencies(self, ignore_dependencies): + def iter_dependencies(self, with_requires): # type: (bool) -> Iterable[Optional[Requirement]] - if ignore_dependencies: + if not with_requires: return for r in self.dist.requires(): yield self._factory.make_requirement_from_spec(str(r), self._ireq) @@ -523,14 +523,14 @@ class ExtrasCandidate(Candidate): # type: () -> Optional[Link] return self.base.source_link - def iter_dependencies(self, ignore_dependencies): + def iter_dependencies(self, with_requires): # type: (bool) -> Iterable[Optional[Requirement]] factory = self.base._factory # Add a dependency on the exact base # (See note 2b in the class docstring) yield factory.make_requirement_from_candidate(self.base) - if ignore_dependencies: + if not with_requires: return # The user may have specified extras that the candidate doesn't @@ -591,7 +591,7 @@ class RequiresPythonCandidate(Candidate): # type: () -> str return "Python {}".format(self.version) - def iter_dependencies(self, ignore_dependencies): + def iter_dependencies(self, with_requires): # type: (bool) -> Iterable[Optional[Requirement]] return () diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index 50b16a12e..b2eb9d06e 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -145,8 +145,9 @@ class PipProvider(AbstractProvider): def get_dependencies(self, candidate): # type: (Candidate) -> Sequence[Requirement] + with_requires = not self._ignore_dependencies return [ r - for r in candidate.iter_dependencies(self._ignore_dependencies) + for r in candidate.iter_dependencies(with_requires) if r is not None ] From d957cc94c8ff9e83503f0a509ecbc90b61180775 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 11:04:27 -0400 Subject: [PATCH 033/209] Don't set _dist until it has been validated Previously a call to `_fetch_metadata` could result in several possible outcomes: 1. `_dist` set, `_provided` not set, dist returned - for lazy wheels 2. `_dist` set, `_provided` not set, exception - for bad lazy wheels 3. `_dist` not set, `_provided` not set, exception - for non-lazy req exceptions 4. `_dist` set, `_provided` not set, exception - for bad non-lazy reqs 5. `_dist` set, `_provided` set, dist returned - for non-lazy reqs and probably more. Our intent is to use `_dist` being set as the indicator of "this requirement has been fully processed successfully" and discard `_prepared`, since we don't actually rely on any of the other states (they simply lead to a failure or in the future a retry). --- .../resolution/resolvelib/candidates.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 3d8e399a7..a203fa3c0 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -203,12 +203,11 @@ class _InstallRequirementBackedCandidate(Candidate): # type: () -> Distribution raise NotImplementedError("Override in subclass") - def _check_metadata_consistency(self): - # type: () -> None + def _check_metadata_consistency(self, dist): + # type: (Distribution) -> None """Check for consistency of project name and version of dist.""" # TODO: (Longer term) Rather than abort, reject this candidate # and backtrack. This would need resolvelib support. - dist = self._dist # type: Distribution name = canonicalize_name(dist.project_name) if self._name is not None and self._name != name: raise MetadataInconsistent(self._ireq, "name", dist.project_name) @@ -221,13 +220,14 @@ class _InstallRequirementBackedCandidate(Candidate): if self._prepared: return try: - self._dist = self._prepare_distribution() + dist = self._prepare_distribution() except HashError as e: e.req = self._ireq raise - assert self._dist is not None, "Distribution already installed" - self._check_metadata_consistency() + assert dist is not None, "Distribution already installed" + self._check_metadata_consistency(dist) + self._dist = dist self._prepared = True def _fetch_metadata(self): @@ -247,8 +247,9 @@ class _InstallRequirementBackedCandidate(Candidate): ) url = self._link.url.split('#', 1)[0] session = preparer.downloader._session - self._dist = dist_from_wheel_url(self._name, url, session) - self._check_metadata_consistency() + dist = dist_from_wheel_url(self._name, url, session) + self._check_metadata_consistency(dist) + self._dist = dist if self._dist is None: self._prepare() From 7289625734e07f425a616727c2c9925a9df724d8 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 11:08:01 -0400 Subject: [PATCH 034/209] Remove redundant guard variable Now that `_dist` is only set on success, we can use it to guard against repeated execution instead of `_prepared`. As a result there are now only two possible outcomes for calling `dist`: 1. `_dist` set and returned - lazy and non-lazy req 2. `_dist` not set and exception raised - bad lazy or bad non-lazy req --- src/pip/_internal/resolution/resolvelib/candidates.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index a203fa3c0..387d6ec26 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -147,7 +147,6 @@ class _InstallRequirementBackedCandidate(Candidate): self._name = name self._version = version self._dist = None # type: Optional[Distribution] - self._prepared = False def __repr__(self): # type: () -> str @@ -217,7 +216,7 @@ class _InstallRequirementBackedCandidate(Candidate): def _prepare(self): # type: () -> None - if self._prepared: + if self._dist is not None: return try: dist = self._prepare_distribution() @@ -228,7 +227,6 @@ class _InstallRequirementBackedCandidate(Candidate): assert dist is not None, "Distribution already installed" self._check_metadata_consistency(dist) self._dist = dist - self._prepared = True def _fetch_metadata(self): # type: () -> None From 7a5e043776d6601f1c425e7a18a4b6a1538baa99 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 11:09:00 -0400 Subject: [PATCH 035/209] Remove unnecessary check for _dist Since `_prepare` now internally validates that `_dist` isn't set, we don't need to. --- src/pip/_internal/resolution/resolvelib/candidates.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 387d6ec26..2fa8e2fcf 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -248,8 +248,7 @@ class _InstallRequirementBackedCandidate(Candidate): dist = dist_from_wheel_url(self._name, url, session) self._check_metadata_consistency(dist) self._dist = dist - if self._dist is None: - self._prepare() + self._prepare() @property def dist(self): From 4d94ae4c40dd6a22bf0062c4e899aa23bc09250d Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 11:10:00 -0400 Subject: [PATCH 036/209] Move non-lazy req fallback outside of `_fetch_metadata` No change in behavior, we just want to unify "requirements processing" and moving this function out is a prereq for moving `_fetch_metadata` in. --- src/pip/_internal/resolution/resolvelib/candidates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 2fa8e2fcf..eefca5ed6 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -248,13 +248,13 @@ class _InstallRequirementBackedCandidate(Candidate): dist = dist_from_wheel_url(self._name, url, session) self._check_metadata_consistency(dist) self._dist = dist - self._prepare() @property def dist(self): # type: () -> Distribution if self._dist is None: self._fetch_metadata() + self._prepare() return self._dist def _get_requires_python_specifier(self): From 8c3c0ade7831d7e82b2390d1fcd1f41c1b0e80f7 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 11:11:00 -0400 Subject: [PATCH 037/209] Move _fetch_metadata into _prepare Since `_prepare` is called in two places, we preserve the `if self._dist is not None` protection above the new call to `_fetch_metadata`. The second `if` in `_prepare` handles the early return required when processing a lazy wheel. --- src/pip/_internal/resolution/resolvelib/candidates.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index eefca5ed6..d810bb2e1 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -216,6 +216,9 @@ class _InstallRequirementBackedCandidate(Candidate): def _prepare(self): # type: () -> None + if self._dist is not None: + return + self._fetch_metadata() if self._dist is not None: return try: @@ -253,7 +256,6 @@ class _InstallRequirementBackedCandidate(Candidate): def dist(self): # type: () -> Distribution if self._dist is None: - self._fetch_metadata() self._prepare() return self._dist From a72d04f734150992328442113ee9df3ce2f2fb00 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 11:35:47 -0400 Subject: [PATCH 038/209] Move common processing out of _fetch_metadata Returning a `Distribution` makes `_fetch_metadata` look more like `_prepare_distribution`, in preparation for moving it there next. --- .../_internal/resolution/resolvelib/candidates.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index d810bb2e1..2a848c276 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -218,8 +218,10 @@ class _InstallRequirementBackedCandidate(Candidate): # type: () -> None if self._dist is not None: return - self._fetch_metadata() - if self._dist is not None: + dist = self._fetch_metadata() + if dist is not None: + self._check_metadata_consistency(dist) + self._dist = dist return try: dist = self._prepare_distribution() @@ -232,7 +234,7 @@ class _InstallRequirementBackedCandidate(Candidate): self._dist = dist def _fetch_metadata(self): - # type: () -> None + # type: () -> Optional[Distribution] """Fetch metadata, using lazy wheel if possible.""" preparer = self._factory.preparer use_lazy_wheel = self._factory.use_lazy_wheel @@ -248,9 +250,8 @@ class _InstallRequirementBackedCandidate(Candidate): ) url = self._link.url.split('#', 1)[0] session = preparer.downloader._session - dist = dist_from_wheel_url(self._name, url, session) - self._check_metadata_consistency(dist) - self._dist = dist + return dist_from_wheel_url(self._name, url, session) + return None @property def dist(self): From ec5b6d7b8091b80533d3f2268eb3df16ad3d19f6 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 11:38:34 -0400 Subject: [PATCH 039/209] Remove extra metadata consistency check Instead of an early return, we fall through to the existing check at the end of this function. This aligns our treatment of `_fetch_metadata` and `_prepare_distribution`. --- src/pip/_internal/resolution/resolvelib/candidates.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 2a848c276..a0bc3d191 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -219,12 +219,9 @@ class _InstallRequirementBackedCandidate(Candidate): if self._dist is not None: return dist = self._fetch_metadata() - if dist is not None: - self._check_metadata_consistency(dist) - self._dist = dist - return try: - dist = self._prepare_distribution() + if dist is None: + dist = self._prepare_distribution() except HashError as e: e.req = self._ireq raise From 45ab317610a0e713c515d405d39f79c8746a5056 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 11:44:42 -0400 Subject: [PATCH 040/209] Move call to _fetch_metadata next to call to RequirementPreparer Since wheels can't be editable, we can move this into LinkCandidate, closer to `RequirementPreparer.prepare_linked_requirement` into which we want to integrate `_fetch_metadata`. --- src/pip/_internal/resolution/resolvelib/candidates.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index a0bc3d191..d7f031476 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -218,10 +218,8 @@ class _InstallRequirementBackedCandidate(Candidate): # type: () -> None if self._dist is not None: return - dist = self._fetch_metadata() try: - if dist is None: - dist = self._prepare_distribution() + dist = self._prepare_distribution() except HashError as e: e.req = self._ireq raise @@ -322,6 +320,9 @@ class LinkCandidate(_InstallRequirementBackedCandidate): def _prepare_distribution(self): # type: () -> Distribution + dist = self._fetch_metadata() + if dist is not None: + return dist return self._factory.preparer.prepare_linked_requirement( self._ireq, parallel_builds=True, ) From e49dcfdc35e7c941df1fc7388adb78c1981496ce Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 11:52:49 -0400 Subject: [PATCH 041/209] Move lazy_wheel warning out of Resolver This warning just needs to be traced in one place for all commands, there's no need for the resolver to know about it. Moving the warning out of the Resolver will make it easier to change how we provide the option. --- src/pip/_internal/cli/req_command.py | 13 ++++++++++++- src/pip/_internal/resolution/resolvelib/resolver.py | 8 -------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 78b5ce6a1..fcbdb70c2 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -259,6 +259,17 @@ class RequirementCommand(IndexGroupCommand): # "Resolver" class being redefined. if '2020-resolver' in options.features_enabled: import pip._internal.resolution.resolvelib.resolver + + lazy_wheel = 'fast-deps' in options.features_enabled + if lazy_wheel: + logger.warning( + 'pip is using lazily downloaded wheels using HTTP ' + 'range requests to obtain dependency information. ' + 'This experimental feature is enabled through ' + '--use-feature=fast-deps and it is not ready for ' + 'production.' + ) + return pip._internal.resolution.resolvelib.resolver.Resolver( preparer=preparer, finder=finder, @@ -271,7 +282,7 @@ class RequirementCommand(IndexGroupCommand): force_reinstall=force_reinstall, upgrade_strategy=upgrade_strategy, py_version_info=py_version_info, - lazy_wheel='fast-deps' in options.features_enabled, + lazy_wheel=lazy_wheel, ) import pip._internal.resolution.legacy.resolver return pip._internal.resolution.legacy.resolver.Resolver( diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py index 43ea24863..db0300e3b 100644 --- a/src/pip/_internal/resolution/resolvelib/resolver.py +++ b/src/pip/_internal/resolution/resolvelib/resolver.py @@ -52,14 +52,6 @@ class Resolver(BaseResolver): lazy_wheel=False, # type: bool ): super(Resolver, self).__init__() - if lazy_wheel: - logger.warning( - 'pip is using lazily downloaded wheels using HTTP ' - 'range requests to obtain dependency information. ' - 'This experimental feature is enabled through ' - '--use-feature=fast-deps and it is not ready for production.' - ) - assert upgrade_strategy in self._allowed_strategies self.factory = Factory( From f0d4df10eb8f757f6bfb66aa5f3ed34629866447 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 11:58:19 -0400 Subject: [PATCH 042/209] Propagate lazy_wheel option through RequirementPreparer Reduces dependence on Candidate (and Resolver (and Factory)). --- src/pip/_internal/cli/req_command.py | 25 +++++++++++-------- src/pip/_internal/operations/prepare.py | 4 +++ .../resolution/resolvelib/candidates.py | 2 +- .../resolution/resolvelib/factory.py | 2 -- .../resolution/resolvelib/resolver.py | 2 -- tests/unit/test_req.py | 1 + 6 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index fcbdb70c2..76d83896c 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -218,6 +218,19 @@ class RequirementCommand(IndexGroupCommand): temp_build_dir_path = temp_build_dir.path assert temp_build_dir_path is not None + if '2020-resolver' in options.features_enabled: + lazy_wheel = 'fast-deps' in options.features_enabled + if lazy_wheel: + logger.warning( + 'pip is using lazily downloaded wheels using HTTP ' + 'range requests to obtain dependency information. ' + 'This experimental feature is enabled through ' + '--use-feature=fast-deps and it is not ready for ' + 'production.' + ) + else: + lazy_wheel = False + return RequirementPreparer( build_dir=temp_build_dir_path, src_dir=options.src_dir, @@ -229,6 +242,7 @@ class RequirementCommand(IndexGroupCommand): finder=finder, require_hashes=options.require_hashes, use_user_site=use_user_site, + lazy_wheel=lazy_wheel, ) @staticmethod @@ -260,16 +274,6 @@ class RequirementCommand(IndexGroupCommand): if '2020-resolver' in options.features_enabled: import pip._internal.resolution.resolvelib.resolver - lazy_wheel = 'fast-deps' in options.features_enabled - if lazy_wheel: - logger.warning( - 'pip is using lazily downloaded wheels using HTTP ' - 'range requests to obtain dependency information. ' - 'This experimental feature is enabled through ' - '--use-feature=fast-deps and it is not ready for ' - 'production.' - ) - return pip._internal.resolution.resolvelib.resolver.Resolver( preparer=preparer, finder=finder, @@ -282,7 +286,6 @@ class RequirementCommand(IndexGroupCommand): force_reinstall=force_reinstall, upgrade_strategy=upgrade_strategy, py_version_info=py_version_info, - lazy_wheel=lazy_wheel, ) import pip._internal.resolution.legacy.resolver return pip._internal.resolution.legacy.resolver.Resolver( diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index e6a34172d..6c8ceae18 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -329,6 +329,7 @@ class RequirementPreparer(object): finder, # type: PackageFinder require_hashes, # type: bool use_user_site, # type: bool + lazy_wheel, # type: bool ): # type: (...) -> None super(RequirementPreparer, self).__init__() @@ -362,6 +363,9 @@ class RequirementPreparer(object): # Should install in user site-packages? self.use_user_site = use_user_site + # Should wheels be downloaded lazily? + self.use_lazy_wheel = lazy_wheel + @property def _download_should_save(self): # type: () -> bool diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index d7f031476..e780dafd5 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -232,7 +232,7 @@ class _InstallRequirementBackedCandidate(Candidate): # type: () -> Optional[Distribution] """Fetch metadata, using lazy wheel if possible.""" preparer = self._factory.preparer - use_lazy_wheel = self._factory.use_lazy_wheel + use_lazy_wheel = preparer.use_lazy_wheel remote_wheel = self._link.is_wheel and not self._link.is_file if use_lazy_wheel and remote_wheel and not preparer.require_hashes: assert self._name is not None diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index bd7e3efd9..c36c45200 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -82,7 +82,6 @@ class Factory(object): ignore_installed, # type: bool ignore_requires_python, # type: bool py_version_info=None, # type: Optional[Tuple[int, ...]] - lazy_wheel=False, # type: bool ): # type: (...) -> None self._finder = finder @@ -93,7 +92,6 @@ class Factory(object): self._use_user_site = use_user_site self._force_reinstall = force_reinstall self._ignore_requires_python = ignore_requires_python - self.use_lazy_wheel = lazy_wheel self._link_candidate_cache = {} # type: Cache[LinkCandidate] self._editable_candidate_cache = {} # type: Cache[EditableCandidate] diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py index db0300e3b..e83f35ca7 100644 --- a/src/pip/_internal/resolution/resolvelib/resolver.py +++ b/src/pip/_internal/resolution/resolvelib/resolver.py @@ -49,7 +49,6 @@ class Resolver(BaseResolver): force_reinstall, # type: bool upgrade_strategy, # type: str py_version_info=None, # type: Optional[Tuple[int, ...]] - lazy_wheel=False, # type: bool ): super(Resolver, self).__init__() assert upgrade_strategy in self._allowed_strategies @@ -64,7 +63,6 @@ class Resolver(BaseResolver): ignore_installed=ignore_installed, ignore_requires_python=ignore_requires_python, py_version_info=py_version_info, - lazy_wheel=lazy_wheel, ) self.ignore_dependencies = ignore_dependencies self.upgrade_strategy = upgrade_strategy diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index 1aee7fcdf..b8da86399 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -89,6 +89,7 @@ class TestRequirementSet(object): finder=finder, require_hashes=require_hashes, use_user_site=False, + lazy_wheel=False, ) yield Resolver( preparer=preparer, From f4603078cf0ba6415612a3130dc85c57f5337b0e Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 12:11:05 -0400 Subject: [PATCH 043/209] Pass InstallRequirement to _fetch_metadata Reduces dependence on Candidate. --- src/pip/_internal/resolution/resolvelib/candidates.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index e780dafd5..54ade93db 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -228,15 +228,15 @@ class _InstallRequirementBackedCandidate(Candidate): self._check_metadata_consistency(dist) self._dist = dist - def _fetch_metadata(self): - # type: () -> Optional[Distribution] + def _fetch_metadata(self, req): + # type: (InstallRequirement) -> Optional[Distribution] """Fetch metadata, using lazy wheel if possible.""" preparer = self._factory.preparer use_lazy_wheel = preparer.use_lazy_wheel remote_wheel = self._link.is_wheel and not self._link.is_file if use_lazy_wheel and remote_wheel and not preparer.require_hashes: assert self._name is not None - logger.info('Collecting %s', self._ireq.req or self._ireq) + logger.info('Collecting %s', req.req or req) # If HTTPRangeRequestUnsupported is raised, fallback silently. with indent_log(), suppress(HTTPRangeRequestUnsupported): logger.info( @@ -320,7 +320,7 @@ class LinkCandidate(_InstallRequirementBackedCandidate): def _prepare_distribution(self): # type: () -> Distribution - dist = self._fetch_metadata() + dist = self._fetch_metadata(self._ireq) if dist is not None: return dist return self._factory.preparer.prepare_linked_requirement( From defbf82a8fba7c1e3e8cc0e4dcf83fbd313b33df Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 12:13:01 -0400 Subject: [PATCH 044/209] Use link from InstallRequirement Since when we generate the InstallRequirement we set the link, these must be the same. Reduces dependence on Candidate. --- src/pip/_internal/resolution/resolvelib/candidates.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 54ade93db..5da4f08a1 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -233,7 +233,9 @@ class _InstallRequirementBackedCandidate(Candidate): """Fetch metadata, using lazy wheel if possible.""" preparer = self._factory.preparer use_lazy_wheel = preparer.use_lazy_wheel - remote_wheel = self._link.is_wheel and not self._link.is_file + assert self._link == req.link + link = req.link + remote_wheel = link.is_wheel and not link.is_file if use_lazy_wheel and remote_wheel and not preparer.require_hashes: assert self._name is not None logger.info('Collecting %s', req.req or req) @@ -243,7 +245,7 @@ class _InstallRequirementBackedCandidate(Candidate): 'Obtaining dependency information from %s %s', self._name, self._version, ) - url = self._link.url.split('#', 1)[0] + url = link.url.split('#', 1)[0] session = preparer.downloader._session return dist_from_wheel_url(self._name, url, session) return None From 9e463916d0077d8db6f616c72ec35b80fa289af5 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 12:33:14 -0400 Subject: [PATCH 045/209] Extract name and version from Wheel link We happen to know that this is the same treatment that gave us `_name` and `_version` for Wheels in the first place (in `LinkEvaluator`). It's not ideal, however the metadata consistency check that occurs in `Candidate` after creation of a `Distribution` guards us against any deviation in the name and version during our processing. Reduces dependence on Candidate. --- .../_internal/resolution/resolvelib/candidates.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 5da4f08a1..12128c690 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -7,6 +7,7 @@ from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.packaging.version import Version from pip._internal.exceptions import HashError, MetadataInconsistent +from pip._internal.models.wheel import Wheel from pip._internal.network.lazy_wheel import ( HTTPRangeRequestUnsupported, dist_from_wheel_url, @@ -237,17 +238,22 @@ class _InstallRequirementBackedCandidate(Candidate): link = req.link remote_wheel = link.is_wheel and not link.is_file if use_lazy_wheel and remote_wheel and not preparer.require_hashes: - assert self._name is not None + wheel = Wheel(link.filename) + name = canonicalize_name(wheel.name) + assert self._name == name + # Version may not be present for PEP 508 direct URLs + if self._version is not None: + assert self._version == wheel.version logger.info('Collecting %s', req.req or req) # If HTTPRangeRequestUnsupported is raised, fallback silently. with indent_log(), suppress(HTTPRangeRequestUnsupported): logger.info( 'Obtaining dependency information from %s %s', - self._name, self._version, + name, wheel.version, ) url = link.url.split('#', 1)[0] session = preparer.downloader._session - return dist_from_wheel_url(self._name, url, session) + return dist_from_wheel_url(name, url, session) return None @property From 4e1bff741dba2a15035a949c380b440bbce13927 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 14:07:06 -0400 Subject: [PATCH 046/209] Promote Wheel-related assertions to LinkCandidate constructor These are things we know will be true because of the existing wheel processing. In the future we may delegate the extraction of these to the LinkCandidate itself so it doesn't have to be an assertion. --- .../resolution/resolvelib/candidates.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 12128c690..5e5e8ac72 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -234,16 +234,12 @@ class _InstallRequirementBackedCandidate(Candidate): """Fetch metadata, using lazy wheel if possible.""" preparer = self._factory.preparer use_lazy_wheel = preparer.use_lazy_wheel - assert self._link == req.link + assert req.link link = req.link remote_wheel = link.is_wheel and not link.is_file if use_lazy_wheel and remote_wheel and not preparer.require_hashes: wheel = Wheel(link.filename) name = canonicalize_name(wheel.name) - assert self._name == name - # Version may not be present for PEP 508 direct URLs - if self._version is not None: - assert self._version == wheel.version logger.info('Collecting %s', req.req or req) # If HTTPRangeRequestUnsupported is raised, fallback silently. with indent_log(), suppress(HTTPRangeRequestUnsupported): @@ -311,6 +307,20 @@ class LinkCandidate(_InstallRequirementBackedCandidate): logger.debug("Using cached wheel link: %s", cache_entry.link) link = cache_entry.link ireq = make_install_req_from_link(link, template) + assert ireq.link == link + if ireq.link.is_wheel and not ireq.link.is_file: + wheel = Wheel(ireq.link.filename) + wheel_name = canonicalize_name(wheel.name) + assert name == wheel_name, ( + "{!r} != {!r} for wheel".format(name, wheel_name) + ) + # Version may not be present for PEP 508 direct URLs + if version is not None: + assert str(version) == wheel.version, ( + "{!r} != {!r} for wheel {}".format( + version, wheel.version, name + ) + ) if (cache_entry is not None and cache_entry.persistent and From 6c4d4f3b78f54fe305da9882b37a1d88769a4d23 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 14:21:49 -0400 Subject: [PATCH 047/209] Move _fetch_metadata to RequirementPreparer The fact that all of this functionality can be put in terms of the `RequirementPreparer` indicates that, at least at this point, this is the cleanest place to put this functionality. --- src/pip/_internal/operations/prepare.py | 32 +++++++++++++++++++ .../resolution/resolvelib/candidates.py | 32 ------------------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 6c8ceae18..8516ea75b 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -9,6 +9,8 @@ import mimetypes import os import shutil +from pip._vendor.contextlib2 import suppress +from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.six import PY2 from pip._internal.distributions import ( @@ -24,6 +26,11 @@ from pip._internal.exceptions import ( PreviousBuildDirError, VcsHashUnsupported, ) +from pip._internal.models.wheel import Wheel +from pip._internal.network.lazy_wheel import ( + HTTPRangeRequestUnsupported, + dist_from_wheel_url, +) from pip._internal.utils.filesystem import copy2_fixed from pip._internal.utils.hashes import MissingHashes from pip._internal.utils.logging import indent_log @@ -452,9 +459,34 @@ class RequirementPreparer(object): # showing the user what the hash should be. return req.hashes(trust_internet=False) or MissingHashes() + def _fetch_metadata(preparer, req): + # type: (InstallRequirement) -> Optional[Distribution] + """Fetch metadata, using lazy wheel if possible.""" + use_lazy_wheel = preparer.use_lazy_wheel + assert req.link + link = req.link + remote_wheel = link.is_wheel and not link.is_file + if use_lazy_wheel and remote_wheel and not preparer.require_hashes: + wheel = Wheel(link.filename) + name = canonicalize_name(wheel.name) + logger.info('Collecting %s', req.req or req) + # If HTTPRangeRequestUnsupported is raised, fallback silently. + with indent_log(), suppress(HTTPRangeRequestUnsupported): + logger.info( + 'Obtaining dependency information from %s %s', + name, wheel.version, + ) + url = link.url.split('#', 1)[0] + session = preparer.downloader._session + return dist_from_wheel_url(name, url, session) + return None + def prepare_linked_requirement(self, req, parallel_builds=False): # type: (InstallRequirement, bool) -> Distribution """Prepare a requirement to be obtained from req.link.""" + wheel_dist = self._fetch_metadata(req) + if wheel_dist is not None: + return wheel_dist assert req.link link = req.link self._log_preparing_link(req) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 5e5e8ac72..3d5d8b8e8 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -1,23 +1,17 @@ import logging import sys -from pip._vendor.contextlib2 import suppress from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.packaging.version import Version from pip._internal.exceptions import HashError, MetadataInconsistent from pip._internal.models.wheel import Wheel -from pip._internal.network.lazy_wheel import ( - HTTPRangeRequestUnsupported, - dist_from_wheel_url, -) from pip._internal.req.constructors import ( install_req_from_editable, install_req_from_line, ) from pip._internal.req.req_install import InstallRequirement -from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import dist_is_editable, normalize_version_info from pip._internal.utils.packaging import get_requires_python from pip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -229,29 +223,6 @@ class _InstallRequirementBackedCandidate(Candidate): self._check_metadata_consistency(dist) self._dist = dist - def _fetch_metadata(self, req): - # type: (InstallRequirement) -> Optional[Distribution] - """Fetch metadata, using lazy wheel if possible.""" - preparer = self._factory.preparer - use_lazy_wheel = preparer.use_lazy_wheel - assert req.link - link = req.link - remote_wheel = link.is_wheel and not link.is_file - if use_lazy_wheel and remote_wheel and not preparer.require_hashes: - wheel = Wheel(link.filename) - name = canonicalize_name(wheel.name) - logger.info('Collecting %s', req.req or req) - # If HTTPRangeRequestUnsupported is raised, fallback silently. - with indent_log(), suppress(HTTPRangeRequestUnsupported): - logger.info( - 'Obtaining dependency information from %s %s', - name, wheel.version, - ) - url = link.url.split('#', 1)[0] - session = preparer.downloader._session - return dist_from_wheel_url(name, url, session) - return None - @property def dist(self): # type: () -> Distribution @@ -338,9 +309,6 @@ class LinkCandidate(_InstallRequirementBackedCandidate): def _prepare_distribution(self): # type: () -> Distribution - dist = self._fetch_metadata(self._ireq) - if dist is not None: - return dist return self._factory.preparer.prepare_linked_requirement( self._ireq, parallel_builds=True, ) From 21db4f3096c3a907a7168152872d36bba97e27f1 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 14:24:27 -0400 Subject: [PATCH 048/209] Log in one common location Reduces dependence on `InstallRequirement` being passed to `_fetch_metadata`. --- src/pip/_internal/operations/prepare.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 8516ea75b..55a2feff5 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -469,7 +469,6 @@ class RequirementPreparer(object): if use_lazy_wheel and remote_wheel and not preparer.require_hashes: wheel = Wheel(link.filename) name = canonicalize_name(wheel.name) - logger.info('Collecting %s', req.req or req) # If HTTPRangeRequestUnsupported is raised, fallback silently. with indent_log(), suppress(HTTPRangeRequestUnsupported): logger.info( @@ -484,12 +483,12 @@ class RequirementPreparer(object): def prepare_linked_requirement(self, req, parallel_builds=False): # type: (InstallRequirement, bool) -> Distribution """Prepare a requirement to be obtained from req.link.""" - wheel_dist = self._fetch_metadata(req) - if wheel_dist is not None: - return wheel_dist assert req.link link = req.link self._log_preparing_link(req) + wheel_dist = self._fetch_metadata(req) + if wheel_dist is not None: + return wheel_dist if link.is_wheel and self.wheel_download_dir: # Download wheels to a dedicated dir when doing `pip wheel`. download_dir = self.wheel_download_dir From c7ade159d46ba0b2ffc0cd1c2b61f721678d00bd Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 14:26:26 -0400 Subject: [PATCH 049/209] Pass link to _fetch_metadata instead of req Removes dependence on `InstallRequirement`. --- src/pip/_internal/operations/prepare.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 55a2feff5..d6f136d0a 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -459,12 +459,10 @@ class RequirementPreparer(object): # showing the user what the hash should be. return req.hashes(trust_internet=False) or MissingHashes() - def _fetch_metadata(preparer, req): - # type: (InstallRequirement) -> Optional[Distribution] + def _fetch_metadata(preparer, link): + # type: (Link) -> Optional[Distribution] """Fetch metadata, using lazy wheel if possible.""" use_lazy_wheel = preparer.use_lazy_wheel - assert req.link - link = req.link remote_wheel = link.is_wheel and not link.is_file if use_lazy_wheel and remote_wheel and not preparer.require_hashes: wheel = Wheel(link.filename) @@ -486,7 +484,7 @@ class RequirementPreparer(object): assert req.link link = req.link self._log_preparing_link(req) - wheel_dist = self._fetch_metadata(req) + wheel_dist = self._fetch_metadata(link) if wheel_dist is not None: return wheel_dist if link.is_wheel and self.wheel_download_dir: From 8b838ebb89c521cd92dd7a1c7cec075c3b30d72f Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sun, 2 Aug 2020 18:32:54 -0400 Subject: [PATCH 050/209] Prepare lazy wheels more so they are downloaded This keeps all knowledge about preparation and types of requirements in `RequirementPreparer`, so there's one place to look when we're ready to start breaking it apart later. --- src/pip/_internal/operations/prepare.py | 14 ++++++++++++++ src/pip/_internal/req/req_install.py | 3 +++ .../_internal/resolution/resolvelib/resolver.py | 3 +++ 3 files changed, 20 insertions(+) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index d6f136d0a..ddfa42989 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -486,7 +486,21 @@ class RequirementPreparer(object): self._log_preparing_link(req) wheel_dist = self._fetch_metadata(link) if wheel_dist is not None: + req.needs_more_preparation = True return wheel_dist + return self._prepare_linked_requirement(req, parallel_builds) + + def prepare_linked_requirement_more(self, req, parallel_builds=False): + # type: (InstallRequirement, bool) -> None + """Prepare a linked requirement more, if needed.""" + if not req.needs_more_preparation: + return + self._prepare_linked_requirement(req, parallel_builds) + + def _prepare_linked_requirement(self, req, parallel_builds): + # type: (InstallRequirement, bool) -> Distribution + assert req.link + link = req.link if link.is_wheel and self.wheel_download_dir: # Download wheels to a dedicated dir when doing `pip wheel`. download_dir = self.wheel_download_dir diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 4759f4af6..816969f8e 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -213,6 +213,9 @@ class InstallRequirement(object): # but after loading this flag should be treated as read only. self.use_pep517 = use_pep517 + # This requirement needs more preparation before it can be built + self.needs_more_preparation = False + def __str__(self): # type: () -> str if self.req: diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py index e83f35ca7..6a38ffa5c 100644 --- a/src/pip/_internal/resolution/resolvelib/resolver.py +++ b/src/pip/_internal/resolution/resolvelib/resolver.py @@ -159,6 +159,9 @@ class Resolver(BaseResolver): req_set.add_named_requirement(ireq) + for actual_req in req_set.all_requirements: + self.factory.preparer.prepare_linked_requirement_more(actual_req) + return req_set def get_installation_order(self, req_set): From 95efbbe588d66d99bc28b04f03290630a22c1a35 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 3 Aug 2020 09:27:50 +0800 Subject: [PATCH 051/209] News --- news/8684.bugfix | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 news/8684.bugfix diff --git a/news/8684.bugfix b/news/8684.bugfix new file mode 100644 index 000000000..18e6ed9bc --- /dev/null +++ b/news/8684.bugfix @@ -0,0 +1,2 @@ +Use the same encoding logic from Python 3 to handle ZIP archive entries on +Python 2, so non-ASCII paths can be resolved as expected. From d4995cb89eed0a2d348e220c6ef061b3d816e0f4 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 3 Aug 2020 06:38:36 +0800 Subject: [PATCH 052/209] Implement heuristics to get non-ASCII ZIP entries --- src/pip/_internal/operations/install/wheel.py | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py index 681fc0aa8..f2fde0b08 100644 --- a/src/pip/_internal/operations/install/wheel.py +++ b/src/pip/_internal/operations/install/wheel.py @@ -78,6 +78,7 @@ else: Union, cast, ) + from zipfile import ZipInfo from pip._vendor.pkg_resources import Distribution @@ -420,6 +421,28 @@ class ZipBackedFile(object): self._zip_file = zip_file self.changed = False + def _getinfo(self): + # type: () -> ZipInfo + if not PY2: + return self._zip_file.getinfo(self.src_record_path) + + # Python 2 does not expose a way to detect a ZIP's encoding, so we + # "guess" with the heuristics below: + # 1. Try encoding the path with UTF-8. + # 2. Check the matching info's flags for language encoding (bit 11). + # 3. If the flag is set, assume UTF-8 is correct. + # 4. If any of the above steps fails, fallback to getting an info with + # CP437 (matching Python 3). + try: + arcname = self.src_record_path.encode("utf-8") + info = self._zip_file.getinfo(arcname) + if info.flag_bits & 0x800: + return info + except (KeyError, UnicodeEncodeError): + pass + arcname = self.src_record_path.encode("cp437") + return self._zip_file.getinfo(arcname) + def save(self): # type: () -> None # directory creation is lazy and after file filtering @@ -439,11 +462,12 @@ class ZipBackedFile(object): if os.path.exists(self.dest_path): os.unlink(self.dest_path) - with self._zip_file.open(self.src_record_path) as f: + zipinfo = self._getinfo() + + with self._zip_file.open(zipinfo) as f: with open(self.dest_path, "wb") as dest: shutil.copyfileobj(f, dest) - zipinfo = self._zip_file.getinfo(self.src_record_path) if zip_item_is_executable(zipinfo): set_extracted_file_to_default_mode_plus_executable(self.dest_path) From a12e2f147997dd0932727150ae596ca489a41e78 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 3 Aug 2020 14:59:42 +0800 Subject: [PATCH 053/209] PEP 427 mandates UTF-8, we don't need the fallback --- news/8684.bugfix | 4 ++-- src/pip/_internal/operations/install/wheel.py | 21 ++++--------------- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/news/8684.bugfix b/news/8684.bugfix index 18e6ed9bc..528291d73 100644 --- a/news/8684.bugfix +++ b/news/8684.bugfix @@ -1,2 +1,2 @@ -Use the same encoding logic from Python 3 to handle ZIP archive entries on -Python 2, so non-ASCII paths can be resolved as expected. +Use UTF-8 to handle ZIP archive entries on Python 2 according to PEP 427, so +non-ASCII paths can be resolved as expected. diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py index f2fde0b08..e91b1b8d5 100644 --- a/src/pip/_internal/operations/install/wheel.py +++ b/src/pip/_internal/operations/install/wheel.py @@ -425,23 +425,10 @@ class ZipBackedFile(object): # type: () -> ZipInfo if not PY2: return self._zip_file.getinfo(self.src_record_path) - - # Python 2 does not expose a way to detect a ZIP's encoding, so we - # "guess" with the heuristics below: - # 1. Try encoding the path with UTF-8. - # 2. Check the matching info's flags for language encoding (bit 11). - # 3. If the flag is set, assume UTF-8 is correct. - # 4. If any of the above steps fails, fallback to getting an info with - # CP437 (matching Python 3). - try: - arcname = self.src_record_path.encode("utf-8") - info = self._zip_file.getinfo(arcname) - if info.flag_bits & 0x800: - return info - except (KeyError, UnicodeEncodeError): - pass - arcname = self.src_record_path.encode("cp437") - return self._zip_file.getinfo(arcname) + # Python 2 does not expose a way to detect a ZIP's encoding, but the + # wheel specification (PEP 427) explicitly mandates that paths should + # use UTF-8, so we assume it is true. + return self._zip_file.getinfo(self.src_record_path.encode("utf-8")) def save(self): # type: () -> None From ba062c3ed068a82b36d818f6c434fe5a91afb645 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 3 Aug 2020 10:41:03 +0200 Subject: [PATCH 054/209] When one keyring attempt fails, don't bother with more This makes https://github.com/pypa/pip/issues/8090 much less painful. --- news/8090.bugfix | 3 +++ src/pip/_internal/network/auth.py | 2 ++ tests/unit/test_network_auth.py | 26 ++++++++++++++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 news/8090.bugfix diff --git a/news/8090.bugfix b/news/8090.bugfix new file mode 100644 index 000000000..e9f2b7cbb --- /dev/null +++ b/news/8090.bugfix @@ -0,0 +1,3 @@ +Only attempt to use the keyring once and if it fails, don't try again. +This prevents spamming users with several keyring unlock prompts when they +cannot unlock or don't want to do so. diff --git a/src/pip/_internal/network/auth.py b/src/pip/_internal/network/auth.py index ca729fcdf..c49deaaf1 100644 --- a/src/pip/_internal/network/auth.py +++ b/src/pip/_internal/network/auth.py @@ -44,6 +44,7 @@ except Exception as exc: def get_keyring_auth(url, username): # type: (str, str) -> Optional[AuthInfo] """Return the tuple auth for a given url from keyring.""" + global keyring if not url or not keyring: return None @@ -69,6 +70,7 @@ def get_keyring_auth(url, username): logger.warning( "Keyring is skipped due to an exception: %s", str(exc), ) + keyring = None return None diff --git a/tests/unit/test_network_auth.py b/tests/unit/test_network_auth.py index 08320cfa1..8116b627f 100644 --- a/tests/unit/test_network_auth.py +++ b/tests/unit/test_network_auth.py @@ -242,3 +242,29 @@ def test_keyring_get_credential(monkeypatch, url, expect): assert auth._get_new_credentials( url, allow_netrc=False, allow_keyring=True ) == expect + + +class KeyringModuleBroken(object): + """Represents the current supported API of keyring, but broken""" + + def __init__(self): + self._call_count = 0 + + def get_credential(self, system, username): + self._call_count += 1 + raise Exception("This keyring is broken!") + + +def test_broken_keyring_disables_keyring(monkeypatch): + keyring_broken = KeyringModuleBroken() + monkeypatch.setattr(pip._internal.network.auth, 'keyring', keyring_broken) + + auth = MultiDomainBasicAuth(index_urls=["http://example.com/"]) + + assert keyring_broken._call_count == 0 + for i in range(5): + url = "http://example.com/path" + str(i) + assert auth._get_new_credentials( + url, allow_netrc=False, allow_keyring=True + ) == (None, None) + assert keyring_broken._call_count == 1 From 20663fc993951ccabe41b4d3744418c37f4a3644 Mon Sep 17 00:00:00 2001 From: Sumana Harihareswara Date: Mon, 3 Aug 2020 10:27:28 -0400 Subject: [PATCH 055/209] Docs: Add details on old resolver deprecation and removal Relevant to #8371, #6536, #8076. Signed-off-by: Sumana Harihareswara --- docs/html/development/release-process.rst | 3 +++ docs/html/user_guide.rst | 30 +++++++++++++++++++---- news/8371.doc | 1 + 3 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 news/8371.doc diff --git a/docs/html/development/release-process.rst b/docs/html/development/release-process.rst index cbfbce4ad..44197955e 100644 --- a/docs/html/development/release-process.rst +++ b/docs/html/development/release-process.rst @@ -2,6 +2,7 @@ Release process =============== +.. _`Release Cadence`: Release Cadence =============== @@ -72,6 +73,8 @@ only bugs will be considered, and merged (subject to normal review processes). Note that there may be delays due to the lack of developer resources for reviewing such pull requests. +.. _`Feature Flags`: + Feature Flags ============= diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 31887a288..11bdc5818 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -1223,18 +1223,37 @@ Specific things we'd love to get feedback on: Please let us know through the `resolver testing survey`_. +Deprecation timeline +-------------------- + +We plan for the resolver changeover to proceed as follows, using +:ref:`Feature Flags` and following our :ref:`Release Cadence`: + +* pip 20.2: a beta of the new resolver is available, opt-in, using + the flag ``--use-feature=2020-resolver``. pip defaults to + legacy behavior. + +* pip 20.3: pip defaults to the new resolver, but a user can opt-out + and choose the old resolver behavior, using the flag + ``--use-deprecated=legacy-resolver``. + +* pip 21.0: pip uses new resolver, and the old resolver is no longer + available. + +Since this work will not change user-visible behavior described in the +pip documentation, this change is not covered by the :ref:`Deprecation +Policy`. + Context and followup -------------------- As discussed in `our announcement on the PSF blog`_, the pip team are in the process of developing a new "dependency resolver" (the part of -pip that works out what to install based on your requirements). Since -this work will not change user-visible behavior described in the pip -documentation, this change is not covered by the :ref:`Deprecation -Policy`. +pip that works out what to install based on your requirements). We're tracking our rollout in :issue:`6536` and you can watch for -announcements on the `low-traffic packaging announcements list`_. +announcements on the `low-traffic packaging announcements list`_ and +`the official Python blog`_. .. _freeze: https://pip.pypa.io/en/latest/reference/pip_freeze/ .. _resolver testing survey: https://tools.simplysecure.org/survey/index.php?r=survey/index&sid=989272&lang=en @@ -1242,3 +1261,4 @@ announcements on the `low-traffic packaging announcements list`_. .. _tensorflow: https://pypi.org/project/tensorflow/ .. _low-traffic packaging announcements list: https://mail.python.org/mailman3/lists/pypi-announce.python.org/ .. _our survey on upgrades that create conflicts: https://docs.google.com/forms/d/e/1FAIpQLSeBkbhuIlSofXqCyhi3kGkLmtrpPOEBwr6iJA6SzHdxWKfqdA/viewform +.. _the official Python blog: https://blog.python.org/ diff --git a/news/8371.doc b/news/8371.doc new file mode 100644 index 000000000..ffd991950 --- /dev/null +++ b/news/8371.doc @@ -0,0 +1 @@ +Add details on old resolver deprecation and removal to migration documentation. From d98ff19c270667d4a52d714676608094d096c6f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Sun, 2 Aug 2020 16:33:04 +0700 Subject: [PATCH 056/209] [fast-deps] Make range requests closer to chunk size --- news/b7b40802-1aae-4295-99f4-a0dd48c96e69.trivial | 0 src/pip/_internal/network/lazy_wheel.py | 4 +++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 news/b7b40802-1aae-4295-99f4-a0dd48c96e69.trivial diff --git a/news/b7b40802-1aae-4295-99f4-a0dd48c96e69.trivial b/news/b7b40802-1aae-4295-99f4-a0dd48c96e69.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/network/lazy_wheel.py b/src/pip/_internal/network/lazy_wheel.py index c2371bf5c..16be0d297 100644 --- a/src/pip/_internal/network/lazy_wheel.py +++ b/src/pip/_internal/network/lazy_wheel.py @@ -109,8 +109,10 @@ class LazyZipOverHTTP(object): all bytes until EOF are returned. Fewer than size bytes may be returned if EOF is reached. """ + download_size = max(size, self._chunk_size) start, length = self.tell(), self._length - stop = start + size if 0 <= size <= length-start else length + stop = length if size < 0 else min(start+download_size, length) + start = max(0, stop-download_size) self._download(start, stop-1) return self._file.read(size) From 57ee51c2b187728d7fbff268907c33c369718a11 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com> Date: Tue, 4 Aug 2020 06:55:17 +0530 Subject: [PATCH 057/209] Un-rewrap lines --- docs/html/user_guide.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index b0b25677e..9680486b0 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -838,9 +838,8 @@ specifying package versions (e.g. ``~=`` or ``*``): semantic versioning.", "``~=3.1``: version ``3.1`` or later, but not version ``4.0`` or later. ``~=3.1.2``: version ``3.1.2`` or later, but not version ``3.2.0`` or later." - ``*``, Can be used at the end of a version number to represent "all", - "``==3.1.*``: any version that starts with ``3.1``. - Equivalent to ``~=3.1.0``." + ``*``,Can be used at the end of a version number to represent "all", "``== 3. + 1.*``: any version that starts with ``3.1``. Equivalent to ``~=3.1.0``." The detailed specification of supported comparison operators can be found in :pep:`440`. From 175371c7e3f366668521dfa681641f59fa3f1d69 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 4 Aug 2020 08:55:12 +0530 Subject: [PATCH 058/209] Mention ResolutionImpossible in 20.2 changelog --- NEWS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.rst b/NEWS.rst index 755ca709e..852a7bdb9 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -43,6 +43,7 @@ Features break. More details about how to test and migrate, and how to report issues, at :ref:`Resolver changes 2020` . Maintainers are preparing to release pip 20.3, with the new resolver on by default, in October. (`#6536 `_) +- Introduce a new ResolutionImpossible error, raised when pip encounters un-satisfiable dependency conflicts (`#8546 `_, `#8377 `_) - Add a subcommand ``debug`` to ``pip config`` to list available configuration sources and the key-value pairs defined in them. (`#6741 `_) - Warn if index pages have unexpected content-type (`#6754 `_) - Allow specifying ``--prefer-binary`` option in a requirements file (`#7693 `_) From 68cdfa93d95565cbc40f1a158828ac4a477467c4 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 4 Aug 2020 08:55:59 +0530 Subject: [PATCH 059/209] Mention ResolutionImpossible in changelog for 8459 --- NEWS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.rst b/NEWS.rst index 852a7bdb9..d2e25b18c 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -93,7 +93,7 @@ Improved Documentation - Fix pip config docstring so that the subcommands render correctly in the docs (`#8072 `_) - replace links to the old pypa-dev mailing list with https://mail.python.org/mailman3/lists/distutils-sig.python.org/ (`#8353 `_) - Fix example for defining multiple values for options which support them (`#8373 `_) -- Add documentation that helps the user fix dependency conflicts (`#8459 `_) +- Add documentation for the ResolutionImpossible error, that helps the user fix dependency conflicts (`#8459 `_) - Add feature flags to docs (`#8512 `_) - Document how to install package extras from git branch and source distributions. (`#8576 `_) From 18aa718a58b23c6577be18923a777d5f74620d7d Mon Sep 17 00:00:00 2001 From: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com> Date: Tue, 4 Aug 2020 09:50:40 +0530 Subject: [PATCH 060/209] Drop punctuation Co-authored-by: Sumana Harihareswara --- NEWS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.rst b/NEWS.rst index d2e25b18c..56ea38f15 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -93,7 +93,7 @@ Improved Documentation - Fix pip config docstring so that the subcommands render correctly in the docs (`#8072 `_) - replace links to the old pypa-dev mailing list with https://mail.python.org/mailman3/lists/distutils-sig.python.org/ (`#8353 `_) - Fix example for defining multiple values for options which support them (`#8373 `_) -- Add documentation for the ResolutionImpossible error, that helps the user fix dependency conflicts (`#8459 `_) +- Add documentation for the ResolutionImpossible error that helps the user fix dependency conflicts (`#8459 `_) - Add feature flags to docs (`#8512 `_) - Document how to install package extras from git branch and source distributions. (`#8576 `_) From bc86c7c33bd68351622af7ddb5a714a5aa0e09fc Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 4 Aug 2020 10:46:12 +0530 Subject: [PATCH 061/209] Bump for development --- src/pip/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/__init__.py b/src/pip/__init__.py index 6a730519c..5a2f3c317 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -4,7 +4,7 @@ if MYPY_CHECK_RUNNING: from typing import List, Optional -__version__ = "20.2.1" +__version__ = "20.3.dev0" def main(args=None): From ae10b82f8cd086fb14ed8b99cbba212294f517c2 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 4 Aug 2020 16:45:12 +0530 Subject: [PATCH 062/209] Remove news/ files for 20.2.1 --- news/094ae133-3a6d-4a6c-a4c3-fe8d78223498.trivial | 0 news/2581396b-ebe4-46a9-b38f-e6b0da97d53a.trivial | 0 news/8371.doc | 1 - news/8603.feature | 1 - news/8645.bugfix | 2 -- news/8654.bugfix | 2 -- news/8660.doc | 1 - news/8665.bugfix | 1 - news/8677.bugfix | 2 -- news/8684.bugfix | 2 -- news/AE707F60-0ABE-4DBA-98AA-59CE8F989386.trivial | 0 news/b7b40802-1aae-4295-99f4-a0dd48c96e69.trivial | 0 12 files changed, 12 deletions(-) delete mode 100644 news/094ae133-3a6d-4a6c-a4c3-fe8d78223498.trivial delete mode 100644 news/2581396b-ebe4-46a9-b38f-e6b0da97d53a.trivial delete mode 100644 news/8371.doc delete mode 100644 news/8603.feature delete mode 100644 news/8645.bugfix delete mode 100644 news/8654.bugfix delete mode 100644 news/8660.doc delete mode 100644 news/8665.bugfix delete mode 100644 news/8677.bugfix delete mode 100644 news/8684.bugfix delete mode 100644 news/AE707F60-0ABE-4DBA-98AA-59CE8F989386.trivial delete mode 100644 news/b7b40802-1aae-4295-99f4-a0dd48c96e69.trivial diff --git a/news/094ae133-3a6d-4a6c-a4c3-fe8d78223498.trivial b/news/094ae133-3a6d-4a6c-a4c3-fe8d78223498.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/2581396b-ebe4-46a9-b38f-e6b0da97d53a.trivial b/news/2581396b-ebe4-46a9-b38f-e6b0da97d53a.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/8371.doc b/news/8371.doc deleted file mode 100644 index ffd991950..000000000 --- a/news/8371.doc +++ /dev/null @@ -1 +0,0 @@ -Add details on old resolver deprecation and removal to migration documentation. diff --git a/news/8603.feature b/news/8603.feature deleted file mode 100644 index 1f8480baa..000000000 --- a/news/8603.feature +++ /dev/null @@ -1 +0,0 @@ -Ignore require-virtualenv in ``pip list`` diff --git a/news/8645.bugfix b/news/8645.bugfix deleted file mode 100644 index a388d24e4..000000000 --- a/news/8645.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Correctly find already-installed distributions with dot (``.``) in the name -and uninstall them when needed. diff --git a/news/8654.bugfix b/news/8654.bugfix deleted file mode 100644 index ec0df7a90..000000000 --- a/news/8654.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Trace a better error message on installation failure due to invalid ``.data`` -files in wheels. diff --git a/news/8660.doc b/news/8660.doc deleted file mode 100644 index 45b71cc26..000000000 --- a/news/8660.doc +++ /dev/null @@ -1 +0,0 @@ -Fix feature flag name in docs. diff --git a/news/8665.bugfix b/news/8665.bugfix deleted file mode 100644 index 0ce458463..000000000 --- a/news/8665.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix SVN version detection for alternative SVN distributions. diff --git a/news/8677.bugfix b/news/8677.bugfix deleted file mode 100644 index e9efd8279..000000000 --- a/news/8677.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -New resolver: Correctly include the base package when specified with extras -in ``--no-deps`` mode. diff --git a/news/8684.bugfix b/news/8684.bugfix deleted file mode 100644 index 528291d73..000000000 --- a/news/8684.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Use UTF-8 to handle ZIP archive entries on Python 2 according to PEP 427, so -non-ASCII paths can be resolved as expected. diff --git a/news/AE707F60-0ABE-4DBA-98AA-59CE8F989386.trivial b/news/AE707F60-0ABE-4DBA-98AA-59CE8F989386.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/b7b40802-1aae-4295-99f4-a0dd48c96e69.trivial b/news/b7b40802-1aae-4295-99f4-a0dd48c96e69.trivial deleted file mode 100644 index e69de29bb..000000000 From ff494f524e2d59865cebe71f21acaa311a3a55ab Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 5 Aug 2020 03:06:31 +0800 Subject: [PATCH 063/209] News --- news/8695.bugfix | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 news/8695.bugfix diff --git a/news/8695.bugfix b/news/8695.bugfix new file mode 100644 index 000000000..668e4672e --- /dev/null +++ b/news/8695.bugfix @@ -0,0 +1,3 @@ +Fix regression that distributions in system site-packages are not correctly +found when a virtual environment is configured with ``system-site-packages`` +on. From e459763d814e0dbb72ad4a4f77835b8c8a615d68 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 5 Aug 2020 02:43:40 +0800 Subject: [PATCH 064/209] Also look for non-local when searching for dists This matches the behavior of pkg_resources.get_distribution(), which this function intends to replace. --- src/pip/_internal/utils/misc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index 24a745562..31167c10c 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -484,17 +484,19 @@ def get_installed_distributions( def search_distribution(req_name): + # type: (str) -> Optional[Distribution] # Canonicalize the name before searching in the list of # installed distributions and also while creating the package # dictionary to get the Distribution object req_name = canonicalize_name(req_name) - packages = get_installed_distributions(skip=()) + packages = get_installed_distributions(local_only=False, skip=()) pkg_dict = {canonicalize_name(p.key): p for p in packages} return pkg_dict.get(req_name) def get_distribution(req_name): + # type: (str) -> Optional[Distribution] """Given a requirement name, return the installed Distribution object""" # Search the distribution by looking through the working set From 8491ce77233a49b71c878a2818600a159217aa57 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 5 Aug 2020 02:45:14 +0800 Subject: [PATCH 065/209] Refactor and clarify get_distribution() behavior The call to get_installed_distributions() now passes all flags excplicitly so they are more obvious and less likely to be misunderstood in the future. The behavior also documented in the function docstring. The search_distribution() helper function is renamed with a leading underscore to make it clear that it is intended as a helper function to get_distribution(). --- src/pip/_internal/utils/misc.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index 31167c10c..5629c60c1 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -483,24 +483,39 @@ def get_installed_distributions( ] -def search_distribution(req_name): +def _search_distribution(req_name): # type: (str) -> Optional[Distribution] + """Find a distribution matching the ``req_name`` in the environment. + This searches from *all* distributions available in the environment, to + match the behavior of ``pkg_resources.get_distribution()``. + """ # Canonicalize the name before searching in the list of # installed distributions and also while creating the package # dictionary to get the Distribution object req_name = canonicalize_name(req_name) - packages = get_installed_distributions(local_only=False, skip=()) + packages = get_installed_distributions( + local_only=False, + skip=(), + include_editables=True, + editables_only=False, + user_only=False, + paths=None, + ) pkg_dict = {canonicalize_name(p.key): p for p in packages} return pkg_dict.get(req_name) def get_distribution(req_name): # type: (str) -> Optional[Distribution] - """Given a requirement name, return the installed Distribution object""" + """Given a requirement name, return the installed Distribution object. + + This searches from *all* distributions available in the environment, to + match the behavior of ``pkg_resources.get_distribution()``. + """ # Search the distribution by looking through the working set - dist = search_distribution(req_name) + dist = _search_distribution(req_name) # If distribution could not be found, call working_set.require # to update the working set, and try to find the distribution @@ -516,7 +531,7 @@ def get_distribution(req_name): pkg_resources.working_set.require(req_name) except pkg_resources.DistributionNotFound: return None - return search_distribution(req_name) + return _search_distribution(req_name) def egg_link_path(dist): From c04182893ac6de24c9dda028cb2fd72111ca66a9 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 5 Aug 2020 04:55:56 +0800 Subject: [PATCH 066/209] Work around lax semantics in commands.search --- src/pip/_internal/commands/search.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pip/_internal/commands/search.py b/src/pip/_internal/commands/search.py index e906ce766..ff0947202 100644 --- a/src/pip/_internal/commands/search.py +++ b/src/pip/_internal/commands/search.py @@ -140,6 +140,7 @@ def print_results(hits, name_column_width=None, terminal_width=None): write_output(line) if name in installed_packages: dist = get_distribution(name) + assert dist is not None with indent_log(): if dist.version == latest: write_output('INSTALLED: %s (latest)', dist.version) From ffd6a38646a32811cd150790e9f762aca545438b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Wed, 5 Aug 2020 22:19:49 +0700 Subject: [PATCH 067/209] Disable caching for range requests --- news/50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial | 0 src/pip/_internal/network/lazy_wheel.py | 6 ++++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 news/50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial diff --git a/news/50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial b/news/50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/network/lazy_wheel.py b/src/pip/_internal/network/lazy_wheel.py index 16be0d297..a0f9e151d 100644 --- a/src/pip/_internal/network/lazy_wheel.py +++ b/src/pip/_internal/network/lazy_wheel.py @@ -194,8 +194,10 @@ class LazyZipOverHTTP(object): def _stream_response(self, start, end, base_headers=HEADERS): # type: (int, int, Dict[str, str]) -> Response """Return HTTP response to a range request from start to end.""" - headers = {'Range': 'bytes={}-{}'.format(start, end)} - headers.update(base_headers) + headers = base_headers.copy() + headers['Range'] = 'bytes={}-{}'.format(start, end) + # TODO: Get range requests to be correctly cached + headers['Cache-Control'] = 'no-cache' return self._session.get(self._url, headers=headers, stream=True) def _merge(self, start, end, left, right): From 4fce2ea88c85a483e4250cef1c04dfdb013942c9 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 6 Aug 2020 10:59:07 +0800 Subject: [PATCH 068/209] Add test to ensure get_distribution() behavior --- tests/unit/test_utils.py | 70 +++++++++++++++++++++++++++++++++------- 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ebabd29e2..0a1c47cd7 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -5,6 +5,7 @@ util tests """ import codecs +import itertools import os import shutil import stat @@ -34,6 +35,7 @@ from pip._internal.utils.misc import ( build_url_from_netloc, egg_link_path, format_size, + get_distribution, get_installed_distributions, get_prog, hide_url, @@ -192,26 +194,30 @@ class Tests_EgglinkPath: @patch('pip._internal.utils.misc.dist_in_usersite') @patch('pip._internal.utils.misc.dist_is_local') @patch('pip._internal.utils.misc.dist_is_editable') -class Tests_get_installed_distributions: - """test util.get_installed_distributions""" +class TestsGetDistributions(object): + """Test get_installed_distributions() and get_distribution(). + """ + class MockWorkingSet(list): + def require(self, name): + pass - workingset = [ - Mock(test_name="global"), - Mock(test_name="editable"), - Mock(test_name="normal"), - Mock(test_name="user"), - ] + workingset = MockWorkingSet(( + Mock(test_name="global", key="global"), + Mock(test_name="editable", key="editable"), + Mock(test_name="normal", key="normal"), + Mock(test_name="user", key="user"), + )) - workingset_stdlib = [ + workingset_stdlib = MockWorkingSet(( Mock(test_name='normal', key='argparse'), Mock(test_name='normal', key='wsgiref') - ] + )) - workingset_freeze = [ + workingset_freeze = MockWorkingSet(( Mock(test_name='normal', key='pip'), Mock(test_name='normal', key='setuptools'), Mock(test_name='normal', key='distribute') - ] + )) def dist_is_editable(self, dist): return dist.test_name == "editable" @@ -287,6 +293,46 @@ class Tests_get_installed_distributions: skip=('setuptools', 'pip', 'distribute')) assert len(dists) == 0 + @pytest.mark.parametrize( + "working_set, req_name", + itertools.chain( + itertools.product([workingset], (d.key for d in workingset)), + itertools.product( + [workingset_stdlib], (d.key for d in workingset_stdlib), + ), + ), + ) + def test_get_distribution( + self, + mock_dist_is_editable, + mock_dist_is_local, + mock_dist_in_usersite, + working_set, + req_name, + ): + """Ensure get_distribution() finds all kinds of distributions. + """ + mock_dist_is_editable.side_effect = self.dist_is_editable + mock_dist_is_local.side_effect = self.dist_is_local + mock_dist_in_usersite.side_effect = self.dist_in_usersite + with patch("pip._vendor.pkg_resources.working_set", working_set): + dist = get_distribution(req_name) + assert dist is not None + assert dist.key == req_name + + @patch('pip._vendor.pkg_resources.working_set', workingset) + def test_get_distribution_nonexist( + self, + mock_dist_is_editable, + mock_dist_is_local, + mock_dist_in_usersite, + ): + mock_dist_is_editable.side_effect = self.dist_is_editable + mock_dist_is_local.side_effect = self.dist_is_local + mock_dist_in_usersite.side_effect = self.dist_in_usersite + dist = get_distribution("non-exist") + assert dist is None + def test_rmtree_errorhandler_nonexistent_directory(tmpdir): """ From 810385b971bc101182ded21e83457d83790e413a Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 6 Aug 2020 11:45:29 +0800 Subject: [PATCH 069/209] Always use UTF-8 to read pyvenv.cfg --- news/8717.bugfix | 1 + src/pip/_internal/utils/virtualenv.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 news/8717.bugfix diff --git a/news/8717.bugfix b/news/8717.bugfix new file mode 100644 index 000000000..e8c8533c4 --- /dev/null +++ b/news/8717.bugfix @@ -0,0 +1 @@ +Always use UTF-8 to read ``pyvenv.cfg`` to match the built-in ``venv``. diff --git a/src/pip/_internal/utils/virtualenv.py b/src/pip/_internal/utils/virtualenv.py index 596a69a7d..4a7812873 100644 --- a/src/pip/_internal/utils/virtualenv.py +++ b/src/pip/_internal/utils/virtualenv.py @@ -1,5 +1,6 @@ from __future__ import absolute_import +import io import logging import os import re @@ -51,7 +52,9 @@ def _get_pyvenv_cfg_lines(): """ pyvenv_cfg_file = os.path.join(sys.prefix, 'pyvenv.cfg') try: - with open(pyvenv_cfg_file) as f: + # Although PEP 405 does not specify, the built-in venv module always + # writes with UTF-8. (pypa/pip#8717) + with io.open(pyvenv_cfg_file, encoding='utf-8') as f: return f.read().splitlines() # avoids trailing newlines except IOError: return None From 4f210f36089398fccce91c1a5769a8b5e7258cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Wed, 5 Aug 2020 15:04:07 +0700 Subject: [PATCH 070/209] [2020-resolver] List downloaded distributions before exiting This unifies the behavior of pip download for both legacy and new resolvers. InstallRequirement.successfully_download is no longer needed for this task and is thus retired. --- news/8696.bugfix | 3 +++ src/pip/_internal/commands/download.py | 10 ++++++---- src/pip/_internal/req/req_install.py | 9 --------- src/pip/_internal/resolution/legacy/resolver.py | 6 ------ 4 files changed, 9 insertions(+), 19 deletions(-) create mode 100644 news/8696.bugfix diff --git a/news/8696.bugfix b/news/8696.bugfix new file mode 100644 index 000000000..989d2d029 --- /dev/null +++ b/news/8696.bugfix @@ -0,0 +1,3 @@ +List downloaded distributions before exiting ``pip download`` +when using the new resolver to make the behavior the same as +that on the legacy resolver. diff --git a/src/pip/_internal/commands/download.py b/src/pip/_internal/commands/download.py index 46e837126..0861d9e67 100644 --- a/src/pip/_internal/commands/download.py +++ b/src/pip/_internal/commands/download.py @@ -134,10 +134,12 @@ class DownloadCommand(RequirementCommand): reqs, check_supported_wheels=True ) - downloaded = ' '.join([req.name # type: ignore - for req in requirement_set.requirements.values() - if req.successfully_downloaded]) + downloaded = [] # type: List[str] + for req in requirement_set.requirements.values(): + if not req.editable and req.satisfied_by is None: + assert req.name is not None + downloaded.append(req.name) if downloaded: - write_output('Successfully downloaded %s', downloaded) + write_output('Successfully downloaded %s', ' '.join(downloaded)) return SUCCESS diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 816969f8e..907c40249 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -179,15 +179,6 @@ class InstallRequirement(object): # e.g. dependencies, extras or constraints. self.user_supplied = user_supplied - # Set by the legacy resolver when the requirement has been downloaded - # TODO: This introduces a strong coupling between the resolver and the - # requirement (the coupling was previously between the resolver - # and the requirement set). This should be refactored to allow - # the requirement to decide for itself when it has been - # successfully downloaded - but that is more tricky to get right, - # se we are making the change in stages. - self.successfully_downloaded = False - self.isolated = isolated self.build_env = NoOpBuildEnvironment() # type: BuildEnvironment diff --git a/src/pip/_internal/resolution/legacy/resolver.py b/src/pip/_internal/resolution/legacy/resolver.py index a743b5696..d2dafa77f 100644 --- a/src/pip/_internal/resolution/legacy/resolver.py +++ b/src/pip/_internal/resolution/legacy/resolver.py @@ -444,12 +444,6 @@ class Resolver(BaseResolver): for subreq in dist.requires(available_requested): add_req(subreq, extras_requested=available_requested) - if not req_to_install.editable and not req_to_install.satisfied_by: - # XXX: --no-install leads this to report 'Successfully - # downloaded' for only non-editable reqs, even though we took - # action on them. - req_to_install.successfully_downloaded = True - return more_reqs def get_installation_order(self, req_set): From 709ad37b2f064e6e48604e8a8c40c04c88f0c338 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Thu, 6 Aug 2020 17:10:13 +0700 Subject: [PATCH 071/209] Make assertion failure give better message (#8692) --- news/9f8da1d9-dd18-47e9-b334-5eb862054409.trivial | 0 tests/functional/test_install_check.py | 3 +-- 2 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 news/9f8da1d9-dd18-47e9-b334-5eb862054409.trivial diff --git a/news/9f8da1d9-dd18-47e9-b334-5eb862054409.trivial b/news/9f8da1d9-dd18-47e9-b334-5eb862054409.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/test_install_check.py b/tests/functional/test_install_check.py index 96ed6b6e1..a173cb550 100644 --- a/tests/functional/test_install_check.py +++ b/tests/functional/test_install_check.py @@ -2,9 +2,8 @@ from tests.lib import create_test_package_with_setup def assert_contains_expected_lines(string, expected_lines): - lines = string.splitlines() for expected_line in expected_lines: - assert any(line.endswith(expected_line) for line in lines) + assert (expected_line + '\n') in string def test_check_install_canonicalization(script): From 487d00295ce0409ab59fd162e328654c6f93afc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Wed, 5 Aug 2020 15:39:21 +0700 Subject: [PATCH 072/209] Define RequirementPreparer._session --- ...182139-edb4-4bf6-bc3f-2d37cb5759ad.trivial | 0 src/pip/_internal/cli/req_command.py | 1 + src/pip/_internal/operations/prepare.py | 45 ++++++++++++------- tests/unit/test_req.py | 4 +- 4 files changed, 33 insertions(+), 17 deletions(-) create mode 100644 news/c6182139-edb4-4bf6-bc3f-2d37cb5759ad.trivial diff --git a/news/c6182139-edb4-4bf6-bc3f-2d37cb5759ad.trivial b/news/c6182139-edb4-4bf6-bc3f-2d37cb5759ad.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 76d83896c..6562fe918 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -238,6 +238,7 @@ class RequirementCommand(IndexGroupCommand): wheel_download_dir=wheel_download_dir, build_isolation=options.build_isolation, req_tracker=req_tracker, + session=session, downloader=downloader, finder=finder, require_hashes=options.require_hashes, diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index ddfa42989..209a6069b 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -56,6 +56,7 @@ if MYPY_CHECK_RUNNING: from pip._internal.index.package_finder import PackageFinder from pip._internal.models.link import Link from pip._internal.network.download import Downloader + from pip._internal.network.session import PipSession from pip._internal.req.req_install import InstallRequirement from pip._internal.req.req_tracker import RequirementTracker from pip._internal.utils.hashes import Hashes @@ -332,6 +333,7 @@ class RequirementPreparer(object): wheel_download_dir, # type: Optional[str] build_isolation, # type: bool req_tracker, # type: RequirementTracker + session, # type: PipSession downloader, # type: Downloader finder, # type: PackageFinder require_hashes, # type: bool @@ -344,6 +346,7 @@ class RequirementPreparer(object): self.src_dir = src_dir self.build_dir = build_dir self.req_tracker = req_tracker + self._session = session self.downloader = downloader self.finder = finder @@ -461,22 +464,32 @@ class RequirementPreparer(object): def _fetch_metadata(preparer, link): # type: (Link) -> Optional[Distribution] - """Fetch metadata, using lazy wheel if possible.""" - use_lazy_wheel = preparer.use_lazy_wheel - remote_wheel = link.is_wheel and not link.is_file - if use_lazy_wheel and remote_wheel and not preparer.require_hashes: - wheel = Wheel(link.filename) - name = canonicalize_name(wheel.name) - # If HTTPRangeRequestUnsupported is raised, fallback silently. - with indent_log(), suppress(HTTPRangeRequestUnsupported): - logger.info( - 'Obtaining dependency information from %s %s', - name, wheel.version, - ) - url = link.url.split('#', 1)[0] - session = preparer.downloader._session - return dist_from_wheel_url(name, url, session) - return None + """Fetch metadata using lazy wheel, if possible.""" + if not self.use_lazy_wheel: + return None + if self.require_hashes: + logger.debug('Lazy wheel is not used as hash checking is required') + return None + if link.is_file or not link.is_wheel: + logger.debug( + 'Lazy wheel is not used as ' + '%r does not points to a remote wheel', + link, + ) + return None + + wheel = Wheel(link.filename) + name = canonicalize_name(wheel.name) + logger.info( + 'Obtaining dependency information from %s %s', + name, wheel.version, + ) + url = link.url.split('#', 1)[0] + try: + return dist_from_wheel_url(name, url, self._session) + except HTTPRangeRequestUnsupported: + logger.debug('%s does not support range requests', url) + return None def prepare_linked_requirement(self, req, parallel_builds=False): # type: (InstallRequirement, bool) -> Distribution diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index b8da86399..ff1b51ae4 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -76,6 +76,7 @@ class TestRequirementSet(object): isolated=False, use_pep517=None, ) + session = PipSession() with get_requirement_tracker() as tracker: preparer = RequirementPreparer( @@ -85,7 +86,8 @@ class TestRequirementSet(object): wheel_download_dir=None, build_isolation=True, req_tracker=tracker, - downloader=Downloader(PipSession(), progress_bar="on"), + session=session, + downloader=Downloader(session, progress_bar="on"), finder=finder, require_hashes=require_hashes, use_user_site=False, From 11f7994a6690fcce5a1fdaddc9e3c779255cb334 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Thu, 6 Aug 2020 17:31:10 +0700 Subject: [PATCH 073/209] Revise method fetching metadata using lazy wheels * Rename it to fit the fact that it no longer handle fetching _not_ using lazy wheels * Use self as the first parameter * Unnest the checks with additional logs showing reason when lazy wheel is not used --- src/pip/_internal/operations/prepare.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 209a6069b..e9e534e32 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -9,7 +9,6 @@ import mimetypes import os import shutil -from pip._vendor.contextlib2 import suppress from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.six import PY2 @@ -462,7 +461,7 @@ class RequirementPreparer(object): # showing the user what the hash should be. return req.hashes(trust_internet=False) or MissingHashes() - def _fetch_metadata(preparer, link): + def _fetch_metadata_using_lazy_wheel(self, link): # type: (Link) -> Optional[Distribution] """Fetch metadata using lazy wheel, if possible.""" if not self.use_lazy_wheel: @@ -497,7 +496,8 @@ class RequirementPreparer(object): assert req.link link = req.link self._log_preparing_link(req) - wheel_dist = self._fetch_metadata(link) + with indent_log(): + wheel_dist = self._fetch_metadata_using_lazy_wheel(link) if wheel_dist is not None: req.needs_more_preparation = True return wheel_dist From aae63795b2bf5bcde9e846215fd62fa1addd60a4 Mon Sep 17 00:00:00 2001 From: Noah Gorny Date: Thu, 2 Jul 2020 21:59:43 +0300 Subject: [PATCH 074/209] reqfile: Update extra-index-url/index-url in session from requirements file Also update the relevant tests --- src/pip/_internal/network/session.py | 8 ++++++++ src/pip/_internal/req/req_file.py | 4 ++++ tests/unit/test_req_file.py | 13 +++++++++---- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/network/session.py b/src/pip/_internal/network/session.py index 39a4a546e..68ef68db5 100644 --- a/src/pip/_internal/network/session.py +++ b/src/pip/_internal/network/session.py @@ -305,6 +305,14 @@ class PipSession(requests.Session): for host in trusted_hosts: self.add_trusted_host(host, suppress_logging=True) + def update_index_urls(self, new_index_urls): + # type: (List[str]) -> None + """ + :param new_index_urls: New index urls to update the authentication + handler with. + """ + self.auth.index_urls = new_index_urls + def add_trusted_host(self, host, source=None, suppress_logging=False): # type: (str, Optional[str], bool) -> None """ diff --git a/src/pip/_internal/req/req_file.py b/src/pip/_internal/req/req_file.py index 105058228..72a568bdf 100644 --- a/src/pip/_internal/req/req_file.py +++ b/src/pip/_internal/req/req_file.py @@ -256,6 +256,10 @@ def handle_option_line( value = relative_to_reqs_file find_links.append(value) + if session: + # We need to update the auth urls in session + session.update_index_urls(index_urls) + search_scope = SearchScope( find_links=find_links, index_urls=index_urls, diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index 879f088a4..69d93a0cd 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -341,17 +341,22 @@ class TestProcessLine(object): line_processor("--no-index", "file", 1, finder=finder) assert finder.index_urls == [] - def test_set_finder_index_url(self, line_processor, finder): - line_processor("--index-url=url", "file", 1, finder=finder) + def test_set_finder_index_url(self, line_processor, finder, session): + line_processor( + "--index-url=url", "file", 1, finder=finder, session=session) assert finder.index_urls == ['url'] + assert session.auth.index_urls == ['url'] def test_set_finder_find_links(self, line_processor, finder): line_processor("--find-links=url", "file", 1, finder=finder) assert finder.find_links == ['url'] - def test_set_finder_extra_index_urls(self, line_processor, finder): - line_processor("--extra-index-url=url", "file", 1, finder=finder) + def test_set_finder_extra_index_urls( + self, line_processor, finder, session): + line_processor( + "--extra-index-url=url", "file", 1, finder=finder, session=session) assert finder.index_urls == ['url'] + assert session.auth.index_urls == ['url'] def test_set_finder_trusted_host( self, line_processor, caplog, session, finder From 3e70cbe5714010effd6c3c4867d41cec07c3faff Mon Sep 17 00:00:00 2001 From: Noah Gorny Date: Thu, 2 Jul 2020 22:11:29 +0300 Subject: [PATCH 075/209] news: Add --extra-index-url req file bugfix news --- news/8103.bugfix | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 news/8103.bugfix diff --git a/news/8103.bugfix b/news/8103.bugfix new file mode 100644 index 000000000..76f17e136 --- /dev/null +++ b/news/8103.bugfix @@ -0,0 +1,2 @@ +Propagate ``--extra-index-url`` from requirements file properly to session auth, +in order that keyrings and other auths will work as expected. From 312d1d0473e573a46fd857f0094adb6409a4ff06 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 7 Aug 2020 13:43:49 +0800 Subject: [PATCH 076/209] Add failing test for constraints with markers --- tests/functional/test_new_resolver.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/functional/test_new_resolver.py b/tests/functional/test_new_resolver.py index 932918033..46e32ddd3 100644 --- a/tests/functional/test_new_resolver.py +++ b/tests/functional/test_new_resolver.py @@ -1,6 +1,7 @@ import json import os import sys +import textwrap import pytest from pip._vendor.packaging.utils import canonicalize_name @@ -729,6 +730,30 @@ def test_new_resolver_constraint_on_path(script): assert msg in result.stderr, str(result) +def test_new_resolver_constraint_only_marker_match(script): + create_basic_wheel_for_package(script, "pkg", "1.0") + create_basic_wheel_for_package(script, "pkg", "2.0") + create_basic_wheel_for_package(script, "pkg", "3.0") + + constrants_content = textwrap.dedent( + """ + pkg==1.0; python_version == "{ver[0]}.{ver[1]}" # Always satisfies. + pkg==2.0; python_version < "0" # Never satisfies. + """ + ).format(ver=sys.version_info) + constraints_txt = script.scratch_path / "constraints.txt" + constraints_txt.write_text(constrants_content) + + script.pip( + "install", "--use-feature=2020-resolver", + "--no-cache-dir", "--no-index", + "-c", constraints_txt, + "--find-links", script.scratch_path, + "pkg", + ) + assert_installed(script, pkg="1.0") + + def test_new_resolver_upgrade_needs_option(script): # Install pkg 1.0.0 create_basic_wheel_for_package(script, "pkg", "1.0.0") From 4683ad02e3133cca4fa677fa4efcb52c9ac51f20 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 7 Aug 2020 13:51:43 +0800 Subject: [PATCH 077/209] Allow filtering constraints with markers --- news/8724.bugfix | 2 ++ src/pip/_internal/resolution/resolvelib/resolver.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 news/8724.bugfix diff --git a/news/8724.bugfix b/news/8724.bugfix new file mode 100644 index 000000000..8641098dd --- /dev/null +++ b/news/8724.bugfix @@ -0,0 +1,2 @@ +2020 Resolver: Correctly handle marker evaluation in constraints and exclude +them if their markers do not match the current environment. diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py index 6a38ffa5c..fde86413d 100644 --- a/src/pip/_internal/resolution/resolvelib/resolver.py +++ b/src/pip/_internal/resolution/resolvelib/resolver.py @@ -80,7 +80,8 @@ class Resolver(BaseResolver): problem = check_invalid_constraint_type(req) if problem: raise InstallationError(problem) - + if not req.match_markers(): + continue name = canonicalize_name(req.name) if name in constraints: constraints[name] = constraints[name] & req.specifier From 8dc0d9c8d916b0371fa61d67916ffc5155a90e92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Sat, 8 Aug 2020 00:14:27 +0700 Subject: [PATCH 078/209] Add news for disabling range response caching --- news/8701.bugfix | 2 ++ news/8716.bugfix | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 news/8701.bugfix create mode 100644 news/8716.bugfix diff --git a/news/8701.bugfix b/news/8701.bugfix new file mode 100644 index 000000000..086a8f2eb --- /dev/null +++ b/news/8701.bugfix @@ -0,0 +1,2 @@ +Disable caching for range requests, which causes corrupted wheels +when pip tries to obtain metadata using the feature ``fast-deps``. diff --git a/news/8716.bugfix b/news/8716.bugfix new file mode 100644 index 000000000..086a8f2eb --- /dev/null +++ b/news/8716.bugfix @@ -0,0 +1,2 @@ +Disable caching for range requests, which causes corrupted wheels +when pip tries to obtain metadata using the feature ``fast-deps``. From cef8abd268d7516dc98a6dc5fe3e61b3fbd6944f Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 11 Aug 2020 16:56:42 +0530 Subject: [PATCH 079/209] Bump for development --- src/pip/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/__init__.py b/src/pip/__init__.py index 611753fed..5a2f3c317 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -4,7 +4,7 @@ if MYPY_CHECK_RUNNING: from typing import List, Optional -__version__ = "20.2.2" +__version__ = "20.3.dev0" def main(args=None): From e62f16e96938ee24e7a57168b829942526be56e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Sun, 9 Aug 2020 15:55:33 +0700 Subject: [PATCH 080/209] Make Downloader perform the download --- src/pip/_internal/network/download.py | 30 +++++++----------------- src/pip/_internal/operations/prepare.py | 31 ++++--------------------- tests/unit/test_operations_prepare.py | 13 ++--------- 3 files changed, 15 insertions(+), 59 deletions(-) diff --git a/src/pip/_internal/network/download.py b/src/pip/_internal/network/download.py index 44f9985a3..a4d4bd2a5 100644 --- a/src/pip/_internal/network/download.py +++ b/src/pip/_internal/network/download.py @@ -24,7 +24,7 @@ from pip._internal.utils.misc import ( from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Iterable, Optional + from typing import Iterable, Optional, Tuple from pip._vendor.requests.models import Response @@ -141,19 +141,6 @@ def _http_get_download(session, link): return resp -class Download(object): - def __init__( - self, - response, # type: Response - filename, # type: str - chunks, # type: Iterable[bytes] - ): - # type: (...) -> None - self.response = response - self.filename = filename - self.chunks = chunks - - class Downloader(object): def __init__( self, @@ -164,8 +151,8 @@ class Downloader(object): self._session = session self._progress_bar = progress_bar - def __call__(self, link): - # type: (Link) -> Download + def __call__(self, link, location): + # type: (Link, str) -> Tuple[str, str] try: resp = _http_get_download(self._session, link) except NetworkConnectionError as e: @@ -175,8 +162,9 @@ class Downloader(object): ) raise - return Download( - resp, - _get_http_response_filename(resp, link), - _prepare_download(resp, link, self._progress_bar), - ) + filename = _get_http_response_filename(resp, link) + chunks = _prepare_download(resp, link, self._progress_bar) + with open(os.path.join(location, filename), 'wb') as content_file: + for chunk in chunks: + content_file.write(chunk) + return content_file.name, resp.headers.get('Content-Type', '') diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index e9e534e32..e4de1be4a 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -45,9 +45,7 @@ from pip._internal.utils.unpacking import unpack_file from pip._internal.vcs import vcs if MYPY_CHECK_RUNNING: - from typing import ( - Callable, List, Optional, Tuple, - ) + from typing import Callable, List, Optional from mypy_extensions import TypedDict from pip._vendor.pkg_resources import Distribution @@ -132,9 +130,9 @@ def get_http_url( content_type = mimetypes.guess_type(from_path)[0] else: # let's download to a tmp dir - from_path, content_type = _download_http_url( - link, downloader, temp_dir.path, hashes - ) + from_path, content_type = downloader(link, temp_dir.path) + if hashes: + hashes.check_against_path(from_path) return File(from_path, content_type) @@ -273,27 +271,6 @@ def unpack_url( return file -def _download_http_url( - link, # type: Link - downloader, # type: Downloader - temp_dir, # type: str - hashes, # type: Optional[Hashes] -): - # type: (...) -> Tuple[str, str] - """Download link url into temp_dir using provided session""" - download = downloader(link) - - file_path = os.path.join(temp_dir, download.filename) - with open(file_path, 'wb') as content_file: - for chunk in download.chunks: - content_file.write(chunk) - - if hashes: - hashes.check_against_path(file_path) - - return file_path, download.response.headers.get('content-type', '') - - def _check_download_dir(link, download_dir, hashes): # type: (Link, str, Optional[Hashes]) -> Optional[str] """ Check download_dir for previously downloaded file with correct hash diff --git a/tests/unit/test_operations_prepare.py b/tests/unit/test_operations_prepare.py index 41d8be260..d2e4d6091 100644 --- a/tests/unit/test_operations_prepare.py +++ b/tests/unit/test_operations_prepare.py @@ -10,11 +10,7 @@ from pip._internal.exceptions import HashMismatch from pip._internal.models.link import Link from pip._internal.network.download import Downloader from pip._internal.network.session import PipSession -from pip._internal.operations.prepare import ( - _copy_source_tree, - _download_http_url, - unpack_url, -) +from pip._internal.operations.prepare import _copy_source_tree, unpack_url from pip._internal.utils.hashes import Hashes from pip._internal.utils.urls import path_to_url from tests.lib.filesystem import ( @@ -83,12 +79,7 @@ def test_download_http_url__no_directory_traversal(mock_raise_for_status, download_dir = tmpdir.joinpath('download') os.mkdir(download_dir) - file_path, content_type = _download_http_url( - link, - downloader, - download_dir, - hashes=None, - ) + file_path, content_type = downloader(link, download_dir) # The file should be downloaded to download_dir. actual = os.listdir(download_dir) assert actual == ['out_dir_file'] From 078e0effb72b1078bab3d268aa5b4e374505e18a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Sun, 9 Aug 2020 22:44:20 +0700 Subject: [PATCH 081/209] Add memoization mechanism for file download This is intentionally dependent from caching, which relies on cache dir. --- ...a2b1b7-744e-4533-b3ff-6e7a1843d573.trivial | 0 src/pip/_internal/network/download.py | 9 +++- src/pip/_internal/operations/prepare.py | 43 +++++++++++-------- .../resolution/resolvelib/resolver.py | 6 ++- tests/unit/test_operations_prepare.py | 2 +- 5 files changed, 39 insertions(+), 21 deletions(-) create mode 100644 news/a3a2b1b7-744e-4533-b3ff-6e7a1843d573.trivial diff --git a/news/a3a2b1b7-744e-4533-b3ff-6e7a1843d573.trivial b/news/a3a2b1b7-744e-4533-b3ff-6e7a1843d573.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/network/download.py b/src/pip/_internal/network/download.py index a4d4bd2a5..0eb4fd9ce 100644 --- a/src/pip/_internal/network/download.py +++ b/src/pip/_internal/network/download.py @@ -151,8 +151,9 @@ class Downloader(object): self._session = session self._progress_bar = progress_bar - def __call__(self, link, location): + def download_one(self, link, location): # type: (Link, str) -> Tuple[str, str] + """Download the file given by link into location.""" try: resp = _http_get_download(self._session, link) except NetworkConnectionError as e: @@ -168,3 +169,9 @@ class Downloader(object): for chunk in chunks: content_file.write(chunk) return content_file.name, resp.headers.get('Content-Type', '') + + def download_many(self, links, location): + # type: (Iterable[Link], str) -> Iterable[Tuple[str, Tuple[str, str]]] + """Download the files given by links into location.""" + for link in links: + yield link.url, self.download_one(link, location) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index e4de1be4a..5fdbd674b 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -45,7 +45,7 @@ from pip._internal.utils.unpacking import unpack_file from pip._internal.vcs import vcs if MYPY_CHECK_RUNNING: - from typing import Callable, List, Optional + from typing import Callable, Dict, List, Optional, Tuple from mypy_extensions import TypedDict from pip._vendor.pkg_resources import Distribution @@ -130,7 +130,7 @@ def get_http_url( content_type = mimetypes.guess_type(from_path)[0] else: # let's download to a tmp dir - from_path, content_type = downloader(link, temp_dir.path) + from_path, content_type = downloader.download_one(link, temp_dir.path) if hashes: hashes.check_against_path(from_path) @@ -352,6 +352,9 @@ class RequirementPreparer(object): # Should wheels be downloaded lazily? self.use_lazy_wheel = lazy_wheel + # Memoized downloaded files, as mapping of url: (path, mime type) + self._downloaded = {} # type: Dict[str, Tuple[str, str]] + @property def _download_should_save(self): # type: () -> bool @@ -480,12 +483,15 @@ class RequirementPreparer(object): return wheel_dist return self._prepare_linked_requirement(req, parallel_builds) - def prepare_linked_requirement_more(self, req, parallel_builds=False): - # type: (InstallRequirement, bool) -> None + def prepare_linked_requirements_more(self, reqs, parallel_builds=False): + # type: (List[InstallRequirement], bool) -> None """Prepare a linked requirement more, if needed.""" - if not req.needs_more_preparation: - return - self._prepare_linked_requirement(req, parallel_builds) + # Let's download to a temporary directory. + tmpdir = TempDirectory(kind="unpack", globally_managed=True).path + links = (req.link for req in reqs) + self._downloaded.update(self.downloader.download_many(links, tmpdir)) + for req in reqs: + self._prepare_linked_requirement(req, parallel_builds) def _prepare_linked_requirement(self, req, parallel_builds): # type: (InstallRequirement, bool) -> Distribution @@ -499,16 +505,19 @@ class RequirementPreparer(object): with indent_log(): self._ensure_link_req_src_dir(req, download_dir, parallel_builds) - try: - local_file = unpack_url( - link, req.source_dir, self.downloader, download_dir, - hashes=self._get_linked_req_hashes(req) - ) - except NetworkConnectionError as exc: - raise InstallationError( - 'Could not install requirement {} because of HTTP ' - 'error {} for URL {}'.format(req, exc, link) - ) + if link.url in self._downloaded: + local_file = File(*self._downloaded[link.url]) + else: + try: + local_file = unpack_url( + link, req.source_dir, self.downloader, download_dir, + hashes=self._get_linked_req_hashes(req) + ) + except NetworkConnectionError as exc: + raise InstallationError( + 'Could not install requirement {} because of HTTP ' + 'error {} for URL {}'.format(req, exc, link) + ) # For use in later processing, preserve the file path on the # requirement. diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py index fde86413d..1cabe236d 100644 --- a/src/pip/_internal/resolution/resolvelib/resolver.py +++ b/src/pip/_internal/resolution/resolvelib/resolver.py @@ -160,8 +160,10 @@ class Resolver(BaseResolver): req_set.add_named_requirement(ireq) - for actual_req in req_set.all_requirements: - self.factory.preparer.prepare_linked_requirement_more(actual_req) + self.factory.preparer.prepare_linked_requirements_more([ + req for req in req_set.all_requirements + if req.needs_more_preparation + ]) return req_set diff --git a/tests/unit/test_operations_prepare.py b/tests/unit/test_operations_prepare.py index d2e4d6091..e90eab8d7 100644 --- a/tests/unit/test_operations_prepare.py +++ b/tests/unit/test_operations_prepare.py @@ -79,7 +79,7 @@ def test_download_http_url__no_directory_traversal(mock_raise_for_status, download_dir = tmpdir.joinpath('download') os.mkdir(download_dir) - file_path, content_type = downloader(link, download_dir) + file_path, content_type = downloader.download_one(link, download_dir) # The file should be downloaded to download_dir. actual = os.listdir(download_dir) assert actual == ['out_dir_file'] From 39d296eeb8cb8b0f8e09493bc8f1cb6eb5940a46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Mon, 10 Aug 2020 22:24:00 +0700 Subject: [PATCH 082/209] Clean up code style and internal interface Co-Authored-By: Pradyun Gedam Co-Authored-By: Chris Hunt --- src/pip/_internal/network/download.py | 7 +++++-- src/pip/_internal/operations/prepare.py | 12 +++++++----- src/pip/_internal/resolution/resolvelib/resolver.py | 7 ++----- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/pip/_internal/network/download.py b/src/pip/_internal/network/download.py index 0eb4fd9ce..5fd277330 100644 --- a/src/pip/_internal/network/download.py +++ b/src/pip/_internal/network/download.py @@ -164,11 +164,14 @@ class Downloader(object): raise filename = _get_http_response_filename(resp, link) + filepath = os.path.join(location, filename) + chunks = _prepare_download(resp, link, self._progress_bar) - with open(os.path.join(location, filename), 'wb') as content_file: + with open(filepath, 'wb') as content_file: for chunk in chunks: content_file.write(chunk) - return content_file.name, resp.headers.get('Content-Type', '') + content_type = resp.headers.get('Content-Type', '') + return filepath, content_type def download_many(self, links, location): # type: (Iterable[Link], str) -> Iterable[Tuple[str, Tuple[str, str]]] diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 5fdbd674b..5f2a0c74f 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -45,7 +45,7 @@ from pip._internal.utils.unpacking import unpack_file from pip._internal.vcs import vcs if MYPY_CHECK_RUNNING: - from typing import Callable, Dict, List, Optional, Tuple + from typing import Callable, Dict, Iterable, List, Optional, Tuple from mypy_extensions import TypedDict from pip._vendor.pkg_resources import Distribution @@ -484,8 +484,10 @@ class RequirementPreparer(object): return self._prepare_linked_requirement(req, parallel_builds) def prepare_linked_requirements_more(self, reqs, parallel_builds=False): - # type: (List[InstallRequirement], bool) -> None + # type: (Iterable[InstallRequirement], bool) -> None """Prepare a linked requirement more, if needed.""" + reqs = [req for req in reqs if req.needs_more_preparation] + # Let's download to a temporary directory. tmpdir = TempDirectory(kind="unpack", globally_managed=True).path links = (req.link for req in reqs) @@ -505,9 +507,7 @@ class RequirementPreparer(object): with indent_log(): self._ensure_link_req_src_dir(req, download_dir, parallel_builds) - if link.url in self._downloaded: - local_file = File(*self._downloaded[link.url]) - else: + if link.url not in self._downloaded: try: local_file = unpack_url( link, req.source_dir, self.downloader, download_dir, @@ -518,6 +518,8 @@ class RequirementPreparer(object): 'Could not install requirement {} because of HTTP ' 'error {} for URL {}'.format(req, exc, link) ) + else: + local_file = File(*self._downloaded[link.url]) # For use in later processing, preserve the file path on the # requirement. diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py index 1cabe236d..031d2f107 100644 --- a/src/pip/_internal/resolution/resolvelib/resolver.py +++ b/src/pip/_internal/resolution/resolvelib/resolver.py @@ -160,11 +160,8 @@ class Resolver(BaseResolver): req_set.add_named_requirement(ireq) - self.factory.preparer.prepare_linked_requirements_more([ - req for req in req_set.all_requirements - if req.needs_more_preparation - ]) - + reqs = req_set.all_requirements + self.factory.preparer.prepare_linked_requirements_more(reqs) return req_set def get_installation_order(self, req_set): From 18c803a41363612569a57b26376693f8b7d72eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Mon, 10 Aug 2020 22:54:09 +0700 Subject: [PATCH 083/209] Check hashes of memoized downloads --- src/pip/_internal/operations/prepare.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 5f2a0c74f..82362a2cf 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -507,11 +507,12 @@ class RequirementPreparer(object): with indent_log(): self._ensure_link_req_src_dir(req, download_dir, parallel_builds) + hashes = self._get_linked_req_hashes(req) if link.url not in self._downloaded: try: local_file = unpack_url( - link, req.source_dir, self.downloader, download_dir, - hashes=self._get_linked_req_hashes(req) + link, req.source_dir, self.downloader, + download_dir, hashes, ) except NetworkConnectionError as exc: raise InstallationError( @@ -519,7 +520,10 @@ class RequirementPreparer(object): 'error {} for URL {}'.format(req, exc, link) ) else: - local_file = File(*self._downloaded[link.url]) + file_path, content_type = self._downloaded[link.url] + if hashes: + hashes.check_against_path(file_path) + local_file = File(file_path, content_type) # For use in later processing, preserve the file path on the # requirement. From a1aeb4ce01cf8753bcca89ea3f83eb30abf51ae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Tue, 11 Aug 2020 17:43:13 +0700 Subject: [PATCH 084/209] Check download folder for files to be downloaded in batch --- src/pip/_internal/operations/prepare.py | 39 ++++++++++++++++--------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 82362a2cf..13a7df65a 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -104,10 +104,14 @@ def unpack_vcs_link(link, location): class File(object): + def __init__(self, path, content_type): - # type: (str, str) -> None + # type: (str, Optional[str]) -> None self.path = path - self.content_type = content_type + if content_type is None: + self.content_type = mimetypes.guess_type(path)[0] + else: + self.content_type = content_type def get_http_url( @@ -127,7 +131,7 @@ def get_http_url( if already_downloaded_path: from_path = already_downloaded_path - content_type = mimetypes.guess_type(from_path)[0] + content_type = None else: # let's download to a tmp dir from_path, content_type = downloader.download_one(link, temp_dir.path) @@ -217,10 +221,7 @@ def get_file_url( # one; no internet-sourced hash will be in `hashes`. if hashes: hashes.check_against_path(from_path) - - content_type = mimetypes.guess_type(from_path)[0] - - return File(from_path, content_type) + return File(from_path, None) def unpack_url( @@ -378,6 +379,13 @@ class RequirementPreparer(object): else: logger.info('Collecting %s', req.req or req) + def _get_download_dir(self, link): + # type: (Link) -> Optional[str] + if link.is_wheel and self.wheel_download_dir: + # Download wheels to a dedicated dir when doing `pip wheel`. + return self.wheel_download_dir + return self.download_dir + def _ensure_link_req_src_dir(self, req, download_dir, parallel_builds): # type: (InstallRequirement, Optional[str], bool) -> None """Ensure source_dir of a linked InstallRequirement.""" @@ -487,10 +495,19 @@ class RequirementPreparer(object): # type: (Iterable[InstallRequirement], bool) -> None """Prepare a linked requirement more, if needed.""" reqs = [req for req in reqs if req.needs_more_preparation] + links = [] # type: List[Link] + for req in reqs: + download_dir = self._get_download_dir(req.link) + if download_dir is not None: + hashes = self._get_linked_req_hashes(req) + file_path = _check_download_dir(req.link, download_dir, hashes) + if download_dir is None or file_path is None: + links.append(req.link) + else: + self._downloaded[req.link.url] = file_path, None # Let's download to a temporary directory. tmpdir = TempDirectory(kind="unpack", globally_managed=True).path - links = (req.link for req in reqs) self._downloaded.update(self.downloader.download_many(links, tmpdir)) for req in reqs: self._prepare_linked_requirement(req, parallel_builds) @@ -499,11 +516,7 @@ class RequirementPreparer(object): # type: (InstallRequirement, bool) -> Distribution assert req.link link = req.link - if link.is_wheel and self.wheel_download_dir: - # Download wheels to a dedicated dir when doing `pip wheel`. - download_dir = self.wheel_download_dir - else: - download_dir = self.download_dir + download_dir = self._get_download_dir(link) with indent_log(): self._ensure_link_req_src_dir(req, download_dir, parallel_builds) From b46576d9336ba2c081b486adb49ec773847c16ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Tue, 11 Aug 2020 22:56:37 +0700 Subject: [PATCH 085/209] Give batch downloader a separate class --- src/pip/_internal/cli/req_command.py | 5 +--- src/pip/_internal/network/download.py | 36 ++++++++++++++++++++++--- src/pip/_internal/operations/prepare.py | 19 ++++++------- tests/unit/test_operations_prepare.py | 16 +++++------ tests/unit/test_req.py | 3 +-- 5 files changed, 53 insertions(+), 26 deletions(-) diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 6562fe918..76abce5ac 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -16,7 +16,6 @@ from pip._internal.exceptions import CommandError, PreviousBuildDirError from pip._internal.index.collector import LinkCollector from pip._internal.index.package_finder import PackageFinder from pip._internal.models.selection_prefs import SelectionPreferences -from pip._internal.network.download import Downloader from pip._internal.network.session import PipSession from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req.constructors import ( @@ -213,8 +212,6 @@ class RequirementCommand(IndexGroupCommand): """ Create a RequirementPreparer instance for the given parameters. """ - downloader = Downloader(session, progress_bar=options.progress_bar) - temp_build_dir_path = temp_build_dir.path assert temp_build_dir_path is not None @@ -239,7 +236,7 @@ class RequirementCommand(IndexGroupCommand): build_isolation=options.build_isolation, req_tracker=req_tracker, session=session, - downloader=downloader, + progress_bar=options.progress_bar, finder=finder, require_hashes=options.require_hashes, use_user_site=use_user_site, diff --git a/src/pip/_internal/network/download.py b/src/pip/_internal/network/download.py index 5fd277330..56feaabac 100644 --- a/src/pip/_internal/network/download.py +++ b/src/pip/_internal/network/download.py @@ -151,7 +151,7 @@ class Downloader(object): self._session = session self._progress_bar = progress_bar - def download_one(self, link, location): + def __call__(self, link, location): # type: (Link, str) -> Tuple[str, str] """Download the file given by link into location.""" try: @@ -173,8 +173,38 @@ class Downloader(object): content_type = resp.headers.get('Content-Type', '') return filepath, content_type - def download_many(self, links, location): + +class BatchDownloader(object): + + def __init__( + self, + session, # type: PipSession + progress_bar, # type: str + ): + # type: (...) -> None + self._session = session + self._progress_bar = progress_bar + + def __call__(self, links, location): # type: (Iterable[Link], str) -> Iterable[Tuple[str, Tuple[str, str]]] """Download the files given by links into location.""" for link in links: - yield link.url, self.download_one(link, location) + try: + resp = _http_get_download(self._session, link) + except NetworkConnectionError as e: + assert e.response is not None + logger.critical( + "HTTP error %s while getting %s", + e.response.status_code, link, + ) + raise + + filename = _get_http_response_filename(resp, link) + filepath = os.path.join(location, filename) + + chunks = _prepare_download(resp, link, self._progress_bar) + with open(filepath, 'wb') as content_file: + for chunk in chunks: + content_file.write(chunk) + content_type = resp.headers.get('Content-Type', '') + yield link.url, (filepath, content_type) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 13a7df65a..5eb71ce07 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -26,6 +26,7 @@ from pip._internal.exceptions import ( VcsHashUnsupported, ) from pip._internal.models.wheel import Wheel +from pip._internal.network.download import BatchDownloader, Downloader from pip._internal.network.lazy_wheel import ( HTTPRangeRequestUnsupported, dist_from_wheel_url, @@ -52,7 +53,6 @@ if MYPY_CHECK_RUNNING: from pip._internal.index.package_finder import PackageFinder from pip._internal.models.link import Link - from pip._internal.network.download import Downloader from pip._internal.network.session import PipSession from pip._internal.req.req_install import InstallRequirement from pip._internal.req.req_tracker import RequirementTracker @@ -116,7 +116,7 @@ class File(object): def get_http_url( link, # type: Link - downloader, # type: Downloader + download, # type: Downloader download_dir=None, # type: Optional[str] hashes=None, # type: Optional[Hashes] ): @@ -134,7 +134,7 @@ def get_http_url( content_type = None else: # let's download to a tmp dir - from_path, content_type = downloader.download_one(link, temp_dir.path) + from_path, content_type = download(link, temp_dir.path) if hashes: hashes.check_against_path(from_path) @@ -227,7 +227,7 @@ def get_file_url( def unpack_url( link, # type: Link location, # type: str - downloader, # type: Downloader + download, # type: Downloader download_dir=None, # type: Optional[str] hashes=None, # type: Optional[Hashes] ): @@ -259,7 +259,7 @@ def unpack_url( else: file = get_http_url( link, - downloader, + download, download_dir, hashes=hashes, ) @@ -311,7 +311,7 @@ class RequirementPreparer(object): build_isolation, # type: bool req_tracker, # type: RequirementTracker session, # type: PipSession - downloader, # type: Downloader + progress_bar, # type: str finder, # type: PackageFinder require_hashes, # type: bool use_user_site, # type: bool @@ -324,7 +324,8 @@ class RequirementPreparer(object): self.build_dir = build_dir self.req_tracker = req_tracker self._session = session - self.downloader = downloader + self._download = Downloader(session, progress_bar) + self._batch_download = BatchDownloader(session, progress_bar) self.finder = finder # Where still-packed archives should be written to. If None, they are @@ -508,7 +509,7 @@ class RequirementPreparer(object): # Let's download to a temporary directory. tmpdir = TempDirectory(kind="unpack", globally_managed=True).path - self._downloaded.update(self.downloader.download_many(links, tmpdir)) + self._downloaded.update(self._batch_download(links, tmpdir)) for req in reqs: self._prepare_linked_requirement(req, parallel_builds) @@ -524,7 +525,7 @@ class RequirementPreparer(object): if link.url not in self._downloaded: try: local_file = unpack_url( - link, req.source_dir, self.downloader, + link, req.source_dir, self._download, download_dir, hashes, ) except NetworkConnectionError as exc: diff --git a/tests/unit/test_operations_prepare.py b/tests/unit/test_operations_prepare.py index e90eab8d7..ab6aaf6aa 100644 --- a/tests/unit/test_operations_prepare.py +++ b/tests/unit/test_operations_prepare.py @@ -35,7 +35,7 @@ def test_unpack_url_with_urllib_response_without_content_type(data): session = Mock() session.get = _fake_session_get - downloader = Downloader(session, progress_bar="on") + download = Downloader(session, progress_bar="on") uri = path_to_url(data.packages.joinpath("simple-1.0.tar.gz")) link = Link(uri) @@ -44,7 +44,7 @@ def test_unpack_url_with_urllib_response_without_content_type(data): unpack_url( link, temp_dir, - downloader=downloader, + download=download, download_dir=None, ) assert set(os.listdir(temp_dir)) == { @@ -75,11 +75,11 @@ def test_download_http_url__no_directory_traversal(mock_raise_for_status, 'content-disposition': 'attachment;filename="../out_dir_file"' } session.get.return_value = resp - downloader = Downloader(session, progress_bar="on") + download = Downloader(session, progress_bar="on") download_dir = tmpdir.joinpath('download') os.mkdir(download_dir) - file_path, content_type = downloader.download_one(link, download_dir) + file_path, content_type = download(link, download_dir) # The file should be downloaded to download_dir. actual = os.listdir(download_dir) assert actual == ['out_dir_file'] @@ -178,11 +178,11 @@ class Test_unpack_url(object): self.dist_path2 = data.packages.joinpath(self.dist_file2) self.dist_url = Link(path_to_url(self.dist_path)) self.dist_url2 = Link(path_to_url(self.dist_path2)) - self.no_downloader = Mock(side_effect=AssertionError) + self.no_download = Mock(side_effect=AssertionError) def test_unpack_url_no_download(self, tmpdir, data): self.prep(tmpdir, data) - unpack_url(self.dist_url, self.build_dir, self.no_downloader) + unpack_url(self.dist_url, self.build_dir, self.no_download) assert os.path.isdir(os.path.join(self.build_dir, 'simple')) assert not os.path.isfile( os.path.join(self.download_dir, self.dist_file)) @@ -198,7 +198,7 @@ class Test_unpack_url(object): with pytest.raises(HashMismatch): unpack_url(dist_url, self.build_dir, - downloader=self.no_downloader, + download=self.no_download, hashes=Hashes({'md5': ['bogus']})) def test_unpack_url_thats_a_dir(self, tmpdir, data): @@ -206,7 +206,7 @@ class Test_unpack_url(object): dist_path = data.packages.joinpath("FSPkg") dist_url = Link(path_to_url(dist_path)) unpack_url(dist_url, self.build_dir, - downloader=self.no_downloader, + download=self.no_download, download_dir=self.download_dir) assert os.path.isdir(os.path.join(self.build_dir, 'fspkg')) diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index ff1b51ae4..a5a9d4bae 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -18,7 +18,6 @@ from pip._internal.exceptions import ( InvalidWheelFilename, PreviousBuildDirError, ) -from pip._internal.network.download import Downloader from pip._internal.network.session import PipSession from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import InstallRequirement, RequirementSet @@ -87,7 +86,7 @@ class TestRequirementSet(object): build_isolation=True, req_tracker=tracker, session=session, - downloader=Downloader(session, progress_bar="on"), + progress_bar='on', finder=finder, require_hashes=require_hashes, use_user_site=False, From 9c4a88b0a067ccf551279c889c90eec1adfc6144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Wed, 12 Aug 2020 11:07:36 +0200 Subject: [PATCH 086/209] Improve deprecation message for issue 8368 --- src/pip/_internal/commands/install.py | 21 +++------------------ src/pip/_internal/req/req_install.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 77ec210d6..e41660070 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -21,7 +21,6 @@ from pip._internal.locations import distutils_scheme from pip._internal.operations.check import check_install_conflicts from pip._internal.req import install_given_reqs from pip._internal.req.req_tracker import get_requirement_tracker -from pip._internal.utils.deprecation import deprecated from pip._internal.utils.distutils_args import parse_distutils_args from pip._internal.utils.filesystem import test_writable_dir from pip._internal.utils.misc import ( @@ -371,23 +370,9 @@ class InstallCommand(RequirementCommand): # For now, we just warn about failures building legacy # requirements, as we'll fall through to a direct # install for those. - legacy_build_failure_names = [ - r.name # type: ignore - for r in build_failures if not r.use_pep517 - ] # type: List[str] - if legacy_build_failure_names: - deprecated( - reason=( - "Could not build wheels for {} which do not use " - "PEP 517. pip will fall back to legacy 'setup.py " - "install' for these.".format( - ", ".join(legacy_build_failure_names) - ) - ), - replacement="to fix the wheel build issue reported above", - gone_in="21.0", - issue=8368, - ) + for r in build_failures: + if not r.use_pep517: + r.legacy_install_reason = 8368 to_install = resolver.get_installation_order( requirement_set diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 816969f8e..8f1b66124 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -121,6 +121,7 @@ class InstallRequirement(object): self.comes_from = comes_from self.constraint = constraint self.editable = editable + self.legacy_install_reason = None # type: Optional[int] # source_dir is the local directory where the linked requirement is # located, or unpacked. In case unpacking is needed, creating and @@ -859,6 +860,18 @@ class InstallRequirement(object): except Exception: self.install_succeeded = True raise + else: + if self.legacy_install_reason == 8368: + deprecated( + reason=( + "{} was installed using the legacy 'setup.py install' " + "method, because a wheel could not be built for it.". + format(self.name) + ), + replacement="to fix the wheel build issue reported above", + gone_in="21.0", + issue=8368, + ) self.install_succeeded = success From 03d49da397a40d6652493340dde6f6a1661f512b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 13 Aug 2020 09:31:20 +0200 Subject: [PATCH 087/209] Add news --- news/8752.feature | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 news/8752.feature diff --git a/news/8752.feature b/news/8752.feature new file mode 100644 index 000000000..d2560da18 --- /dev/null +++ b/news/8752.feature @@ -0,0 +1,3 @@ +Make the ``setup.py install`` deprecation warning less noisy. We warn only +when ``setup.py install`` succeeded and ``setup.py bdist_wheel`` failed, as +situations where both fails are most probably irrelevant to this deprecation. From 4c348cf3a08f05f498e5a3ad19cc268752de3b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 13 Aug 2020 09:38:56 +0200 Subject: [PATCH 088/209] Consider success flag instead of absence of exception --- src/pip/_internal/req/req_install.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 8f1b66124..4e306be01 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -860,21 +860,21 @@ class InstallRequirement(object): except Exception: self.install_succeeded = True raise - else: - if self.legacy_install_reason == 8368: - deprecated( - reason=( - "{} was installed using the legacy 'setup.py install' " - "method, because a wheel could not be built for it.". - format(self.name) - ), - replacement="to fix the wheel build issue reported above", - gone_in="21.0", - issue=8368, - ) self.install_succeeded = success + if success and self.legacy_install_reason == 8368: + deprecated( + reason=( + "{} was installed using the legacy 'setup.py install' " + "method, because a wheel could not be built for it.". + format(self.name) + ), + replacement="to fix the wheel build issue reported above", + gone_in="21.0", + issue=8368, + ) + def check_invalid_constraint_type(req): # type: (InstallRequirement) -> str From 24d5fe86c04a8e6f728c34eab8aabd13e1a8cc8a Mon Sep 17 00:00:00 2001 From: Noah Date: Thu, 13 Aug 2020 15:34:00 +0300 Subject: [PATCH 089/209] Update news/8103.bugfix Co-authored-by: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com> --- news/8103.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/8103.bugfix b/news/8103.bugfix index 76f17e136..55e4d6571 100644 --- a/news/8103.bugfix +++ b/news/8103.bugfix @@ -1,2 +1,2 @@ Propagate ``--extra-index-url`` from requirements file properly to session auth, -in order that keyrings and other auths will work as expected. +so that keyring auth will work as expected. From 46b938349abc8b57a8f132b9859c24b9620cb380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Fri, 14 Aug 2020 17:38:44 +0700 Subject: [PATCH 090/209] Allow py2 deprecation warning from setuptools --- news/d90a40c1-15b7-46b9-9162-335bb346b53f.trivial | 0 tests/functional/test_download.py | 2 ++ tests/functional/test_install.py | 5 ++++- 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 news/d90a40c1-15b7-46b9-9162-335bb346b53f.trivial diff --git a/news/d90a40c1-15b7-46b9-9162-335bb346b53f.trivial b/news/d90a40c1-15b7-46b9-9162-335bb346b53f.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py index 3e80fa5c5..3291d580d 100644 --- a/tests/functional/test_download.py +++ b/tests/functional/test_download.py @@ -4,6 +4,7 @@ import textwrap from hashlib import sha256 import pytest +from pip._vendor.six import PY2 from pip._internal.cli.status_codes import ERROR from pip._internal.utils.urls import path_to_url @@ -474,6 +475,7 @@ def make_wheel_with_python_requires(script, package_name, python_requires): package_dir.joinpath('setup.py').write_text(text) script.run( 'python', 'setup.py', 'bdist_wheel', '--universal', cwd=package_dir, + allow_stderr_warning=PY2, ) file_name = '{}-1.0-py2.py3-none-any.whl'.format(package_name) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 2185251d2..abf5a8d07 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -9,6 +9,7 @@ import textwrap from os.path import curdir, join, pardir import pytest +from pip._vendor.six import PY2 from pip._internal.cli.status_codes import ERROR, SUCCESS from pip._internal.models.index import PyPI, TestPyPI @@ -1565,7 +1566,9 @@ def test_install_incompatible_python_requires_wheel(script, with_wheel): version='0.1') """)) script.run( - 'python', 'setup.py', 'bdist_wheel', '--universal', cwd=pkga_path) + 'python', 'setup.py', 'bdist_wheel', '--universal', + cwd=pkga_path, allow_stderr_warning=PY2, + ) result = script.pip('install', './pkga/dist/pkga-0.1-py2.py3-none-any.whl', expect_error=True) assert _get_expected_error_text() in result.stderr, str(result) From 530463879ed5d988038fcb481a043f5092a30ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Sat, 15 Aug 2020 21:22:13 +0700 Subject: [PATCH 091/209] Use --durations instead of --duration for pytest Newer pytest no longer accepts --duration as an alias for --durations. --- .azure-pipelines/steps/run-tests-windows.yml | 2 +- .azure-pipelines/steps/run-tests.yml | 4 ++-- news/946beace-6164-4d1a-a05d-e9bebf43ccd0.trivial | 0 tools/travis/run.sh | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 news/946beace-6164-4d1a-a05d-e9bebf43ccd0.trivial diff --git a/.azure-pipelines/steps/run-tests-windows.yml b/.azure-pipelines/steps/run-tests-windows.yml index a65136289..39282a3cc 100644 --- a/.azure-pipelines/steps/run-tests-windows.yml +++ b/.azure-pipelines/steps/run-tests-windows.yml @@ -43,7 +43,7 @@ steps: # https://bugs.python.org/issue18199 $env:TEMP = "R:\Temp" - tox -e py -- -m integration -n auto --duration=5 --junit-xml=junit/integration-test.xml + tox -e py -- -m integration -n auto --durations=5 --junit-xml=junit/integration-test.xml displayName: Tox run integration tests - task: PublishTestResults@2 diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index 11ea22727..5b9a9c50c 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -11,10 +11,10 @@ steps: displayName: Tox run unit tests # Run integration tests in two groups so we will fail faster if there is a failure in the first group -- script: tox -e py -- -m integration -n auto --duration=5 -k "not test_install" --junit-xml=junit/integration-test-group0.xml +- script: tox -e py -- -m integration -n auto --durations=5 -k "not test_install" --junit-xml=junit/integration-test-group0.xml displayName: Tox run Group 0 integration tests -- script: tox -e py -- -m integration -n auto --duration=5 -k "test_install" --junit-xml=junit/integration-test-group1.xml +- script: tox -e py -- -m integration -n auto --durations=5 -k "test_install" --junit-xml=junit/integration-test-group1.xml displayName: Tox run Group 1 integration tests - task: PublishTestResults@2 diff --git a/news/946beace-6164-4d1a-a05d-e9bebf43ccd0.trivial b/news/946beace-6164-4d1a-a05d-e9bebf43ccd0.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/tools/travis/run.sh b/tools/travis/run.sh index a531cbb56..df8f03e7a 100755 --- a/tools/travis/run.sh +++ b/tools/travis/run.sh @@ -49,15 +49,15 @@ if [[ "$GROUP" == "1" ]]; then # Unit tests tox -- --use-venv -m unit -n auto # Integration tests (not the ones for 'pip install') - tox -- -m integration -n auto --duration=5 -k "not test_install" \ + tox -- -m integration -n auto --durations=5 -k "not test_install" \ --use-venv $RESOLVER_SWITCH elif [[ "$GROUP" == "2" ]]; then # Separate Job for running integration tests for 'pip install' - tox -- -m integration -n auto --duration=5 -k "test_install" \ + tox -- -m integration -n auto --durations=5 -k "test_install" \ --use-venv $RESOLVER_SWITCH elif [[ "$GROUP" == "3" ]]; then # Separate Job for tests that fail with the new resolver - tox -- -m fails_on_new_resolver -n auto --duration=5 \ + tox -- -m fails_on_new_resolver -n auto --durations=5 \ --use-venv $RESOLVER_SWITCH --new-resolver-runtests else # Non-Testing Jobs should run once From 0c0223765ae7ba6eda5d6317d83155c227e2002e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Sun, 16 Aug 2020 14:24:41 +0700 Subject: [PATCH 092/209] Unpin pytest and its plugins This works around the incompatibility of pytest-xdist 2.0.0 with older pytest: https://github.com/pytest-dev/pytest-xdist/issues/580 --- tools/requirements/tests.txt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tools/requirements/tests.txt b/tools/requirements/tests.txt index 0c84f20aa..d24c973de 100644 --- a/tools/requirements/tests.txt +++ b/tools/requirements/tests.txt @@ -4,16 +4,14 @@ enum34; python_version < '3.4' freezegun mock pretend -# pytest 5.x only supports python 3.5+ -pytest<5.0.0 +pytest pytest-cov -# Prevent installing 9.0 which has install_requires "pytest >= 5.0". -pytest-rerunfailures<9.0 +pytest-rerunfailures pytest-timeout pytest-xdist pyyaml -setuptools>=39.2.0 # Needed for `setuptools.wheel.Wheel` support. scripttest +setuptools>=39.2.0 # Needed for `setuptools.wheel.Wheel` support. https://github.com/pypa/virtualenv/archive/legacy.zip#egg=virtualenv werkzeug==0.16.0 wheel From 15e5680d8a66a96ded6a578c554292cc654f0b11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Sun, 16 Aug 2020 14:25:32 +0700 Subject: [PATCH 093/209] Use the new resolver for test requirements --- tools/requirements/tests.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/requirements/tests.txt b/tools/requirements/tests.txt index d24c973de..ef87225d6 100644 --- a/tools/requirements/tests.txt +++ b/tools/requirements/tests.txt @@ -1,3 +1,4 @@ +--use-feature=2020-resolver cryptography==2.8 csv23 enum34; python_version < '3.4' From 14397418d178bdc20c8084d39c1219d08425dbd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Mon, 10 Aug 2020 22:08:48 +0700 Subject: [PATCH 094/209] Test hash checking for fast-deps --- ...494986-202e-4275-b7ec-d6f046c0aa05.trivial | 0 tests/functional/test_fast_deps.py | 29 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 news/0e494986-202e-4275-b7ec-d6f046c0aa05.trivial diff --git a/news/0e494986-202e-4275-b7ec-d6f046c0aa05.trivial b/news/0e494986-202e-4275-b7ec-d6f046c0aa05.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/test_fast_deps.py b/tests/functional/test_fast_deps.py index b41055c56..655440b88 100644 --- a/tests/functional/test_fast_deps.py +++ b/tests/functional/test_fast_deps.py @@ -48,3 +48,32 @@ def test_build_wheel_with_deps(data, script): assert fnmatch.filter(created, 'requiresPaste-3.1.4-*.whl') assert fnmatch.filter(created, 'Paste-3.4.2-*.whl') assert fnmatch.filter(created, 'six-*.whl') + + +@mark.network +def test_require_hash(script, tmp_path): + reqs = tmp_path / 'requirements.txt' + reqs.write_text( + u'idna==2.10' + ' --hash=sha256:' + 'b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0' + ' --hash=sha256:' + 'b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6' + ) + result = script.pip( + 'download', '--use-feature=fast-deps', '-r', str(reqs), + allow_stderr_warning=True, + ) + created = list(map(basename, result.files_created)) + assert fnmatch.filter(created, 'idna-2.10*') + + +@mark.network +def test_hash_mismatch(script, tmp_path): + reqs = tmp_path / 'requirements.txt' + reqs.write_text(u'idna==2.10 --hash=sha256:irna') + result = script.pip( + 'download', '--use-feature=fast-deps', '-r', str(reqs), + expect_error=True, + ) + assert 'DO NOT MATCH THE HASHES' in result.stderr From e93257c080221da6d607a958573c4ab0e007b34c Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 18 Aug 2020 15:22:16 +0300 Subject: [PATCH 095/209] Warn Python 3.5 support is deprecated and will be removed in pip 21.0, Jan 2021 --- news/8181.removal | 1 + src/pip/_internal/cli/base_command.py | 14 +++++++++++++- tests/conftest.py | 4 ++-- 3 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 news/8181.removal diff --git a/news/8181.removal b/news/8181.removal new file mode 100644 index 000000000..ae6bbe9f8 --- /dev/null +++ b/news/8181.removal @@ -0,0 +1 @@ +Deprecate support for Python 3.5 diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py index c3b6a856b..197400a72 100644 --- a/src/pip/_internal/cli/base_command.py +++ b/src/pip/_internal/cli/base_command.py @@ -158,7 +158,19 @@ class Command(CommandContextMixIn): "1st, 2020. Please upgrade your Python as Python 2.7 " "is no longer maintained. " ) + message - deprecated(message, replacement=None, gone_in=None) + deprecated(message, replacement=None, gone_in="21.0") + + if ( + sys.version_info[:2] == (3, 5) and + not options.no_python_version_warning + ): + message = ( + "Python 3.5 reached the end of its life on September " + "13th, 2020. Please upgrade your Python as Python 3.5 " + "is no longer maintained. pip 21.0 will drop support " + "for Python 3.5 in January 2021." + ) + deprecated(message, replacement=None, gone_in="21.0") # TODO: Try to get these passing down from the command? # without resorting to os.environ to hold these. diff --git a/tests/conftest.py b/tests/conftest.py index 2aab50207..32b6e6926 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -470,8 +470,8 @@ def in_memory_pip(): @pytest.fixture(scope="session") def deprecated_python(): - """Used to indicate whether pip deprecated this python version""" - return sys.version_info[:2] in [(2, 7)] + """Used to indicate whether pip deprecated this Python version""" + return sys.version_info[:2] in [(2, 7), (3, 5)] @pytest.fixture(scope="session") From 1f0ace9a2e0ba1ea1551f5d0c217482368b33ca8 Mon Sep 17 00:00:00 2001 From: wim glenn Date: Tue, 18 Aug 2020 20:57:08 -0500 Subject: [PATCH 096/209] restore a broken slug anchor in user guide --- docs/html/user_guide.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 73a264322..d672be02e 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -788,8 +788,6 @@ archives are built with identical packages. to use such a package, see :ref:`Controlling setup_requires`. -.. _`Using pip from your program`: - Fixing conflicting dependencies =============================== @@ -971,6 +969,8 @@ issue tracker`_ if you believe that your problem has exposed a bug in pip. .. _"How do I ask a good question?": https://stackoverflow.com/help/how-to-ask .. _pip issue tracker: https://github.com/pypa/pip/issues +.. _`Using pip from your program`: + Using pip from your program =========================== From 4c1dcbba9cc77eba7c6596317a1ebedbe328afef Mon Sep 17 00:00:00 2001 From: wim glenn Date: Wed, 19 Aug 2020 12:21:16 -0500 Subject: [PATCH 097/209] Create 8781.doc --- news/8781.doc | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/8781.doc diff --git a/news/8781.doc b/news/8781.doc new file mode 100644 index 000000000..108cf1a56 --- /dev/null +++ b/news/8781.doc @@ -0,0 +1 @@ +Fixed a broken slug anchor in user guide (https://github.com/pypa/pip/pull/8781) From ce85775155df227dec3bb3992db64a5f239861dc Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Thu, 20 Aug 2020 07:47:36 -0400 Subject: [PATCH 098/209] Bump Sphinx to v3.1.2 and fix rst issues --- docs/html/development/architecture/configuration-files.rst | 1 + news/629892ca-55da-4ca9-9cff-c15373e97ad1.trivial | 0 tools/requirements/docs.txt | 2 +- 3 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 news/629892ca-55da-4ca9-9cff-c15373e97ad1.trivial diff --git a/docs/html/development/architecture/configuration-files.rst b/docs/html/development/architecture/configuration-files.rst index ce0ef40ee..2f96ea5ca 100644 --- a/docs/html/development/architecture/configuration-files.rst +++ b/docs/html/development/architecture/configuration-files.rst @@ -109,6 +109,7 @@ manipulated. In addition to the methods discussed in the previous section, the methods used would be: .. py:class:: Configuration + :noindex: .. py:method:: get_file_to_edit() diff --git a/news/629892ca-55da-4ca9-9cff-c15373e97ad1.trivial b/news/629892ca-55da-4ca9-9cff-c15373e97ad1.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/tools/requirements/docs.txt b/tools/requirements/docs.txt index acbd33906..e787ceca6 100644 --- a/tools/requirements/docs.txt +++ b/tools/requirements/docs.txt @@ -1,4 +1,4 @@ -sphinx == 2.4.3 +sphinx == 3.1.2 git+https://github.com/python/python-docs-theme.git#egg=python-docs-theme git+https://github.com/pypa/pypa-docs-theme.git#egg=pypa-docs-theme From e51de6ecb40203f733734ec6957505abcf7fac34 Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Thu, 20 Aug 2020 07:54:34 -0400 Subject: [PATCH 099/209] Bump to latest version 3.2.1 --- tools/requirements/docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/requirements/docs.txt b/tools/requirements/docs.txt index e787ceca6..267d5b1dd 100644 --- a/tools/requirements/docs.txt +++ b/tools/requirements/docs.txt @@ -1,4 +1,4 @@ -sphinx == 3.1.2 +sphinx == 3.2.1 git+https://github.com/python/python-docs-theme.git#egg=python-docs-theme git+https://github.com/pypa/pypa-docs-theme.git#egg=pypa-docs-theme From 4e34f6530a97ca16ee524f5ac97b90fa46475e26 Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Thu, 20 Aug 2020 12:24:58 -0400 Subject: [PATCH 100/209] add theme config to set background color and text color for code-blocks --- docs/html/conf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/html/conf.py b/docs/html/conf.py index bd44b9bff..2ef2647ce 100644 --- a/docs/html/conf.py +++ b/docs/html/conf.py @@ -143,7 +143,9 @@ html_theme_options = { 'collapsiblesidebar': True, 'externalrefs': True, 'navigation_depth': 3, - 'issues_url': 'https://github.com/pypa/pip/issues' + 'issues_url': 'https://github.com/pypa/pip/issues', + 'codebgcolor': '#eeffcc', + 'codetextcolor': '#333333', } # Add any paths that contain custom themes here, relative to this directory. From c84ef7a67c2bb3b38a4e0616a433b143e0ddc87c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Thu, 20 Aug 2020 17:10:59 +0700 Subject: [PATCH 101/209] Mark tests using remote svn and hg as xfail The source repositories for testing is no longer available. --- news/559bb022-21ae-498c-a2ce-2c354d880f5e.trivial | 0 tests/functional/test_install.py | 1 + tests/functional/test_install_reqs.py | 1 + tests/functional/test_install_user.py | 1 + tests/functional/test_uninstall.py | 2 ++ 5 files changed, 5 insertions(+) create mode 100644 news/559bb022-21ae-498c-a2ce-2c354d880f5e.trivial diff --git a/news/559bb022-21ae-498c-a2ce-2c354d880f5e.trivial b/news/559bb022-21ae-498c-a2ce-2c354d880f5e.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index abf5a8d07..17a72bca8 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -758,6 +758,7 @@ def test_install_using_install_option_and_editable(script, tmpdir): result.did_create(script_file) +@pytest.mark.xfail @pytest.mark.network @need_mercurial @windows_workaround_7667 diff --git a/tests/functional/test_install_reqs.py b/tests/functional/test_install_reqs.py index d12e19b21..c879b6903 100644 --- a/tests/functional/test_install_reqs.py +++ b/tests/functional/test_install_reqs.py @@ -159,6 +159,7 @@ def test_relative_requirements_file( result.did_create(egg_link_file) +@pytest.mark.xfail @pytest.mark.network @need_svn def test_multiple_requirements_files(script, tmpdir, with_wheel): diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index 24169470a..c5d7acced 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -45,6 +45,7 @@ class Tests_UserSite: project_name = result.stdout.strip() assert 'INITools' == project_name, project_name + @pytest.mark.xfail @pytest.mark.network @need_svn @pytest.mark.incompatible_with_test_venv diff --git a/tests/functional/test_uninstall.py b/tests/functional/test_uninstall.py index 1f2fe6912..6e4aec0e5 100644 --- a/tests/functional/test_uninstall.py +++ b/tests/functional/test_uninstall.py @@ -307,6 +307,7 @@ def test_uninstall_easy_installed_console_scripts(script): ) +@pytest.mark.xfail @pytest.mark.network @need_svn def test_uninstall_editable_from_svn(script, tmpdir): @@ -372,6 +373,7 @@ def _test_uninstall_editable_with_source_outside_venv( ) +@pytest.mark.xfail @pytest.mark.network @need_svn def test_uninstall_from_reqs_file(script, tmpdir): From f060669e05931e6d69c315b4e718ea38f91bf84d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Mon, 24 Aug 2020 15:42:31 +0700 Subject: [PATCH 102/209] Fix indentation of lists and literal blocks --- docs/html/user_guide.rst | 38 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 73a264322..bce2d5e65 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -36,9 +36,7 @@ directly from distribution files. The most common scenario is to install from `PyPI`_ using :ref:`Requirement -Specifiers` - - :: +Specifiers` :: $ pip install SomePackage # latest version $ pip install SomePackage==1.0.4 # specific version @@ -119,9 +117,7 @@ Requirements Files ================== "Requirements files" are files containing a list of items to be -installed using :ref:`pip install` like so: - - :: +installed using :ref:`pip install` like so:: pip install -r requirements.txt @@ -207,9 +203,7 @@ contents is nearly identical to :ref:`Requirements Files`. There is one key difference: Including a package in a constraints file does not trigger installation of the package. -Use a constraints file like so: - - :: +Use a constraints file like so:: pip install -c constraints.txt @@ -807,16 +801,14 @@ Understanding your error message When you get a ``ResolutionImpossible`` error, you might see something like this: -:: +.. code-block:: console - pip install package_coffee==0.44.1 package_tea==4.3.0 - -:: - - Due to conflicting dependencies pip cannot install package_coffee and - package_tea: - - package_coffee depends on package_water<3.0.0,>=2.4.2 - - package_tea depends on package_water==2.3.1 + $ pip install package_coffee==0.44.1 package_tea==4.3.0 + ... + Due to conflicting dependencies pip cannot install + package_coffee and package_tea: + - package_coffee depends on package_water<3.0.0,>=2.4.2 + - package_tea depends on package_water==2.3.1 In this example, pip cannot install the packages you have requested, because they each depend on different versions of the same package @@ -1138,11 +1130,11 @@ How to test - If you use pip to install your software, try out the new resolver and let us know if it works for you with ``pip install``. Try: - - installing several packages simultaneously - - re-creating an environment using a ``requirements.txt`` file - - using ``pip install --force-reinstall`` to check whether - it does what you think it should - - using constraints files + - installing several packages simultaneously + - re-creating an environment using a ``requirements.txt`` file + - using ``pip install --force-reinstall`` to check whether + it does what you think it should + - using constraints files - If you have a build pipeline that depends on pip installing your dependencies for you, check that the new resolver does what you From fd48624ea95eea4978caf13166b31badb2fb6a4e Mon Sep 17 00:00:00 2001 From: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com> Date: Mon, 24 Aug 2020 14:19:02 +0530 Subject: [PATCH 103/209] Update and rename 8781.doc to 8781.trivial --- news/8781.doc | 1 - news/8781.trivial | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 news/8781.doc create mode 100644 news/8781.trivial diff --git a/news/8781.doc b/news/8781.doc deleted file mode 100644 index 108cf1a56..000000000 --- a/news/8781.doc +++ /dev/null @@ -1 +0,0 @@ -Fixed a broken slug anchor in user guide (https://github.com/pypa/pip/pull/8781) diff --git a/news/8781.trivial b/news/8781.trivial new file mode 100644 index 000000000..e6044f52f --- /dev/null +++ b/news/8781.trivial @@ -0,0 +1 @@ +Fix a broken slug anchor in user guide. From 984fa3c66419c69cadc9b45daae54522666cc5ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Mon, 24 Aug 2020 15:48:39 +0700 Subject: [PATCH 104/209] Make version specifier explanation easier to read in reST --- docs/html/user_guide.rst | 52 +++++++++++-------- ...3f7456-cc25-4df9-9518-4732b1e07fe5.trivial | 0 2 files changed, 31 insertions(+), 21 deletions(-) create mode 100644 news/093f7456-cc25-4df9-9518-4732b1e07fe5.trivial diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index bce2d5e65..4699f95a1 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -827,27 +827,37 @@ commonly understood comparison operators to specify the required version However, Python packaging also supports some more complex ways for specifying package versions (e.g. ``~=`` or ``*``): -.. csv-table:: - :header: "Operator", "Description", "Example" - - ``>``, "Any version greater than the specified version", "``>3.1``: any - version greater than 3.1" - ``<``, "Any version less than the specified version", "``<3.1``: any version - less than ``3.1``" - ``<=``, "Any version less than or equal to the specified version", "``<=3.1``: - any version less than or equal to ``3.1``" - ``>=``, "Any version greater than or equal to the specified - version", "``>=3.1``: version ``3.1`` and greater" - ``==``, "Exactly the specified version", ``==3.1``: only version ``3.1`` - ``!=``, "Any version not equal to the specified version", "``!=3.1``: any - version other than ``3.1``" - ``~=``, "Any compatible release. Compatible releases are releases that are - within the same major or minor version, assuming the package author is using - semantic versioning.", "``~=3.1``: version ``3.1`` or later, but not version - ``4.0`` or later. ``~=3.1.2``: version ``3.1.2`` or later, but not - version ``3.2.0`` or later." - ``*``,Can be used at the end of a version number to represent "all", "``== 3. - 1.*``: any version that starts with ``3.1``. Equivalent to ``~=3.1.0``." ++----------+---------------------------------+--------------------------------+ +| Operator | Description | Example | ++==========+=================================+================================+ +| ``>`` | Any version greater than | ``>3.1``: any version | +| | the specified version. | greater than ``3.1``. | ++----------+---------------------------------+--------------------------------+ +| ``<`` | Any version less than | ``<3.1``: any version | +| | the specified version. | less than ``3.1``. | ++----------+---------------------------------+--------------------------------+ +| ``<=`` | Any version less than or | ``<=3.1``: any version | +| | equal to the specified version. | less than or equal to ``3.1``. | ++----------+---------------------------------+--------------------------------+ +| ``>=`` | Any version greater than or | ``>=3.1``: | +| | equal to the specified version. | version ``3.1`` and greater. | ++----------+---------------------------------+--------------------------------+ +| ``==`` | Exactly the specified version. | ``==3.1``: only ``3.1``. | ++----------+---------------------------------+--------------------------------+ +| ``!=`` | Any version not equal | ``!=3.1``: any version | +| | to the specified version. | other than ``3.1``. | ++----------+---------------------------------+--------------------------------+ +| ``~=`` | Any compatible release. | ``~=3.1``: version ``3.1`` | +| | Compatible releases are | or later, but not | +| | releases that are within the | version ``4.0`` or later. | +| | same major or minor version, | ``~=3.1.2``: version ``3.1.2`` | +| | assuming the package author | or later, but not | +| | is using semantic versioning. | version ``3.2.0`` or later. | ++----------+---------------------------------+--------------------------------+ +| ``*`` | Can be used at the end of | ``==3.1.*``: any version | +| | a version number to represent | that starts with ``3.1``. | +| | *all*, | Equivalent to ``~=3.1.0``. | ++----------+---------------------------------+--------------------------------+ The detailed specification of supported comparison operators can be found in :pep:`440`. diff --git a/news/093f7456-cc25-4df9-9518-4732b1e07fe5.trivial b/news/093f7456-cc25-4df9-9518-4732b1e07fe5.trivial new file mode 100644 index 000000000..e69de29bb From 8b2b92485c497accbb3f0169142629ab83fd16e5 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 14 Aug 2020 08:41:42 +0800 Subject: [PATCH 105/209] Include Requires-Python dep even with --no-deps --- .../resolution/resolvelib/candidates.py | 24 ++++++-------- tests/functional/test_new_resolver.py | 32 +++++++++++++++++++ 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index ea2a8686b..ff2b336d9 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -230,31 +230,25 @@ class _InstallRequirementBackedCandidate(Candidate): self._prepare() return self._dist - def _get_requires_python_specifier(self): - # type: () -> Optional[SpecifierSet] + def _get_requires_python_dependency(self): + # type: () -> Optional[Requirement] requires_python = get_requires_python(self.dist) if requires_python is None: return None try: spec = SpecifierSet(requires_python) except InvalidSpecifier as e: - logger.warning( - "Package %r has an invalid Requires-Python: %s", self.name, e, - ) + message = "Package %r has an invalid Requires-Python: %s" + logger.warning(message, self.name, e) return None - return spec + return self._factory.make_requires_python_requirement(spec) def iter_dependencies(self, with_requires): # type: (bool) -> Iterable[Optional[Requirement]] - if not with_requires: - return - for r in self.dist.requires(): + requires = self.dist.requires() if with_requires else () + for r in requires: yield self._factory.make_requirement_from_spec(str(r), self._ireq) - python_dep = self._factory.make_requires_python_requirement( - self._get_requires_python_specifier(), - ) - if python_dep: - yield python_dep + yield self._get_requires_python_dependency() def get_install_requirement(self): # type: () -> Optional[InstallRequirement] @@ -285,7 +279,7 @@ class LinkCandidate(_InstallRequirementBackedCandidate): wheel = Wheel(ireq.link.filename) wheel_name = canonicalize_name(wheel.name) assert name == wheel_name, ( - "{!r} != {!r} for wheel".format(name, wheel_name) + "{!r} != {!r} for wheel".format(name, wheel_name) ) # Version may not be present for PEP 508 direct URLs if version is not None: diff --git a/tests/functional/test_new_resolver.py b/tests/functional/test_new_resolver.py index 46e32ddd3..1dab8d470 100644 --- a/tests/functional/test_new_resolver.py +++ b/tests/functional/test_new_resolver.py @@ -988,3 +988,35 @@ def test_new_resolver_local_and_req(script): source_dir, "pkg!=0.1.0", expect_error=True, ) + + +def test_new_resolver_no_deps_checks_requires_python(script): + create_basic_wheel_for_package( + script, + "base", + "0.1.0", + depends=["dep"], + requires_python="<2", # Something that always fails. + ) + create_basic_wheel_for_package( + script, + "dep", + "0.2.0", + ) + + result = script.pip( + "install", + "--use-feature=2020-resolver", + "--no-cache-dir", + "--no-index", + "--no-deps", + "--find-links", script.scratch_path, + "base", + expect_error=True, + ) + + message = ( + "Package 'base' requires a different Python: " + "{}.{}.{} not in '<2'".format(*sys.version_info[:3]) + ) + assert message in result.stderr From 5401cc6e1003eeef68421b284697f3082b9085e6 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 14 Aug 2020 08:45:26 +0800 Subject: [PATCH 106/209] News --- news/8758.bugfix | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 news/8758.bugfix diff --git a/news/8758.bugfix b/news/8758.bugfix new file mode 100644 index 000000000..9f44b7e47 --- /dev/null +++ b/news/8758.bugfix @@ -0,0 +1,2 @@ +New resolver: Correctly respect ``Requires-Python`` metadata to reject +incompatible packages in ``--no-deps`` mode. From 46bdaa1ece0f727b35fe646e238211f70d16bf8b Mon Sep 17 00:00:00 2001 From: Nicole Harris Date: Wed, 26 Aug 2020 10:42:22 +0100 Subject: [PATCH 107/209] Add ux docs to pip documentation --- docs/html/index.rst | 1 + docs/html/ux_research_design.rst | 67 ++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 docs/html/ux_research_design.rst diff --git a/docs/html/index.rst b/docs/html/index.rst index 1ec915585..860abcac9 100644 --- a/docs/html/index.rst +++ b/docs/html/index.rst @@ -14,6 +14,7 @@ Please take a look at our documentation for how to install and use pip: user_guide reference/index development/index + ux_research_design news In 2020, we're working on improvements to the heart of pip. Please `learn more and take our survey`_ to help us do it right. diff --git a/docs/html/ux_research_design.rst b/docs/html/ux_research_design.rst new file mode 100644 index 000000000..302091917 --- /dev/null +++ b/docs/html/ux_research_design.rst @@ -0,0 +1,67 @@ +==================== +UX Research & Design +==================== + +Over the course of 2020, the pip team has been working on improving pip's user +experience. + +Currently, our focus is on: + +1. `Understanding who uses pip`_ +2. `Understanding how pip compares to other package managers, and how pip supports other Python packaging tools`_ +3. `Understanding how pip's functionality is used, and how it could be improved`_ +4. `Understanding how pip's documentation is used, and how it could be improved`_ + +You can read the `overall plan`_ and the `mid-year update`_ to learn more about +our work. + +How to contribute +----------------- + +It is important that we hear from pip users so that we can: + +- Understand how pip is currently used by the Python community +- Understand how pip users would *like* pip to behave +- Understand pip's strengths and shortcomings +- Make useful design recommendations for improving pip + +If you are interested in participating in pip user research, please +`join pip's user panel`_. +You can `read more information about the user panel here`_. + +We are also looking for users to: + +- `Give us feedback about pip's new resolver`_ +- `Tell us how pip should handle conflicts with already installed packages when updating other packages`_ + + +Other ways to contribute +======================== + +You can also help by: + +- Reporting UX issues (or suggesting ideas for improvement) on the `pip issue tracker`_ +- `Working on UX issues`_ +- Testing new features. Currently, we are looking for users to `test pip's new dependency resolver`_. + +Next steps +---------- + +In the coming months we will extend this documentation to include: + +1. Summaries of our user research, including recommendations for how to improve pip +2. Tools for the pip team to continue to practice user centered design (e.g. user personas, etc.) + +.. _Understanding who uses pip: https://github.com/pypa/pip/issues/8518 +.. _Understanding how pip compares to other package managers, and how pip supports other Python packaging tools: https://github.com/pypa/pip/issues/8515 +.. _Understanding how pip's functionality is used, and how it could be improved: https://github.com/pypa/pip/issues/8516 +.. _Understanding how pip's documentation is used, and how it could be improved: https://github.com/pypa/pip/issues/8517 +.. _overall plan: https://wiki.python.org/psf/Pip2020DonorFundedRoadmap +.. _mid-year update: http://pyfound.blogspot.com/2020/07/pip-team-midyear-report.html +.. _join pip's user panel: https://tools.simplysecure.org/survey/index.php?r=survey/index&sid=827389&lang=en +.. _read more information about the user panel here: https://bit.ly/pip-ux-studies +.. _Give us feedback about pip's new resolver: https://tools.simplysecure.org/survey/index.php?r=survey/index&sid=989272&lang=en +.. _Tell us how pip should handle conflicts with already installed packages when updating other packages: https://docs.google.com/forms/d/1KtejgZnK-6NPTmAJ-7aWox4iktcezQauW-Mh3gbnydQ/edit +.. _pip issue tracker: https://github.com/pypa/pip/issues/new +.. _Working on UX issues: https://github.com/pypa/pip/issues?q=is%3Aissue+is%3Aopen+label%3A%22K%3A+UX%22 +.. _test pip's new dependency resolver: https://pip.pypa.io/en/latest/user_guide/#changes-to-the-pip-dependency-resolver-in-20-2-2020 From 3f40b10fe004eb01208b853a427fd674b4097cce Mon Sep 17 00:00:00 2001 From: Nicole Harris Date: Wed, 26 Aug 2020 10:44:20 +0100 Subject: [PATCH 108/209] Add news entry --- news/8807.doc | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/8807.doc diff --git a/news/8807.doc b/news/8807.doc new file mode 100644 index 000000000..6ef1a123a --- /dev/null +++ b/news/8807.doc @@ -0,0 +1 @@ +Add ux documentation From 1c61b4679fe875fdd6b3e17339d006041ae8438c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 28 Aug 2020 11:47:29 +0200 Subject: [PATCH 109/209] Add failing test fetching a git commit in refs --- tests/functional/test_vcs_git.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/functional/test_vcs_git.py b/tests/functional/test_vcs_git.py index 37c35c4b5..8dd0ffcd9 100644 --- a/tests/functional/test_vcs_git.py +++ b/tests/functional/test_vcs_git.py @@ -250,3 +250,30 @@ def test_get_repository_root(script): root2 = Git.get_repository_root(version_pkg_path.joinpath("tests")) assert os.path.normcase(root2) == os.path.normcase(version_pkg_path) + + +def test_resolve_commit_not_on_branch(script, tmp_path): + repo_path = tmp_path / "repo" + repo_file = repo_path / "file.txt" + clone_path = repo_path / "clone" + repo_path.mkdir() + script.run("git", "init", cwd=str(repo_path)) + repo_file.write_text(u".") + script.run("git", "add", "file.txt", cwd=str(repo_path)) + script.run("git", "commit", "-m", "initial commit", cwd=str(repo_path)) + script.run("git", "checkout", "-b", "abranch", cwd=str(repo_path)) + # create a commit + repo_file.write_text(u"..") + script.run("git", "commit", "-a", "-m", "commit 1", cwd=str(repo_path)) + commit = script.run( + "git", "rev-parse", "HEAD", cwd=str(repo_path) + ).stdout.strip() + # make sure our commit is not on a branch + script.run("git", "checkout", "master", cwd=str(repo_path)) + script.run("git", "branch", "-D", "abranch", cwd=str(repo_path)) + # create a ref that points to our commit + (repo_path / ".git" / "refs" / "myrefs").mkdir(parents=True) + (repo_path / ".git" / "refs" / "myrefs" / "myref").write_text(commit) + # check we can fetch our commit + rev_options = Git.make_rev_options(commit) + Git().fetch_new(str(clone_path), repo_path.as_uri(), rev_options) From 3aa0c2ed914b26ab1bc4379caf98ab7e68a3000e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 30 Aug 2020 11:46:05 +0200 Subject: [PATCH 110/209] Git fetch more aggressively Before we were fetching only revisions starting with refs/. Now we also fetch revisions that look like commit that we don't have locally. --- news/8815.feature | 2 ++ src/pip/_internal/vcs/git.py | 41 ++++++++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 news/8815.feature diff --git a/news/8815.feature b/news/8815.feature new file mode 100644 index 000000000..7d9149d69 --- /dev/null +++ b/news/8815.feature @@ -0,0 +1,2 @@ +When installing a git URL that refers to a commit that is not available locally +after git clone, attempt to fetch it from the remote. diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index a9c7fb66e..308a87dfa 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -163,6 +163,29 @@ class Git(VersionControl): return (sha, False) + @classmethod + def _should_fetch(cls, dest, rev): + """ + Return true if rev is a ref or is a commit that we don't have locally. + + Branches and tags are not considered in this method because they are + assumed to be always available locally (which is a normal outcome of + ``git clone`` and ``git fetch --tags``). + """ + if rev.startswith("refs/"): + # Always fetch remote refs. + return True + + if not looks_like_hash(rev): + # Git fetch would fail with abbreviated commits. + return False + + if cls.has_commit(dest, rev): + # Don't fetch if we have the commit locally. + return False + + return True + @classmethod def resolve_revision(cls, dest, url, rev_options): # type: (str, HiddenText, RevOptions) -> RevOptions @@ -194,10 +217,10 @@ class Git(VersionControl): rev, ) - if not rev.startswith('refs/'): + if not cls._should_fetch(dest, rev): return rev_options - # If it looks like a ref, we have to fetch it explicitly. + # fetch the requested revision cls.run_command( make_command('fetch', '-q', url, rev_options.to_args()), cwd=dest, @@ -306,6 +329,20 @@ class Git(VersionControl): url = found_remote.split(' ')[1] return url.strip() + @classmethod + def has_commit(cls, location, rev): + """ + Check if rev is a commit that is available in the local repository. + """ + try: + cls.run_command( + ['rev-parse', '-q', '--verify', "sha^" + rev], cwd=location + ) + except SubProcessError: + return False + else: + return True + @classmethod def get_revision(cls, location, rev=None): if rev is None: From 5797a080b60962113ed743d5085e76b226faceae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 28 Aug 2020 11:53:14 +0200 Subject: [PATCH 111/209] Fix test that now requires a working git repo --- tests/unit/test_vcs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 93598c367..b6ed86b62 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -173,8 +173,9 @@ def test_git_resolve_revision_not_found_warning(get_sha_mock, caplog): sha = 40 * 'a' rev_options = Git.make_rev_options(sha) - new_options = Git.resolve_revision('.', url, rev_options) - assert new_options.rev == sha + # resolve_revision with a full sha would fail here because + # it attempts a git fetch. This case is now covered by + # test_resolve_commit_not_on_branch. rev_options = Git.make_rev_options(sha[:6]) new_options = Git.resolve_revision('.', url, rev_options) From eff32df28abe05c5f25515faf587bf98e3791978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 30 Aug 2020 18:15:35 +0200 Subject: [PATCH 112/209] Add spacing in test Co-authored-by: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com> --- tests/functional/test_vcs_git.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/functional/test_vcs_git.py b/tests/functional/test_vcs_git.py index 8dd0ffcd9..8b07ae667 100644 --- a/tests/functional/test_vcs_git.py +++ b/tests/functional/test_vcs_git.py @@ -258,22 +258,27 @@ def test_resolve_commit_not_on_branch(script, tmp_path): clone_path = repo_path / "clone" repo_path.mkdir() script.run("git", "init", cwd=str(repo_path)) + repo_file.write_text(u".") script.run("git", "add", "file.txt", cwd=str(repo_path)) script.run("git", "commit", "-m", "initial commit", cwd=str(repo_path)) script.run("git", "checkout", "-b", "abranch", cwd=str(repo_path)) + # create a commit repo_file.write_text(u"..") script.run("git", "commit", "-a", "-m", "commit 1", cwd=str(repo_path)) commit = script.run( "git", "rev-parse", "HEAD", cwd=str(repo_path) ).stdout.strip() + # make sure our commit is not on a branch script.run("git", "checkout", "master", cwd=str(repo_path)) script.run("git", "branch", "-D", "abranch", cwd=str(repo_path)) + # create a ref that points to our commit (repo_path / ".git" / "refs" / "myrefs").mkdir(parents=True) (repo_path / ".git" / "refs" / "myrefs" / "myref").write_text(commit) + # check we can fetch our commit rev_options = Git.make_rev_options(commit) Git().fetch_new(str(clone_path), repo_path.as_uri(), rev_options) From 87d129a801c126f340273256dd355f6b5cd32763 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 1 Sep 2020 15:37:10 +0800 Subject: [PATCH 113/209] Replace custom URL parsing with url_to_path() --- src/pip/_internal/req/req_file.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/pip/_internal/req/req_file.py b/src/pip/_internal/req/req_file.py index 72a568bdf..c8d7a0a5a 100644 --- a/src/pip/_internal/req/req_file.py +++ b/src/pip/_internal/req/req_file.py @@ -21,7 +21,7 @@ from pip._internal.models.search_scope import SearchScope from pip._internal.network.utils import raise_for_status from pip._internal.utils.encoding import auto_decode from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from pip._internal.utils.urls import get_url_scheme +from pip._internal.utils.urls import get_url_scheme, url_to_path if MYPY_CHECK_RUNNING: from optparse import Values @@ -572,16 +572,7 @@ def get_file_content(url, session, comes_from=None): 'Requirements file {} references URL {}, ' 'which is local'.format(comes_from, url) ) - - path = url.split(':', 1)[1] - path = path.replace('\\', '/') - match = _url_slash_drive_re.match(path) - if match: - path = match.group(1) + ':' + path.split('|', 1)[1] - path = urllib_parse.unquote(path) - if path.startswith('/'): - path = '/' + path.lstrip('/') - url = path + url = url_to_path(url) try: with open(url, 'rb') as f: @@ -591,6 +582,3 @@ def get_file_content(url, session, comes_from=None): 'Could not open requirements file: {}'.format(exc) ) return url, content - - -_url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I) From d570e504026b84570b7c707f68da7fc5b6b8283a Mon Sep 17 00:00:00 2001 From: Nicole Harris Date: Tue, 1 Sep 2020 18:43:11 +0100 Subject: [PATCH 114/209] Update docs based on feedback from UX team --- docs/html/ux_research_design.rst | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/docs/html/ux_research_design.rst b/docs/html/ux_research_design.rst index 302091917..165b69496 100644 --- a/docs/html/ux_research_design.rst +++ b/docs/html/ux_research_design.rst @@ -18,10 +18,14 @@ our work. How to contribute ----------------- +Participate in UX research +========================== + It is important that we hear from pip users so that we can: - Understand how pip is currently used by the Python community -- Understand how pip users would *like* pip to behave +- Understand how pip users *need* pip to behave +- Understand how pip users *would like* pip to behave - Understand pip's strengths and shortcomings - Make useful design recommendations for improving pip @@ -34,15 +38,25 @@ We are also looking for users to: - `Give us feedback about pip's new resolver`_ - `Tell us how pip should handle conflicts with already installed packages when updating other packages`_ +Report UX issues +================ -Other ways to contribute -======================== +If you believe that you have found a user experience bug in pip, or you have +ideas for how pip could be made better for all users, you please file an issue +on the `pip issue tracker`_. -You can also help by: +Work on UX issues +================= -- Reporting UX issues (or suggesting ideas for improvement) on the `pip issue tracker`_ -- `Working on UX issues`_ -- Testing new features. Currently, we are looking for users to `test pip's new dependency resolver`_. +You can help improve pip's user experience by `working on UX issues`_. +Issues that are ideal for new contributors are marked with "good first issue". + +Test new features +================= + +You can help the team by testing new features as they are released to the +community. Currently, we are looking for users to +`test pip's new dependency resolver`_. Next steps ---------- @@ -63,5 +77,5 @@ In the coming months we will extend this documentation to include: .. _Give us feedback about pip's new resolver: https://tools.simplysecure.org/survey/index.php?r=survey/index&sid=989272&lang=en .. _Tell us how pip should handle conflicts with already installed packages when updating other packages: https://docs.google.com/forms/d/1KtejgZnK-6NPTmAJ-7aWox4iktcezQauW-Mh3gbnydQ/edit .. _pip issue tracker: https://github.com/pypa/pip/issues/new -.. _Working on UX issues: https://github.com/pypa/pip/issues?q=is%3Aissue+is%3Aopen+label%3A%22K%3A+UX%22 +.. _working on UX issues: https://github.com/pypa/pip/issues?q=is%3Aissue+is%3Aopen+label%3A%22K%3A+UX%22 .. _test pip's new dependency resolver: https://pip.pypa.io/en/latest/user_guide/#changes-to-the-pip-dependency-resolver-in-20-2-2020 From 700eb7734fc10174eff3d198adad0db326047032 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 3 Sep 2020 16:33:52 +0800 Subject: [PATCH 115/209] Hashes from lines should intersect, not union --- news/8839.bugfix | 3 + .../resolution/resolvelib/factory.py | 2 +- src/pip/_internal/utils/hashes.py | 20 +++-- tests/functional/test_new_resolver_hashes.py | 87 +++++++++++++++++++ 4 files changed, 105 insertions(+), 7 deletions(-) create mode 100644 news/8839.bugfix create mode 100644 tests/functional/test_new_resolver_hashes.py diff --git a/news/8839.bugfix b/news/8839.bugfix new file mode 100644 index 000000000..987b801e9 --- /dev/null +++ b/news/8839.bugfix @@ -0,0 +1,3 @@ +New resolver: If a package appears multiple times in user specification with +different ``--hash`` options, only hashes that present in all specifications +should be allowed. diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index e56304949..7209f8c94 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -168,7 +168,7 @@ class Factory(object): extras = frozenset() # type: FrozenSet[str] for ireq in ireqs: specifier &= ireq.req.specifier - hashes |= ireq.hashes(trust_internet=False) + hashes &= ireq.hashes(trust_internet=False) extras |= frozenset(ireq.extras) # We use this to ensure that we only yield a single candidate for diff --git a/src/pip/_internal/utils/hashes.py b/src/pip/_internal/utils/hashes.py index d1b062fed..d9f74a640 100644 --- a/src/pip/_internal/utils/hashes.py +++ b/src/pip/_internal/utils/hashes.py @@ -46,16 +46,24 @@ class Hashes(object): """ self._allowed = {} if hashes is None else hashes - def __or__(self, other): + def __and__(self, other): # type: (Hashes) -> Hashes if not isinstance(other, Hashes): return NotImplemented - new = self._allowed.copy() + + # If either of the Hashes object is entirely empty (i.e. no hash + # specified at all), all hashes from the other object are allowed. + if not other: + return self + if not self: + return other + + # Otherwise only hashes that present in both objects are allowed. + new = {} for alg, values in iteritems(other._allowed): - try: - new[alg] += values - except KeyError: - new[alg] = values + if alg not in self._allowed: + continue + new[alg] = [v for v in values if v in self._allowed[alg]] return Hashes(new) @property diff --git a/tests/functional/test_new_resolver_hashes.py b/tests/functional/test_new_resolver_hashes.py new file mode 100644 index 000000000..130f86b62 --- /dev/null +++ b/tests/functional/test_new_resolver_hashes.py @@ -0,0 +1,87 @@ +import collections +import hashlib + +import pytest + +from pip._internal.utils.urls import path_to_url +from tests.lib import ( + create_basic_sdist_for_package, + create_basic_wheel_for_package, +) + +_FindLinks = collections.namedtuple( + "_FindLinks", "index_html sdist_hash wheel_hash", +) + + +def _create_find_links(script): + sdist_path = create_basic_sdist_for_package(script, "base", "0.1.0") + wheel_path = create_basic_wheel_for_package(script, "base", "0.1.0") + + sdist_hash = hashlib.sha256(sdist_path.read_bytes()).hexdigest() + wheel_hash = hashlib.sha256(wheel_path.read_bytes()).hexdigest() + + index_html = script.scratch_path / "index.html" + index_html.write_text( + """ + {sdist_path.stem} + {wheel_path.stem} + """.format( + sdist_url=path_to_url(sdist_path), + sdist_hash=sdist_hash, + sdist_path=sdist_path, + wheel_url=path_to_url(wheel_path), + wheel_hash=wheel_hash, + wheel_path=wheel_path, + ) + ) + + return _FindLinks(index_html, sdist_hash, wheel_hash) + + +@pytest.mark.parametrize( + "requirements_template, message", + [ + ( + """ + base==0.1.0 --hash=sha256:{sdist_hash} --hash=sha256:{wheel_hash} + base==0.1.0 --hash=sha256:{sdist_hash} --hash=sha256:{wheel_hash} + """, + "Checked 2 links for project 'base' against 2 hashes " + "(2 matches, 0 no digest): discarding no candidates", + ), + ( + # Different hash lists are intersected. + """ + base==0.1.0 --hash=sha256:{sdist_hash} --hash=sha256:{wheel_hash} + base==0.1.0 --hash=sha256:{sdist_hash} + """, + "Checked 2 links for project 'base' against 1 hashes " + "(1 matches, 0 no digest): discarding 1 non-matches", + ), + ], + ids=["identical", "intersect"], +) +def test_new_resolver_hash_intersect(script, requirements_template, message): + find_links = _create_find_links(script) + + requirements_txt = script.scratch_path / "requirements.txt" + requirements_txt.write_text( + requirements_template.format( + sdist_hash=find_links.sdist_hash, + wheel_hash=find_links.wheel_hash, + ), + ) + + result = script.pip( + "install", + "--use-feature=2020-resolver", + "--no-cache-dir", + "--no-deps", + "--no-index", + "--find-links", find_links.index_html, + "--verbose", + "--requirement", requirements_txt, + ) + + assert message in result.stdout, str(result) From dfaa1110047165f4baf65428bf332e6912994252 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 3 Sep 2020 17:07:51 +0800 Subject: [PATCH 116/209] Pull in hashes from constraint files --- news/8792.bugfix | 2 + .../_internal/resolution/resolvelib/base.py | 37 +++++++++++++++++- .../resolution/resolvelib/factory.py | 11 ++++-- .../resolution/resolvelib/provider.py | 7 ++-- .../resolution/resolvelib/resolver.py | 8 ++-- tests/functional/test_new_resolver_hashes.py | 38 +++++++++++++++++++ .../resolution_resolvelib/test_requirement.py | 7 ++-- 7 files changed, 95 insertions(+), 15 deletions(-) create mode 100644 news/8792.bugfix diff --git a/news/8792.bugfix b/news/8792.bugfix new file mode 100644 index 000000000..e83bdb09c --- /dev/null +++ b/news/8792.bugfix @@ -0,0 +1,2 @@ +New resolver: Pick up hash declarations in constraints files and use them to +filter available distributions. diff --git a/src/pip/_internal/resolution/resolvelib/base.py b/src/pip/_internal/resolution/resolvelib/base.py index 9245747bf..7c09cd70b 100644 --- a/src/pip/_internal/resolution/resolvelib/base.py +++ b/src/pip/_internal/resolution/resolvelib/base.py @@ -1,5 +1,8 @@ +from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.utils import canonicalize_name +from pip._internal.req.req_install import InstallRequirement +from pip._internal.utils.hashes import Hashes from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: @@ -8,7 +11,6 @@ if MYPY_CHECK_RUNNING: from pip._vendor.packaging.version import _BaseVersion from pip._internal.models.link import Link - from pip._internal.req.req_install import InstallRequirement CandidateLookup = Tuple[ Optional["Candidate"], @@ -24,6 +26,39 @@ def format_name(project, extras): return "{}[{}]".format(project, ",".join(canonical_extras)) +class Constraint(object): + def __init__(self, specifier, hashes): + # type: (SpecifierSet, Hashes) -> None + self.specifier = specifier + self.hashes = hashes + + @classmethod + def empty(cls): + # type: () -> Constraint + return Constraint(SpecifierSet(), Hashes()) + + @classmethod + def from_ireq(cls, ireq): + # type: (InstallRequirement) -> Constraint + return Constraint(ireq.specifier, ireq.hashes(trust_internet=False)) + + def __nonzero__(self): + # type: () -> bool + return bool(self.specifier) or bool(self.hashes) + + def __bool__(self): + # type: () -> bool + return self.__nonzero__() + + def __and__(self, other): + # type: (InstallRequirement) -> Constraint + if not isinstance(other, InstallRequirement): + return NotImplemented + specifier = self.specifier & other.specifier + hashes = self.hashes & other.hashes(trust_internet=False) + return Constraint(specifier, hashes) + + class Requirement(object): @property def name(self): diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 7209f8c94..ed310dd8c 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -22,6 +22,7 @@ from pip._internal.utils.misc import ( from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.virtualenv import running_under_virtualenv +from .base import Constraint from .candidates import ( AlreadyInstalledCandidate, EditableCandidate, @@ -152,6 +153,7 @@ class Factory(object): self, ireqs, # type: Sequence[InstallRequirement] specifier, # type: SpecifierSet + hashes, # type: Hashes ): # type: (...) -> Iterable[Candidate] if not ireqs: @@ -164,7 +166,6 @@ class Factory(object): template = ireqs[0] name = canonicalize_name(template.req.name) - hashes = Hashes() extras = frozenset() # type: FrozenSet[str] for ireq in ireqs: specifier &= ireq.req.specifier @@ -218,7 +219,7 @@ class Factory(object): return six.itervalues(candidates) def find_candidates(self, requirements, constraint): - # type: (Sequence[Requirement], SpecifierSet) -> Iterable[Candidate] + # type: (Sequence[Requirement], Constraint) -> Iterable[Candidate] explicit_candidates = set() # type: Set[Candidate] ireqs = [] # type: List[InstallRequirement] for req in requirements: @@ -231,7 +232,11 @@ class Factory(object): # If none of the requirements want an explicit candidate, we can ask # the finder for candidates. if not explicit_candidates: - return self._iter_found_candidates(ireqs, constraint) + return self._iter_found_candidates( + ireqs, + constraint.specifier, + constraint.hashes, + ) if constraint: name = explicit_candidates.pop().name diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index b2eb9d06e..80577a61c 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -1,8 +1,9 @@ -from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.resolvelib.providers import AbstractProvider from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from .base import Constraint + if MYPY_CHECK_RUNNING: from typing import ( Any, @@ -41,7 +42,7 @@ class PipProvider(AbstractProvider): def __init__( self, factory, # type: Factory - constraints, # type: Dict[str, SpecifierSet] + constraints, # type: Dict[str, Constraint] ignore_dependencies, # type: bool upgrade_strategy, # type: str user_requested, # type: Set[str] @@ -134,7 +135,7 @@ class PipProvider(AbstractProvider): if not requirements: return [] constraint = self._constraints.get( - requirements[0].name, SpecifierSet(), + requirements[0].name, Constraint.empty(), ) candidates = self._factory.find_candidates(requirements, constraint) return reversed(self._sort_matches(candidates)) diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py index 031d2f107..cb7d1ae8a 100644 --- a/src/pip/_internal/resolution/resolvelib/resolver.py +++ b/src/pip/_internal/resolution/resolvelib/resolver.py @@ -14,12 +14,12 @@ from pip._internal.resolution.resolvelib.provider import PipProvider from pip._internal.utils.misc import dist_is_editable from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from .base import Constraint from .factory import Factory if MYPY_CHECK_RUNNING: from typing import Dict, List, Optional, Set, Tuple - from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.resolvelib.resolvers import Result from pip._vendor.resolvelib.structs import Graph @@ -71,7 +71,7 @@ class Resolver(BaseResolver): def resolve(self, root_reqs, check_supported_wheels): # type: (List[InstallRequirement], bool) -> RequirementSet - constraints = {} # type: Dict[str, SpecifierSet] + constraints = {} # type: Dict[str, Constraint] user_requested = set() # type: Set[str] requirements = [] for req in root_reqs: @@ -84,9 +84,9 @@ class Resolver(BaseResolver): continue name = canonicalize_name(req.name) if name in constraints: - constraints[name] = constraints[name] & req.specifier + constraints[name] &= req else: - constraints[name] = req.specifier + constraints[name] = Constraint.from_ireq(req) else: if req.user_supplied and req.name: user_requested.add(canonicalize_name(req.name)) diff --git a/tests/functional/test_new_resolver_hashes.py b/tests/functional/test_new_resolver_hashes.py index 130f86b62..703d4b400 100644 --- a/tests/functional/test_new_resolver_hashes.py +++ b/tests/functional/test_new_resolver_hashes.py @@ -85,3 +85,41 @@ def test_new_resolver_hash_intersect(script, requirements_template, message): ) assert message in result.stdout, str(result) + + +def test_new_resolver_hash_intersect_from_constraint(script): + find_links = _create_find_links(script) + + constraints_txt = script.scratch_path / "constraints.txt" + constraints_txt.write_text( + "base==0.1.0 --hash=sha256:{sdist_hash}".format( + sdist_hash=find_links.sdist_hash, + ), + ) + requirements_txt = script.scratch_path / "requirements.txt" + requirements_txt.write_text( + """ + base==0.1.0 --hash=sha256:{sdist_hash} --hash=sha256:{wheel_hash} + """.format( + sdist_hash=find_links.sdist_hash, + wheel_hash=find_links.wheel_hash, + ), + ) + + result = script.pip( + "install", + "--use-feature=2020-resolver", + "--no-cache-dir", + "--no-deps", + "--no-index", + "--find-links", find_links.index_html, + "--verbose", + "--constraint", constraints_txt, + "--requirement", requirements_txt, + ) + + message = ( + "Checked 2 links for project 'base' against 1 hashes " + "(1 matches, 0 no digest): discarding 1 non-matches" + ) + assert message in result.stdout, str(result) diff --git a/tests/unit/resolution_resolvelib/test_requirement.py b/tests/unit/resolution_resolvelib/test_requirement.py index 21de3df4a..a03edb6f7 100644 --- a/tests/unit/resolution_resolvelib/test_requirement.py +++ b/tests/unit/resolution_resolvelib/test_requirement.py @@ -1,8 +1,7 @@ import pytest -from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.resolvelib import BaseReporter, Resolver -from pip._internal.resolution.resolvelib.base import Candidate +from pip._internal.resolution.resolvelib.base import Candidate, Constraint from pip._internal.utils.urls import path_to_url # NOTE: All tests are prefixed `test_rlr` (for "test resolvelib resolver"). @@ -59,7 +58,7 @@ def test_new_resolver_correct_number_of_matches(test_cases, factory): """Requirements should return the correct number of candidates""" for spec, _, match_count in test_cases: req = factory.make_requirement_from_spec(spec, comes_from=None) - matches = factory.find_candidates([req], SpecifierSet()) + matches = factory.find_candidates([req], Constraint.empty()) assert len(list(matches)) == match_count @@ -68,7 +67,7 @@ def test_new_resolver_candidates_match_requirement(test_cases, factory): """ for spec, _, _ in test_cases: req = factory.make_requirement_from_spec(spec, comes_from=None) - for c in factory.find_candidates([req], SpecifierSet()): + for c in factory.find_candidates([req], Constraint.empty()): assert isinstance(c, Candidate) assert req.is_satisfied_by(c) From 753ad4d23008c77dada3da91bb0a9486841c9cea Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 3 Sep 2020 17:35:22 +0800 Subject: [PATCH 117/209] F Python 2 --- tests/functional/test_new_resolver_hashes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/functional/test_new_resolver_hashes.py b/tests/functional/test_new_resolver_hashes.py index 703d4b400..20a700ac8 100644 --- a/tests/functional/test_new_resolver_hashes.py +++ b/tests/functional/test_new_resolver_hashes.py @@ -47,7 +47,7 @@ def _create_find_links(script): base==0.1.0 --hash=sha256:{sdist_hash} --hash=sha256:{wheel_hash} base==0.1.0 --hash=sha256:{sdist_hash} --hash=sha256:{wheel_hash} """, - "Checked 2 links for project 'base' against 2 hashes " + "Checked 2 links for project {name!r} against 2 hashes " "(2 matches, 0 no digest): discarding no candidates", ), ( @@ -56,7 +56,7 @@ def _create_find_links(script): base==0.1.0 --hash=sha256:{sdist_hash} --hash=sha256:{wheel_hash} base==0.1.0 --hash=sha256:{sdist_hash} """, - "Checked 2 links for project 'base' against 1 hashes " + "Checked 2 links for project {name!r} against 1 hashes " "(1 matches, 0 no digest): discarding 1 non-matches", ), ], @@ -84,7 +84,7 @@ def test_new_resolver_hash_intersect(script, requirements_template, message): "--requirement", requirements_txt, ) - assert message in result.stdout, str(result) + assert message.format(name=u"base") in result.stdout, str(result) def test_new_resolver_hash_intersect_from_constraint(script): @@ -119,7 +119,7 @@ def test_new_resolver_hash_intersect_from_constraint(script): ) message = ( - "Checked 2 links for project 'base' against 1 hashes " + "Checked 2 links for project {name!r} against 1 hashes " "(1 matches, 0 no digest): discarding 1 non-matches" - ) + ).format(name=u"base") assert message in result.stdout, str(result) From cd549eb7f1f8de518929381de698bfa32c33ba0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Thu, 23 Jul 2020 16:32:26 +0700 Subject: [PATCH 118/209] Use mock to patch for TZ env var --- tests/conftest.py | 11 ++++ tests/unit/test_base_command.py | 100 +++++++++++++------------------- tests/unit/test_logging.py | 31 ++-------- 3 files changed, 56 insertions(+), 86 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 1ca4d45d0..bdfe2f9ff 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,7 @@ import re import shutil import subprocess import sys +import time from contextlib import contextmanager import pytest @@ -556,3 +557,13 @@ def mock_server(): test_server = MockServer(server) with test_server.context: yield test_server + + +@pytest.fixture +def utc(): + # time.tzset() is not implemented on some platforms, e.g. Windows. + tzset = getattr(time, 'tzset', lambda: None) + with patch.dict(os.environ, {'TZ': 'UTC'}): + tzset() + yield + tzset() diff --git a/tests/unit/test_base_command.py b/tests/unit/test_base_command.py index 27147e769..8ba3d9e25 100644 --- a/tests/unit/test_base_command.py +++ b/tests/unit/test_base_command.py @@ -1,6 +1,5 @@ import logging import os -import time import pytest from mock import Mock, patch @@ -12,6 +11,12 @@ from pip._internal.utils.logging import BrokenStdoutLoggingError from pip._internal.utils.temp_dir import TempDirectory +@pytest.fixture +def fixed_time(utc): + with patch('time.time', lambda: 1547704837.040001): + yield + + class FakeCommand(Command): _name = 'fake' @@ -91,67 +96,40 @@ def test_handle_pip_version_check_called(mock_handle_version_check): mock_handle_version_check.assert_called_once() -class Test_base_command_logging(object): +def test_log_command_success(fixed_time, tmpdir): + """Test the --log option logs when command succeeds.""" + cmd = FakeCommand() + log_path = tmpdir.joinpath('log') + cmd.main(['fake', '--log', log_path]) + with open(log_path) as f: + assert f.read().rstrip() == '2019-01-17T06:00:37,040 fake' + + +def test_log_command_error(fixed_time, tmpdir): + """Test the --log option logs when command fails.""" + cmd = FakeCommand(error=True) + log_path = tmpdir.joinpath('log') + cmd.main(['fake', '--log', log_path]) + with open(log_path) as f: + assert f.read().startswith('2019-01-17T06:00:37,040 fake') + + +def test_log_file_command_error(fixed_time, tmpdir): + """Test the --log-file option logs (when there's an error).""" + cmd = FakeCommand(error=True) + log_file_path = tmpdir.joinpath('log_file') + cmd.main(['fake', '--log-file', log_file_path]) + with open(log_file_path) as f: + assert f.read().startswith('2019-01-17T06:00:37,040 fake') + + +def test_log_unicode_messages(fixed_time, tmpdir): + """Tests that logging bytestrings and unicode objects + don't break logging. """ - Test `pip.base_command.Command` setting up logging consumers based on - options - """ - - def setup(self): - self.old_time = time.time - time.time = lambda: 1547704837.040001 - self.old_tz = os.environ.get('TZ') - os.environ['TZ'] = 'UTC' - # time.tzset() is not implemented on some platforms (notably, Windows). - if hasattr(time, 'tzset'): - time.tzset() - - def teardown(self): - if self.old_tz: - os.environ['TZ'] = self.old_tz - else: - del os.environ['TZ'] - if 'tzset' in dir(time): - time.tzset() - time.time = self.old_time - - def test_log_command_success(self, tmpdir): - """ - Test the --log option logs when command succeeds - """ - cmd = FakeCommand() - log_path = tmpdir.joinpath('log') - cmd.main(['fake', '--log', log_path]) - with open(log_path) as f: - assert f.read().rstrip() == '2019-01-17T06:00:37,040 fake' - - def test_log_command_error(self, tmpdir): - """ - Test the --log option logs when command fails - """ - cmd = FakeCommand(error=True) - log_path = tmpdir.joinpath('log') - cmd.main(['fake', '--log', log_path]) - with open(log_path) as f: - assert f.read().startswith('2019-01-17T06:00:37,040 fake') - - def test_log_file_command_error(self, tmpdir): - """ - Test the --log-file option logs (when there's an error). - """ - cmd = FakeCommand(error=True) - log_file_path = tmpdir.joinpath('log_file') - cmd.main(['fake', '--log-file', log_file_path]) - with open(log_file_path) as f: - assert f.read().startswith('2019-01-17T06:00:37,040 fake') - - def test_unicode_messages(self, tmpdir): - """ - Tests that logging bytestrings and unicode objects don't break logging - """ - cmd = FakeCommandWithUnicode() - log_path = tmpdir.joinpath('log') - cmd.main(['fake_unicode', '--log', log_path]) + cmd = FakeCommandWithUnicode() + log_path = tmpdir.joinpath('log') + cmd.main(['fake_unicode', '--log', log_path]) @pytest.mark.no_auto_tempdir_manager diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py index a62c18c77..10d47eb61 100644 --- a/tests/unit/test_logging.py +++ b/tests/unit/test_logging.py @@ -1,7 +1,5 @@ import errno import logging -import os -import time from threading import Thread import pytest @@ -33,24 +31,7 @@ def _make_broken_pipe_error(): class TestIndentingFormatter(object): - """ - Test `pip._internal.utils.logging.IndentingFormatter`. - """ - - def setup(self): - self.old_tz = os.environ.get('TZ') - os.environ['TZ'] = 'UTC' - # time.tzset() is not implemented on some platforms (notably, Windows). - if hasattr(time, 'tzset'): - time.tzset() - - def teardown(self): - if self.old_tz: - os.environ['TZ'] = self.old_tz - else: - del os.environ['TZ'] - if 'tzset' in dir(time): - time.tzset() + """Test ``pip._internal.utils.logging.IndentingFormatter``.""" def make_record(self, msg, level_name): level_number = getattr(logging, level_name) @@ -72,7 +53,7 @@ class TestIndentingFormatter(object): ('ERROR', 'ERROR: hello\nworld'), ('CRITICAL', 'ERROR: hello\nworld'), ]) - def test_format(self, level_name, expected): + def test_format(self, level_name, expected, utc): """ Args: level_name: a logging level name (e.g. "WARNING"). @@ -89,7 +70,7 @@ class TestIndentingFormatter(object): '2019-01-17T06:00:37,040 WARNING: hello\n' '2019-01-17T06:00:37,040 world'), ]) - def test_format_with_timestamp(self, level_name, expected): + def test_format_with_timestamp(self, level_name, expected, utc): record = self.make_record('hello\nworld', level_name=level_name) f = IndentingFormatter(fmt="%(message)s", add_timestamp=True) assert f.format(record) == expected @@ -99,7 +80,7 @@ class TestIndentingFormatter(object): ('ERROR', 'DEPRECATION: hello\nworld'), ('CRITICAL', 'DEPRECATION: hello\nworld'), ]) - def test_format_deprecated(self, level_name, expected): + def test_format_deprecated(self, level_name, expected, utc): """ Test that logged deprecation warnings coming from deprecated() don't get another prefix. @@ -110,7 +91,7 @@ class TestIndentingFormatter(object): f = IndentingFormatter(fmt="%(message)s") assert f.format(record) == expected - def test_thread_safety_base(self): + def test_thread_safety_base(self, utc): record = self.make_record( 'DEPRECATION: hello\nworld', level_name='WARNING', ) @@ -126,7 +107,7 @@ class TestIndentingFormatter(object): thread.join() assert results[0] == results[1] - def test_thread_safety_indent_log(self): + def test_thread_safety_indent_log(self, utc): record = self.make_record( 'DEPRECATION: hello\nworld', level_name='WARNING', ) From 4a2e03c4ff293e6295b854819ab574873778f94d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Thu, 23 Jul 2020 16:32:56 +0700 Subject: [PATCH 119/209] Use monkeypatch for env var in wheel unit tests --- tests/conftest.py | 2 +- tests/unit/test_wheel.py | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index bdfe2f9ff..81bd55353 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -111,7 +111,7 @@ def use_new_resolver(request): features.add("2020-resolver") else: features.discard("2020-resolver") - with patch.dict(os.environ, {"PIP_USE_FEATURE", " ".join(features)}): + with patch.dict(os.environ, {"PIP_USE_FEATURE": " ".join(features)}): yield new_resolver diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index 15dff94ca..6c9fe02b3 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -600,16 +600,14 @@ class TestMessageAboutScriptsNotOnPATH(object): ) assert retval is None - def test_missing_PATH_env_treated_as_empty_PATH_env(self): + def test_missing_PATH_env_treated_as_empty_PATH_env(self, monkeypatch): scripts = ['a/b/foo'] - env = os.environ.copy() - del env['PATH'] - with patch.dict('os.environ', env, clear=True): - retval_missing = wheel.message_about_scripts_not_on_PATH(scripts) + monkeypatch.delenv('PATH') + retval_missing = wheel.message_about_scripts_not_on_PATH(scripts) - with patch.dict('os.environ', {'PATH': ''}): - retval_empty = wheel.message_about_scripts_not_on_PATH(scripts) + monkeypatch.setenv('PATH', '') + retval_empty = wheel.message_about_scripts_not_on_PATH(scripts) assert retval_missing == retval_empty From cca500f05352b1e599f44f7fef955d8b465337f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Fri, 4 Sep 2020 17:37:43 +0700 Subject: [PATCH 120/209] Remove unused definitions --- news/6f8eff8d-9886-4e00-b431-5c809500e6bf.trivial | 0 src/pip/_internal/commands/list.py | 1 - src/pip/_internal/resolution/resolvelib/factory.py | 4 ---- src/pip/_internal/vcs/git.py | 1 - 4 files changed, 6 deletions(-) create mode 100644 news/6f8eff8d-9886-4e00-b431-5c809500e6bf.trivial diff --git a/news/6f8eff8d-9886-4e00-b431-5c809500e6bf.trivial b/news/6f8eff8d-9886-4e00-b431-5c809500e6bf.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index 20e9bff2b..7c18a7d9f 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -201,7 +201,6 @@ class ListCommand(IndexGroupCommand): def latest_info(dist): # type: (Distribution) -> Distribution - typ = 'unknown' all_candidates = finder.find_all_candidates(dist.key) if not options.pre: # Remove prereleases diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index e56304949..a0ec3d433 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -400,10 +400,6 @@ class Factory(object): return ", ".join(parts[:-1]) + " and " + parts[-1] - def readable_form(cand): - # type: (Candidate) -> str - return "{} {}".format(cand.name, cand.version) - def describe_trigger(parent): # type: (Candidate) -> str ireq = parent.get_install_requirement() diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index 308a87dfa..db8c72349 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -386,7 +386,6 @@ class Git(VersionControl): urllib_request.url2pathname(path) .replace('\\', '/').lstrip('/') ) - url = urlunsplit((scheme, netloc, newpath, query, fragment)) after_plus = scheme.find('+') + 1 url = scheme[:after_plus] + urlunsplit( (scheme[after_plus:], netloc, newpath, query, fragment), From 1b0517e68b872a6ba5171261198c0e9d9bed58c4 Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Mon, 31 Aug 2020 11:27:34 -0400 Subject: [PATCH 121/209] Add documentation for netrc support --- docs/html/user_guide.rst | 27 +++++++++++++++++++++++++++ news/7231.doc | 1 + 2 files changed, 28 insertions(+) create mode 100644 news/7231.doc diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index b92e8b0a3..61dfaf0d5 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -79,6 +79,31 @@ as the "username" and do not provide a password, for example - ``https://0123456789abcdef@pypi.company.com`` +netrc Support +------------- + +If no credentials are part of the URL, pip will attempt to get authentication credentials +for the URL’s hostname from the user’s .netrc file. This behaviour comes from the underlying +use of `requests`_ which in turn delegates it to the `Python standard library`_. + +The .netrc file contains login and initialization information used by the auto-login process. +It resides in the user's home directory. The .netrc file format is simple. You specify lines +with a machine name and follow that with lines for the login and password that are +associated with that machine. Machine name is the hostname in your URL. + +An example .netrc for the host example.com with a user named 'daniel', using the password +'qwerty' would look like: + +.. code-block:: shell + + machine example.com + login daniel + password qwerty + +As mentioned in the `standard library docs `_, +whitespace and non-printable characters are not allowed in passwords. + + Keyring Support --------------- @@ -1296,3 +1321,5 @@ announcements on the `low-traffic packaging announcements list`_ and .. _low-traffic packaging announcements list: https://mail.python.org/mailman3/lists/pypi-announce.python.org/ .. _our survey on upgrades that create conflicts: https://docs.google.com/forms/d/e/1FAIpQLSeBkbhuIlSofXqCyhi3kGkLmtrpPOEBwr6iJA6SzHdxWKfqdA/viewform .. _the official Python blog: https://blog.python.org/ +.. _requests: https://requests.readthedocs.io/en/master/user/authentication/#netrc-authentication +.. _Python standard library: https://docs.python.org/3/library/netrc.html diff --git a/news/7231.doc b/news/7231.doc new file mode 100644 index 000000000..bef9bf3e6 --- /dev/null +++ b/news/7231.doc @@ -0,0 +1 @@ +Add documentation for '.netrc' support. From 8e622390d06cafb4d179152ed19d23d81aed45f4 Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Mon, 31 Aug 2020 11:43:26 -0400 Subject: [PATCH 122/209] mention using only ASCII for password --- docs/html/user_guide.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 61dfaf0d5..3ef99e01b 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -101,7 +101,7 @@ An example .netrc for the host example.com with a user named 'daniel', using the password qwerty As mentioned in the `standard library docs `_, -whitespace and non-printable characters are not allowed in passwords. +only ASCII characters are allowed. Whitespace and non-printable characters are not allowed in passwords. Keyring Support From 65bddac52f99afd2c1f4c0964394f7c91b88756f Mon Sep 17 00:00:00 2001 From: Emmanuel Arias Date: Tue, 18 Aug 2020 17:14:23 -0300 Subject: [PATCH 123/209] Update Caching documentation Any headers from serve are ignored. So, this PR remove the paragraph that mention that. See #8009 and #5670 Close #8009 --- docs/html/reference/pip_install.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/html/reference/pip_install.rst b/docs/html/reference/pip_install.rst index 7be7bcd25..fd962cd36 100644 --- a/docs/html/reference/pip_install.rst +++ b/docs/html/reference/pip_install.rst @@ -550,11 +550,6 @@ response telling pip to simply use the cached item (and refresh the expiration timer) or it will return a whole new response which pip can then store in the cache. -When storing items in the cache, pip will respect the ``CacheControl`` header -if it exists, or it will fall back to the ``Expires`` header if that exists. -This allows pip to function as a browser would, and allows the index server -to communicate to pip how long it is reasonable to cache any particular item. - While this cache attempts to minimize network activity, it does not prevent network access altogether. If you want a local install solution that circumvents accessing PyPI, see :ref:`Installing from local packages`. From e046092068018c186195485092ee4e8911554d27 Mon Sep 17 00:00:00 2001 From: Emmanuel Arias Date: Tue, 18 Aug 2020 18:08:07 -0300 Subject: [PATCH 124/209] Add trivial news --- news/5e8c60c2-d540-4a25-af03-100d848acbc0.trivial | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 news/5e8c60c2-d540-4a25-af03-100d848acbc0.trivial diff --git a/news/5e8c60c2-d540-4a25-af03-100d848acbc0.trivial b/news/5e8c60c2-d540-4a25-af03-100d848acbc0.trivial new file mode 100644 index 000000000..e69de29bb From e3015eb3a51b84a63d946784e8a0b03283106df5 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 8 Sep 2020 16:43:35 +0530 Subject: [PATCH 125/209] Bump for development --- src/pip/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/__init__.py b/src/pip/__init__.py index 9fb68d403..5a2f3c317 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -4,7 +4,7 @@ if MYPY_CHECK_RUNNING: from typing import List, Optional -__version__ = "20.2.3" +__version__ = "20.3.dev0" def main(args=None): From c07ef581e729ea26110afeed5f084d3162a87f1f Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 10 Sep 2020 17:15:32 +0530 Subject: [PATCH 126/209] Factor out logger.into into a single call This makes it easier to conditionally print this information. --- src/pip/_internal/operations/prepare.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 5eb71ce07..95ffec562 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -373,12 +373,15 @@ class RequirementPreparer(object): def _log_preparing_link(self, req): # type: (InstallRequirement) -> None - """Log the way the link prepared.""" + """Provide context for the requirement being prepared.""" if req.link.is_file: - path = req.link.file_path - logger.info('Processing %s', display_path(path)) + message = "Processing %s" + information = str(display_path(req.link.file_path)) else: - logger.info('Collecting %s', req.req or req) + message = "Collecting %s" + information = str(req.req or req) + + logger.info(message, information) def _get_download_dir(self, link): # type: (Link) -> Optional[str] From 963e390abe3c7ac00c90a189b3557981e92f5dde Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 10 Sep 2020 17:21:04 +0530 Subject: [PATCH 127/209] Improve how cached wheels are presented This is specifically for the case of look ups done in the new resolver. --- src/pip/_internal/operations/prepare.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 95ffec562..756adfa43 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -374,7 +374,7 @@ class RequirementPreparer(object): def _log_preparing_link(self, req): # type: (InstallRequirement) -> None """Provide context for the requirement being prepared.""" - if req.link.is_file: + if req.link.is_file and not req.original_link_is_in_wheel_cache: message = "Processing %s" information = str(display_path(req.link.file_path)) else: @@ -383,6 +383,10 @@ class RequirementPreparer(object): logger.info(message, information) + if req.original_link_is_in_wheel_cache: + with indent_log(): + logger.info("Using cached %s", req.link.filename) + def _get_download_dir(self, link): # type: (Link) -> Optional[str] if link.is_wheel and self.wheel_download_dir: From 3d32960c80ffed7946d84c636318153619fa5bf6 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 10 Sep 2020 17:21:42 +0530 Subject: [PATCH 128/209] Only Print "Collecting ..." when the requirement changes --- src/pip/_internal/operations/prepare.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 756adfa43..6f899e6ff 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -357,6 +357,9 @@ class RequirementPreparer(object): # Memoized downloaded files, as mapping of url: (path, mime type) self._downloaded = {} # type: Dict[str, Tuple[str, str]] + # Previous "header" printed for a link-based InstallRequirement + self._previous_requirement_header = None + @property def _download_should_save(self): # type: () -> bool @@ -381,7 +384,9 @@ class RequirementPreparer(object): message = "Collecting %s" information = str(req.req or req) - logger.info(message, information) + if (message, information) != self._previous_requirement_header: + self._previous_requirement_header = (message, information) + logger.info(message, information) if req.original_link_is_in_wheel_cache: with indent_log(): From cc472fd54ae4cc860b014fcb23378ba73a538a3d Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 10 Sep 2020 17:32:58 +0530 Subject: [PATCH 129/209] Use a symmetric type and make mypy happy --- src/pip/_internal/operations/prepare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 6f899e6ff..bf89091aa 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -358,7 +358,7 @@ class RequirementPreparer(object): self._downloaded = {} # type: Dict[str, Tuple[str, str]] # Previous "header" printed for a link-based InstallRequirement - self._previous_requirement_header = None + self._previous_requirement_header = ("", "") @property def _download_should_save(self): From 0b3ba87bbfb46cd12778b2ca5b9f6e74be0b770a Mon Sep 17 00:00:00 2001 From: Devesh Kumar Singh Date: Fri, 3 Jul 2020 19:55:35 +0530 Subject: [PATCH 130/209] Add option to output full path of cache enty --- src/pip/_internal/commands/cache.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/commands/cache.py b/src/pip/_internal/commands/cache.py index 747277f6e..31e196ee3 100644 --- a/src/pip/_internal/commands/cache.py +++ b/src/pip/_internal/commands/cache.py @@ -37,11 +37,21 @@ class CacheCommand(Command): usage = """ %prog dir %prog info - %prog list [] + %prog list [] [--abspath] %prog remove %prog purge """ + def add_options(self): + # type: () -> None + self.cmd_opts.add_option( + '--abspath', + dest='abspath', + action='store_true', + help='List the absolute path of wheels') + + self.parser.insert_option_group(0, self.cmd_opts) + def run(self, options, args): # type: (Values, List[Any]) -> int handlers = { @@ -118,15 +128,20 @@ class CacheCommand(Command): files = self._find_wheels(options, pattern) if not files: - logger.info('Nothing cached.') + if not options.abspath: + logger.info('Nothing cached.') return results = [] for filename in files: wheel = os.path.basename(filename) size = filesystem.format_file_size(filename) - results.append(' - {} ({})'.format(wheel, size)) - logger.info('Cache contents:\n') + if options.abspath: + results.append(filename) + else: + results.append(' - {} ({})'.format(wheel, size)) + if not options.abspath: + logger.info('Cache contents:\n') logger.info('\n'.join(sorted(results))) def remove_cache_items(self, options, args): From d22becc073c4717a6f57ec56b9acd5f46e168cd3 Mon Sep 17 00:00:00 2001 From: Devesh Kumar Singh Date: Sat, 18 Jul 2020 14:06:44 +0530 Subject: [PATCH 131/209] Add unit tests for --abspath flag --- tests/functional/test_cache.py | 57 ++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/functional/test_cache.py b/tests/functional/test_cache.py index e30b2c079..be165661a 100644 --- a/tests/functional/test_cache.py +++ b/tests/functional/test_cache.py @@ -70,6 +70,19 @@ def list_matches_wheel(wheel_name, result): return any(map(lambda l: l.startswith(expected), lines)) +def list_matches_wheel_abspath(wheel_name, result): + """Returns True if any line in `result`, which should be the output of + a `pip cache list --abspath` call, is a valid path and belongs to + `wheel_name`. + + E.g., If wheel_name is `foo-1.2.3` it searches for a line starting with + `foo-1.2.3-py3-none-any.whl`.""" + lines = result.stdout.splitlines() + expected = '{}-py3-none-any.whl'.format(wheel_name) + return any(map(lambda l: os.path.basename(l).startswith(expected) + and os.path.exists(l), lines)) + + @pytest.fixture def remove_matches_wheel(wheel_cache_dir): """Returns True if any line in `result`, which should be the output of @@ -132,6 +145,18 @@ def test_cache_list(script): assert list_matches_wheel('zzz-7.8.9', result) +@pytest.mark.usefixtures("populate_wheel_cache") +def test_cache_list_abspath(script): + """Running `pip cache list --abspath` should return full + paths of exactly what the populate_wheel_cache fixture adds.""" + result = script.pip('cache', 'list', '--abspath') + + assert list_matches_wheel_abspath('yyy-1.2.3', result) + assert list_matches_wheel_abspath('zzz-4.5.6', result) + assert list_matches_wheel_abspath('zzz-4.5.7', result) + assert list_matches_wheel_abspath('zzz-7.8.9', result) + + @pytest.mark.usefixtures("empty_wheel_cache") def test_cache_list_with_empty_cache(script): """Running `pip cache list` with an empty cache should print @@ -140,6 +165,14 @@ def test_cache_list_with_empty_cache(script): assert result.stdout == "Nothing cached.\n" +@pytest.mark.usefixtures("empty_wheel_cache") +def test_cache_list_with_empty_cache_abspath(script): + """Running `pip cache list --abspath` with an empty cache should not + print anything and exit.""" + result = script.pip('cache', 'list', '--abspath') + assert result.stdout.strip() == "" + + def test_cache_list_too_many_args(script): """Passing `pip cache list` too many arguments should cause an error.""" script.pip('cache', 'list', 'aaa', 'bbb', @@ -158,6 +191,18 @@ def test_cache_list_name_match(script): assert list_matches_wheel('zzz-7.8.9', result) +@pytest.mark.usefixtures("populate_wheel_cache") +def test_cache_list_name_match_abspath(script): + """Running `pip cache list zzz --abspath` should list paths of + zzz-4.5.6, zzz-4.5.7, zzz-7.8.9, but nothing else.""" + result = script.pip('cache', 'list', 'zzz', '--abspath', '--verbose') + + assert not list_matches_wheel_abspath('yyy-1.2.3', result) + assert list_matches_wheel_abspath('zzz-4.5.6', result) + assert list_matches_wheel_abspath('zzz-4.5.7', result) + assert list_matches_wheel_abspath('zzz-7.8.9', result) + + @pytest.mark.usefixtures("populate_wheel_cache") def test_cache_list_name_and_version_match(script): """Running `pip cache list zzz-4.5.6` should list zzz-4.5.6, but @@ -170,6 +215,18 @@ def test_cache_list_name_and_version_match(script): assert not list_matches_wheel('zzz-7.8.9', result) +@pytest.mark.usefixtures("populate_wheel_cache") +def test_cache_list_name_and_version_match_abspath(script): + """Running `pip cache list zzz-4.5.6 --abspath` should list path of + zzz-4.5.6, but nothing else.""" + result = script.pip('cache', 'list', 'zzz-4.5.6', '--abspath', '--verbose') + + assert not list_matches_wheel_abspath('yyy-1.2.3', result) + assert list_matches_wheel_abspath('zzz-4.5.6', result) + assert not list_matches_wheel_abspath('zzz-4.5.7', result) + assert not list_matches_wheel_abspath('zzz-7.8.9', result) + + @pytest.mark.usefixtures("populate_wheel_cache") def test_cache_remove_no_arguments(script): """Running `pip cache remove` with no arguments should cause an error.""" From b76a4c0153c3f5d49a1324a97429404853da9fe8 Mon Sep 17 00:00:00 2001 From: Devesh Kumar Singh Date: Sat, 18 Jul 2020 14:10:06 +0530 Subject: [PATCH 132/209] Add feature news entry --- news/8355.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/8355.feature diff --git a/news/8355.feature b/news/8355.feature new file mode 100644 index 000000000..3a7fb5375 --- /dev/null +++ b/news/8355.feature @@ -0,0 +1 @@ +Add option ``--format`` to subcommand ``list`` of ``pip cache``, with ``abspath`` choice to output the full path of a wheel file. From 9450f8837af334b7daa798c40623a388db9b96e6 Mon Sep 17 00:00:00 2001 From: Devesh Kumar Singh Date: Fri, 31 Jul 2020 02:54:58 +0530 Subject: [PATCH 133/209] Use format options for abspath --- src/pip/_internal/commands/cache.py | 42 ++++++++++++++++++++--------- tests/functional/test_cache.py | 20 +++++++------- 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/src/pip/_internal/commands/cache.py b/src/pip/_internal/commands/cache.py index 31e196ee3..b9d3ed410 100644 --- a/src/pip/_internal/commands/cache.py +++ b/src/pip/_internal/commands/cache.py @@ -37,18 +37,22 @@ class CacheCommand(Command): usage = """ %prog dir %prog info - %prog list [] [--abspath] + %prog list [] [--format=[human, abspath]] %prog remove %prog purge """ def add_options(self): # type: () -> None + self.cmd_opts.add_option( - '--abspath', - dest='abspath', - action='store_true', - help='List the absolute path of wheels') + '--format', + action='store', + dest='list_format', + default="human", + choices=('human', 'abspath'), + help="Select the output format among: human (default) or abspath" + ) self.parser.insert_option_group(0, self.cmd_opts) @@ -126,22 +130,34 @@ class CacheCommand(Command): pattern = '*' files = self._find_wheels(options, pattern) + if options.list_format == 'human': + self.format_for_human(files) + else: + self.format_for_abspath(files) + def format_for_human(self, files): + # type: (List[str]) -> None if not files: - if not options.abspath: - logger.info('Nothing cached.') + logger.info('Nothing cached.') return results = [] for filename in files: wheel = os.path.basename(filename) size = filesystem.format_file_size(filename) - if options.abspath: - results.append(filename) - else: - results.append(' - {} ({})'.format(wheel, size)) - if not options.abspath: - logger.info('Cache contents:\n') + results.append(' - {} ({})'.format(wheel, size)) + logger.info('Cache contents:\n') + logger.info('\n'.join(sorted(results))) + + def format_for_abspath(self, files): + # type: (List[str]) -> None + if not files: + return + + results = [] + for filename in files: + results.append(filename) + logger.info('\n'.join(sorted(results))) def remove_cache_items(self, options, args): diff --git a/tests/functional/test_cache.py b/tests/functional/test_cache.py index be165661a..603e11b5b 100644 --- a/tests/functional/test_cache.py +++ b/tests/functional/test_cache.py @@ -72,7 +72,7 @@ def list_matches_wheel(wheel_name, result): def list_matches_wheel_abspath(wheel_name, result): """Returns True if any line in `result`, which should be the output of - a `pip cache list --abspath` call, is a valid path and belongs to + a `pip cache list --format=abspath` call, is a valid path and belongs to `wheel_name`. E.g., If wheel_name is `foo-1.2.3` it searches for a line starting with @@ -147,9 +147,9 @@ def test_cache_list(script): @pytest.mark.usefixtures("populate_wheel_cache") def test_cache_list_abspath(script): - """Running `pip cache list --abspath` should return full + """Running `pip cache list --format=abspath` should return full paths of exactly what the populate_wheel_cache fixture adds.""" - result = script.pip('cache', 'list', '--abspath') + result = script.pip('cache', 'list', '--format=abspath') assert list_matches_wheel_abspath('yyy-1.2.3', result) assert list_matches_wheel_abspath('zzz-4.5.6', result) @@ -167,9 +167,9 @@ def test_cache_list_with_empty_cache(script): @pytest.mark.usefixtures("empty_wheel_cache") def test_cache_list_with_empty_cache_abspath(script): - """Running `pip cache list --abspath` with an empty cache should not + """Running `pip cache list --format=abspath` with an empty cache should not print anything and exit.""" - result = script.pip('cache', 'list', '--abspath') + result = script.pip('cache', 'list', '--format=abspath') assert result.stdout.strip() == "" @@ -193,9 +193,10 @@ def test_cache_list_name_match(script): @pytest.mark.usefixtures("populate_wheel_cache") def test_cache_list_name_match_abspath(script): - """Running `pip cache list zzz --abspath` should list paths of + """Running `pip cache list zzz --format=abspath` should list paths of zzz-4.5.6, zzz-4.5.7, zzz-7.8.9, but nothing else.""" - result = script.pip('cache', 'list', 'zzz', '--abspath', '--verbose') + result = script.pip('cache', 'list', 'zzz', '--format=abspath', + '--verbose') assert not list_matches_wheel_abspath('yyy-1.2.3', result) assert list_matches_wheel_abspath('zzz-4.5.6', result) @@ -217,9 +218,10 @@ def test_cache_list_name_and_version_match(script): @pytest.mark.usefixtures("populate_wheel_cache") def test_cache_list_name_and_version_match_abspath(script): - """Running `pip cache list zzz-4.5.6 --abspath` should list path of + """Running `pip cache list zzz-4.5.6 --format=abspath` should list path of zzz-4.5.6, but nothing else.""" - result = script.pip('cache', 'list', 'zzz-4.5.6', '--abspath', '--verbose') + result = script.pip('cache', 'list', 'zzz-4.5.6', '--format=abspath', + '--verbose') assert not list_matches_wheel_abspath('yyy-1.2.3', result) assert list_matches_wheel_abspath('zzz-4.5.6', result) From 179ccbe6f5a28a02361cf5d1eef10de7c943d5bb Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 11 Sep 2020 15:01:50 +0800 Subject: [PATCH 134/209] Add tests for empty hash intersection --- tests/functional/test_new_resolver_hashes.py | 88 ++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/tests/functional/test_new_resolver_hashes.py b/tests/functional/test_new_resolver_hashes.py index 20a700ac8..4b13ebc30 100644 --- a/tests/functional/test_new_resolver_hashes.py +++ b/tests/functional/test_new_resolver_hashes.py @@ -123,3 +123,91 @@ def test_new_resolver_hash_intersect_from_constraint(script): "(1 matches, 0 no digest): discarding 1 non-matches" ).format(name=u"base") assert message in result.stdout, str(result) + + +@pytest.mark.parametrize( + "requirements_template, constraints_template", + [ + ( + """ + base==0.1.0 --hash=sha256:{sdist_hash} + base==0.1.0 --hash=sha256:{wheel_hash} + """, + "", + ), + ( + "base==0.1.0 --hash=sha256:{sdist_hash}", + "base==0.1.0 --hash=sha256:{wheel_hash}", + ), + ], + ids=["both-requirements", "one-each"], +) +def test_new_resolver_hash_intersect_empty( + script, requirements_template, constraints_template, +): + find_links = _create_find_links(script) + + constraints_txt = script.scratch_path / "constraints.txt" + constraints_txt.write_text( + constraints_template.format( + sdist_hash=find_links.sdist_hash, + wheel_hash=find_links.wheel_hash, + ), + ) + + requirements_txt = script.scratch_path / "requirements.txt" + requirements_txt.write_text( + requirements_template.format( + sdist_hash=find_links.sdist_hash, + wheel_hash=find_links.wheel_hash, + ), + ) + + result = script.pip( + "install", + "--use-feature=2020-resolver", + "--no-cache-dir", + "--no-deps", + "--no-index", + "--find-links", find_links.index_html, + "--constraint", constraints_txt, + "--requirement", requirements_txt, + expect_error=True, + ) + + assert ( + "THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS FILE." + ) in result.stderr, str(result) + + +def test_new_resolver_hash_intersect_empty_from_constraint(script): + find_links = _create_find_links(script) + + constraints_txt = script.scratch_path / "constraints.txt" + constraints_txt.write_text( + """ + base==0.1.0 --hash=sha256:{sdist_hash} + base==0.1.0 --hash=sha256:{wheel_hash} + """.format( + sdist_hash=find_links.sdist_hash, + wheel_hash=find_links.wheel_hash, + ), + ) + + result = script.pip( + "install", + "--use-feature=2020-resolver", + "--no-cache-dir", + "--no-deps", + "--no-index", + "--find-links", find_links.index_html, + "--constraint", constraints_txt, + "base==0.1.0", + expect_error=True, + ) + + message = ( + "Hashes are required in --require-hashes mode, but they are missing " + "from some requirements." + ) + assert message in result.stderr, str(result) From 8385cb5250e72129210b9b3f4f78665325741003 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 13 Sep 2020 10:14:03 +0300 Subject: [PATCH 135/209] Document fast-track deprecation policy --- docs/html/development/release-process.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/html/development/release-process.rst b/docs/html/development/release-process.rst index 44197955e..82358b3eb 100644 --- a/docs/html/development/release-process.rst +++ b/docs/html/development/release-process.rst @@ -38,11 +38,17 @@ Deprecation Policy Any change to pip that removes or significantly alters user-visible behavior that is described in the pip documentation will be deprecated for a minimum of -6 months before the change occurs. Deprecation will take the form of a warning +6 months before the change occurs. + +Certain changes may be fast tracked and have a deprecation period of 3 months. +This requires at least two members of the pip team to be in favor of doing so, +and no pip maintainers opposing. + +Deprecation will take the form of a warning being issued by pip when the feature is used. Longer deprecation periods, or deprecation warnings for behavior changes that would not normally be covered by this policy, are also possible depending on circumstances, but this is at the -discretion of the pip developers. +discretion of the pip maintainers. Note that the documentation is the sole reference for what counts as agreed behavior. If something isn't explicitly mentioned in the documentation, it can From b6fb8b80e247c096d0f96b5980c1f57f3e53902e Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 13 Sep 2020 10:15:34 +0300 Subject: [PATCH 136/209] Reflow paragraph --- docs/html/development/release-process.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/html/development/release-process.rst b/docs/html/development/release-process.rst index 82358b3eb..66ff3ca7c 100644 --- a/docs/html/development/release-process.rst +++ b/docs/html/development/release-process.rst @@ -44,11 +44,11 @@ Certain changes may be fast tracked and have a deprecation period of 3 months. This requires at least two members of the pip team to be in favor of doing so, and no pip maintainers opposing. -Deprecation will take the form of a warning -being issued by pip when the feature is used. Longer deprecation periods, or -deprecation warnings for behavior changes that would not normally be covered by -this policy, are also possible depending on circumstances, but this is at the -discretion of the pip maintainers. +Deprecation will take the form of a warning being issued by pip when the +feature is used. Longer deprecation periods, or deprecation warnings for +behavior changes that would not normally be covered by this policy, are also +possible depending on circumstances, but this is at the discretion of the pip +maintainers. Note that the documentation is the sole reference for what counts as agreed behavior. If something isn't explicitly mentioned in the documentation, it can From 8f8a1d65b2d6b68dcfad6946ee9e24f76d867fe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Thu, 6 Aug 2020 21:56:34 +0700 Subject: [PATCH 137/209] Dedent late download logs --- ...82ffb4-cde3-4dd5-8f37-6f4ef53e028b.trivial | 0 src/pip/_internal/operations/prepare.py | 87 +++++++++---------- 2 files changed, 42 insertions(+), 45 deletions(-) create mode 100644 news/4c82ffb4-cde3-4dd5-8f37-6f4ef53e028b.trivial diff --git a/news/4c82ffb4-cde3-4dd5-8f37-6f4ef53e028b.trivial b/news/4c82ffb4-cde3-4dd5-8f37-6f4ef53e028b.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 5eb71ce07..0a2540526 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -487,10 +487,10 @@ class RequirementPreparer(object): self._log_preparing_link(req) with indent_log(): wheel_dist = self._fetch_metadata_using_lazy_wheel(link) - if wheel_dist is not None: - req.needs_more_preparation = True - return wheel_dist - return self._prepare_linked_requirement(req, parallel_builds) + if wheel_dist is not None: + req.needs_more_preparation = True + return wheel_dist + return self._prepare_linked_requirement(req, parallel_builds) def prepare_linked_requirements_more(self, reqs, parallel_builds=False): # type: (Iterable[InstallRequirement], bool) -> None @@ -519,51 +519,48 @@ class RequirementPreparer(object): link = req.link download_dir = self._get_download_dir(link) - with indent_log(): - self._ensure_link_req_src_dir(req, download_dir, parallel_builds) - hashes = self._get_linked_req_hashes(req) - if link.url not in self._downloaded: - try: - local_file = unpack_url( - link, req.source_dir, self._download, - download_dir, hashes, - ) - except NetworkConnectionError as exc: - raise InstallationError( - 'Could not install requirement {} because of HTTP ' - 'error {} for URL {}'.format(req, exc, link) - ) - else: - file_path, content_type = self._downloaded[link.url] - if hashes: - hashes.check_against_path(file_path) - local_file = File(file_path, content_type) + self._ensure_link_req_src_dir(req, download_dir, parallel_builds) + hashes = self._get_linked_req_hashes(req) + if link.url not in self._downloaded: + try: + local_file = unpack_url( + link, req.source_dir, self._download, + download_dir, hashes, + ) + except NetworkConnectionError as exc: + raise InstallationError( + 'Could not install requirement {} because of HTTP ' + 'error {} for URL {}'.format(req, exc, link) + ) + else: + file_path, content_type = self._downloaded[link.url] + if hashes: + hashes.check_against_path(file_path) + local_file = File(file_path, content_type) - # For use in later processing, preserve the file path on the - # requirement. - if local_file: - req.local_file_path = local_file.path + # For use in later processing, + # preserve the file path on the requirement. + if local_file: + req.local_file_path = local_file.path - dist = _get_prepared_distribution( - req, self.req_tracker, self.finder, self.build_isolation, - ) + dist = _get_prepared_distribution( + req, self.req_tracker, self.finder, self.build_isolation, + ) - if download_dir: - if link.is_existing_dir(): - logger.info('Link is a directory, ignoring download_dir') - elif local_file: - download_location = os.path.join( - download_dir, link.filename - ) - if not os.path.exists(download_location): - shutil.copy(local_file.path, download_location) - download_path = display_path(download_location) - logger.info('Saved %s', download_path) + if download_dir: + if link.is_existing_dir(): + logger.info('Link is a directory, ignoring download_dir') + elif local_file: + download_location = os.path.join(download_dir, link.filename) + if not os.path.exists(download_location): + shutil.copy(local_file.path, download_location) + download_path = display_path(download_location) + logger.info('Saved %s', download_path) - if self._download_should_save: - # Make a .zip of the source_dir we already created. - if link.is_vcs: - req.archive(self.download_dir) + if self._download_should_save: + # Make a .zip of the source_dir we already created. + if link.is_vcs: + req.archive(self.download_dir) return dist def prepare_editable_requirement( From 2f0fa53e95edceb5b45d7cd17cb508cc0aaed49f Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 13 Sep 2020 21:03:16 +0300 Subject: [PATCH 138/209] Add news fragment --- news/8417.removal | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/8417.removal diff --git a/news/8417.removal b/news/8417.removal new file mode 100644 index 000000000..8f280b535 --- /dev/null +++ b/news/8417.removal @@ -0,0 +1 @@ +Document that certain removals can be fast tracked. From 12fe2872ccdb023dbe1fb1aaa2aa03eb8102bacc Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Thu, 16 Jul 2020 08:53:18 -0400 Subject: [PATCH 139/209] first set of changes to use python -m to run pip in the docs --- docs/html/installing.rst | 9 +-------- docs/html/quickstart.rst | 12 ++++++------ news/7311.doc | 1 + 3 files changed, 8 insertions(+), 14 deletions(-) create mode 100644 news/7311.doc diff --git a/docs/html/installing.rst b/docs/html/installing.rst index 0a263ac41..0f83bdd11 100644 --- a/docs/html/installing.rst +++ b/docs/html/installing.rst @@ -97,12 +97,7 @@ the `Python Packaging User Guide Upgrading pip ============= -On Linux or macOS:: - - pip install -U pip - - -On Windows [4]_:: +On Windows, Linux or macOS:: python -m pip install -U pip @@ -134,5 +129,3 @@ pip works on Unix/Linux, macOS, and Windows. ``--user`` installs for pip itself, should not be considered to be fully tested or endorsed. For discussion, see `Issue 1668 `_. - -.. [4] https://github.com/pypa/pip/issues/1299 diff --git a/docs/html/quickstart.rst b/docs/html/quickstart.rst index c2250399c..88d207ebc 100644 --- a/docs/html/quickstart.rst +++ b/docs/html/quickstart.rst @@ -8,7 +8,7 @@ Install a package from `PyPI`_: :: - $ pip install SomePackage + $ python -m pip install SomePackage [...] Successfully installed SomePackage @@ -18,7 +18,7 @@ network connection: :: - $ pip install SomePackage-1.0-py2.py3-none-any.whl + $ python -m pip install SomePackage-1.0-py2.py3-none-any.whl [...] Successfully installed SomePackage @@ -26,7 +26,7 @@ Show what files were installed: :: - $ pip show --files SomePackage + $ python -m pip show --files SomePackage Name: SomePackage Version: 1.0 Location: /my/env/lib/pythonx.x/site-packages @@ -38,14 +38,14 @@ List what packages are outdated: :: - $ pip list --outdated + $ python -m pip list --outdated SomePackage (Current: 1.0 Latest: 2.0) Upgrade a package: :: - $ pip install --upgrade SomePackage + $ python -m pip install --upgrade SomePackage [...] Found existing installation: SomePackage 1.0 Uninstalling SomePackage: @@ -57,7 +57,7 @@ Uninstall a package: :: - $ pip uninstall SomePackage + $ python -m pip uninstall SomePackage Uninstalling SomePackage: /my/env/lib/pythonx.x/site-packages/somepackage Proceed (y/n)? y diff --git a/news/7311.doc b/news/7311.doc new file mode 100644 index 000000000..f08d30a06 --- /dev/null +++ b/news/7311.doc @@ -0,0 +1 @@ +Use ``python -m`` to run pip in the docs From 2329d796897867cacf6b7b35b44ca16f89ae79bb Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Thu, 16 Jul 2020 09:49:21 -0400 Subject: [PATCH 140/209] use shell instead of mentioning specific platform in installing doc --- docs/html/installing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/html/installing.rst b/docs/html/installing.rst index 0f83bdd11..4c2c2eca3 100644 --- a/docs/html/installing.rst +++ b/docs/html/installing.rst @@ -97,7 +97,7 @@ the `Python Packaging User Guide Upgrading pip ============= -On Windows, Linux or macOS:: +In a shell:: python -m pip install -U pip From 119d8666b4b3e59c531afcf970cd0ae2d0f3cdf0 Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Sun, 19 Jul 2020 23:56:47 -0400 Subject: [PATCH 141/209] implement sphinx-tabs on installing page of docs --- docs/html/conf.py | 2 +- docs/html/installing.rst | 10 ++++++++-- tools/requirements/docs.txt | 1 + 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/html/conf.py b/docs/html/conf.py index 2ef2647ce..c5c2e02e6 100644 --- a/docs/html/conf.py +++ b/docs/html/conf.py @@ -30,7 +30,7 @@ sys.path.insert(0, docs_dir) # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. # extensions = ['sphinx.ext.autodoc'] -extensions = ['sphinx.ext.extlinks', 'pip_sphinxext', 'sphinx.ext.intersphinx'] +extensions = ['sphinx.ext.extlinks', 'pip_sphinxext', 'sphinx.ext.intersphinx', 'sphinx_tabs.tabs'] # intersphinx intersphinx_cache_limit = 0 diff --git a/docs/html/installing.rst b/docs/html/installing.rst index 4c2c2eca3..9653331a4 100644 --- a/docs/html/installing.rst +++ b/docs/html/installing.rst @@ -97,9 +97,15 @@ the `Python Packaging User Guide Upgrading pip ============= -In a shell:: +.. tabs:: - python -m pip install -U pip + .. tab:: Linux/MacOS + + python -m pip install -U pip + + .. tab:: Windows + + py -m pip install -U pip .. _compatibility-requirements: diff --git a/tools/requirements/docs.txt b/tools/requirements/docs.txt index 267d5b1dd..10a9b0c60 100644 --- a/tools/requirements/docs.txt +++ b/tools/requirements/docs.txt @@ -1,6 +1,7 @@ sphinx == 3.2.1 git+https://github.com/python/python-docs-theme.git#egg=python-docs-theme git+https://github.com/pypa/pypa-docs-theme.git#egg=pypa-docs-theme +https://github.com/djungelorm/sphinx-tabs/releases/download/v1.1.13/sphinx-tabs-1.1.13.tar.gz # `docs.pipext` uses pip's internals to generate documentation. So, we install # the current directory to make it work. From dccf555813a744c6aa5807bbfadaa00448ea3d30 Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Mon, 20 Jul 2020 09:18:26 -0400 Subject: [PATCH 142/209] update docs requirements --- tools/requirements/docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/requirements/docs.txt b/tools/requirements/docs.txt index 10a9b0c60..dc93a60ff 100644 --- a/tools/requirements/docs.txt +++ b/tools/requirements/docs.txt @@ -1,7 +1,7 @@ sphinx == 3.2.1 git+https://github.com/python/python-docs-theme.git#egg=python-docs-theme git+https://github.com/pypa/pypa-docs-theme.git#egg=pypa-docs-theme -https://github.com/djungelorm/sphinx-tabs/releases/download/v1.1.13/sphinx-tabs-1.1.13.tar.gz +sphinx-tabs == 1.1.13 # `docs.pipext` uses pip's internals to generate documentation. So, we install # the current directory to make it work. From 28f3f2cd2ce21ed6e0f354a3ca924d9e9aac5ce7 Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Tue, 21 Jul 2020 09:58:00 -0400 Subject: [PATCH 143/209] ignore sphinx-tab extension for man builds --- docs/html/conf.py | 7 ++++++- tox.ini | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/html/conf.py b/docs/html/conf.py index c5c2e02e6..f0e880f5e 100644 --- a/docs/html/conf.py +++ b/docs/html/conf.py @@ -30,7 +30,12 @@ sys.path.insert(0, docs_dir) # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. # extensions = ['sphinx.ext.autodoc'] -extensions = ['sphinx.ext.extlinks', 'pip_sphinxext', 'sphinx.ext.intersphinx', 'sphinx_tabs.tabs'] +extensions = [ + 'sphinx.ext.extlinks', + 'pip_sphinxext', + 'sphinx.ext.intersphinx', + 'sphinx_tabs.tabs' +] # intersphinx intersphinx_cache_limit = 0 diff --git a/tox.ini b/tox.ini index 82e9abc68..a673b0f59 100644 --- a/tox.ini +++ b/tox.ini @@ -56,7 +56,7 @@ commands = # can not use a different configuration directory vs source directory on RTD # currently -- https://github.com/rtfd/readthedocs.org/issues/1543. # That is why we have a "-c docs/html" in the next line. - sphinx-build -W -d {envtmpdir}/doctrees/man -b man docs/man docs/build/man -c docs/html + sphinx-build -W -d {envtmpdir}/doctrees/man -b man docs/man docs/build/man -c docs/html -D extensions=sphinx.ext.extlinks,pip_sphinxext,sphinx.ext.intersphinx [testenv:lint] skip_install = True From ad8cc27b8c53bb93115e50974ced130875e05d2e Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Tue, 28 Jul 2020 12:44:17 -0400 Subject: [PATCH 144/209] update quickstart.rst to use tabs --- docs/html/quickstart.rst | 125 ++++++++++++++++++++++++++++----------- 1 file changed, 92 insertions(+), 33 deletions(-) diff --git a/docs/html/quickstart.rst b/docs/html/quickstart.rst index 88d207ebc..41a9275e9 100644 --- a/docs/html/quickstart.rst +++ b/docs/html/quickstart.rst @@ -6,62 +6,121 @@ First, :doc:`install pip `. Install a package from `PyPI`_: -:: +.. tabs:: + + .. code-tab:: Bash Unix + + $ python -m pip install SomePackage + [...] + Successfully installed SomePackage + + .. code-tab:: Bash Windows + + $ py -m pip install SomePackage + [...] + Successfully installed SomePackage - $ python -m pip install SomePackage - [...] - Successfully installed SomePackage Install a package that's already been downloaded from `PyPI`_ or obtained from elsewhere. This is useful if the target machine does not have a network connection: -:: +.. tabs:: + + .. code-tab:: Bash Unix + + $ python -m pip install SomePackage-1.0-py2.py3-none-any.whl + [...] + Successfully installed SomePackage + + .. code-tab:: Bash Windows + + $ py -m pip install SomePackage-1.0-py2.py3-none-any.whl + [...] + Successfully installed SomePackage - $ python -m pip install SomePackage-1.0-py2.py3-none-any.whl - [...] - Successfully installed SomePackage Show what files were installed: -:: +.. tabs:: - $ python -m pip show --files SomePackage - Name: SomePackage - Version: 1.0 - Location: /my/env/lib/pythonx.x/site-packages - Files: - ../somepackage/__init__.py - [...] + .. code-tab:: Bash Unix + + $ python -m pip show --files SomePackage + Name: SomePackage + Version: 1.0 + Location: /my/env/lib/pythonx.x/site-packages + Files: + ../somepackage/__init__.py + [...] + + .. code-tab:: Bash Windows + + $ py -m pip show --files SomePackage + Name: SomePackage + Version: 1.0 + Location: /my/env/lib/pythonx.x/site-packages + Files: + ../somepackage/__init__.py + [...] List what packages are outdated: -:: +.. tabs:: - $ python -m pip list --outdated - SomePackage (Current: 1.0 Latest: 2.0) + .. code-tab:: Bash Unix + + $ python -m pip list --outdated + SomePackage (Current: 1.0 Latest: 2.0) + + .. code-tab:: Bash Windows + + $ py -m pip list --outdated + SomePackage (Current: 1.0 Latest: 2.0) Upgrade a package: -:: +.. tabs:: - $ python -m pip install --upgrade SomePackage - [...] - Found existing installation: SomePackage 1.0 - Uninstalling SomePackage: - Successfully uninstalled SomePackage - Running setup.py install for SomePackage - Successfully installed SomePackage + .. code-tab:: Bash Unix + + $ python -m pip install --upgrade SomePackage + [...] + Found existing installation: SomePackage 1.0 + Uninstalling SomePackage: + Successfully uninstalled SomePackage + Running setup.py install for SomePackage + Successfully installed SomePackage + + .. code-tab:: Bash Windows + + $ py -m pip install --upgrade SomePackage + [...] + Found existing installation: SomePackage 1.0 + Uninstalling SomePackage: + Successfully uninstalled SomePackage + Running setup.py install for SomePackage + Successfully installed SomePackage Uninstall a package: -:: +.. tabs:: - $ python -m pip uninstall SomePackage - Uninstalling SomePackage: - /my/env/lib/pythonx.x/site-packages/somepackage - Proceed (y/n)? y - Successfully uninstalled SomePackage + .. code-tab:: Bash Unix + + $ python -m pip uninstall SomePackage + Uninstalling SomePackage: + /my/env/lib/pythonx.x/site-packages/somepackage + Proceed (y/n)? y + Successfully uninstalled SomePackage + + .. code-tab:: Bash Windows + + $ py -m pip uninstall SomePackage + Uninstalling SomePackage: + /my/env/lib/pythonx.x/site-packages/somepackage + Proceed (y/n)? y + Successfully uninstalled SomePackage .. _PyPI: https://pypi.org/ From f1abd651e3b3c0ca393efc396dd6cace874b15ab Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Wed, 29 Jul 2020 07:07:32 -0400 Subject: [PATCH 145/209] WIP change user guide + make consistent changes across --- docs/html/installing.rst | 8 +- docs/html/quickstart.rst | 38 ++-- docs/html/user_guide.rst | 465 +++++++++++++++++++++++++++++---------- news/7311.doc | 2 +- 4 files changed, 375 insertions(+), 138 deletions(-) diff --git a/docs/html/installing.rst b/docs/html/installing.rst index 9653331a4..511e4d02e 100644 --- a/docs/html/installing.rst +++ b/docs/html/installing.rst @@ -99,13 +99,13 @@ Upgrading pip .. tabs:: - .. tab:: Linux/MacOS + .. code-tab:: shell Unix/macOS - python -m pip install -U pip + $ python -m pip install -U pip - .. tab:: Windows + .. code-tab:: shell Windows - py -m pip install -U pip + C:\> py -m pip install -U pip .. _compatibility-requirements: diff --git a/docs/html/quickstart.rst b/docs/html/quickstart.rst index 41a9275e9..e8754d661 100644 --- a/docs/html/quickstart.rst +++ b/docs/html/quickstart.rst @@ -8,34 +8,34 @@ Install a package from `PyPI`_: .. tabs:: - .. code-tab:: Bash Unix + .. code-tab:: shell Unix/macOS $ python -m pip install SomePackage [...] Successfully installed SomePackage - .. code-tab:: Bash Windows + .. code-tab:: shell Windows - $ py -m pip install SomePackage + C:\> py -m pip install SomePackage [...] Successfully installed SomePackage Install a package that's already been downloaded from `PyPI`_ or -obtained from elsewhere. This is useful if the target machine does not have a +obtained from elsewhere. This is useful if the target macOShine does not have a network connection: .. tabs:: - .. code-tab:: Bash Unix + .. code-tab:: shell Unix/macOS $ python -m pip install SomePackage-1.0-py2.py3-none-any.whl [...] Successfully installed SomePackage - .. code-tab:: Bash Windows + .. code-tab:: shell Windows - $ py -m pip install SomePackage-1.0-py2.py3-none-any.whl + C:\> py -m pip install SomePackage-1.0-py2.py3-none-any.whl [...] Successfully installed SomePackage @@ -44,7 +44,7 @@ Show what files were installed: .. tabs:: - .. code-tab:: Bash Unix + .. code-tab:: shell Unix/macOS $ python -m pip show --files SomePackage Name: SomePackage @@ -54,9 +54,9 @@ Show what files were installed: ../somepackage/__init__.py [...] - .. code-tab:: Bash Windows + .. code-tab:: shell Windows - $ py -m pip show --files SomePackage + C:\> py -m pip show --files SomePackage Name: SomePackage Version: 1.0 Location: /my/env/lib/pythonx.x/site-packages @@ -68,21 +68,21 @@ List what packages are outdated: .. tabs:: - .. code-tab:: Bash Unix + .. code-tab:: shell Unix/macOS $ python -m pip list --outdated SomePackage (Current: 1.0 Latest: 2.0) - .. code-tab:: Bash Windows + .. code-tab:: shell Windows - $ py -m pip list --outdated + C:\> py -m pip list --outdated SomePackage (Current: 1.0 Latest: 2.0) Upgrade a package: .. tabs:: - .. code-tab:: Bash Unix + .. code-tab:: shell Unix/macOS $ python -m pip install --upgrade SomePackage [...] @@ -92,9 +92,9 @@ Upgrade a package: Running setup.py install for SomePackage Successfully installed SomePackage - .. code-tab:: Bash Windows + .. code-tab:: shell Windows - $ py -m pip install --upgrade SomePackage + C:\> py -m pip install --upgrade SomePackage [...] Found existing installation: SomePackage 1.0 Uninstalling SomePackage: @@ -106,7 +106,7 @@ Uninstall a package: .. tabs:: - .. code-tab:: Bash Unix + .. code-tab:: shell Unix/macOS $ python -m pip uninstall SomePackage Uninstalling SomePackage: @@ -114,9 +114,9 @@ Uninstall a package: Proceed (y/n)? y Successfully uninstalled SomePackage - .. code-tab:: Bash Windows + .. code-tab:: shell Windows - $ py -m pip uninstall SomePackage + C:\> py -m pip uninstall SomePackage Uninstalling SomePackage: /my/env/lib/pythonx.x/site-packages/somepackage Proceed (y/n)? y diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index ce013c2a3..99162ccf2 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -9,23 +9,24 @@ Running pip =========== pip is a command line program. When you install pip, a ``pip`` command is added -to your system, which can be run from the command prompt as follows:: +to your system, which can be run from the command prompt as follows: - $ pip +.. tabs:: -If you cannot run the ``pip`` command directly (possibly because the location -where it was installed isn't on your operating system's ``PATH``) then you can -run pip via the Python interpreter:: + .. code-tab:: shell Unix/macOS - $ python -m pip + $ python -m pip -On Windows, the ``py`` launcher can be used:: + # python -m pip executes pip using the Python interpreter you + # specified as python. So '/usr/bin/python3.7 -m pip' means + # you are executing pip for your interpreter located at /usr/bin/python3.7. - $ py -m pip + .. code-tab:: shell Windows -Even though pip is available from your Python installation as an importable -module, via ``import pip``, it is *not supported* to use pip in this way. For -more details, see :ref:`Using pip from your program`. + C:\> py -m pip + + # py -m pip executes pip using the latest Python interpreter you + # have installed. For more details, see https://docs.python.org/3/using/windows.html#launcher. Installing Packages @@ -36,12 +37,21 @@ directly from distribution files. The most common scenario is to install from `PyPI`_ using :ref:`Requirement -Specifiers` :: +Specifiers` - $ pip install SomePackage # latest version - $ pip install SomePackage==1.0.4 # specific version - $ pip install 'SomePackage>=1.0.4' # minimum version +.. tabs:: + .. code-tab:: shell Unix/macOS + + $ python -m pip install SomePackage # latest version + $ python -m pip install SomePackage==1.0.4 # specific version + $ python -m pip install 'SomePackage>=1.0.4' # minimum version + + .. code-tab:: shell Windows + + C:\> py -m pip install SomePackage # latest version + C:\> py -m pip install SomePackage==1.0.4 # specific version + C:\> py -m pip install 'SomePackage>=1.0.4' # minimum version For more information and examples, see the :ref:`pip install` reference. @@ -142,10 +152,17 @@ Requirements Files ================== "Requirements files" are files containing a list of items to be -installed using :ref:`pip install` like so:: +installed using :ref:`pip install` like so: - pip install -r requirements.txt +.. tabs:: + .. code-tab:: shell Unix/macOS + + $ python -m pip install -r requirements.txt + + .. code-tab:: shell Windows + + C:\> py -m pip install -r requirements.txt Details on the format of the files are here: :ref:`Requirements File Format`. @@ -160,10 +177,17 @@ In practice, there are 4 common uses of Requirements files: this case, your requirement file contains a pinned version of everything that was installed when ``pip freeze`` was run. - :: +.. tabs:: - pip freeze > requirements.txt - pip install -r requirements.txt + .. code-tab:: shell Unix/macOS + + $ python -m pip freeze > requirements.txt + $ python -m pip install -r requirements.txt + + .. code-tab:: shell Windows + + C:\> py -m pip freeze > requirements.txt + C:\> py -m pip install -r requirements.txt 2. Requirements files are used to force pip to properly resolve dependencies. As it is now, pip `doesn't have true dependency resolution @@ -228,9 +252,17 @@ contents is nearly identical to :ref:`Requirements Files`. There is one key difference: Including a package in a constraints file does not trigger installation of the package. -Use a constraints file like so:: +Use a constraints file like so: - pip install -c constraints.txt +.. tabs:: + + .. code-tab:: shell Unix/macOS + + $ python -m pip install -c constraints.txt + + .. code-tab:: shell Windows + + C:\> py -m pip install -c constraints.txt Constraints files are used for exactly the same reason as requirements files when you don't know exactly what things you want to install. For instance, say @@ -268,9 +300,15 @@ archives. To install directly from a wheel archive: -:: +.. tabs:: - pip install SomePackage-1.0-py2.py3-none-any.whl + .. code-tab:: shell Unix/macOS + + $ python -m pip install SomePackage-1.0-py2.py3-none-any.whl + + .. code-tab:: shell Windows + + C:\> py -m pip install SomePackage-1.0-py2.py3-none-any.whl For the cases where wheels are not available, pip offers :ref:`pip wheel` as a @@ -283,17 +321,30 @@ convenience, to build wheels for all your requirements and dependencies. To build wheels for your requirements and all their dependencies to a local directory: -:: +.. tabs:: - pip install wheel - pip wheel --wheel-dir=/local/wheels -r requirements.txt + .. code-tab:: shell Unix/macOS + + $ python -m pip install wheel + $ python -m pip wheel --wheel-dir=/local/wheels -r requirements.txt + + .. code-tab:: shell Windows + + C:\> py -m pip install wheel + C:\> py -m pip wheel --wheel-dir=/local/wheels -r requirements.txt And *then* to install those requirements just using your local directory of wheels (and not from PyPI): -:: +.. tabs:: - pip install --no-index --find-links=/local/wheels -r requirements.txt + .. code-tab:: shell Unix/macOS + + $ python -m pip install --no-index --find-links=/local/wheels -r requirements.txt + + .. code-tab:: shell Windows + + C:\> py -m pip install --no-index --find-links=/local/wheels -r requirements.txt Uninstalling Packages @@ -301,9 +352,16 @@ Uninstalling Packages pip is able to uninstall most packages like so: -:: +.. tabs:: + + .. code-tab:: shell Unix/macOS + + $ python -m pip uninstall SomePackage + + .. code-tab:: shell Windows + + C:\> py -m pip uninstall SomePackage - $ pip uninstall SomePackage pip also performs an automatic uninstall of an old version of a package before upgrading to a newer version. @@ -316,33 +374,62 @@ Listing Packages To list installed packages: -:: +.. tabs:: + + .. code-tab:: shell Unix/macOS + + $ python -m pip list + docutils (0.9.1) + Jinja2 (2.6) + Pygments (1.5) + Sphinx (1.1.2) + + .. code-tab:: shell Windows + + C:\> py -m pip list + docutils (0.9.1) + Jinja2 (2.6) + Pygments (1.5) + Sphinx (1.1.2) - $ pip list - docutils (0.9.1) - Jinja2 (2.6) - Pygments (1.5) - Sphinx (1.1.2) To list outdated packages, and show the latest version available: -:: +.. tabs:: - $ pip list --outdated - docutils (Current: 0.9.1 Latest: 0.10) - Sphinx (Current: 1.1.2 Latest: 1.1.3) + .. code-tab:: shell Unix/macOS + $ python -m pip list --outdated + docutils (Current: 0.9.1 Latest: 0.10) + Sphinx (Current: 1.1.2 Latest: 1.1.3) + + .. code-tab:: shell Windows + + C:\> py -m pip list --outdated + docutils (Current: 0.9.1 Latest: 0.10) + Sphinx (Current: 1.1.2 Latest: 1.1.3) To show details about an installed package: -:: +.. tabs:: - $ pip show sphinx - --- - Name: Sphinx - Version: 1.1.3 - Location: /my/env/lib/pythonx.x/site-packages - Requires: Pygments, Jinja2, docutils + .. code-tab:: shell Unix/macOS + + $ python -m pip show sphinx + --- + Name: Sphinx + Version: 1.1.3 + Location: /my/env/lib/pythonx.x/site-packages + Requires: Pygments, Jinja2, docutils + + .. code-tab:: shell Windows + + C:\> py -m pip show sphinx + --- + Name: Sphinx + Version: 1.1.3 + Location: /my/env/lib/pythonx.x/site-packages + Requires: Pygments, Jinja2, docutils For more information and examples, see the :ref:`pip list` and :ref:`pip show` @@ -353,9 +440,17 @@ Searching for Packages ====================== pip can search `PyPI`_ for packages using the ``pip search`` -command:: +command: - $ pip search "query" +.. tabs:: + + .. code-tab:: shell Unix/macOS + + $ python -m pip search "query" + + .. code-tab:: shell Windows + + C:\> py -m pip search "query" The query will be used to search the names and summaries of all packages. @@ -384,7 +479,7 @@ all users) configuration: * On Unix the default configuration file is: :file:`$HOME/.config/pip/pip.conf` which respects the ``XDG_CONFIG_HOME`` environment variable. -* On macOS the configuration file is +* On macOSOS the configuration file is :file:`$HOME/Library/Application Support/pip/pip.conf` if directory ``$HOME/Library/Application Support/pip`` exists else :file:`$HOME/.config/pip/pip.conf`. @@ -393,7 +488,7 @@ all users) configuration: There are also a legacy per-user configuration file which is also respected, these are located at: -* On Unix and macOS the configuration file is: :file:`$HOME/.pip/pip.conf` +* On Unix and macOSOS the configuration file is: :file:`$HOME/.pip/pip.conf` * On Windows the configuration file is: :file:`%HOME%\\pip\\pip.ini` You can set a custom path location for this config file using the environment @@ -401,7 +496,7 @@ variable ``PIP_CONFIG_FILE``. **Inside a virtualenv**: -* On Unix and macOS the file is :file:`$VIRTUAL_ENV/pip.conf` +* On Unix and macOSOS the file is :file:`$VIRTUAL_ENV/pip.conf` * On Windows the file is: :file:`%VIRTUAL_ENV%\\pip.ini` **Global**: @@ -410,7 +505,7 @@ variable ``PIP_CONFIG_FILE``. it may be in a "pip" subdirectory of any of the paths set in the environment variable ``XDG_CONFIG_DIRS`` (if it exists), for example :file:`/etc/xdg/pip/pip.conf`. -* On macOS the file is: :file:`/Library/Application Support/pip/pip.conf` +* On macOSOS the file is: :file:`/Library/Application Support/pip/pip.conf` * On Windows XP the file is: :file:`C:\\Documents and Settings\\All Users\\Application Data\\pip\\pip.ini` * On Windows 7 and later the file is hidden, but writeable at @@ -515,22 +610,56 @@ pip's command line options can be set with environment variables using the format ``PIP_`` . Dashes (``-``) have to be replaced with underscores (``_``). -For example, to set the default timeout:: +For example, to set the default timeout: - export PIP_DEFAULT_TIMEOUT=60 +.. tabs:: -This is the same as passing the option to pip directly:: + .. code-tab:: shell Unix/macOS - pip --default-timeout=60 [...] + $ export PIP_DEFAULT_TIMEOUT=60 + + .. code-tab:: shell Windows + + C:\> set PIP_DEFAULT_TIMEOUT=60 + +This is the same as passing the option to pip directly: + +.. tabs:: + + .. code-tab:: shell Unix/macOS + + $ python -m pip --default-timeout=60 [...] + + .. code-tab:: shell Windows + + C:\> py -m pip --default-timeout=60 [...] For command line options which can be repeated, use a space to separate -multiple values. For example:: +multiple values. For example: - export PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" +.. tabs:: -is the same as calling:: + .. code-tab:: shell Unix/macOS + + $ export PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" + + .. code-tab:: shell Windows + + C:\> set PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" + + +is the same as calling: + +.. tabs:: + + .. code-tab:: shell Unix/macOS + + $ python -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com + + .. code-tab:: shell Windows + + C:\> py -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com - pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com Options that do not take a value, but can be repeated (such as ``--verbose``) can be specified using the number of repetitions, so:: @@ -573,15 +702,15 @@ pip comes with support for command line completion in bash, zsh and fish. To setup for bash:: - $ pip completion --bash >> ~/.profile + $ python -m pip completion --bash >> ~/.profile To setup for zsh:: - $ pip completion --zsh >> ~/.zprofile + $ python -m pip completion --zsh >> ~/.zprofile To setup for fish:: -$ pip completion --fish > ~/.config/fish/completions/pip.fish + $ python -m pip completion --fish > ~/.config/fish/completions/pip.fish Alternatively, you can use the result of the ``completion`` command directly with the eval function of your shell, e.g. by adding the following to your @@ -600,24 +729,47 @@ Installing from local packages In some cases, you may want to install from local packages only, with no traffic to PyPI. -First, download the archives that fulfill your requirements:: +First, download the archives that fulfill your requirements: -$ pip download --destination-directory DIR -r requirements.txt +.. tabs:: + + .. code-tab:: shell Unix/macOS + + $ python -m pip download --destination-directory DIR -r requirements.txt + + .. code-tab:: shell Windows + + C:\> py -m pip download --destination-directory DIR -r requirements.txt Note that ``pip download`` will look in your wheel cache first, before trying to download from PyPI. If you've never installed your requirements before, you won't have a wheel cache for those items. In that case, if some of your requirements don't come as wheels from PyPI, and you want wheels, then run -this instead:: +this instead: -$ pip wheel --wheel-dir DIR -r requirements.txt +.. tabs:: + .. code-tab:: shell Unix/macOS + + $ python -m pip wheel --wheel-dir DIR -r requirements.txt + + .. code-tab:: shell Windows + + C:\> py -m pip wheel --wheel-dir DIR -r requirements.txt Then, to install from local only, you'll be using :ref:`--find-links -` and :ref:`--no-index ` like so:: +` and :ref:`--no-index ` like so: -$ pip install --no-index --find-links=DIR -r requirements.txt +.. tabs:: + + .. code-tab:: shell Unix/macOS + + $ python -m pip install --no-index --find-links=DIR -r requirements.txt + + .. code-tab:: shell Windows + + C:\> -m pip install --no-index --find-links=DIR -r requirements.txt "Only if needed" Recursive Upgrade @@ -636,10 +788,20 @@ The default strategy is ``only-if-needed``. This was changed in pip 10.0 due to the breaking nature of ``eager`` when upgrading conflicting dependencies. As an historic note, an earlier "fix" for getting the ``only-if-needed`` -behaviour was:: +behaviour was: + +.. tabs:: + + .. code-tab:: shell Unix/macOS + + $ python -m pip install --upgrade --no-deps SomePackage + $ python -m pip install SomePackage + + .. code-tab:: shell Windows + + C:\> py -m pip install --upgrade --no-deps SomePackage + C:\> py -m pip install SomePackage - pip install --upgrade --no-deps SomePackage - pip install SomePackage A proposal for an ``upgrade-all`` command is being considered as a safer alternative to the behaviour of eager upgrading. @@ -662,11 +824,19 @@ Moreover, the "user scheme" can be customized by setting the ``site.USER_BASE``. To install "SomePackage" into an environment with site.USER_BASE customized to -'/myappenv', do the following:: +'/myappenv', do the following: - export PYTHONUSERBASE=/myappenv - pip install --user SomePackage +.. tabs:: + .. code-tab:: shell Unix/macOS + + $ export PYTHONUSERBASE=/myappenv + $ python -m pip install --user SomePackage + + .. code-tab:: shell Windows + + C:\> set PYTHONUSERBASE=c:/myappenv + C:\> py -m pip install --user SomePackage ``pip install --user`` follows four rules: @@ -689,54 +859,105 @@ To install "SomePackage" into an environment with site.USER_BASE customized to To make the rules clearer, here are some examples: -From within a ``--no-site-packages`` virtualenv (i.e. the default kind):: +From within a ``--no-site-packages`` virtualenv (i.e. the default kind): - $ pip install --user SomePackage - Can not perform a '--user' install. User site-packages are not visible in this virtualenv. +.. tabs:: + + .. code-tab:: shell Unix/macOS + + $ python -m pip install --user SomePackage + Can not perform a '--user' install. User site-packages are not visible in this virtualenv. + + .. code-tab:: shell Windows + + C:\> py -m pip install --user SomePackage + Can not perform a '--user' install. User site-packages are not visible in this virtualenv. From within a ``--system-site-packages`` virtualenv where ``SomePackage==0.3`` -is already installed in the virtualenv:: +is already installed in the virtualenv: - $ pip install --user SomePackage==0.4 - Will not install to the user site because it will lack sys.path precedence +.. tabs:: + .. code-tab:: shell Unix/macOS -From within a real python, where ``SomePackage`` is *not* installed globally:: + $ python -m pip install --user SomePackage==0.4 + Will not install to the user site because it will lack sys.path precedence - $ pip install --user SomePackage - [...] - Successfully installed SomePackage + .. code-tab:: shell Windows + C:\> py -m pip install --user SomePackage==0.4 + Will not install to the user site because it will lack sys.path precedence + +From within a real python, where ``SomePackage`` is *not* installed globally: + +.. tabs:: + + .. code-tab:: shell Unix/macOS + + $ python -m pip install --user SomePackage + [...] + Successfully installed SomePackage + + .. code-tab:: shell Windows + + C:\> py -m pip install --user SomePackage + [...] + Successfully installed SomePackage From within a real python, where ``SomePackage`` *is* installed globally, but -is *not* the latest version:: +is *not* the latest version: - $ pip install --user SomePackage - [...] - Requirement already satisfied (use --upgrade to upgrade) +.. tabs:: - $ pip install --user --upgrade SomePackage - [...] - Successfully installed SomePackage + .. code-tab:: shell Unix/macOS + $ python -m pip install --user SomePackage + [...] + Requirement already satisfied (use --upgrade to upgrade) + $ python -m pip install --user --upgrade SomePackage + [...] + Successfully installed SomePackage + + .. code-tab:: shell Windows + + C:\> py -m pip install --user SomePackage + [...] + Requirement already satisfied (use --upgrade to upgrade) + C:\> py -m pip install --user --upgrade SomePackage + [...] + Successfully installed SomePackage From within a real python, where ``SomePackage`` *is* installed globally, and -is the latest version:: +is the latest version: - $ pip install --user SomePackage - [...] - Requirement already satisfied (use --upgrade to upgrade) +.. tabs:: - $ pip install --user --upgrade SomePackage - [...] - Requirement already up-to-date: SomePackage + .. code-tab:: shell Unix/macOS - # force the install - $ pip install --user --ignore-installed SomePackage - [...] - Successfully installed SomePackage + $ python -m pip install --user SomePackage + [...] + Requirement already satisfied (use --upgrade to upgrade) + $ python -m pip install --user --upgrade SomePackage + [...] + Requirement already up-to-date: SomePackage + # force the install + $ python -m pip install --user --ignore-installed SomePackage + [...] + Successfully installed SomePackage + .. code-tab:: shell Windows + + C:\> py -m pip install --user SomePackage + [...] + Requirement already satisfied (use --upgrade to upgrade) + C:\> py -m pip install --user --upgrade SomePackage + [...] + Requirement already up-to-date: SomePackage + # force the install + C:\> py -m pip install --user --ignore-installed SomePackage + [...] + Successfully installed SomePackage .. _`Repeatability`: @@ -801,7 +1022,7 @@ index servers are unavailable and avoids time-consuming recompilation. Create an archive like this:: $ tempdir=$(mktemp -d /tmp/wheelhouse-XXXXX) - $ pip wheel -r requirements.txt --wheel-dir=$tempdir + $ python -m pip wheel -r requirements.txt --wheel-dir=$tempdir $ cwd=`pwd` $ (cd "$tempdir"; tar -cjvf "$cwd/bundled.tar.bz2" *) @@ -809,10 +1030,10 @@ You can then install from the archive like this:: $ tempdir=$(mktemp -d /tmp/wheelhouse-XXXXX) $ (cd $tempdir; tar -xvf /path/to/bundled.tar.bz2) - $ pip install --force-reinstall --ignore-installed --upgrade --no-index --no-deps $tempdir/* + $ python -m pip install --force-reinstall --ignore-installed --upgrade --no-index --no-deps $tempdir/* Note that compiled packages are typically OS- and architecture-specific, so -these archives are not necessarily portable across machines. +these archives are not necessarily portable across macOShines. Hash-checking mode can be used along with this method to ensure that future archives are built with identical packages. @@ -842,10 +1063,18 @@ Understanding your error message When you get a ``ResolutionImpossible`` error, you might see something like this: -.. code-block:: console +.. tabs:: + + .. code-tab:: shell Unix/macOS + + $ python -m pip install package_coffee==0.44.1 package_tea==4.3.0 + + .. code-tab:: shell Windows + + C:\> py -m pip install package_coffee==0.44.1 package_tea==4.3.0 + +:: - $ pip install package_coffee==0.44.1 package_tea==4.3.0 - ... Due to conflicting dependencies pip cannot install package_coffee and package_tea: - package_coffee depends on package_water<3.0.0,>=2.4.2 @@ -936,7 +1165,7 @@ the same version of ``package_water``, you might consider: (e.g. ``pip install "package_coffee>0.44.*" "package_tea>4.0.0"``) - Asking pip to install *any* version of ``package_coffee`` and ``package_tea`` by removing the version specifiers altogether (e.g. - ``pip install package_coffee package_tea``) + ``python -m pip install package_coffee package_tea``) In the second case, pip will automatically find a version of both ``package_coffee`` and ``package_tea`` that depend on the same version of @@ -946,9 +1175,17 @@ In the second case, pip will automatically find a version of both - ``package_tea 4.3.0`` which *also* depends on ``package_water 2.6.1`` If you want to prioritize one package over another, you can add version -specifiers to *only* the more important package:: +specifiers to *only* the more important package: - pip install package_coffee==0.44.1b0 package_tea +.. tabs:: + + .. code-tab:: shell Unix/macOS + + $ python -m pip install package_coffee==0.44.1b0 package_tea + + .. code-tab:: shell Windows + + C:\> py -m pip install package_coffee==0.44.1b0 package_tea This will result in: diff --git a/news/7311.doc b/news/7311.doc index f08d30a06..6ed2c4204 100644 --- a/news/7311.doc +++ b/news/7311.doc @@ -1 +1 @@ -Use ``python -m`` to run pip in the docs +Add OS tabs for OS-specific commands. From 55c06d181b0844e3efcd9c21339cf05f8875083c Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Thu, 30 Jul 2020 03:03:16 -0400 Subject: [PATCH 146/209] WIP - use code-block within group-tab for consistent look --- docs/html/installing.rst | 12 +- docs/html/quickstart.rst | 156 ++++++------ docs/html/user_guide.rst | 499 ++++++++++++++++++++++++--------------- 3 files changed, 406 insertions(+), 261 deletions(-) diff --git a/docs/html/installing.rst b/docs/html/installing.rst index 511e4d02e..919f47606 100644 --- a/docs/html/installing.rst +++ b/docs/html/installing.rst @@ -99,13 +99,17 @@ Upgrading pip .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install -U pip + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install -U pip - C:\> py -m pip install -U pip + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install -U pip .. _compatibility-requirements: diff --git a/docs/html/quickstart.rst b/docs/html/quickstart.rst index e8754d661..bae9e2286 100644 --- a/docs/html/quickstart.rst +++ b/docs/html/quickstart.rst @@ -8,17 +8,21 @@ Install a package from `PyPI`_: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install SomePackage - [...] - Successfully installed SomePackage + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install SomePackage + [...] + Successfully installed SomePackage - C:\> py -m pip install SomePackage - [...] - Successfully installed SomePackage + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install SomePackage + [...] + Successfully installed SomePackage Install a package that's already been downloaded from `PyPI`_ or @@ -27,100 +31,120 @@ network connection: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install SomePackage-1.0-py2.py3-none-any.whl - [...] - Successfully installed SomePackage + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install SomePackage-1.0-py2.py3-none-any.whl + [...] + Successfully installed SomePackage - C:\> py -m pip install SomePackage-1.0-py2.py3-none-any.whl - [...] - Successfully installed SomePackage + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install SomePackage-1.0-py2.py3-none-any.whl + [...] + Successfully installed SomePackage Show what files were installed: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip show --files SomePackage - Name: SomePackage - Version: 1.0 - Location: /my/env/lib/pythonx.x/site-packages - Files: - ../somepackage/__init__.py - [...] + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip show --files SomePackage + Name: SomePackage + Version: 1.0 + Location: /my/env/lib/pythonx.x/site-packages + Files: + ../somepackage/__init__.py + [...] - C:\> py -m pip show --files SomePackage - Name: SomePackage - Version: 1.0 - Location: /my/env/lib/pythonx.x/site-packages - Files: - ../somepackage/__init__.py - [...] + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip show --files SomePackage + Name: SomePackage + Version: 1.0 + Location: /my/env/lib/pythonx.x/site-packages + Files: + ../somepackage/__init__.py + [...] List what packages are outdated: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip list --outdated - SomePackage (Current: 1.0 Latest: 2.0) + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip list --outdated + SomePackage (Current: 1.0 Latest: 2.0) - C:\> py -m pip list --outdated - SomePackage (Current: 1.0 Latest: 2.0) + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip list --outdated + SomePackage (Current: 1.0 Latest: 2.0) Upgrade a package: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install --upgrade SomePackage - [...] - Found existing installation: SomePackage 1.0 - Uninstalling SomePackage: - Successfully uninstalled SomePackage - Running setup.py install for SomePackage - Successfully installed SomePackage + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install --upgrade SomePackage + [...] + Found existing installation: SomePackage 1.0 + Uninstalling SomePackage: + Successfully uninstalled SomePackage + Running setup.py install for SomePackage + Successfully installed SomePackage - C:\> py -m pip install --upgrade SomePackage - [...] - Found existing installation: SomePackage 1.0 - Uninstalling SomePackage: - Successfully uninstalled SomePackage - Running setup.py install for SomePackage - Successfully installed SomePackage + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install --upgrade SomePackage + [...] + Found existing installation: SomePackage 1.0 + Uninstalling SomePackage: + Successfully uninstalled SomePackage + Running setup.py install for SomePackage + Successfully installed SomePackage Uninstall a package: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip uninstall SomePackage - Uninstalling SomePackage: - /my/env/lib/pythonx.x/site-packages/somepackage - Proceed (y/n)? y - Successfully uninstalled SomePackage + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip uninstall SomePackage + Uninstalling SomePackage: + /my/env/lib/pythonx.x/site-packages/somepackage + Proceed (y/n)? y + Successfully uninstalled SomePackage - C:\> py -m pip uninstall SomePackage - Uninstalling SomePackage: - /my/env/lib/pythonx.x/site-packages/somepackage - Proceed (y/n)? y - Successfully uninstalled SomePackage + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip uninstall SomePackage + Uninstalling SomePackage: + /my/env/lib/pythonx.x/site-packages/somepackage + Proceed (y/n)? y + Successfully uninstalled SomePackage .. _PyPI: https://pypi.org/ diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 99162ccf2..177baec3c 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -13,20 +13,24 @@ to your system, which can be run from the command prompt as follows: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip + .. code-block:: shell - # python -m pip executes pip using the Python interpreter you - # specified as python. So '/usr/bin/python3.7 -m pip' means - # you are executing pip for your interpreter located at /usr/bin/python3.7. + $ python -m pip - .. code-tab:: shell Windows + ``python -m pip`` executes pip using the Python interpreter you + specified as python. So ``/usr/bin/python3.7 -m pip`` means + you are executing pip for your interpreter located at /usr/bin/python3.7. - C:\> py -m pip + .. group-tab:: Windows - # py -m pip executes pip using the latest Python interpreter you - # have installed. For more details, see https://docs.python.org/3/using/windows.html#launcher. + .. code-block:: shell + + C:\> py -m pip + + ``py -m pip`` executes pip using the latest Python interpreter you + have installed. For more details, read the `Python Windows launcher`_ docs. Installing Packages @@ -41,17 +45,21 @@ Specifiers` .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install SomePackage # latest version - $ python -m pip install SomePackage==1.0.4 # specific version - $ python -m pip install 'SomePackage>=1.0.4' # minimum version + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install SomePackage # latest version + $ python -m pip install SomePackage==1.0.4 # specific version + $ python -m pip install 'SomePackage>=1.0.4' # minimum version - C:\> py -m pip install SomePackage # latest version - C:\> py -m pip install SomePackage==1.0.4 # specific version - C:\> py -m pip install 'SomePackage>=1.0.4' # minimum version + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install SomePackage # latest version + C:\> py -m pip install SomePackage==1.0.4 # specific version + C:\> py -m pip install 'SomePackage>=1.0.4' # minimum version For more information and examples, see the :ref:`pip install` reference. @@ -156,13 +164,17 @@ installed using :ref:`pip install` like so: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install -r requirements.txt + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install -r requirements.txt - C:\> py -m pip install -r requirements.txt + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install -r requirements.txt Details on the format of the files are here: :ref:`Requirements File Format`. @@ -179,15 +191,19 @@ In practice, there are 4 common uses of Requirements files: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip freeze > requirements.txt - $ python -m pip install -r requirements.txt + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip freeze > requirements.txt + $ python -m pip install -r requirements.txt - C:\> py -m pip freeze > requirements.txt - C:\> py -m pip install -r requirements.txt + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip freeze > requirements.txt + C:\> py -m pip install -r requirements.txt 2. Requirements files are used to force pip to properly resolve dependencies. As it is now, pip `doesn't have true dependency resolution @@ -256,13 +272,17 @@ Use a constraints file like so: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install -c constraints.txt + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install -c constraints.txt - C:\> py -m pip install -c constraints.txt + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install -c constraints.txt Constraints files are used for exactly the same reason as requirements files when you don't know exactly what things you want to install. For instance, say @@ -302,13 +322,17 @@ To install directly from a wheel archive: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install SomePackage-1.0-py2.py3-none-any.whl + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install SomePackage-1.0-py2.py3-none-any.whl - C:\> py -m pip install SomePackage-1.0-py2.py3-none-any.whl + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install SomePackage-1.0-py2.py3-none-any.whl For the cases where wheels are not available, pip offers :ref:`pip wheel` as a @@ -323,28 +347,36 @@ directory: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install wheel - $ python -m pip wheel --wheel-dir=/local/wheels -r requirements.txt + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install wheel + $ python -m pip wheel --wheel-dir=/local/wheels -r requirements.txt - C:\> py -m pip install wheel - C:\> py -m pip wheel --wheel-dir=/local/wheels -r requirements.txt + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install wheel + C:\> py -m pip wheel --wheel-dir=/local/wheels -r requirements.txt And *then* to install those requirements just using your local directory of wheels (and not from PyPI): .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install --no-index --find-links=/local/wheels -r requirements.txt + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install --no-index --find-links=/local/wheels -r requirements.txt - C:\> py -m pip install --no-index --find-links=/local/wheels -r requirements.txt + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install --no-index --find-links=/local/wheels -r requirements.txt Uninstalling Packages @@ -354,13 +386,17 @@ pip is able to uninstall most packages like so: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip uninstall SomePackage + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip uninstall SomePackage - C:\> py -m pip uninstall SomePackage + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip uninstall SomePackage pip also performs an automatic uninstall of an old version of a package @@ -376,60 +412,72 @@ To list installed packages: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip list - docutils (0.9.1) - Jinja2 (2.6) - Pygments (1.5) - Sphinx (1.1.2) + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip list + docutils (0.9.1) + Jinja2 (2.6) + Pygments (1.5) + Sphinx (1.1.2) - C:\> py -m pip list - docutils (0.9.1) - Jinja2 (2.6) - Pygments (1.5) - Sphinx (1.1.2) + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip list + docutils (0.9.1) + Jinja2 (2.6) + Pygments (1.5) + Sphinx (1.1.2) To list outdated packages, and show the latest version available: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip list --outdated - docutils (Current: 0.9.1 Latest: 0.10) - Sphinx (Current: 1.1.2 Latest: 1.1.3) + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip list --outdated + docutils (Current: 0.9.1 Latest: 0.10) + Sphinx (Current: 1.1.2 Latest: 1.1.3) - C:\> py -m pip list --outdated - docutils (Current: 0.9.1 Latest: 0.10) - Sphinx (Current: 1.1.2 Latest: 1.1.3) + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip list --outdated + docutils (Current: 0.9.1 Latest: 0.10) + Sphinx (Current: 1.1.2 Latest: 1.1.3) To show details about an installed package: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip show sphinx - --- - Name: Sphinx - Version: 1.1.3 - Location: /my/env/lib/pythonx.x/site-packages - Requires: Pygments, Jinja2, docutils + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip show sphinx + --- + Name: Sphinx + Version: 1.1.3 + Location: /my/env/lib/pythonx.x/site-packages + Requires: Pygments, Jinja2, docutils - C:\> py -m pip show sphinx - --- - Name: Sphinx - Version: 1.1.3 - Location: /my/env/lib/pythonx.x/site-packages - Requires: Pygments, Jinja2, docutils + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip show sphinx + --- + Name: Sphinx + Version: 1.1.3 + Location: /my/env/lib/pythonx.x/site-packages + Requires: Pygments, Jinja2, docutils For more information and examples, see the :ref:`pip list` and :ref:`pip show` @@ -444,13 +492,17 @@ command: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip search "query" + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip search "query" - C:\> py -m pip search "query" + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip search "query" The query will be used to search the names and summaries of all packages. @@ -614,51 +666,67 @@ For example, to set the default timeout: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ export PIP_DEFAULT_TIMEOUT=60 + .. code-block:: shell - .. code-tab:: shell Windows + $ export PIP_DEFAULT_TIMEOUT=60 - C:\> set PIP_DEFAULT_TIMEOUT=60 + .. group-tab:: Windows + + .. code-block:: shell + + C:\> set PIP_DEFAULT_TIMEOUT=60 This is the same as passing the option to pip directly: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip --default-timeout=60 [...] + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip --default-timeout=60 [...] - C:\> py -m pip --default-timeout=60 [...] + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip --default-timeout=60 [...] For command line options which can be repeated, use a space to separate multiple values. For example: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ export PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" + .. code-block:: shell - .. code-tab:: shell Windows + $ export PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" - C:\> set PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" + .. group-tab:: Windows + + .. code-block:: shell + + C:\> set PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" is the same as calling: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com - C:\> py -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com Options that do not take a value, but can be repeated (such as ``--verbose``) @@ -733,13 +801,17 @@ First, download the archives that fulfill your requirements: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip download --destination-directory DIR -r requirements.txt + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip download --destination-directory DIR -r requirements.txt - C:\> py -m pip download --destination-directory DIR -r requirements.txt + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip download --destination-directory DIR -r requirements.txt Note that ``pip download`` will look in your wheel cache first, before @@ -750,26 +822,34 @@ this instead: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip wheel --wheel-dir DIR -r requirements.txt + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip wheel --wheel-dir DIR -r requirements.txt - C:\> py -m pip wheel --wheel-dir DIR -r requirements.txt + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip wheel --wheel-dir DIR -r requirements.txt Then, to install from local only, you'll be using :ref:`--find-links ` and :ref:`--no-index ` like so: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install --no-index --find-links=DIR -r requirements.txt + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install --no-index --find-links=DIR -r requirements.txt - C:\> -m pip install --no-index --find-links=DIR -r requirements.txt + .. group-tab:: Windows + + .. code-block:: shell + + C:\> -m pip install --no-index --find-links=DIR -r requirements.txt "Only if needed" Recursive Upgrade @@ -792,15 +872,19 @@ behaviour was: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install --upgrade --no-deps SomePackage - $ python -m pip install SomePackage + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install --upgrade --no-deps SomePackage + $ python -m pip install SomePackage - C:\> py -m pip install --upgrade --no-deps SomePackage - C:\> py -m pip install SomePackage + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install --upgrade --no-deps SomePackage + C:\> py -m pip install SomePackage A proposal for an ``upgrade-all`` command is being considered as a safer @@ -828,15 +912,19 @@ To install "SomePackage" into an environment with site.USER_BASE customized to .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ export PYTHONUSERBASE=/myappenv - $ python -m pip install --user SomePackage + .. code-block:: shell - .. code-tab:: shell Windows + $ export PYTHONUSERBASE=/myappenv + $ python -m pip install --user SomePackage - C:\> set PYTHONUSERBASE=c:/myappenv - C:\> py -m pip install --user SomePackage + .. group-tab:: Windows + + .. code-block:: shell + + C:\> set PYTHONUSERBASE=c:/myappenv + C:\> py -m pip install --user SomePackage ``pip install --user`` follows four rules: @@ -863,15 +951,19 @@ From within a ``--no-site-packages`` virtualenv (i.e. the default kind): .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install --user SomePackage - Can not perform a '--user' install. User site-packages are not visible in this virtualenv. + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install --user SomePackage + Can not perform a '--user' install. User site-packages are not visible in this virtualenv. - C:\> py -m pip install --user SomePackage - Can not perform a '--user' install. User site-packages are not visible in this virtualenv. + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install --user SomePackage + Can not perform a '--user' install. User site-packages are not visible in this virtualenv. From within a ``--system-site-packages`` virtualenv where ``SomePackage==0.3`` @@ -879,85 +971,101 @@ is already installed in the virtualenv: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install --user SomePackage==0.4 - Will not install to the user site because it will lack sys.path precedence + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install --user SomePackage==0.4 + Will not install to the user site because it will lack sys.path precedence - C:\> py -m pip install --user SomePackage==0.4 - Will not install to the user site because it will lack sys.path precedence + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install --user SomePackage==0.4 + Will not install to the user site because it will lack sys.path precedence From within a real python, where ``SomePackage`` is *not* installed globally: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install --user SomePackage - [...] - Successfully installed SomePackage + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install --user SomePackage + [...] + Successfully installed SomePackage - C:\> py -m pip install --user SomePackage - [...] - Successfully installed SomePackage + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install --user SomePackage + [...] + Successfully installed SomePackage From within a real python, where ``SomePackage`` *is* installed globally, but is *not* the latest version: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install --user SomePackage - [...] - Requirement already satisfied (use --upgrade to upgrade) - $ python -m pip install --user --upgrade SomePackage - [...] - Successfully installed SomePackage + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install --user SomePackage + [...] + Requirement already satisfied (use --upgrade to upgrade) + $ python -m pip install --user --upgrade SomePackage + [...] + Successfully installed SomePackage - C:\> py -m pip install --user SomePackage - [...] - Requirement already satisfied (use --upgrade to upgrade) - C:\> py -m pip install --user --upgrade SomePackage - [...] - Successfully installed SomePackage + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install --user SomePackage + [...] + Requirement already satisfied (use --upgrade to upgrade) + C:\> py -m pip install --user --upgrade SomePackage + [...] + Successfully installed SomePackage From within a real python, where ``SomePackage`` *is* installed globally, and is the latest version: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install --user SomePackage - [...] - Requirement already satisfied (use --upgrade to upgrade) - $ python -m pip install --user --upgrade SomePackage - [...] - Requirement already up-to-date: SomePackage - # force the install - $ python -m pip install --user --ignore-installed SomePackage - [...] - Successfully installed SomePackage + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install --user SomePackage + [...] + Requirement already satisfied (use --upgrade to upgrade) + $ python -m pip install --user --upgrade SomePackage + [...] + Requirement already up-to-date: SomePackage + # force the install + $ python -m pip install --user --ignore-installed SomePackage + [...] + Successfully installed SomePackage - C:\> py -m pip install --user SomePackage - [...] - Requirement already satisfied (use --upgrade to upgrade) - C:\> py -m pip install --user --upgrade SomePackage - [...] - Requirement already up-to-date: SomePackage - # force the install - C:\> py -m pip install --user --ignore-installed SomePackage - [...] - Successfully installed SomePackage + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install --user SomePackage + [...] + Requirement already satisfied (use --upgrade to upgrade) + C:\> py -m pip install --user --upgrade SomePackage + [...] + Requirement already up-to-date: SomePackage + # force the install + C:\> py -m pip install --user --ignore-installed SomePackage + [...] + Successfully installed SomePackage .. _`Repeatability`: @@ -1065,13 +1173,17 @@ like this: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install package_coffee==0.44.1 package_tea==4.3.0 + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install package_coffee==0.44.1 package_tea==4.3.0 - C:\> py -m pip install package_coffee==0.44.1 package_tea==4.3.0 + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install package_coffee==0.44.1 package_tea==4.3.0 :: @@ -1179,13 +1291,17 @@ specifiers to *only* the more important package: .. tabs:: - .. code-tab:: shell Unix/macOS + .. group-tab:: Unix/macOS - $ python -m pip install package_coffee==0.44.1b0 package_tea + .. code-block:: shell - .. code-tab:: shell Windows + $ python -m pip install package_coffee==0.44.1b0 package_tea - C:\> py -m pip install package_coffee==0.44.1b0 package_tea + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install package_coffee==0.44.1b0 package_tea This will result in: @@ -1562,3 +1678,4 @@ announcements on the `low-traffic packaging announcements list`_ and .. _the official Python blog: https://blog.python.org/ .. _requests: https://requests.readthedocs.io/en/master/user/authentication/#netrc-authentication .. _Python standard library: https://docs.python.org/3/library/netrc.html +.. _Python Windows launcher: https://docs.python.org/3/using/windows.html#launcher From bcd0450158455049cc644b8ad439a1d296b02edc Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Mon, 3 Aug 2020 23:30:43 -0400 Subject: [PATCH 147/209] WIP - update getting-started and pip ref docs --- docs/html/development/getting-started.rst | 22 +++++++++++++++++----- docs/html/reference/pip.rst | 13 +++++++++++-- docs/html/user_guide.rst | 8 ++++---- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/docs/html/development/getting-started.rst b/docs/html/development/getting-started.rst index 326543202..9810bfaa4 100644 --- a/docs/html/development/getting-started.rst +++ b/docs/html/development/getting-started.rst @@ -38,13 +38,25 @@ To run the pip executable from your source tree during development, install pip locally using editable installation (inside a virtualenv). You can then invoke your local source tree pip normally. -.. code-block:: console +.. tabs:: - $ virtualenv venv # You can also use "python -m venv venv" from python3.3+ - $ source venv/bin/activate - $ python -m pip install -e . - $ python -m pip --version + .. group-tab:: Unix/macOS + .. code-block:: shell + + $ virtualenv venv # You can also use "python -m venv venv" from python3.3+ + $ source venv/bin/activate + $ python -m pip install -e . + $ python -m pip --version + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> virtualenv venv # You can also use "py -m venv venv" from python3.3+ + C:\> source venv/bin/activate + C:\> py -m pip install -e . + C:\> py -m pip --version Running Tests ============= diff --git a/docs/html/reference/pip.rst b/docs/html/reference/pip.rst index 9c218f355..9ca06e2c7 100644 --- a/docs/html/reference/pip.rst +++ b/docs/html/reference/pip.rst @@ -7,10 +7,19 @@ pip Usage ***** -:: +.. tabs:: - pip [options] + .. group-tab:: Unix/macOS + .. code-block:: shell + + $ python -m pip [options] + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip [options] Description *********** diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 177baec3c..41cbb6404 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -531,7 +531,7 @@ all users) configuration: * On Unix the default configuration file is: :file:`$HOME/.config/pip/pip.conf` which respects the ``XDG_CONFIG_HOME`` environment variable. -* On macOSOS the configuration file is +* On macOS the configuration file is :file:`$HOME/Library/Application Support/pip/pip.conf` if directory ``$HOME/Library/Application Support/pip`` exists else :file:`$HOME/.config/pip/pip.conf`. @@ -540,7 +540,7 @@ all users) configuration: There are also a legacy per-user configuration file which is also respected, these are located at: -* On Unix and macOSOS the configuration file is: :file:`$HOME/.pip/pip.conf` +* On Unix and macOS the configuration file is: :file:`$HOME/.pip/pip.conf` * On Windows the configuration file is: :file:`%HOME%\\pip\\pip.ini` You can set a custom path location for this config file using the environment @@ -548,7 +548,7 @@ variable ``PIP_CONFIG_FILE``. **Inside a virtualenv**: -* On Unix and macOSOS the file is :file:`$VIRTUAL_ENV/pip.conf` +* On Unix and macOS the file is :file:`$VIRTUAL_ENV/pip.conf` * On Windows the file is: :file:`%VIRTUAL_ENV%\\pip.ini` **Global**: @@ -557,7 +557,7 @@ variable ``PIP_CONFIG_FILE``. it may be in a "pip" subdirectory of any of the paths set in the environment variable ``XDG_CONFIG_DIRS`` (if it exists), for example :file:`/etc/xdg/pip/pip.conf`. -* On macOSOS the file is: :file:`/Library/Application Support/pip/pip.conf` +* On macOS the file is: :file:`/Library/Application Support/pip/pip.conf` * On Windows XP the file is: :file:`C:\\Documents and Settings\\All Users\\Application Data\\pip\\pip.ini` * On Windows 7 and later the file is hidden, but writeable at From da3b7e05786efda511c275ce7e6f0bae1c91c005 Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Tue, 4 Aug 2020 12:59:10 -0400 Subject: [PATCH 148/209] WIP - update pip-command-usage to take optional arguments and update pip install docs --- docs/html/reference/pip_install.rst | 440 +++++++++++++++++++++++----- docs/pip_sphinxext.py | 6 +- 2 files changed, 370 insertions(+), 76 deletions(-) diff --git a/docs/html/reference/pip_install.rst b/docs/html/reference/pip_install.rst index fd962cd36..69ea76275 100644 --- a/docs/html/reference/pip_install.rst +++ b/docs/html/reference/pip_install.rst @@ -10,7 +10,16 @@ pip install Usage ===== -.. pip-command-usage:: install +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: install $ python -m pip + + .. group-tab:: Windows + + .. pip-command-usage:: install C:\> py -m pip + Description @@ -89,15 +98,33 @@ implementation (which might possibly change later) has it such that the first encountered member of the cycle is installed last. For instance, if quux depends on foo which depends on bar which depends on baz, -which depends on foo:: +which depends on foo: - pip install quux - ... - Installing collected packages baz, bar, foo, quux +.. tabs:: - pip install bar - ... - Installing collected packages foo, baz, bar + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install quux + ... + Installing collected packages baz, bar, foo, quux + + $ python -m pip install bar + ... + Installing collected packages foo, baz, bar + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install quux + ... + Installing collected packages baz, bar, foo, quux + + C:\> py -m pip install bar + ... + Installing collected packages foo, baz, bar Prior to v6.1.0, pip made no commitments about install order. @@ -387,9 +414,21 @@ If your repository layout is:: └── some_file some_other_file -Then, to install from this repository, the syntax would be:: +Then, to install from this repository, the syntax would be: - $ pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir" +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir" + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir" Git @@ -636,17 +675,38 @@ against any requirement not only checks that hash but also activates a global .. _`--require-hashes`: Hash-checking mode can be forced on with the ``--require-hashes`` command-line -option:: +option: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install --require-hashes -r requirements.txt + ... + Hashes are required in --require-hashes mode (implicitly on when a hash is + specified for any package). These requirements were missing hashes, + leaving them open to tampering. These are the hashes the downloaded + archives actually had. You can add lines like these to your requirements + files to prevent tampering. + pyelasticsearch==1.0 --hash=sha256:44ddfb1225054d7d6b1d02e9338e7d4809be94edbe9929a2ec0807d38df993fa + more-itertools==2.2 --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install --require-hashes -r requirements.txt + ... + Hashes are required in --require-hashes mode (implicitly on when a hash is + specified for any package). These requirements were missing hashes, + leaving them open to tampering. These are the hashes the downloaded + archives actually had. You can add lines like these to your requirements + files to prevent tampering. + pyelasticsearch==1.0 --hash=sha256:44ddfb1225054d7d6b1d02e9338e7d4809be94edbe9929a2ec0807d38df993fa + more-itertools==2.2 --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 - $ pip install --require-hashes -r requirements.txt - ... - Hashes are required in --require-hashes mode (implicitly on when a hash is - specified for any package). These requirements were missing hashes, - leaving them open to tampering. These are the hashes the downloaded - archives actually had. You can add lines like these to your requirements - files to prevent tampering. - pyelasticsearch==1.0 --hash=sha256:44ddfb1225054d7d6b1d02e9338e7d4809be94edbe9929a2ec0807d38df993fa - more-itertools==2.2 --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 This can be useful in deploy scripts, to ensure that the author of the requirements file provided hashes. It is also a convenient way to bootstrap @@ -692,14 +752,38 @@ Hash-checking mode also works with :ref:`pip download` and :ref:`pip wheel`. A as your project evolves. To be safe, install your project using pip and :ref:`--no-deps `. - Instead of ``python setup.py develop``, use... :: + Instead of ``python setup.py develop``, use... - pip install --no-deps -e . + .. tabs:: - Instead of ``python setup.py install``, use... :: + .. group-tab:: Unix/macOS - pip install --no-deps . + .. code-block:: shell + $ python -m pip install --no-deps -e . + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install --no-deps -e . + + + Instead of ``python setup.py install``, use... + + .. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install --no-deps . + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install --no-deps . Hashes from PyPI ^^^^^^^^^^^^^^^^ @@ -717,9 +801,22 @@ Local project installs ---------------------- pip supports installing local project in both regular mode and editable mode. -You can install local projects by specifying the project path to pip:: +You can install local projects by specifying the project path to pip: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install path/to/SomeProject + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install path/to/SomeProject -$ pip install path/to/SomeProject During regular installation, pip will copy the entire project directory to a temporary location and install from there. The exception is that pip will @@ -736,10 +833,24 @@ being copied. `_ installs. -You can install local projects or VCS projects in "editable" mode:: +You can install local projects or VCS projects in "editable" mode: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install -e path/to/SomeProject + $ python -m pip install -e git+http://repo/my_project.git#egg=SomeProject + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install -e path/to/SomeProject + C:\> py -m pip install -e git+http://repo/my_project.git#egg=SomeProject -$ pip install -e path/to/SomeProject -$ pip install -e git+http://repo/my_project.git#egg=SomeProject (See the :ref:`VCS Support` section above for more information on VCS-related syntax.) @@ -846,113 +957,292 @@ Examples #. Install ``SomePackage`` and its dependencies from `PyPI`_ using :ref:`Requirement Specifiers` - :: + .. tabs:: - $ pip install SomePackage # latest version - $ pip install SomePackage==1.0.4 # specific version - $ pip install 'SomePackage>=1.0.4' # minimum version + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install SomePackage # latest version + $ python -m pip install SomePackage==1.0.4 # specific version + $ python -m pip install 'SomePackage>=1.0.4' # minimum version + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install SomePackage # latest version + C:\> py -m pip install SomePackage==1.0.4 # specific version + C:\> py -m pip install 'SomePackage>=1.0.4' # minimum version #. Install a list of requirements specified in a file. See the :ref:`Requirements files `. - :: + .. tabs:: - $ pip install -r requirements.txt + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install -r requirements.txt + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install -r requirements.txt #. Upgrade an already installed ``SomePackage`` to the latest from PyPI. - :: + .. tabs:: - $ pip install --upgrade SomePackage + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install --upgrade SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install --upgrade SomePackage #. Install a local project in "editable" mode. See the section on :ref:`Editable Installs `. - :: + .. tabs:: - $ pip install -e . # project in current directory - $ pip install -e path/to/project # project in another directory + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install -e . # project in current directory + $ python -m pip install -e path/to/project # project in another directory + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install -e . # project in current directory + C:\> py -m pip install -e path/to/project # project in another directory #. Install a project from VCS - :: + .. tabs:: - $ pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1 + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1 + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1 #. Install a project from VCS in "editable" mode. See the sections on :ref:`VCS Support ` and :ref:`Editable Installs `. - :: + .. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage # from git + $ python -m pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage # from mercurial + $ python -m python -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage # from svn + $ python -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage # from 'feature' branch + $ python -m pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage # from git + C:\> py -m pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage # from mercurial + C:\> py -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage # from svn + C:\> py -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage # from 'feature' branch + C:\> py -m pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory - $ pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage # from git - $ pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage # from mercurial - $ pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage # from svn - $ pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage # from 'feature' branch - $ pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory #. Install a package with `setuptools extras`_. - :: + .. tabs:: - $ pip install SomePackage[PDF] - $ pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@master#subdirectory=subdir_path" - $ pip install .[PDF] # project in current directory - $ pip install SomePackage[PDF]==3.0 - $ pip install SomePackage[PDF,EPUB] # multiple extras + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install SomePackage[PDF] + $ python -m pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@master#subdirectory=subdir_path" + $ python -m pip install .[PDF] # project in current directory + $ python -m pip install SomePackage[PDF]==3.0 + $ python -m pip install SomePackage[PDF,EPUB] # multiple extras + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install SomePackage[PDF] + C:\> py -m pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@master#subdirectory=subdir_path" + C:\> py -m pip install .[PDF] # project in current directory + C:\> py -m pip install SomePackage[PDF]==3.0 + C:\> py -m pip install SomePackage[PDF,EPUB] # multiple extras #. Install a particular source archive file. - :: + .. tabs:: - $ pip install ./downloads/SomePackage-1.0.4.tar.gz - $ pip install http://my.package.repo/SomePackage-1.0.4.zip + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install ./downloads/SomePackage-1.0.4.tar.gz + $ python -m pip install http://my.package.repo/SomePackage-1.0.4.zip + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install ./downloads/SomePackage-1.0.4.tar.gz + C:\> py -m pip install http://my.package.repo/SomePackage-1.0.4.zip #. Install a particular source archive file following :pep:`440` direct references. - :: + .. tabs:: - $ pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl - $ pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl" - $ pip install SomeProject@http://my.package.repo/1.2.3.tar.gz + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl + $ python -m pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl" + $ python -m pip install SomeProject@http://my.package.repo/1.2.3.tar.gz + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl + C:\> py -m pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl" + C:\> py -m pip install SomeProject@http://my.package.repo/1.2.3.tar.gz #. Install from alternative package repositories. - Install from a different index, and not `PyPI`_ :: + Install from a different index, and not `PyPI`_ - $ pip install --index-url http://my.package.repo/simple/ SomePackage + .. tabs:: - Search an additional index during install, in addition to `PyPI`_ :: + .. group-tab:: Unix/macOS - $ pip install --extra-index-url http://my.package.repo/simple SomePackage + .. code-block:: shell - Install from a local flat directory containing archives (and don't scan indexes):: + $ python -m pip install --index-url http://my.package.repo/simple/ SomePackage - $ pip install --no-index --find-links=file:///local/dir/ SomePackage - $ pip install --no-index --find-links=/local/dir/ SomePackage - $ pip install --no-index --find-links=relative/dir/ SomePackage + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install --index-url http://my.package.repo/simple/ SomePackage + + + Search an additional index during install, in addition to `PyPI`_ + + .. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install --extra-index-url http://my.package.repo/simple SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install --extra-index-url http://my.package.repo/simple SomePackage + + + Install from a local flat directory containing archives (and don't scan indexes): + + .. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install --no-index --find-links=file:///local/dir/ SomePackage + $ python -m pip install --no-index --find-links=/local/dir/ SomePackage + $ python -m pip install --no-index --find-links=relative/dir/ SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install --no-index --find-links=file:///local/dir/ SomePackage + C:\> py -m pip install --no-index --find-links=/local/dir/ SomePackage + C:\> py -m pip install --no-index --find-links=relative/dir/ SomePackage #. Find pre-release and development versions, in addition to stable versions. By default, pip only finds stable versions. - :: + .. tabs:: - $ pip install --pre SomePackage + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install --pre SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install --pre SomePackage #. Install packages from source. - Do not use any binary packages:: + Do not use any binary packages - $ pip install SomePackage1 SomePackage2 --no-binary :all: + .. tabs:: - Specify ``SomePackage1`` to be installed from source:: + .. group-tab:: Unix/macOS - $ pip install SomePackage1 SomePackage2 --no-binary SomePackage1 + .. code-block:: shell + + $ python -m pip install SomePackage1 SomePackage2 --no-binary :all: + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install SomePackage1 SomePackage2 --no-binary :all: + + Specify ``SomePackage1`` to be installed from source: + + .. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip install SomePackage1 SomePackage2 --no-binary SomePackage1 + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip install SomePackage1 SomePackage2 --no-binary SomePackage1 ---- diff --git a/docs/pip_sphinxext.py b/docs/pip_sphinxext.py index 6cc7a2c82..0e7571eea 100644 --- a/docs/pip_sphinxext.py +++ b/docs/pip_sphinxext.py @@ -15,11 +15,15 @@ from pip._internal.req.req_file import SUPPORTED_OPTIONS class PipCommandUsage(rst.Directive): required_arguments = 1 + optional_arguments = 4 def run(self): cmd = create_command(self.arguments[0]) + pip_cmd = '$ python -m pip' + if len(self.arguments) > 1: + pip_cmd = " ".join(self.arguments[1:]) usage = dedent( - cmd.usage.replace('%prog', 'pip {}'.format(cmd.name)) + cmd.usage.replace('%prog', '{} {}'.format(pip_cmd, cmd.name)) ).strip() node = nodes.literal_block(usage, usage) return [node] From 423ccfd4f13e43fb0bdee16ee0220e95813ff45f Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Wed, 5 Aug 2020 23:18:37 -0400 Subject: [PATCH 149/209] WIP upd directive to use cmd prefix + upd ref docs --- docs/html/reference/pip_download.rst | 164 ++++++++++++++++++++------ docs/html/reference/pip_freeze.rst | 56 +++++++-- docs/html/reference/pip_install.rst | 4 +- docs/html/reference/pip_uninstall.rst | 39 ++++-- docs/pip_sphinxext.py | 10 +- 5 files changed, 210 insertions(+), 63 deletions(-) diff --git a/docs/html/reference/pip_download.rst b/docs/html/reference/pip_download.rst index b74b1d240..d3c217b94 100644 --- a/docs/html/reference/pip_download.rst +++ b/docs/html/reference/pip_download.rst @@ -11,7 +11,15 @@ pip download Usage ===== -.. pip-command-usage:: download +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: download "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: download "py -m pip" Description @@ -56,11 +64,24 @@ Examples #. Download a package and all of its dependencies - :: + .. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip download SomePackage + $ python -m pip download -d . SomePackage # equivalent to above + $ python -m pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip download SomePackage + C:\> py -m pip download -d . SomePackage # equivalent to above + C:\> py -m pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage - $ pip download SomePackage - $ pip download -d . SomePackage # equivalent to above - $ pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage #. Download a package and all of its dependencies with OSX specific interpreter constraints. This forces OSX 10.10 or lower compatibility. Since OSX deps are forward compatible, @@ -69,51 +90,118 @@ Examples It will also match deps with platform ``any``. Also force the interpreter version to ``27`` (or more generic, i.e. ``2``) and implementation to ``cp`` (or more generic, i.e. ``py``). - :: + .. tabs:: - $ pip download \ - --only-binary=:all: \ - --platform macosx-10_10_x86_64 \ - --python-version 27 \ - --implementation cp \ - SomePackage + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip download \ + --only-binary=:all: \ + --platform macosx-10_10_x86_64 \ + --python-version 27 \ + --implementation cp \ + SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip download ^ + --only-binary=:all: ^ + --platform macosx-10_10_x86_64 ^ + --python-version 27 ^ + --implementation cp ^ + SomePackage #. Download a package and its dependencies with linux specific constraints. Force the interpreter to be any minor version of py3k, and only accept ``cp34m`` or ``none`` as the abi. - :: + .. tabs:: - $ pip download \ - --only-binary=:all: \ - --platform linux_x86_64 \ - --python-version 3 \ - --implementation cp \ - --abi cp34m \ - SomePackage + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip download \ + --only-binary=:all: \ + --platform linux_x86_64 \ + --python-version 3 \ + --implementation cp \ + --abi cp34m \ + SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip download ^ + --only-binary=:all: ^ + --platform linux_x86_64 ^ + --python-version 3 ^ + --implementation cp ^ + --abi cp34m ^ + SomePackage #. Force platform, implementation, and abi agnostic deps. - :: + .. tabs:: - $ pip download \ - --only-binary=:all: \ - --platform any \ - --python-version 3 \ - --implementation py \ - --abi none \ - SomePackage + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip download \ + --only-binary=:all: \ + --platform any \ + --python-version 3 \ + --implementation py \ + --abi none \ + SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip download ^ + --only-binary=:all: ^ + --platform any ^ + --python-version 3 ^ + --implementation py ^ + --abi none ^ + SomePackage #. Even when overconstrained, this will still correctly fetch the pip universal wheel. - :: + .. tabs:: - $ pip download \ - --only-binary=:all: \ - --platform linux_x86_64 \ - --python-version 33 \ - --implementation cp \ - --abi cp34m \ - pip>=8 - $ ls pip-8.1.1-py2.py3-none-any.whl - pip-8.1.1-py2.py3-none-any.whl + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip download \ + --only-binary=:all: \ + --platform linux_x86_64 \ + --python-version 33 \ + --implementation cp \ + --abi cp34m \ + pip>=8 + + $ ls pip-8.1.1-py2.py3-none-any.whl + pip-8.1.1-py2.py3-none-any.whl + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip download ^ + --only-binary=:all: ^ + --platform linux_x86_64 ^ + --python-version 33 ^ + --implementation cp ^ + --abi cp34m ^ + pip>=8 + + C:\> dir pip-8.1.1-py2.py3-none-any.whl + pip-8.1.1-py2.py3-none-any.whl diff --git a/docs/html/reference/pip_freeze.rst b/docs/html/reference/pip_freeze.rst index 31efd571b..cd180e095 100644 --- a/docs/html/reference/pip_freeze.rst +++ b/docs/html/reference/pip_freeze.rst @@ -11,7 +11,15 @@ pip freeze Usage ===== -.. pip-command-usage:: freeze +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: freeze "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: freeze "py -m pip" Description @@ -31,19 +39,45 @@ Examples #. Generate output suitable for a requirements file. - :: + .. tabs:: - $ pip freeze - docutils==0.11 - Jinja2==2.7.2 - MarkupSafe==0.19 - Pygments==1.6 - Sphinx==1.2.2 + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip freeze + docutils==0.11 + Jinja2==2.7.2 + MarkupSafe==0.19 + Pygments==1.6 + Sphinx==1.2.2 + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip freeze + docutils==0.11 + Jinja2==2.7.2 + MarkupSafe==0.19 + Pygments==1.6 + Sphinx==1.2.2 #. Generate a requirements file and then install from it in another environment. - :: + .. tabs:: - $ env1/bin/pip freeze > requirements.txt - $ env2/bin/pip install -r requirements.txt + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ env1/bin/python -m pip freeze > requirements.txt + $ env2/bin/python -m pip install -r requirements.txt + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> env1\bin\python -m pip freeze > requirements.txt + C:\> env2\bin\python -m pip install -r requirements.txt diff --git a/docs/html/reference/pip_install.rst b/docs/html/reference/pip_install.rst index 69ea76275..67cfd03c6 100644 --- a/docs/html/reference/pip_install.rst +++ b/docs/html/reference/pip_install.rst @@ -14,11 +14,11 @@ Usage .. group-tab:: Unix/macOS - .. pip-command-usage:: install $ python -m pip + .. pip-command-usage:: install "python -m pip" .. group-tab:: Windows - .. pip-command-usage:: install C:\> py -m pip + .. pip-command-usage:: install "py -m pip" diff --git a/docs/html/reference/pip_uninstall.rst b/docs/html/reference/pip_uninstall.rst index 67d752d6b..165ac4cd5 100644 --- a/docs/html/reference/pip_uninstall.rst +++ b/docs/html/reference/pip_uninstall.rst @@ -10,7 +10,15 @@ pip uninstall Usage ===== -.. pip-command-usage:: uninstall +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: uninstall "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: uninstall "py -m pip" Description @@ -30,11 +38,26 @@ Examples #. Uninstall a package. - :: + .. tabs:: - $ pip uninstall simplejson - Uninstalling simplejson: - /home/me/env/lib/python2.7/site-packages/simplejson - /home/me/env/lib/python2.7/site-packages/simplejson-2.2.1-py2.7.egg-info - Proceed (y/n)? y - Successfully uninstalled simplejson + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip uninstall simplejson + Uninstalling simplejson: + /home/me/env/lib/python2.7/site-packages/simplejson + /home/me/env/lib/python2.7/site-packages/simplejson-2.2.1-py2.7.egg-info + Proceed (y/n)? y + Successfully uninstalled simplejson + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip uninstall simplejson + Uninstalling simplejson: + /home/me/env/lib/python2.7/site-packages/simplejson + /home/me/env/lib/python2.7/site-packages/simplejson-2.2.1-py2.7.egg-info + Proceed (y/n)? y + Successfully uninstalled simplejson diff --git a/docs/pip_sphinxext.py b/docs/pip_sphinxext.py index 0e7571eea..9386d71e7 100644 --- a/docs/pip_sphinxext.py +++ b/docs/pip_sphinxext.py @@ -15,15 +15,17 @@ from pip._internal.req.req_file import SUPPORTED_OPTIONS class PipCommandUsage(rst.Directive): required_arguments = 1 - optional_arguments = 4 + optional_arguments = 3 def run(self): cmd = create_command(self.arguments[0]) - pip_cmd = '$ python -m pip' + cmd_prefix = 'python -m pip' if len(self.arguments) > 1: - pip_cmd = " ".join(self.arguments[1:]) + cmd_prefix = " ".join(self.arguments[1:]) + cmd_prefix = cmd_prefix.strip('"') + cmd_prefix = cmd_prefix.strip("'") usage = dedent( - cmd.usage.replace('%prog', '{} {}'.format(pip_cmd, cmd.name)) + cmd.usage.replace('%prog', '{} {}'.format(cmd_prefix, cmd.name)) ).strip() node = nodes.literal_block(usage, usage) return [node] From 4a6276bfc98d17d523c69e9739e82eeb900a440b Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Thu, 6 Aug 2020 16:06:10 -0400 Subject: [PATCH 150/209] Add tabs to ref docs --- docs/html/reference/pip_cache.rst | 10 +- docs/html/reference/pip_check.rst | 79 +++++++++--- docs/html/reference/pip_config.rst | 10 +- docs/html/reference/pip_debug.rst | 10 +- docs/html/reference/pip_hash.rst | 47 +++++-- docs/html/reference/pip_list.rst | 201 +++++++++++++++++++++++------ docs/html/reference/pip_search.rst | 30 ++++- docs/html/reference/pip_show.rst | 176 +++++++++++++++++-------- docs/html/reference/pip_wheel.rst | 75 +++++++++-- 9 files changed, 500 insertions(+), 138 deletions(-) diff --git a/docs/html/reference/pip_cache.rst b/docs/html/reference/pip_cache.rst index 8ad99f65c..35e0dfcad 100644 --- a/docs/html/reference/pip_cache.rst +++ b/docs/html/reference/pip_cache.rst @@ -9,7 +9,15 @@ pip cache Usage ***** -.. pip-command-usage:: cache +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: cache "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: cache "py -m pip" Description *********** diff --git a/docs/html/reference/pip_check.rst b/docs/html/reference/pip_check.rst index a12d5b3ec..07d700475 100644 --- a/docs/html/reference/pip_check.rst +++ b/docs/html/reference/pip_check.rst @@ -10,7 +10,15 @@ pip check Usage ===== -.. pip-command-usage:: check +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: check "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: check "py -m pip" Description @@ -24,27 +32,66 @@ Examples #. If all dependencies are compatible: - :: + .. tabs:: - $ pip check - No broken requirements found. - $ echo $? - 0 + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip check + No broken requirements found. + $ echo $? + 0 + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip check + No broken requirements found. + C:\> echo %errorlevel% + 0 #. If a package is missing: - :: + .. tabs:: - $ pip check - pyramid 1.5.2 requires WebOb, which is not installed. - $ echo $? - 1 + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip check + pyramid 1.5.2 requires WebOb, which is not installed. + $ echo $? + 1 + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip check + pyramid 1.5.2 requires WebOb, which is not installed. + C:\> echo %errorlevel% + 1 #. If a package has the wrong version: - :: + .. tabs:: - $ pip check - pyramid 1.5.2 has requirement WebOb>=1.3.1, but you have WebOb 0.8. - $ echo $? - 1 + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip check + pyramid 1.5.2 has requirement WebOb>=1.3.1, but you have WebOb 0.8. + $ echo $? + 1 + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip check + pyramid 1.5.2 has requirement WebOb>=1.3.1, but you have WebOb 0.8. + C:\> echo %errorlevel% + 1 diff --git a/docs/html/reference/pip_config.rst b/docs/html/reference/pip_config.rst index 70d9406c5..d9bf0afc8 100644 --- a/docs/html/reference/pip_config.rst +++ b/docs/html/reference/pip_config.rst @@ -11,7 +11,15 @@ pip config Usage ===== -.. pip-command-usage:: config +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: config "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: config "py -m pip" Description diff --git a/docs/html/reference/pip_debug.rst b/docs/html/reference/pip_debug.rst index da147bcf2..2ef98228a 100644 --- a/docs/html/reference/pip_debug.rst +++ b/docs/html/reference/pip_debug.rst @@ -10,7 +10,15 @@ pip debug Usage ===== -.. pip-command-usage:: debug +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: debug "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: debug "py -m pip" .. warning:: diff --git a/docs/html/reference/pip_hash.rst b/docs/html/reference/pip_hash.rst index dbf1f3e94..7ed39280c 100644 --- a/docs/html/reference/pip_hash.rst +++ b/docs/html/reference/pip_hash.rst @@ -10,7 +10,15 @@ pip hash Usage ===== -.. pip-command-usage:: hash +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: hash "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: hash "py -m pip" Description @@ -39,13 +47,32 @@ Options Example ======= -Compute the hash of a downloaded archive:: +Compute the hash of a downloaded archive: - $ pip download SomePackage - Collecting SomePackage - Downloading SomePackage-2.2.tar.gz - Saved ./pip_downloads/SomePackage-2.2.tar.gz - Successfully downloaded SomePackage - $ pip hash ./pip_downloads/SomePackage-2.2.tar.gz - ./pip_downloads/SomePackage-2.2.tar.gz: - --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip download SomePackage + Collecting SomePackage + Downloading SomePackage-2.2.tar.gz + Saved ./pip_downloads/SomePackage-2.2.tar.gz + Successfully downloaded SomePackage + $ python -m pip hash ./pip_downloads/SomePackage-2.2.tar.gz + ./pip_downloads/SomePackage-2.2.tar.gz: + --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip download SomePackage + Collecting SomePackage + Downloading SomePackage-2.2.tar.gz + Saved ./pip_downloads/SomePackage-2.2.tar.gz + Successfully downloaded SomePackage + C:\> py -m pip hash ./pip_downloads/SomePackage-2.2.tar.gz + ./pip_downloads/SomePackage-2.2.tar.gz: + --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 diff --git a/docs/html/reference/pip_list.rst b/docs/html/reference/pip_list.rst index 15d0920a7..bfdef9ebe 100644 --- a/docs/html/reference/pip_list.rst +++ b/docs/html/reference/pip_list.rst @@ -10,7 +10,15 @@ pip list Usage ===== -.. pip-command-usage:: list +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: list "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: list "py -m pip" Description @@ -32,75 +40,182 @@ Examples #. List installed packages. - :: + .. tabs:: - $ pip list - docutils (0.10) - Jinja2 (2.7.2) - MarkupSafe (0.18) - Pygments (1.6) - Sphinx (1.2.1) + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip list + docutils (0.10) + Jinja2 (2.7.2) + MarkupSafe (0.18) + Pygments (1.6) + Sphinx (1.2.1) + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip list + docutils (0.10) + Jinja2 (2.7.2) + MarkupSafe (0.18) + Pygments (1.6) + Sphinx (1.2.1) #. List outdated packages (excluding editables), and the latest version available. - :: + .. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip list --outdated + docutils (Current: 0.10 Latest: 0.11) + Sphinx (Current: 1.2.1 Latest: 1.2.2) + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip list --outdated + docutils (Current: 0.10 Latest: 0.11) + Sphinx (Current: 1.2.1 Latest: 1.2.2) - $ pip list --outdated - docutils (Current: 0.10 Latest: 0.11) - Sphinx (Current: 1.2.1 Latest: 1.2.2) #. List installed packages with column formatting. - :: + .. tabs:: - $ pip list --format columns - Package Version - ------- ------- - docopt 0.6.2 - idlex 1.13 - jedi 0.9.0 + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip list --format columns + Package Version + ------- ------- + docopt 0.6.2 + idlex 1.13 + jedi 0.9.0 + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip list --format columns + Package Version + ------- ------- + docopt 0.6.2 + idlex 1.13 + jedi 0.9.0 #. List outdated packages with column formatting. - :: + .. tabs:: - $ pip list -o --format columns - Package Version Latest Type - ---------- ------- ------ ----- - retry 0.8.1 0.9.1 wheel - setuptools 20.6.7 21.0.0 wheel + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip list -o --format columns + Package Version Latest Type + ---------- ------- ------ ----- + retry 0.8.1 0.9.1 wheel + setuptools 20.6.7 21.0.0 wheel + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip list -o --format columns + Package Version Latest Type + ---------- ------- ------ ----- + retry 0.8.1 0.9.1 wheel + setuptools 20.6.7 21.0.0 wheel #. List packages that are not dependencies of other packages. Can be combined with other options. - :: + .. tabs:: - $ pip list --outdated --not-required - docutils (Current: 0.10 Latest: 0.11) + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip list --outdated --not-required + docutils (Current: 0.10 Latest: 0.11) + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip list --outdated --not-required + docutils (Current: 0.10 Latest: 0.11) #. Use legacy formatting - :: + .. tabs:: - $ pip list --format=legacy - colorama (0.3.7) - docopt (0.6.2) - idlex (1.13) - jedi (0.9.0) + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip list --format=legacy + colorama (0.3.7) + docopt (0.6.2) + idlex (1.13) + jedi (0.9.0) + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip list --format=legacy + colorama (0.3.7) + docopt (0.6.2) + idlex (1.13) + jedi (0.9.0) #. Use json formatting - :: + .. tabs:: - $ pip list --format=json - [{'name': 'colorama', 'version': '0.3.7'}, {'name': 'docopt', 'version': '0.6.2'}, ... + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip list --format=json + [{'name': 'colorama', 'version': '0.3.7'}, {'name': 'docopt', 'version': '0.6.2'}, ... + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip list --format=json + [{'name': 'colorama', 'version': '0.3.7'}, {'name': 'docopt', 'version': '0.6.2'}, ... #. Use freeze formatting - :: + .. tabs:: - $ pip list --format=freeze - colorama==0.3.7 - docopt==0.6.2 - idlex==1.13 - jedi==0.9.0 + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip list --format=freeze + colorama==0.3.7 + docopt==0.6.2 + idlex==1.13 + jedi==0.9.0 + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip list --format=freeze + colorama==0.3.7 + docopt==0.6.2 + idlex==1.13 + jedi==0.9.0 diff --git a/docs/html/reference/pip_search.rst b/docs/html/reference/pip_search.rst index db1bd2be8..e0fc3e25b 100644 --- a/docs/html/reference/pip_search.rst +++ b/docs/html/reference/pip_search.rst @@ -10,7 +10,15 @@ pip search Usage ===== -.. pip-command-usage:: search +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: search "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: search "py -m pip" Description @@ -30,8 +38,20 @@ Examples #. Search for "peppercorn" - :: + .. tabs:: - $ pip search peppercorn - pepperedform - Helpers for using peppercorn with formprocess. - peppercorn - A library for converting a token stream into [...] + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip search peppercorn + pepperedform - Helpers for using peppercorn with formprocess. + peppercorn - A library for converting a token stream into [...] + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip search peppercorn + pepperedform - Helpers for using peppercorn with formprocess. + peppercorn - A library for converting a token stream into [...] diff --git a/docs/html/reference/pip_show.rst b/docs/html/reference/pip_show.rst index e9568b6b0..ae9182f54 100644 --- a/docs/html/reference/pip_show.rst +++ b/docs/html/reference/pip_show.rst @@ -10,7 +10,15 @@ pip show Usage ===== -.. pip-command-usage:: show +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: show "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: show "py -m pip" Description @@ -30,58 +38,124 @@ Examples #. Show information about a package: - :: + .. tabs:: - $ pip show sphinx - Name: Sphinx - Version: 1.4.5 - Summary: Python documentation generator - Home-page: http://sphinx-doc.org/ - Author: Georg Brandl - Author-email: georg@python.org - License: BSD - Location: /my/env/lib/python2.7/site-packages - Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip show sphinx + Name: Sphinx + Version: 1.4.5 + Summary: Python documentation generator + Home-page: http://sphinx-doc.org/ + Author: Georg Brandl + Author-email: georg@python.org + License: BSD + Location: /my/env/lib/python2.7/site-packages + Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip show sphinx + Name: Sphinx + Version: 1.4.5 + Summary: Python documentation generator + Home-page: http://sphinx-doc.org/ + Author: Georg Brandl + Author-email: georg@python.org + License: BSD + Location: /my/env/lib/python2.7/site-packages + Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six #. Show all information about a package - :: + .. tabs:: - $ pip show --verbose sphinx - Name: Sphinx - Version: 1.4.5 - Summary: Python documentation generator - Home-page: http://sphinx-doc.org/ - Author: Georg Brandl - Author-email: georg@python.org - License: BSD - Location: /my/env/lib/python2.7/site-packages - Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six - Metadata-Version: 2.0 - Installer: - Classifiers: - Development Status :: 5 - Production/Stable - Environment :: Console - Environment :: Web Environment - Intended Audience :: Developers - Intended Audience :: Education - License :: OSI Approved :: BSD License - Operating System :: OS Independent - Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 3 - Framework :: Sphinx - Framework :: Sphinx :: Extension - Framework :: Sphinx :: Theme - Topic :: Documentation - Topic :: Documentation :: Sphinx - Topic :: Text Processing - Topic :: Utilities - Entry-points: - [console_scripts] - sphinx-apidoc = sphinx.apidoc:main - sphinx-autogen = sphinx.ext.autosummary.generate:main - sphinx-build = sphinx:main - sphinx-quickstart = sphinx.quickstart:main - [distutils.commands] - build_sphinx = sphinx.setup_command:BuildDoc + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip show --verbose sphinx + Name: Sphinx + Version: 1.4.5 + Summary: Python documentation generator + Home-page: http://sphinx-doc.org/ + Author: Georg Brandl + Author-email: georg@python.org + License: BSD + Location: /my/env/lib/python2.7/site-packages + Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six + Metadata-Version: 2.0 + Installer: + Classifiers: + Development Status :: 5 - Production/Stable + Environment :: Console + Environment :: Web Environment + Intended Audience :: Developers + Intended Audience :: Education + License :: OSI Approved :: BSD License + Operating System :: OS Independent + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 3 + Framework :: Sphinx + Framework :: Sphinx :: Extension + Framework :: Sphinx :: Theme + Topic :: Documentation + Topic :: Documentation :: Sphinx + Topic :: Text Processing + Topic :: Utilities + Entry-points: + [console_scripts] + sphinx-apidoc = sphinx.apidoc:main + sphinx-autogen = sphinx.ext.autosummary.generate:main + sphinx-build = sphinx:main + sphinx-quickstart = sphinx.quickstart:main + [distutils.commands] + build_sphinx = sphinx.setup_command:BuildDoc + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip show --verbose sphinx + Name: Sphinx + Version: 1.4.5 + Summary: Python documentation generator + Home-page: http://sphinx-doc.org/ + Author: Georg Brandl + Author-email: georg@python.org + License: BSD + Location: /my/env/lib/python2.7/site-packages + Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six + Metadata-Version: 2.0 + Installer: + Classifiers: + Development Status :: 5 - Production/Stable + Environment :: Console + Environment :: Web Environment + Intended Audience :: Developers + Intended Audience :: Education + License :: OSI Approved :: BSD License + Operating System :: OS Independent + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 3 + Framework :: Sphinx + Framework :: Sphinx :: Extension + Framework :: Sphinx :: Theme + Topic :: Documentation + Topic :: Documentation :: Sphinx + Topic :: Text Processing + Topic :: Utilities + Entry-points: + [console_scripts] + sphinx-apidoc = sphinx.apidoc:main + sphinx-autogen = sphinx.ext.autosummary.generate:main + sphinx-build = sphinx:main + sphinx-quickstart = sphinx.quickstart:main + [distutils.commands] + build_sphinx = sphinx.setup_command:BuildDoc diff --git a/docs/html/reference/pip_wheel.rst b/docs/html/reference/pip_wheel.rst index dc32dda46..0a11c6e25 100644 --- a/docs/html/reference/pip_wheel.rst +++ b/docs/html/reference/pip_wheel.rst @@ -11,7 +11,15 @@ pip wheel Usage ===== -.. pip-command-usage:: wheel +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: wheel "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: wheel "py -m pip" Description @@ -24,9 +32,22 @@ Build System Interface ---------------------- In order for pip to build a wheel, ``setup.py`` must implement the -``bdist_wheel`` command with the following syntax:: +``bdist_wheel`` command with the following syntax: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python setup.py bdist_wheel -d TARGET + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py setup.py bdist_wheel -d TARGET - python setup.py bdist_wheel -d TARGET This command must create a wheel compatible with the invoking Python interpreter, and save that wheel in the directory TARGET. @@ -39,9 +60,22 @@ Customising the build It is possible using ``--global-option`` to include additional build commands with their arguments in the ``setup.py`` command. This is currently the only way to influence the building of C extensions from the command line. For -example:: +example: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip wheel --global-option bdist_ext --global-option -DFOO wheel + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip wheel --global-option bdist_ext --global-option -DFOO wheel - pip wheel --global-option bdist_ext --global-option -DFOO wheel will result in a build command of @@ -69,13 +103,34 @@ Examples #. Build wheels for a requirement (and all its dependencies), and then install - :: + .. tabs:: - $ pip wheel --wheel-dir=/tmp/wheelhouse SomePackage - $ pip install --no-index --find-links=/tmp/wheelhouse SomePackage + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip wheel --wheel-dir=/tmp/wheelhouse SomePackage + $ python -m pip install --no-index --find-links=/tmp/wheelhouse SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip wheel --wheel-dir=/tmp/wheelhouse SomePackage + C:\> py -m pip install --no-index --find-links=/tmp/wheelhouse SomePackage #. Build a wheel for a package from source - :: + .. tabs:: - $ pip wheel --no-binary SomePackage SomePackage + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python -m pip wheel --no-binary SomePackage SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py -m pip wheel --no-binary SomePackage SomePackage From 7673712d3e0fd4718367c9a183b89a56605f474d Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Fri, 7 Aug 2020 14:14:23 -0400 Subject: [PATCH 151/209] Add tabs to get-pip.py install page --- docs/html/installing.rst | 79 ++++++++++++++++++++++++++++++++----- docs/html/reference/pip.rst | 16 +++++++- 2 files changed, 83 insertions(+), 12 deletions(-) diff --git a/docs/html/installing.rst b/docs/html/installing.rst index 919f47606..14d517160 100644 --- a/docs/html/installing.rst +++ b/docs/html/installing.rst @@ -26,9 +26,21 @@ this link: `get-pip.py curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py Then run the following command in the folder where you -have downloaded ``get-pip.py``:: +have downloaded ``get-pip.py``: - python get-pip.py +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python get-pip.py + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py get-pip.py .. warning:: @@ -67,23 +79,70 @@ get-pip.py options install Options>` and the :ref:`general options `. Below are some examples: -Install from local copies of pip and setuptools:: +Install from local copies of pip and setuptools: - python get-pip.py --no-index --find-links=/local/copies +.. tabs:: -Install to the user site [3]_:: + .. group-tab:: Unix/macOS - python get-pip.py --user + .. code-block:: shell -Install behind a proxy:: + $ python get-pip.py --no-index --find-links=/local/copies - python get-pip.py --proxy="http://[user:passwd@]proxy.server:port" + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py get-pip.py --no-index --find-links=/local/copies + +Install to the user site [3]_: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python get-pip.py --user + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py get-pip.py --user + +Install behind a proxy: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python get-pip.py --proxy="http://[user:passwd@]proxy.server:port" + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py get-pip.py --proxy="http://[user:passwd@]proxy.server:port" ``get-pip.py`` can also be used to install a specified combination of ``pip``, -``setuptools``, and ``wheel`` using the same requirements syntax as pip:: +``setuptools``, and ``wheel`` using the same requirements syntax as pip: - python get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 +.. tabs:: + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 Using Linux Package Managers ============================ diff --git a/docs/html/reference/pip.rst b/docs/html/reference/pip.rst index 9ca06e2c7..3694ca7f5 100644 --- a/docs/html/reference/pip.rst +++ b/docs/html/reference/pip.rst @@ -230,9 +230,21 @@ Build Options The ``--global-option`` and ``--build-option`` arguments to the ``pip install`` and ``pip wheel`` inject additional arguments into the ``setup.py`` command (``--build-option`` is only available in ``pip wheel``). These arguments are -included in the command as follows:: +included in the command as follows: - python setup.py BUILD COMMAND +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + $ python setup.py BUILD COMMAND + + .. group-tab:: Windows + + .. code-block:: shell + + C:\> py setup.py BUILD COMMAND The options are passed unmodified, and presently offer direct access to the distutils command line. Use of ``--global-option`` and ``--build-option`` From c36bd748f3194d885511d58170b24e57f8330b69 Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Tue, 11 Aug 2020 08:50:14 -0400 Subject: [PATCH 152/209] fix tox.ini and conf.py to use sphinx_tabs_nowarn --- docs/html/conf.py | 3 +++ tox.ini | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/html/conf.py b/docs/html/conf.py index f0e880f5e..9d0b7a2fa 100644 --- a/docs/html/conf.py +++ b/docs/html/conf.py @@ -135,6 +135,9 @@ extlinks = { 'pypi': ('https://pypi.org/project/%s/', ''), } +# Turn off sphinx build warnings because of sphinx tabs during man pages build +sphinx_tabs_nowarn = True + # -- Options for HTML output -------------------------------------------------- # The theme to use for HTML and HTML Help pages. Major themes that come with diff --git a/tox.ini b/tox.ini index a673b0f59..82e9abc68 100644 --- a/tox.ini +++ b/tox.ini @@ -56,7 +56,7 @@ commands = # can not use a different configuration directory vs source directory on RTD # currently -- https://github.com/rtfd/readthedocs.org/issues/1543. # That is why we have a "-c docs/html" in the next line. - sphinx-build -W -d {envtmpdir}/doctrees/man -b man docs/man docs/build/man -c docs/html -D extensions=sphinx.ext.extlinks,pip_sphinxext,sphinx.ext.intersphinx + sphinx-build -W -d {envtmpdir}/doctrees/man -b man docs/man docs/build/man -c docs/html [testenv:lint] skip_install = True From 28a00633d8614b8042fa29e50fce6d7c6ba278d7 Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Sun, 16 Aug 2020 14:07:09 -0400 Subject: [PATCH 153/209] Remove platform prompt when not output is displayed --- docs/html/development/getting-started.rst | 16 +-- docs/html/installing.rst | 24 ++-- docs/html/reference/pip.rst | 4 +- docs/html/reference/pip_download.rst | 92 +++++++------- docs/html/reference/pip_freeze.rst | 8 +- docs/html/reference/pip_install.rst | 148 +++++++++++----------- docs/html/reference/pip_wheel.rst | 20 +-- docs/html/user_guide.rst | 114 ++++++++--------- 8 files changed, 213 insertions(+), 213 deletions(-) diff --git a/docs/html/development/getting-started.rst b/docs/html/development/getting-started.rst index 9810bfaa4..1bc4a5516 100644 --- a/docs/html/development/getting-started.rst +++ b/docs/html/development/getting-started.rst @@ -44,19 +44,19 @@ You can then invoke your local source tree pip normally. .. code-block:: shell - $ virtualenv venv # You can also use "python -m venv venv" from python3.3+ - $ source venv/bin/activate - $ python -m pip install -e . - $ python -m pip --version + virtualenv venv # You can also use "python -m venv venv" from python3.3+ + source venv/bin/activate + python -m pip install -e . + python -m pip --version .. group-tab:: Windows .. code-block:: shell - C:\> virtualenv venv # You can also use "py -m venv venv" from python3.3+ - C:\> source venv/bin/activate - C:\> py -m pip install -e . - C:\> py -m pip --version + virtualenv venv # You can also use "py -m venv venv" from python3.3+ + venv\Scripts\activate + py -m pip install -e . + py -m pip --version Running Tests ============= diff --git a/docs/html/installing.rst b/docs/html/installing.rst index 14d517160..e6717881a 100644 --- a/docs/html/installing.rst +++ b/docs/html/installing.rst @@ -34,13 +34,13 @@ have downloaded ``get-pip.py``: .. code-block:: shell - $ python get-pip.py + python get-pip.py .. group-tab:: Windows .. code-block:: shell - C:\> py get-pip.py + py get-pip.py .. warning:: @@ -87,13 +87,13 @@ Install from local copies of pip and setuptools: .. code-block:: shell - $ python get-pip.py --no-index --find-links=/local/copies + python get-pip.py --no-index --find-links=/local/copies .. group-tab:: Windows .. code-block:: shell - C:\> py get-pip.py --no-index --find-links=/local/copies + py get-pip.py --no-index --find-links=/local/copies Install to the user site [3]_: @@ -103,13 +103,13 @@ Install to the user site [3]_: .. code-block:: shell - $ python get-pip.py --user + python get-pip.py --user .. group-tab:: Windows .. code-block:: shell - C:\> py get-pip.py --user + py get-pip.py --user Install behind a proxy: @@ -119,13 +119,13 @@ Install behind a proxy: .. code-block:: shell - $ python get-pip.py --proxy="http://[user:passwd@]proxy.server:port" + python get-pip.py --proxy="http://[user:passwd@]proxy.server:port" .. group-tab:: Windows .. code-block:: shell - C:\> py get-pip.py --proxy="http://[user:passwd@]proxy.server:port" + py get-pip.py --proxy="http://[user:passwd@]proxy.server:port" ``get-pip.py`` can also be used to install a specified combination of ``pip``, ``setuptools``, and ``wheel`` using the same requirements syntax as pip: @@ -136,13 +136,13 @@ Install behind a proxy: .. code-block:: shell - $ python get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 + python get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 .. group-tab:: Windows .. code-block:: shell - C:\> py get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 + py get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 Using Linux Package Managers ============================ @@ -162,13 +162,13 @@ Upgrading pip .. code-block:: shell - $ python -m pip install -U pip + python -m pip install -U pip .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install -U pip + py -m pip install -U pip .. _compatibility-requirements: diff --git a/docs/html/reference/pip.rst b/docs/html/reference/pip.rst index 3694ca7f5..a0c1148bf 100644 --- a/docs/html/reference/pip.rst +++ b/docs/html/reference/pip.rst @@ -238,13 +238,13 @@ included in the command as follows: .. code-block:: shell - $ python setup.py BUILD COMMAND + python setup.py BUILD COMMAND .. group-tab:: Windows .. code-block:: shell - C:\> py setup.py BUILD COMMAND + py setup.py BUILD COMMAND The options are passed unmodified, and presently offer direct access to the distutils command line. Use of ``--global-option`` and ``--build-option`` diff --git a/docs/html/reference/pip_download.rst b/docs/html/reference/pip_download.rst index d3c217b94..141b6e1fa 100644 --- a/docs/html/reference/pip_download.rst +++ b/docs/html/reference/pip_download.rst @@ -70,17 +70,17 @@ Examples .. code-block:: shell - $ python -m pip download SomePackage - $ python -m pip download -d . SomePackage # equivalent to above - $ python -m pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage + python -m pip download SomePackage + python -m pip download -d . SomePackage # equivalent to above + python -m pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip download SomePackage - C:\> py -m pip download -d . SomePackage # equivalent to above - C:\> py -m pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage + py -m pip download SomePackage + py -m pip download -d . SomePackage # equivalent to above + py -m pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage #. Download a package and all of its dependencies with OSX specific interpreter constraints. @@ -96,23 +96,23 @@ Examples .. code-block:: shell - $ python -m pip download \ - --only-binary=:all: \ - --platform macosx-10_10_x86_64 \ - --python-version 27 \ - --implementation cp \ - SomePackage + python -m pip download \ + --only-binary=:all: \ + --platform macosx-10_10_x86_64 \ + --python-version 27 \ + --implementation cp \ + SomePackage .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip download ^ - --only-binary=:all: ^ - --platform macosx-10_10_x86_64 ^ - --python-version 27 ^ - --implementation cp ^ - SomePackage + py -m pip download ^ + --only-binary=:all: ^ + --platform macosx-10_10_x86_64 ^ + --python-version 27 ^ + --implementation cp ^ + SomePackage #. Download a package and its dependencies with linux specific constraints. Force the interpreter to be any minor version of py3k, and only accept @@ -124,25 +124,25 @@ Examples .. code-block:: shell - $ python -m pip download \ - --only-binary=:all: \ - --platform linux_x86_64 \ - --python-version 3 \ - --implementation cp \ - --abi cp34m \ - SomePackage + python -m pip download \ + --only-binary=:all: \ + --platform linux_x86_64 \ + --python-version 3 \ + --implementation cp \ + --abi cp34m \ + SomePackage .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip download ^ - --only-binary=:all: ^ - --platform linux_x86_64 ^ - --python-version 3 ^ - --implementation cp ^ - --abi cp34m ^ - SomePackage + py -m pip download ^ + --only-binary=:all: ^ + --platform linux_x86_64 ^ + --python-version 3 ^ + --implementation cp ^ + --abi cp34m ^ + SomePackage #. Force platform, implementation, and abi agnostic deps. @@ -152,25 +152,25 @@ Examples .. code-block:: shell - $ python -m pip download \ - --only-binary=:all: \ - --platform any \ - --python-version 3 \ - --implementation py \ - --abi none \ - SomePackage + python -m pip download \ + --only-binary=:all: \ + --platform any \ + --python-version 3 \ + --implementation py \ + --abi none \ + SomePackage .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip download ^ - --only-binary=:all: ^ - --platform any ^ - --python-version 3 ^ - --implementation py ^ - --abi none ^ - SomePackage + py -m pip download ^ + --only-binary=:all: ^ + --platform any ^ + --python-version 3 ^ + --implementation py ^ + --abi none ^ + SomePackage #. Even when overconstrained, this will still correctly fetch the pip universal wheel. diff --git a/docs/html/reference/pip_freeze.rst b/docs/html/reference/pip_freeze.rst index cd180e095..91c80979a 100644 --- a/docs/html/reference/pip_freeze.rst +++ b/docs/html/reference/pip_freeze.rst @@ -72,12 +72,12 @@ Examples .. code-block:: shell - $ env1/bin/python -m pip freeze > requirements.txt - $ env2/bin/python -m pip install -r requirements.txt + env1/bin/python -m pip freeze > requirements.txt + env2/bin/python -m pip install -r requirements.txt .. group-tab:: Windows .. code-block:: shell - C:\> env1\bin\python -m pip freeze > requirements.txt - C:\> env2\bin\python -m pip install -r requirements.txt + env1\bin\python -m pip freeze > requirements.txt + env2\bin\python -m pip install -r requirements.txt diff --git a/docs/html/reference/pip_install.rst b/docs/html/reference/pip_install.rst index 67cfd03c6..b2da26240 100644 --- a/docs/html/reference/pip_install.rst +++ b/docs/html/reference/pip_install.rst @@ -422,13 +422,13 @@ Then, to install from this repository, the syntax would be: .. code-block:: shell - $ python -m pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir" + python -m pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir" .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir" + py -m pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir" Git @@ -760,13 +760,13 @@ Hash-checking mode also works with :ref:`pip download` and :ref:`pip wheel`. A .. code-block:: shell - $ python -m pip install --no-deps -e . + python -m pip install --no-deps -e . .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install --no-deps -e . + py -m pip install --no-deps -e . Instead of ``python setup.py install``, use... @@ -777,13 +777,13 @@ Hash-checking mode also works with :ref:`pip download` and :ref:`pip wheel`. A .. code-block:: shell - $ python -m pip install --no-deps . + python -m pip install --no-deps . .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install --no-deps . + py -m pip install --no-deps . Hashes from PyPI ^^^^^^^^^^^^^^^^ @@ -809,13 +809,13 @@ You can install local projects by specifying the project path to pip: .. code-block:: shell - $ python -m pip install path/to/SomeProject + python -m pip install path/to/SomeProject .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install path/to/SomeProject + py -m pip install path/to/SomeProject During regular installation, pip will copy the entire project directory to a @@ -841,15 +841,15 @@ You can install local projects or VCS projects in "editable" mode: .. code-block:: shell - $ python -m pip install -e path/to/SomeProject - $ python -m pip install -e git+http://repo/my_project.git#egg=SomeProject + python -m pip install -e path/to/SomeProject + python -m pip install -e git+http://repo/my_project.git#egg=SomeProject .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install -e path/to/SomeProject - C:\> py -m pip install -e git+http://repo/my_project.git#egg=SomeProject + py -m pip install -e path/to/SomeProject + py -m pip install -e git+http://repo/my_project.git#egg=SomeProject (See the :ref:`VCS Support` section above for more information on VCS-related syntax.) @@ -963,17 +963,17 @@ Examples .. code-block:: shell - $ python -m pip install SomePackage # latest version - $ python -m pip install SomePackage==1.0.4 # specific version - $ python -m pip install 'SomePackage>=1.0.4' # minimum version + python -m pip install SomePackage # latest version + python -m pip install SomePackage==1.0.4 # specific version + python -m pip install 'SomePackage>=1.0.4' # minimum version .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install SomePackage # latest version - C:\> py -m pip install SomePackage==1.0.4 # specific version - C:\> py -m pip install 'SomePackage>=1.0.4' # minimum version + py -m pip install SomePackage # latest version + py -m pip install SomePackage==1.0.4 # specific version + py -m pip install 'SomePackage>=1.0.4' # minimum version #. Install a list of requirements specified in a file. See the :ref:`Requirements files `. @@ -984,13 +984,13 @@ Examples .. code-block:: shell - $ python -m pip install -r requirements.txt + python -m pip install -r requirements.txt .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install -r requirements.txt + py -m pip install -r requirements.txt #. Upgrade an already installed ``SomePackage`` to the latest from PyPI. @@ -1001,13 +1001,13 @@ Examples .. code-block:: shell - $ python -m pip install --upgrade SomePackage + python -m pip install --upgrade SomePackage .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install --upgrade SomePackage + py -m pip install --upgrade SomePackage #. Install a local project in "editable" mode. See the section on :ref:`Editable Installs `. @@ -1018,15 +1018,15 @@ Examples .. code-block:: shell - $ python -m pip install -e . # project in current directory - $ python -m pip install -e path/to/project # project in another directory + python -m pip install -e . # project in current directory + python -m pip install -e path/to/project # project in another directory .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install -e . # project in current directory - C:\> py -m pip install -e path/to/project # project in another directory + py -m pip install -e . # project in current directory + py -m pip install -e path/to/project # project in another directory #. Install a project from VCS @@ -1037,13 +1037,13 @@ Examples .. code-block:: shell - $ python -m pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1 + python -m pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1 .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1 + py -m pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1 #. Install a project from VCS in "editable" mode. See the sections on :ref:`VCS Support ` and :ref:`Editable Installs `. @@ -1054,21 +1054,21 @@ Examples .. code-block:: shell - $ python -m pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage # from git - $ python -m pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage # from mercurial - $ python -m python -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage # from svn - $ python -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage # from 'feature' branch - $ python -m pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory + python -m pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage # from git + python -m pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage # from mercurial + python -m python -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage # from svn + python -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage # from 'feature' branch + python -m pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage # from git - C:\> py -m pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage # from mercurial - C:\> py -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage # from svn - C:\> py -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage # from 'feature' branch - C:\> py -m pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory + py -m pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage # from git + py -m pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage # from mercurial + py -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage # from svn + py -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage # from 'feature' branch + py -m pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory #. Install a package with `setuptools extras`_. @@ -1079,21 +1079,21 @@ Examples .. code-block:: shell - $ python -m pip install SomePackage[PDF] - $ python -m pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@master#subdirectory=subdir_path" - $ python -m pip install .[PDF] # project in current directory - $ python -m pip install SomePackage[PDF]==3.0 - $ python -m pip install SomePackage[PDF,EPUB] # multiple extras + python -m pip install SomePackage[PDF] + python -m pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@master#subdirectory=subdir_path" + python -m pip install .[PDF] # project in current directory + python -m pip install SomePackage[PDF]==3.0 + python -m pip install SomePackage[PDF,EPUB] # multiple extras .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install SomePackage[PDF] - C:\> py -m pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@master#subdirectory=subdir_path" - C:\> py -m pip install .[PDF] # project in current directory - C:\> py -m pip install SomePackage[PDF]==3.0 - C:\> py -m pip install SomePackage[PDF,EPUB] # multiple extras + py -m pip install SomePackage[PDF] + py -m pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@master#subdirectory=subdir_path" + py -m pip install .[PDF] # project in current directory + py -m pip install SomePackage[PDF]==3.0 + py -m pip install SomePackage[PDF,EPUB] # multiple extras #. Install a particular source archive file. @@ -1104,15 +1104,15 @@ Examples .. code-block:: shell - $ python -m pip install ./downloads/SomePackage-1.0.4.tar.gz - $ python -m pip install http://my.package.repo/SomePackage-1.0.4.zip + python -m pip install ./downloads/SomePackage-1.0.4.tar.gz + python -m pip install http://my.package.repo/SomePackage-1.0.4.zip .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install ./downloads/SomePackage-1.0.4.tar.gz - C:\> py -m pip install http://my.package.repo/SomePackage-1.0.4.zip + py -m pip install ./downloads/SomePackage-1.0.4.tar.gz + py -m pip install http://my.package.repo/SomePackage-1.0.4.zip #. Install a particular source archive file following :pep:`440` direct references. @@ -1123,17 +1123,17 @@ Examples .. code-block:: shell - $ python -m pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl - $ python -m pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl" - $ python -m pip install SomeProject@http://my.package.repo/1.2.3.tar.gz + python -m pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl + python -m pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl" + python -m pip install SomeProject@http://my.package.repo/1.2.3.tar.gz .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl - C:\> py -m pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl" - C:\> py -m pip install SomeProject@http://my.package.repo/1.2.3.tar.gz + py -m pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl + py -m pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl" + py -m pip install SomeProject@http://my.package.repo/1.2.3.tar.gz #. Install from alternative package repositories. @@ -1146,13 +1146,13 @@ Examples .. code-block:: shell - $ python -m pip install --index-url http://my.package.repo/simple/ SomePackage + python -m pip install --index-url http://my.package.repo/simple/ SomePackage .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install --index-url http://my.package.repo/simple/ SomePackage + py -m pip install --index-url http://my.package.repo/simple/ SomePackage Search an additional index during install, in addition to `PyPI`_ @@ -1163,13 +1163,13 @@ Examples .. code-block:: shell - $ python -m pip install --extra-index-url http://my.package.repo/simple SomePackage + python -m pip install --extra-index-url http://my.package.repo/simple SomePackage .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install --extra-index-url http://my.package.repo/simple SomePackage + py -m pip install --extra-index-url http://my.package.repo/simple SomePackage Install from a local flat directory containing archives (and don't scan indexes): @@ -1180,17 +1180,17 @@ Examples .. code-block:: shell - $ python -m pip install --no-index --find-links=file:///local/dir/ SomePackage - $ python -m pip install --no-index --find-links=/local/dir/ SomePackage - $ python -m pip install --no-index --find-links=relative/dir/ SomePackage + python -m pip install --no-index --find-links=file:///local/dir/ SomePackage + python -m pip install --no-index --find-links=/local/dir/ SomePackage + python -m pip install --no-index --find-links=relative/dir/ SomePackage .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install --no-index --find-links=file:///local/dir/ SomePackage - C:\> py -m pip install --no-index --find-links=/local/dir/ SomePackage - C:\> py -m pip install --no-index --find-links=relative/dir/ SomePackage + py -m pip install --no-index --find-links=file:///local/dir/ SomePackage + py -m pip install --no-index --find-links=/local/dir/ SomePackage + py -m pip install --no-index --find-links=relative/dir/ SomePackage #. Find pre-release and development versions, in addition to stable versions. By default, pip only finds stable versions. @@ -1201,13 +1201,13 @@ Examples .. code-block:: shell - $ python -m pip install --pre SomePackage + python -m pip install --pre SomePackage .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install --pre SomePackage + py -m pip install --pre SomePackage #. Install packages from source. @@ -1220,13 +1220,13 @@ Examples .. code-block:: shell - $ python -m pip install SomePackage1 SomePackage2 --no-binary :all: + python -m pip install SomePackage1 SomePackage2 --no-binary :all: .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install SomePackage1 SomePackage2 --no-binary :all: + py -m pip install SomePackage1 SomePackage2 --no-binary :all: Specify ``SomePackage1`` to be installed from source: @@ -1236,13 +1236,13 @@ Examples .. code-block:: shell - $ python -m pip install SomePackage1 SomePackage2 --no-binary SomePackage1 + python -m pip install SomePackage1 SomePackage2 --no-binary SomePackage1 .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install SomePackage1 SomePackage2 --no-binary SomePackage1 + py -m pip install SomePackage1 SomePackage2 --no-binary SomePackage1 ---- diff --git a/docs/html/reference/pip_wheel.rst b/docs/html/reference/pip_wheel.rst index 0a11c6e25..5dd4b0328 100644 --- a/docs/html/reference/pip_wheel.rst +++ b/docs/html/reference/pip_wheel.rst @@ -40,13 +40,13 @@ In order for pip to build a wheel, ``setup.py`` must implement the .. code-block:: shell - $ python setup.py bdist_wheel -d TARGET + python setup.py bdist_wheel -d TARGET .. group-tab:: Windows .. code-block:: shell - C:\> py setup.py bdist_wheel -d TARGET + py setup.py bdist_wheel -d TARGET This command must create a wheel compatible with the invoking Python @@ -68,13 +68,13 @@ example: .. code-block:: shell - $ python -m pip wheel --global-option bdist_ext --global-option -DFOO wheel + python -m pip wheel --global-option bdist_ext --global-option -DFOO wheel .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip wheel --global-option bdist_ext --global-option -DFOO wheel + py -m pip wheel --global-option bdist_ext --global-option -DFOO wheel will result in a build command of @@ -109,15 +109,15 @@ Examples .. code-block:: shell - $ python -m pip wheel --wheel-dir=/tmp/wheelhouse SomePackage - $ python -m pip install --no-index --find-links=/tmp/wheelhouse SomePackage + python -m pip wheel --wheel-dir=/tmp/wheelhouse SomePackage + python -m pip install --no-index --find-links=/tmp/wheelhouse SomePackage .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip wheel --wheel-dir=/tmp/wheelhouse SomePackage - C:\> py -m pip install --no-index --find-links=/tmp/wheelhouse SomePackage + py -m pip wheel --wheel-dir=/tmp/wheelhouse SomePackage + py -m pip install --no-index --find-links=/tmp/wheelhouse SomePackage #. Build a wheel for a package from source @@ -127,10 +127,10 @@ Examples .. code-block:: shell - $ python -m pip wheel --no-binary SomePackage SomePackage + python -m pip wheel --no-binary SomePackage SomePackage .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip wheel --no-binary SomePackage SomePackage + py -m pip wheel --no-binary SomePackage SomePackage diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 41cbb6404..aa18a6f66 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -17,7 +17,7 @@ to your system, which can be run from the command prompt as follows: .. code-block:: shell - $ python -m pip + python -m pip ``python -m pip`` executes pip using the Python interpreter you specified as python. So ``/usr/bin/python3.7 -m pip`` means @@ -27,7 +27,7 @@ to your system, which can be run from the command prompt as follows: .. code-block:: shell - C:\> py -m pip + py -m pip ``py -m pip`` executes pip using the latest Python interpreter you have installed. For more details, read the `Python Windows launcher`_ docs. @@ -49,17 +49,17 @@ Specifiers` .. code-block:: shell - $ python -m pip install SomePackage # latest version - $ python -m pip install SomePackage==1.0.4 # specific version - $ python -m pip install 'SomePackage>=1.0.4' # minimum version + python -m pip install SomePackage # latest version + python -m pip install SomePackage==1.0.4 # specific version + python -m pip install 'SomePackage>=1.0.4' # minimum version .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install SomePackage # latest version - C:\> py -m pip install SomePackage==1.0.4 # specific version - C:\> py -m pip install 'SomePackage>=1.0.4' # minimum version + py -m pip install SomePackage # latest version + py -m pip install SomePackage==1.0.4 # specific version + py -m pip install 'SomePackage>=1.0.4' # minimum version For more information and examples, see the :ref:`pip install` reference. @@ -168,13 +168,13 @@ installed using :ref:`pip install` like so: .. code-block:: shell - $ python -m pip install -r requirements.txt + python -m pip install -r requirements.txt .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install -r requirements.txt + py -m pip install -r requirements.txt Details on the format of the files are here: :ref:`Requirements File Format`. @@ -195,15 +195,15 @@ In practice, there are 4 common uses of Requirements files: .. code-block:: shell - $ python -m pip freeze > requirements.txt - $ python -m pip install -r requirements.txt + python -m pip freeze > requirements.txt + python -m pip install -r requirements.txt .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip freeze > requirements.txt - C:\> py -m pip install -r requirements.txt + py -m pip freeze > requirements.txt + py -m pip install -r requirements.txt 2. Requirements files are used to force pip to properly resolve dependencies. As it is now, pip `doesn't have true dependency resolution @@ -276,13 +276,13 @@ Use a constraints file like so: .. code-block:: shell - $ python -m pip install -c constraints.txt + python -m pip install -c constraints.txt .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install -c constraints.txt + py -m pip install -c constraints.txt Constraints files are used for exactly the same reason as requirements files when you don't know exactly what things you want to install. For instance, say @@ -326,13 +326,13 @@ To install directly from a wheel archive: .. code-block:: shell - $ python -m pip install SomePackage-1.0-py2.py3-none-any.whl + python -m pip install SomePackage-1.0-py2.py3-none-any.whl .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install SomePackage-1.0-py2.py3-none-any.whl + py -m pip install SomePackage-1.0-py2.py3-none-any.whl For the cases where wheels are not available, pip offers :ref:`pip wheel` as a @@ -351,15 +351,15 @@ directory: .. code-block:: shell - $ python -m pip install wheel - $ python -m pip wheel --wheel-dir=/local/wheels -r requirements.txt + python -m pip install wheel + python -m pip wheel --wheel-dir=/local/wheels -r requirements.txt .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install wheel - C:\> py -m pip wheel --wheel-dir=/local/wheels -r requirements.txt + py -m pip install wheel + py -m pip wheel --wheel-dir=/local/wheels -r requirements.txt And *then* to install those requirements just using your local directory of wheels (and not from PyPI): @@ -370,13 +370,13 @@ wheels (and not from PyPI): .. code-block:: shell - $ python -m pip install --no-index --find-links=/local/wheels -r requirements.txt + python -m pip install --no-index --find-links=/local/wheels -r requirements.txt .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install --no-index --find-links=/local/wheels -r requirements.txt + py -m pip install --no-index --find-links=/local/wheels -r requirements.txt Uninstalling Packages @@ -390,13 +390,13 @@ pip is able to uninstall most packages like so: .. code-block:: shell - $ python -m pip uninstall SomePackage + python -m pip uninstall SomePackage .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip uninstall SomePackage + py -m pip uninstall SomePackage pip also performs an automatic uninstall of an old version of a package @@ -496,13 +496,13 @@ command: .. code-block:: shell - $ python -m pip search "query" + python -m pip search "query" .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip search "query" + py -m pip search "query" The query will be used to search the names and summaries of all packages. @@ -670,13 +670,13 @@ For example, to set the default timeout: .. code-block:: shell - $ export PIP_DEFAULT_TIMEOUT=60 + export PIP_DEFAULT_TIMEOUT=60 .. group-tab:: Windows .. code-block:: shell - C:\> set PIP_DEFAULT_TIMEOUT=60 + set PIP_DEFAULT_TIMEOUT=60 This is the same as passing the option to pip directly: @@ -686,13 +686,13 @@ This is the same as passing the option to pip directly: .. code-block:: shell - $ python -m pip --default-timeout=60 [...] + python -m pip --default-timeout=60 [...] .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip --default-timeout=60 [...] + py -m pip --default-timeout=60 [...] For command line options which can be repeated, use a space to separate multiple values. For example: @@ -703,13 +703,13 @@ multiple values. For example: .. code-block:: shell - $ export PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" + export PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" .. group-tab:: Windows .. code-block:: shell - C:\> set PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" + set PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" is the same as calling: @@ -720,13 +720,13 @@ is the same as calling: .. code-block:: shell - $ python -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com + python -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com + py -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com Options that do not take a value, but can be repeated (such as ``--verbose``) @@ -770,15 +770,15 @@ pip comes with support for command line completion in bash, zsh and fish. To setup for bash:: - $ python -m pip completion --bash >> ~/.profile + python -m pip completion --bash >> ~/.profile To setup for zsh:: - $ python -m pip completion --zsh >> ~/.zprofile + python -m pip completion --zsh >> ~/.zprofile To setup for fish:: - $ python -m pip completion --fish > ~/.config/fish/completions/pip.fish + python -m pip completion --fish > ~/.config/fish/completions/pip.fish Alternatively, you can use the result of the ``completion`` command directly with the eval function of your shell, e.g. by adding the following to your @@ -805,13 +805,13 @@ First, download the archives that fulfill your requirements: .. code-block:: shell - $ python -m pip download --destination-directory DIR -r requirements.txt + python -m pip download --destination-directory DIR -r requirements.txt .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip download --destination-directory DIR -r requirements.txt + py -m pip download --destination-directory DIR -r requirements.txt Note that ``pip download`` will look in your wheel cache first, before @@ -826,13 +826,13 @@ this instead: .. code-block:: shell - $ python -m pip wheel --wheel-dir DIR -r requirements.txt + python -m pip wheel --wheel-dir DIR -r requirements.txt .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip wheel --wheel-dir DIR -r requirements.txt + py -m pip wheel --wheel-dir DIR -r requirements.txt Then, to install from local only, you'll be using :ref:`--find-links ` and :ref:`--no-index ` like so: @@ -843,13 +843,13 @@ Then, to install from local only, you'll be using :ref:`--find-links .. code-block:: shell - $ python -m pip install --no-index --find-links=DIR -r requirements.txt + python -m pip install --no-index --find-links=DIR -r requirements.txt .. group-tab:: Windows .. code-block:: shell - C:\> -m pip install --no-index --find-links=DIR -r requirements.txt + py -m pip install --no-index --find-links=DIR -r requirements.txt "Only if needed" Recursive Upgrade @@ -876,15 +876,15 @@ behaviour was: .. code-block:: shell - $ python -m pip install --upgrade --no-deps SomePackage - $ python -m pip install SomePackage + python -m pip install --upgrade --no-deps SomePackage + python -m pip install SomePackage .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install --upgrade --no-deps SomePackage - C:\> py -m pip install SomePackage + py -m pip install --upgrade --no-deps SomePackage + py -m pip install SomePackage A proposal for an ``upgrade-all`` command is being considered as a safer @@ -916,15 +916,15 @@ To install "SomePackage" into an environment with site.USER_BASE customized to .. code-block:: shell - $ export PYTHONUSERBASE=/myappenv - $ python -m pip install --user SomePackage + export PYTHONUSERBASE=/myappenv + python -m pip install --user SomePackage .. group-tab:: Windows .. code-block:: shell - C:\> set PYTHONUSERBASE=c:/myappenv - C:\> py -m pip install --user SomePackage + set PYTHONUSERBASE=c:/myappenv + py -m pip install --user SomePackage ``pip install --user`` follows four rules: @@ -1177,13 +1177,13 @@ like this: .. code-block:: shell - $ python -m pip install package_coffee==0.44.1 package_tea==4.3.0 + python -m pip install package_coffee==0.44.1 package_tea==4.3.0 .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install package_coffee==0.44.1 package_tea==4.3.0 + py -m pip install package_coffee==0.44.1 package_tea==4.3.0 :: @@ -1295,13 +1295,13 @@ specifiers to *only* the more important package: .. code-block:: shell - $ python -m pip install package_coffee==0.44.1b0 package_tea + python -m pip install package_coffee==0.44.1b0 package_tea .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip install package_coffee==0.44.1b0 package_tea + py -m pip install package_coffee==0.44.1b0 package_tea This will result in: From a2e2f5d052db75b47420044c807bf24456cde3ea Mon Sep 17 00:00:00 2001 From: Srinivas Nyayapati Date: Sun, 16 Aug 2020 23:21:47 -0400 Subject: [PATCH 154/209] update all tabs with command output be console code-blocks --- docs/html/quickstart.rst | 26 +++++++++++----------- docs/html/reference/pip.rst | 4 ++-- docs/html/reference/pip_check.rst | 12 +++++----- docs/html/reference/pip_download.rst | 4 ++-- docs/html/reference/pip_freeze.rst | 4 ++-- docs/html/reference/pip_hash.rst | 4 ++-- docs/html/reference/pip_install.rst | 8 +++---- docs/html/reference/pip_list.rst | 32 +++++++++++++-------------- docs/html/reference/pip_search.rst | 4 ++-- docs/html/reference/pip_show.rst | 8 +++---- docs/html/reference/pip_uninstall.rst | 4 ++-- docs/html/user_guide.rst | 32 +++++++++++++-------------- 12 files changed, 71 insertions(+), 71 deletions(-) diff --git a/docs/html/quickstart.rst b/docs/html/quickstart.rst index bae9e2286..de0fd711b 100644 --- a/docs/html/quickstart.rst +++ b/docs/html/quickstart.rst @@ -10,7 +10,7 @@ Install a package from `PyPI`_: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip install SomePackage [...] @@ -18,7 +18,7 @@ Install a package from `PyPI`_: .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip install SomePackage [...] @@ -26,14 +26,14 @@ Install a package from `PyPI`_: Install a package that's already been downloaded from `PyPI`_ or -obtained from elsewhere. This is useful if the target macOShine does not have a +obtained from elsewhere. This is useful if the target machine does not have a network connection: .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip install SomePackage-1.0-py2.py3-none-any.whl [...] @@ -41,7 +41,7 @@ network connection: .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip install SomePackage-1.0-py2.py3-none-any.whl [...] @@ -54,7 +54,7 @@ Show what files were installed: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip show --files SomePackage Name: SomePackage @@ -66,7 +66,7 @@ Show what files were installed: .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip show --files SomePackage Name: SomePackage @@ -82,14 +82,14 @@ List what packages are outdated: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip list --outdated SomePackage (Current: 1.0 Latest: 2.0) .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip list --outdated SomePackage (Current: 1.0 Latest: 2.0) @@ -100,7 +100,7 @@ Upgrade a package: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip install --upgrade SomePackage [...] @@ -112,7 +112,7 @@ Upgrade a package: .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip install --upgrade SomePackage [...] @@ -128,7 +128,7 @@ Uninstall a package: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip uninstall SomePackage Uninstalling SomePackage: @@ -138,7 +138,7 @@ Uninstall a package: .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip uninstall SomePackage Uninstalling SomePackage: diff --git a/docs/html/reference/pip.rst b/docs/html/reference/pip.rst index a0c1148bf..ffd6df3d6 100644 --- a/docs/html/reference/pip.rst +++ b/docs/html/reference/pip.rst @@ -13,13 +13,13 @@ Usage .. code-block:: shell - $ python -m pip [options] + python -m pip [options] .. group-tab:: Windows .. code-block:: shell - C:\> py -m pip [options] + py -m pip [options] Description *********** diff --git a/docs/html/reference/pip_check.rst b/docs/html/reference/pip_check.rst index 07d700475..719528f20 100644 --- a/docs/html/reference/pip_check.rst +++ b/docs/html/reference/pip_check.rst @@ -36,7 +36,7 @@ Examples .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip check No broken requirements found. @@ -45,7 +45,7 @@ Examples .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip check No broken requirements found. @@ -58,7 +58,7 @@ Examples .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip check pyramid 1.5.2 requires WebOb, which is not installed. @@ -67,7 +67,7 @@ Examples .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip check pyramid 1.5.2 requires WebOb, which is not installed. @@ -80,7 +80,7 @@ Examples .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip check pyramid 1.5.2 has requirement WebOb>=1.3.1, but you have WebOb 0.8. @@ -89,7 +89,7 @@ Examples .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip check pyramid 1.5.2 has requirement WebOb>=1.3.1, but you have WebOb 0.8. diff --git a/docs/html/reference/pip_download.rst b/docs/html/reference/pip_download.rst index 141b6e1fa..b4321be13 100644 --- a/docs/html/reference/pip_download.rst +++ b/docs/html/reference/pip_download.rst @@ -178,7 +178,7 @@ Examples .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip download \ --only-binary=:all: \ @@ -193,7 +193,7 @@ Examples .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip download ^ --only-binary=:all: ^ diff --git a/docs/html/reference/pip_freeze.rst b/docs/html/reference/pip_freeze.rst index 91c80979a..e7cc14ead 100644 --- a/docs/html/reference/pip_freeze.rst +++ b/docs/html/reference/pip_freeze.rst @@ -43,7 +43,7 @@ Examples .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip freeze docutils==0.11 @@ -54,7 +54,7 @@ Examples .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip freeze docutils==0.11 diff --git a/docs/html/reference/pip_hash.rst b/docs/html/reference/pip_hash.rst index 7ed39280c..2313cae82 100644 --- a/docs/html/reference/pip_hash.rst +++ b/docs/html/reference/pip_hash.rst @@ -53,7 +53,7 @@ Compute the hash of a downloaded archive: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip download SomePackage Collecting SomePackage @@ -66,7 +66,7 @@ Compute the hash of a downloaded archive: .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip download SomePackage Collecting SomePackage diff --git a/docs/html/reference/pip_install.rst b/docs/html/reference/pip_install.rst index b2da26240..68728581f 100644 --- a/docs/html/reference/pip_install.rst +++ b/docs/html/reference/pip_install.rst @@ -104,7 +104,7 @@ which depends on foo: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip install quux ... @@ -116,7 +116,7 @@ which depends on foo: .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip install quux ... @@ -681,7 +681,7 @@ option: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip install --require-hashes -r requirements.txt ... @@ -695,7 +695,7 @@ option: .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip install --require-hashes -r requirements.txt ... diff --git a/docs/html/reference/pip_list.rst b/docs/html/reference/pip_list.rst index bfdef9ebe..cb6064cc0 100644 --- a/docs/html/reference/pip_list.rst +++ b/docs/html/reference/pip_list.rst @@ -44,7 +44,7 @@ Examples .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip list docutils (0.10) @@ -55,7 +55,7 @@ Examples .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip list docutils (0.10) @@ -70,7 +70,7 @@ Examples .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip list --outdated docutils (Current: 0.10 Latest: 0.11) @@ -78,7 +78,7 @@ Examples .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip list --outdated docutils (Current: 0.10 Latest: 0.11) @@ -91,7 +91,7 @@ Examples .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip list --format columns Package Version @@ -102,7 +102,7 @@ Examples .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip list --format columns Package Version @@ -117,7 +117,7 @@ Examples .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip list -o --format columns Package Version Latest Type @@ -127,7 +127,7 @@ Examples .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip list -o --format columns Package Version Latest Type @@ -142,14 +142,14 @@ Examples .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip list --outdated --not-required docutils (Current: 0.10 Latest: 0.11) .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip list --outdated --not-required docutils (Current: 0.10 Latest: 0.11) @@ -160,7 +160,7 @@ Examples .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip list --format=legacy colorama (0.3.7) @@ -170,7 +170,7 @@ Examples .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip list --format=legacy colorama (0.3.7) @@ -184,14 +184,14 @@ Examples .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip list --format=json [{'name': 'colorama', 'version': '0.3.7'}, {'name': 'docopt', 'version': '0.6.2'}, ... .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip list --format=json [{'name': 'colorama', 'version': '0.3.7'}, {'name': 'docopt', 'version': '0.6.2'}, ... @@ -202,7 +202,7 @@ Examples .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip list --format=freeze colorama==0.3.7 @@ -212,7 +212,7 @@ Examples .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip list --format=freeze colorama==0.3.7 diff --git a/docs/html/reference/pip_search.rst b/docs/html/reference/pip_search.rst index e0fc3e25b..c0737aa76 100644 --- a/docs/html/reference/pip_search.rst +++ b/docs/html/reference/pip_search.rst @@ -42,7 +42,7 @@ Examples .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip search peppercorn pepperedform - Helpers for using peppercorn with formprocess. @@ -50,7 +50,7 @@ Examples .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip search peppercorn pepperedform - Helpers for using peppercorn with formprocess. diff --git a/docs/html/reference/pip_show.rst b/docs/html/reference/pip_show.rst index ae9182f54..71aa27e78 100644 --- a/docs/html/reference/pip_show.rst +++ b/docs/html/reference/pip_show.rst @@ -42,7 +42,7 @@ Examples .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip show sphinx Name: Sphinx @@ -57,7 +57,7 @@ Examples .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip show sphinx Name: Sphinx @@ -76,7 +76,7 @@ Examples .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip show --verbose sphinx Name: Sphinx @@ -119,7 +119,7 @@ Examples .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip show --verbose sphinx Name: Sphinx diff --git a/docs/html/reference/pip_uninstall.rst b/docs/html/reference/pip_uninstall.rst index 165ac4cd5..ecc3b9210 100644 --- a/docs/html/reference/pip_uninstall.rst +++ b/docs/html/reference/pip_uninstall.rst @@ -42,7 +42,7 @@ Examples .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip uninstall simplejson Uninstalling simplejson: @@ -53,7 +53,7 @@ Examples .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip uninstall simplejson Uninstalling simplejson: diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index aa18a6f66..64edc0b18 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -414,7 +414,7 @@ To list installed packages: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip list docutils (0.9.1) @@ -424,7 +424,7 @@ To list installed packages: .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip list docutils (0.9.1) @@ -439,7 +439,7 @@ To list outdated packages, and show the latest version available: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip list --outdated docutils (Current: 0.9.1 Latest: 0.10) @@ -447,7 +447,7 @@ To list outdated packages, and show the latest version available: .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip list --outdated docutils (Current: 0.9.1 Latest: 0.10) @@ -459,7 +459,7 @@ To show details about an installed package: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip show sphinx --- @@ -470,7 +470,7 @@ To show details about an installed package: .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip show sphinx --- @@ -953,14 +953,14 @@ From within a ``--no-site-packages`` virtualenv (i.e. the default kind): .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip install --user SomePackage Can not perform a '--user' install. User site-packages are not visible in this virtualenv. .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip install --user SomePackage Can not perform a '--user' install. User site-packages are not visible in this virtualenv. @@ -973,14 +973,14 @@ is already installed in the virtualenv: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip install --user SomePackage==0.4 Will not install to the user site because it will lack sys.path precedence .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip install --user SomePackage==0.4 Will not install to the user site because it will lack sys.path precedence @@ -991,7 +991,7 @@ From within a real python, where ``SomePackage`` is *not* installed globally: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip install --user SomePackage [...] @@ -999,7 +999,7 @@ From within a real python, where ``SomePackage`` is *not* installed globally: .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip install --user SomePackage [...] @@ -1012,7 +1012,7 @@ is *not* the latest version: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip install --user SomePackage [...] @@ -1023,7 +1023,7 @@ is *not* the latest version: .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip install --user SomePackage [...] @@ -1039,7 +1039,7 @@ is the latest version: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console $ python -m pip install --user SomePackage [...] @@ -1054,7 +1054,7 @@ is the latest version: .. group-tab:: Windows - .. code-block:: shell + .. code-block:: console C:\> py -m pip install --user SomePackage [...] From 8f3151324fe971afbdb6fa8fe638c9108492aec0 Mon Sep 17 00:00:00 2001 From: shireenrao Date: Tue, 25 Aug 2020 07:48:04 -0400 Subject: [PATCH 155/209] fix inconsistent indentation of tab blocks --- docs/html/conf.py | 2 +- docs/html/installing.rst | 64 +++--- docs/html/quickstart.rst | 2 +- docs/html/reference/pip.rst | 12 +- docs/html/reference/pip_check.rst | 18 +- docs/html/reference/pip_download.rst | 176 ++++++++------- docs/html/reference/pip_freeze.rst | 12 +- docs/html/reference/pip_hash.rst | 40 ++-- docs/html/reference/pip_install.rst | 308 +++++++++++++------------- docs/html/reference/pip_list.rst | 48 ++-- docs/html/reference/pip_search.rst | 6 +- docs/html/reference/pip_show.rst | 204 ++++++++--------- docs/html/reference/pip_uninstall.rst | 30 +-- docs/html/reference/pip_wheel.rst | 36 +-- 14 files changed, 481 insertions(+), 477 deletions(-) diff --git a/docs/html/conf.py b/docs/html/conf.py index 9d0b7a2fa..a88ac33e2 100644 --- a/docs/html/conf.py +++ b/docs/html/conf.py @@ -34,7 +34,7 @@ extensions = [ 'sphinx.ext.extlinks', 'pip_sphinxext', 'sphinx.ext.intersphinx', - 'sphinx_tabs.tabs' + 'sphinx_tabs.tabs', ] # intersphinx diff --git a/docs/html/installing.rst b/docs/html/installing.rst index e6717881a..5379c1da0 100644 --- a/docs/html/installing.rst +++ b/docs/html/installing.rst @@ -30,17 +30,17 @@ have downloaded ``get-pip.py``: .. tabs:: - .. group-tab:: Unix/macOS + .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python get-pip.py + python get-pip.py - .. group-tab:: Windows + .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py get-pip.py + py get-pip.py .. warning:: @@ -83,66 +83,66 @@ Install from local copies of pip and setuptools: .. tabs:: - .. group-tab:: Unix/macOS + .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python get-pip.py --no-index --find-links=/local/copies + python get-pip.py --no-index --find-links=/local/copies - .. group-tab:: Windows + .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py get-pip.py --no-index --find-links=/local/copies + py get-pip.py --no-index --find-links=/local/copies Install to the user site [3]_: .. tabs:: - .. group-tab:: Unix/macOS + .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python get-pip.py --user + python get-pip.py --user - .. group-tab:: Windows + .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py get-pip.py --user + py get-pip.py --user Install behind a proxy: .. tabs:: - .. group-tab:: Unix/macOS + .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python get-pip.py --proxy="http://[user:passwd@]proxy.server:port" + python get-pip.py --proxy="http://[user:passwd@]proxy.server:port" - .. group-tab:: Windows + .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py get-pip.py --proxy="http://[user:passwd@]proxy.server:port" + py get-pip.py --proxy="http://[user:passwd@]proxy.server:port" ``get-pip.py`` can also be used to install a specified combination of ``pip``, ``setuptools``, and ``wheel`` using the same requirements syntax as pip: .. tabs:: - .. group-tab:: Unix/macOS + .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 + python get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 - .. group-tab:: Windows + .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 + py get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 Using Linux Package Managers ============================ @@ -162,13 +162,13 @@ Upgrading pip .. code-block:: shell - python -m pip install -U pip + python -m pip install -U pip .. group-tab:: Windows .. code-block:: shell - py -m pip install -U pip + py -m pip install -U pip .. _compatibility-requirements: diff --git a/docs/html/quickstart.rst b/docs/html/quickstart.rst index de0fd711b..9591e1127 100644 --- a/docs/html/quickstart.rst +++ b/docs/html/quickstart.rst @@ -142,7 +142,7 @@ Uninstall a package: C:\> py -m pip uninstall SomePackage Uninstalling SomePackage: - /my/env/lib/pythonx.x/site-packages/somepackage + /my/env/lib/pythonx.x/site-packages/somepackage Proceed (y/n)? y Successfully uninstalled SomePackage diff --git a/docs/html/reference/pip.rst b/docs/html/reference/pip.rst index ffd6df3d6..9fd42c676 100644 --- a/docs/html/reference/pip.rst +++ b/docs/html/reference/pip.rst @@ -234,17 +234,17 @@ included in the command as follows: .. tabs:: - .. group-tab:: Unix/macOS + .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: console - python setup.py BUILD COMMAND + python setup.py BUILD COMMAND - .. group-tab:: Windows + .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py setup.py BUILD COMMAND + py setup.py BUILD COMMAND The options are passed unmodified, and presently offer direct access to the distutils command line. Use of ``--global-option`` and ``--build-option`` diff --git a/docs/html/reference/pip_check.rst b/docs/html/reference/pip_check.rst index 719528f20..d3bb457e1 100644 --- a/docs/html/reference/pip_check.rst +++ b/docs/html/reference/pip_check.rst @@ -32,11 +32,11 @@ Examples #. If all dependencies are compatible: - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console $ python -m pip check No broken requirements found. @@ -45,7 +45,7 @@ Examples .. group-tab:: Windows - .. code-block:: console + .. code-block:: console C:\> py -m pip check No broken requirements found. @@ -54,11 +54,11 @@ Examples #. If a package is missing: - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console $ python -m pip check pyramid 1.5.2 requires WebOb, which is not installed. @@ -67,7 +67,7 @@ Examples .. group-tab:: Windows - .. code-block:: console + .. code-block:: console C:\> py -m pip check pyramid 1.5.2 requires WebOb, which is not installed. @@ -76,11 +76,11 @@ Examples #. If a package has the wrong version: - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console $ python -m pip check pyramid 1.5.2 has requirement WebOb>=1.3.1, but you have WebOb 0.8. @@ -89,7 +89,7 @@ Examples .. group-tab:: Windows - .. code-block:: console + .. code-block:: console C:\> py -m pip check pyramid 1.5.2 has requirement WebOb>=1.3.1, but you have WebOb 0.8. diff --git a/docs/html/reference/pip_download.rst b/docs/html/reference/pip_download.rst index b4321be13..7983bb95b 100644 --- a/docs/html/reference/pip_download.rst +++ b/docs/html/reference/pip_download.rst @@ -64,144 +64,148 @@ Examples #. Download a package and all of its dependencies - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip download SomePackage - python -m pip download -d . SomePackage # equivalent to above - python -m pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage + python -m pip download SomePackage + python -m pip download -d . SomePackage # equivalent to above + python -m pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip download SomePackage - py -m pip download -d . SomePackage # equivalent to above - py -m pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage + py -m pip download SomePackage + py -m pip download -d . SomePackage # equivalent to above + py -m pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage #. Download a package and all of its dependencies with OSX specific interpreter constraints. - This forces OSX 10.10 or lower compatibility. Since OSX deps are forward compatible, - this will also match ``macosx-10_9_x86_64``, ``macosx-10_8_x86_64``, ``macosx-10_8_intel``, - etc. - It will also match deps with platform ``any``. Also force the interpreter version to ``27`` - (or more generic, i.e. ``2``) and implementation to ``cp`` (or more generic, i.e. ``py``). + This forces OSX 10.10 or lower compatibility. Since OSX deps are forward compatible, + this will also match ``macosx-10_9_x86_64``, ``macosx-10_8_x86_64``, ``macosx-10_8_intel``, + etc. + It will also match deps with platform ``any``. Also force the interpreter version to ``27`` + (or more generic, i.e. ``2``) and implementation to ``cp`` (or more generic, i.e. ``py``). - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip download \ - --only-binary=:all: \ - --platform macosx-10_10_x86_64 \ - --python-version 27 \ - --implementation cp \ - SomePackage + python -m pip download \ + --only-binary=:all: \ + --platform macosx-10_10_x86_64 \ + --python-version 27 \ + --implementation cp \ + SomePackage .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip download ^ - --only-binary=:all: ^ - --platform macosx-10_10_x86_64 ^ - --python-version 27 ^ - --implementation cp ^ - SomePackage + py -m pip download ^ + --only-binary=:all: ^ + --platform macosx-10_10_x86_64 ^ + --python-version 27 ^ + --implementation cp ^ + SomePackage #. Download a package and its dependencies with linux specific constraints. - Force the interpreter to be any minor version of py3k, and only accept - ``cp34m`` or ``none`` as the abi. + Force the interpreter to be any minor version of py3k, and only accept + ``cp34m`` or ``none`` as the abi. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip download \ - --only-binary=:all: \ - --platform linux_x86_64 \ - --python-version 3 \ - --implementation cp \ - --abi cp34m \ - SomePackage + python -m pip download \ + --only-binary=:all: \ + --platform linux_x86_64 \ + --python-version 3 \ + --implementation cp \ + --abi cp34m \ + SomePackage .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip download ^ - --only-binary=:all: ^ - --platform linux_x86_64 ^ - --python-version 3 ^ - --implementation cp ^ - --abi cp34m ^ - SomePackage + py -m pip download ^ + --only-binary=:all: ^ + --platform linux_x86_64 ^ + --python-version 3 ^ + --implementation cp ^ + --abi cp34m ^ + SomePackage #. Force platform, implementation, and abi agnostic deps. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip download \ - --only-binary=:all: \ - --platform any \ - --python-version 3 \ - --implementation py \ - --abi none \ - SomePackage + python -m pip download \ + --only-binary=:all: \ + --platform any \ + --python-version 3 \ + --implementation py \ + --abi none \ + SomePackage .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip download ^ - --only-binary=:all: ^ - --platform any ^ - --python-version 3 ^ - --implementation py ^ - --abi none ^ - SomePackage + py -m pip download ^ + --only-binary=:all: ^ + --platform any ^ + --python-version 3 ^ + --implementation py ^ + --abi none ^ + SomePackage #. Even when overconstrained, this will still correctly fetch the pip universal wheel. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console - $ python -m pip download \ - --only-binary=:all: \ - --platform linux_x86_64 \ - --python-version 33 \ - --implementation cp \ - --abi cp34m \ - pip>=8 + $ python -m pip download \ + --only-binary=:all: \ + --platform linux_x86_64 \ + --python-version 33 \ + --implementation cp \ + --abi cp34m \ + pip>=8 - $ ls pip-8.1.1-py2.py3-none-any.whl - pip-8.1.1-py2.py3-none-any.whl + .. code-block:: console + + $ ls pip-8.1.1-py2.py3-none-any.whl + pip-8.1.1-py2.py3-none-any.whl .. group-tab:: Windows - .. code-block:: console + .. code-block:: console - C:\> py -m pip download ^ - --only-binary=:all: ^ - --platform linux_x86_64 ^ - --python-version 33 ^ - --implementation cp ^ - --abi cp34m ^ - pip>=8 + C:\> py -m pip download ^ + --only-binary=:all: ^ + --platform linux_x86_64 ^ + --python-version 33 ^ + --implementation cp ^ + --abi cp34m ^ + pip>=8 - C:\> dir pip-8.1.1-py2.py3-none-any.whl - pip-8.1.1-py2.py3-none-any.whl + .. code-block:: console + + C:\> dir pip-8.1.1-py2.py3-none-any.whl + pip-8.1.1-py2.py3-none-any.whl diff --git a/docs/html/reference/pip_freeze.rst b/docs/html/reference/pip_freeze.rst index e7cc14ead..d4ed00bfb 100644 --- a/docs/html/reference/pip_freeze.rst +++ b/docs/html/reference/pip_freeze.rst @@ -39,11 +39,11 @@ Examples #. Generate output suitable for a requirements file. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console $ python -m pip freeze docutils==0.11 @@ -54,7 +54,7 @@ Examples .. group-tab:: Windows - .. code-block:: console + .. code-block:: console C:\> py -m pip freeze docutils==0.11 @@ -66,18 +66,18 @@ Examples #. Generate a requirements file and then install from it in another environment. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell env1/bin/python -m pip freeze > requirements.txt env2/bin/python -m pip install -r requirements.txt .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell env1\bin\python -m pip freeze > requirements.txt env2\bin\python -m pip install -r requirements.txt diff --git a/docs/html/reference/pip_hash.rst b/docs/html/reference/pip_hash.rst index 2313cae82..71e1cf4be 100644 --- a/docs/html/reference/pip_hash.rst +++ b/docs/html/reference/pip_hash.rst @@ -51,28 +51,28 @@ Compute the hash of a downloaded archive: .. tabs:: - .. group-tab:: Unix/macOS + .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console - $ python -m pip download SomePackage - Collecting SomePackage - Downloading SomePackage-2.2.tar.gz - Saved ./pip_downloads/SomePackage-2.2.tar.gz - Successfully downloaded SomePackage - $ python -m pip hash ./pip_downloads/SomePackage-2.2.tar.gz - ./pip_downloads/SomePackage-2.2.tar.gz: - --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 + $ python -m pip download SomePackage + Collecting SomePackage + Downloading SomePackage-2.2.tar.gz + Saved ./pip_downloads/SomePackage-2.2.tar.gz + Successfully downloaded SomePackage + $ python -m pip hash ./pip_downloads/SomePackage-2.2.tar.gz + ./pip_downloads/SomePackage-2.2.tar.gz: + --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 - .. group-tab:: Windows + .. group-tab:: Windows - .. code-block:: console + .. code-block:: console - C:\> py -m pip download SomePackage - Collecting SomePackage - Downloading SomePackage-2.2.tar.gz - Saved ./pip_downloads/SomePackage-2.2.tar.gz - Successfully downloaded SomePackage - C:\> py -m pip hash ./pip_downloads/SomePackage-2.2.tar.gz - ./pip_downloads/SomePackage-2.2.tar.gz: - --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 + C:\> py -m pip download SomePackage + Collecting SomePackage + Downloading SomePackage-2.2.tar.gz + Saved ./pip_downloads/SomePackage-2.2.tar.gz + Successfully downloaded SomePackage + C:\> py -m pip hash ./pip_downloads/SomePackage-2.2.tar.gz + ./pip_downloads/SomePackage-2.2.tar.gz: + --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 diff --git a/docs/html/reference/pip_install.rst b/docs/html/reference/pip_install.rst index 68728581f..cb97c8ee0 100644 --- a/docs/html/reference/pip_install.rst +++ b/docs/html/reference/pip_install.rst @@ -106,25 +106,25 @@ which depends on foo: .. code-block:: console - $ python -m pip install quux - ... - Installing collected packages baz, bar, foo, quux + $ python -m pip install quux + ... + Installing collected packages baz, bar, foo, quux - $ python -m pip install bar - ... - Installing collected packages foo, baz, bar + $ python -m pip install bar + ... + Installing collected packages foo, baz, bar .. group-tab:: Windows .. code-block:: console - C:\> py -m pip install quux - ... - Installing collected packages baz, bar, foo, quux + C:\> py -m pip install quux + ... + Installing collected packages baz, bar, foo, quux - C:\> py -m pip install bar - ... - Installing collected packages foo, baz, bar + C:\> py -m pip install bar + ... + Installing collected packages foo, baz, bar Prior to v6.1.0, pip made no commitments about install order. @@ -422,13 +422,13 @@ Then, to install from this repository, the syntax would be: .. code-block:: shell - python -m pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir" + python -m pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir" .. group-tab:: Windows .. code-block:: shell - py -m pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir" + py -m pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir" Git @@ -683,29 +683,29 @@ option: .. code-block:: console - $ python -m pip install --require-hashes -r requirements.txt - ... - Hashes are required in --require-hashes mode (implicitly on when a hash is - specified for any package). These requirements were missing hashes, - leaving them open to tampering. These are the hashes the downloaded - archives actually had. You can add lines like these to your requirements - files to prevent tampering. - pyelasticsearch==1.0 --hash=sha256:44ddfb1225054d7d6b1d02e9338e7d4809be94edbe9929a2ec0807d38df993fa - more-itertools==2.2 --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 + $ python -m pip install --require-hashes -r requirements.txt + ... + Hashes are required in --require-hashes mode (implicitly on when a hash is + specified for any package). These requirements were missing hashes, + leaving them open to tampering. These are the hashes the downloaded + archives actually had. You can add lines like these to your requirements + files to prevent tampering. + pyelasticsearch==1.0 --hash=sha256:44ddfb1225054d7d6b1d02e9338e7d4809be94edbe9929a2ec0807d38df993fa + more-itertools==2.2 --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 .. group-tab:: Windows .. code-block:: console - C:\> py -m pip install --require-hashes -r requirements.txt - ... - Hashes are required in --require-hashes mode (implicitly on when a hash is - specified for any package). These requirements were missing hashes, - leaving them open to tampering. These are the hashes the downloaded - archives actually had. You can add lines like these to your requirements - files to prevent tampering. - pyelasticsearch==1.0 --hash=sha256:44ddfb1225054d7d6b1d02e9338e7d4809be94edbe9929a2ec0807d38df993fa - more-itertools==2.2 --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 + C:\> py -m pip install --require-hashes -r requirements.txt + ... + Hashes are required in --require-hashes mode (implicitly on when a hash is + specified for any package). These requirements were missing hashes, + leaving them open to tampering. These are the hashes the downloaded + archives actually had. You can add lines like these to your requirements + files to prevent tampering. + pyelasticsearch==1.0 --hash=sha256:44ddfb1225054d7d6b1d02e9338e7d4809be94edbe9929a2ec0807d38df993fa + more-itertools==2.2 --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 This can be useful in deploy scripts, to ensure that the author of the @@ -756,34 +756,34 @@ Hash-checking mode also works with :ref:`pip download` and :ref:`pip wheel`. A .. tabs:: - .. group-tab:: Unix/macOS + .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip install --no-deps -e . + python -m pip install --no-deps -e . - .. group-tab:: Windows + .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip install --no-deps -e . + py -m pip install --no-deps -e . Instead of ``python setup.py install``, use... - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip install --no-deps . + python -m pip install --no-deps . .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip install --no-deps . + py -m pip install --no-deps . Hashes from PyPI ^^^^^^^^^^^^^^^^ @@ -841,15 +841,15 @@ You can install local projects or VCS projects in "editable" mode: .. code-block:: shell - python -m pip install -e path/to/SomeProject - python -m pip install -e git+http://repo/my_project.git#egg=SomeProject + python -m pip install -e path/to/SomeProject + python -m pip install -e git+http://repo/my_project.git#egg=SomeProject .. group-tab:: Windows .. code-block:: shell - py -m pip install -e path/to/SomeProject - py -m pip install -e git+http://repo/my_project.git#egg=SomeProject + py -m pip install -e path/to/SomeProject + py -m pip install -e git+http://repo/my_project.git#egg=SomeProject (See the :ref:`VCS Support` section above for more information on VCS-related syntax.) @@ -957,292 +957,292 @@ Examples #. Install ``SomePackage`` and its dependencies from `PyPI`_ using :ref:`Requirement Specifiers` - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip install SomePackage # latest version - python -m pip install SomePackage==1.0.4 # specific version - python -m pip install 'SomePackage>=1.0.4' # minimum version + python -m pip install SomePackage # latest version + python -m pip install SomePackage==1.0.4 # specific version + python -m pip install 'SomePackage>=1.0.4' # minimum version .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip install SomePackage # latest version - py -m pip install SomePackage==1.0.4 # specific version - py -m pip install 'SomePackage>=1.0.4' # minimum version + py -m pip install SomePackage # latest version + py -m pip install SomePackage==1.0.4 # specific version + py -m pip install 'SomePackage>=1.0.4' # minimum version #. Install a list of requirements specified in a file. See the :ref:`Requirements files `. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip install -r requirements.txt + python -m pip install -r requirements.txt .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip install -r requirements.txt + py -m pip install -r requirements.txt #. Upgrade an already installed ``SomePackage`` to the latest from PyPI. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip install --upgrade SomePackage + python -m pip install --upgrade SomePackage .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip install --upgrade SomePackage + py -m pip install --upgrade SomePackage #. Install a local project in "editable" mode. See the section on :ref:`Editable Installs `. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip install -e . # project in current directory - python -m pip install -e path/to/project # project in another directory + python -m pip install -e . # project in current directory + python -m pip install -e path/to/project # project in another directory .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip install -e . # project in current directory - py -m pip install -e path/to/project # project in another directory + py -m pip install -e . # project in current directory + py -m pip install -e path/to/project # project in another directory #. Install a project from VCS - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1 + python -m pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1 .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1 + py -m pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1 #. Install a project from VCS in "editable" mode. See the sections on :ref:`VCS Support ` and :ref:`Editable Installs `. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage # from git - python -m pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage # from mercurial - python -m python -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage # from svn - python -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage # from 'feature' branch - python -m pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory + python -m pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage # from git + python -m pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage # from mercurial + python -m python -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage # from svn + python -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage # from 'feature' branch + python -m pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage # from git - py -m pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage # from mercurial - py -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage # from svn - py -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage # from 'feature' branch - py -m pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory + py -m pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage # from git + py -m pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage # from mercurial + py -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage # from svn + py -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage # from 'feature' branch + py -m pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory #. Install a package with `setuptools extras`_. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip install SomePackage[PDF] - python -m pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@master#subdirectory=subdir_path" - python -m pip install .[PDF] # project in current directory - python -m pip install SomePackage[PDF]==3.0 - python -m pip install SomePackage[PDF,EPUB] # multiple extras + python -m pip install SomePackage[PDF] + python -m pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@master#subdirectory=subdir_path" + python -m pip install .[PDF] # project in current directory + python -m pip install SomePackage[PDF]==3.0 + python -m pip install SomePackage[PDF,EPUB] # multiple extras .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip install SomePackage[PDF] - py -m pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@master#subdirectory=subdir_path" - py -m pip install .[PDF] # project in current directory - py -m pip install SomePackage[PDF]==3.0 - py -m pip install SomePackage[PDF,EPUB] # multiple extras + py -m pip install SomePackage[PDF] + py -m pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@master#subdirectory=subdir_path" + py -m pip install .[PDF] # project in current directory + py -m pip install SomePackage[PDF]==3.0 + py -m pip install SomePackage[PDF,EPUB] # multiple extras #. Install a particular source archive file. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip install ./downloads/SomePackage-1.0.4.tar.gz - python -m pip install http://my.package.repo/SomePackage-1.0.4.zip + python -m pip install ./downloads/SomePackage-1.0.4.tar.gz + python -m pip install http://my.package.repo/SomePackage-1.0.4.zip .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip install ./downloads/SomePackage-1.0.4.tar.gz - py -m pip install http://my.package.repo/SomePackage-1.0.4.zip + py -m pip install ./downloads/SomePackage-1.0.4.tar.gz + py -m pip install http://my.package.repo/SomePackage-1.0.4.zip #. Install a particular source archive file following :pep:`440` direct references. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl - python -m pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl" - python -m pip install SomeProject@http://my.package.repo/1.2.3.tar.gz + python -m pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl + python -m pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl" + python -m pip install SomeProject@http://my.package.repo/1.2.3.tar.gz .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl - py -m pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl" - py -m pip install SomeProject@http://my.package.repo/1.2.3.tar.gz + py -m pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl + py -m pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl" + py -m pip install SomeProject@http://my.package.repo/1.2.3.tar.gz #. Install from alternative package repositories. Install from a different index, and not `PyPI`_ - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip install --index-url http://my.package.repo/simple/ SomePackage + python -m pip install --index-url http://my.package.repo/simple/ SomePackage .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip install --index-url http://my.package.repo/simple/ SomePackage + py -m pip install --index-url http://my.package.repo/simple/ SomePackage Search an additional index during install, in addition to `PyPI`_ - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip install --extra-index-url http://my.package.repo/simple SomePackage + python -m pip install --extra-index-url http://my.package.repo/simple SomePackage .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip install --extra-index-url http://my.package.repo/simple SomePackage + py -m pip install --extra-index-url http://my.package.repo/simple SomePackage Install from a local flat directory containing archives (and don't scan indexes): - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip install --no-index --find-links=file:///local/dir/ SomePackage - python -m pip install --no-index --find-links=/local/dir/ SomePackage - python -m pip install --no-index --find-links=relative/dir/ SomePackage + python -m pip install --no-index --find-links=file:///local/dir/ SomePackage + python -m pip install --no-index --find-links=/local/dir/ SomePackage + python -m pip install --no-index --find-links=relative/dir/ SomePackage .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip install --no-index --find-links=file:///local/dir/ SomePackage - py -m pip install --no-index --find-links=/local/dir/ SomePackage - py -m pip install --no-index --find-links=relative/dir/ SomePackage + py -m pip install --no-index --find-links=file:///local/dir/ SomePackage + py -m pip install --no-index --find-links=/local/dir/ SomePackage + py -m pip install --no-index --find-links=relative/dir/ SomePackage #. Find pre-release and development versions, in addition to stable versions. By default, pip only finds stable versions. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip install --pre SomePackage + python -m pip install --pre SomePackage .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip install --pre SomePackage + py -m pip install --pre SomePackage #. Install packages from source. Do not use any binary packages - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip install SomePackage1 SomePackage2 --no-binary :all: + python -m pip install SomePackage1 SomePackage2 --no-binary :all: .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip install SomePackage1 SomePackage2 --no-binary :all: + py -m pip install SomePackage1 SomePackage2 --no-binary :all: Specify ``SomePackage1`` to be installed from source: - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip install SomePackage1 SomePackage2 --no-binary SomePackage1 + python -m pip install SomePackage1 SomePackage2 --no-binary SomePackage1 .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip install SomePackage1 SomePackage2 --no-binary SomePackage1 + py -m pip install SomePackage1 SomePackage2 --no-binary SomePackage1 ---- diff --git a/docs/html/reference/pip_list.rst b/docs/html/reference/pip_list.rst index cb6064cc0..1489ed751 100644 --- a/docs/html/reference/pip_list.rst +++ b/docs/html/reference/pip_list.rst @@ -40,11 +40,11 @@ Examples #. List installed packages. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console $ python -m pip list docutils (0.10) @@ -55,7 +55,7 @@ Examples .. group-tab:: Windows - .. code-block:: console + .. code-block:: console C:\> py -m pip list docutils (0.10) @@ -66,11 +66,11 @@ Examples #. List outdated packages (excluding editables), and the latest version available. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console $ python -m pip list --outdated docutils (Current: 0.10 Latest: 0.11) @@ -78,7 +78,7 @@ Examples .. group-tab:: Windows - .. code-block:: console + .. code-block:: console C:\> py -m pip list --outdated docutils (Current: 0.10 Latest: 0.11) @@ -87,11 +87,11 @@ Examples #. List installed packages with column formatting. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console $ python -m pip list --format columns Package Version @@ -102,7 +102,7 @@ Examples .. group-tab:: Windows - .. code-block:: console + .. code-block:: console C:\> py -m pip list --format columns Package Version @@ -113,11 +113,11 @@ Examples #. List outdated packages with column formatting. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console $ python -m pip list -o --format columns Package Version Latest Type @@ -127,7 +127,7 @@ Examples .. group-tab:: Windows - .. code-block:: console + .. code-block:: console C:\> py -m pip list -o --format columns Package Version Latest Type @@ -138,29 +138,29 @@ Examples #. List packages that are not dependencies of other packages. Can be combined with other options. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console $ python -m pip list --outdated --not-required docutils (Current: 0.10 Latest: 0.11) .. group-tab:: Windows - .. code-block:: console + .. code-block:: console C:\> py -m pip list --outdated --not-required docutils (Current: 0.10 Latest: 0.11) #. Use legacy formatting - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console $ python -m pip list --format=legacy colorama (0.3.7) @@ -170,7 +170,7 @@ Examples .. group-tab:: Windows - .. code-block:: console + .. code-block:: console C:\> py -m pip list --format=legacy colorama (0.3.7) @@ -180,29 +180,29 @@ Examples #. Use json formatting - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console $ python -m pip list --format=json [{'name': 'colorama', 'version': '0.3.7'}, {'name': 'docopt', 'version': '0.6.2'}, ... .. group-tab:: Windows - .. code-block:: console + .. code-block:: console C:\> py -m pip list --format=json [{'name': 'colorama', 'version': '0.3.7'}, {'name': 'docopt', 'version': '0.6.2'}, ... #. Use freeze formatting - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console $ python -m pip list --format=freeze colorama==0.3.7 @@ -212,7 +212,7 @@ Examples .. group-tab:: Windows - .. code-block:: console + .. code-block:: console C:\> py -m pip list --format=freeze colorama==0.3.7 diff --git a/docs/html/reference/pip_search.rst b/docs/html/reference/pip_search.rst index c0737aa76..fba629593 100644 --- a/docs/html/reference/pip_search.rst +++ b/docs/html/reference/pip_search.rst @@ -38,11 +38,11 @@ Examples #. Search for "peppercorn" - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console $ python -m pip search peppercorn pepperedform - Helpers for using peppercorn with formprocess. @@ -50,7 +50,7 @@ Examples .. group-tab:: Windows - .. code-block:: console + .. code-block:: console C:\> py -m pip search peppercorn pepperedform - Helpers for using peppercorn with formprocess. diff --git a/docs/html/reference/pip_show.rst b/docs/html/reference/pip_show.rst index 71aa27e78..6bd3718b9 100644 --- a/docs/html/reference/pip_show.rst +++ b/docs/html/reference/pip_show.rst @@ -38,124 +38,124 @@ Examples #. Show information about a package: - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console - $ python -m pip show sphinx - Name: Sphinx - Version: 1.4.5 - Summary: Python documentation generator - Home-page: http://sphinx-doc.org/ - Author: Georg Brandl - Author-email: georg@python.org - License: BSD - Location: /my/env/lib/python2.7/site-packages - Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six + $ python -m pip show sphinx + Name: Sphinx + Version: 1.4.5 + Summary: Python documentation generator + Home-page: http://sphinx-doc.org/ + Author: Georg Brandl + Author-email: georg@python.org + License: BSD + Location: /my/env/lib/python2.7/site-packages + Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six .. group-tab:: Windows - .. code-block:: console + .. code-block:: console - C:\> py -m pip show sphinx - Name: Sphinx - Version: 1.4.5 - Summary: Python documentation generator - Home-page: http://sphinx-doc.org/ - Author: Georg Brandl - Author-email: georg@python.org - License: BSD - Location: /my/env/lib/python2.7/site-packages - Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six + C:\> py -m pip show sphinx + Name: Sphinx + Version: 1.4.5 + Summary: Python documentation generator + Home-page: http://sphinx-doc.org/ + Author: Georg Brandl + Author-email: georg@python.org + License: BSD + Location: /my/env/lib/python2.7/site-packages + Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six #. Show all information about a package - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console - $ python -m pip show --verbose sphinx - Name: Sphinx - Version: 1.4.5 - Summary: Python documentation generator - Home-page: http://sphinx-doc.org/ - Author: Georg Brandl - Author-email: georg@python.org - License: BSD - Location: /my/env/lib/python2.7/site-packages - Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six - Metadata-Version: 2.0 - Installer: - Classifiers: - Development Status :: 5 - Production/Stable - Environment :: Console - Environment :: Web Environment - Intended Audience :: Developers - Intended Audience :: Education - License :: OSI Approved :: BSD License - Operating System :: OS Independent - Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 3 - Framework :: Sphinx - Framework :: Sphinx :: Extension - Framework :: Sphinx :: Theme - Topic :: Documentation - Topic :: Documentation :: Sphinx - Topic :: Text Processing - Topic :: Utilities - Entry-points: - [console_scripts] - sphinx-apidoc = sphinx.apidoc:main - sphinx-autogen = sphinx.ext.autosummary.generate:main - sphinx-build = sphinx:main - sphinx-quickstart = sphinx.quickstart:main - [distutils.commands] - build_sphinx = sphinx.setup_command:BuildDoc + $ python -m pip show --verbose sphinx + Name: Sphinx + Version: 1.4.5 + Summary: Python documentation generator + Home-page: http://sphinx-doc.org/ + Author: Georg Brandl + Author-email: georg@python.org + License: BSD + Location: /my/env/lib/python2.7/site-packages + Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six + Metadata-Version: 2.0 + Installer: + Classifiers: + Development Status :: 5 - Production/Stable + Environment :: Console + Environment :: Web Environment + Intended Audience :: Developers + Intended Audience :: Education + License :: OSI Approved :: BSD License + Operating System :: OS Independent + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 3 + Framework :: Sphinx + Framework :: Sphinx :: Extension + Framework :: Sphinx :: Theme + Topic :: Documentation + Topic :: Documentation :: Sphinx + Topic :: Text Processing + Topic :: Utilities + Entry-points: + [console_scripts] + sphinx-apidoc = sphinx.apidoc:main + sphinx-autogen = sphinx.ext.autosummary.generate:main + sphinx-build = sphinx:main + sphinx-quickstart = sphinx.quickstart:main + [distutils.commands] + build_sphinx = sphinx.setup_command:BuildDoc .. group-tab:: Windows - .. code-block:: console + .. code-block:: console - C:\> py -m pip show --verbose sphinx - Name: Sphinx - Version: 1.4.5 - Summary: Python documentation generator - Home-page: http://sphinx-doc.org/ - Author: Georg Brandl - Author-email: georg@python.org - License: BSD - Location: /my/env/lib/python2.7/site-packages - Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six - Metadata-Version: 2.0 - Installer: - Classifiers: - Development Status :: 5 - Production/Stable - Environment :: Console - Environment :: Web Environment - Intended Audience :: Developers - Intended Audience :: Education - License :: OSI Approved :: BSD License - Operating System :: OS Independent - Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 3 - Framework :: Sphinx - Framework :: Sphinx :: Extension - Framework :: Sphinx :: Theme - Topic :: Documentation - Topic :: Documentation :: Sphinx - Topic :: Text Processing - Topic :: Utilities - Entry-points: - [console_scripts] - sphinx-apidoc = sphinx.apidoc:main - sphinx-autogen = sphinx.ext.autosummary.generate:main - sphinx-build = sphinx:main - sphinx-quickstart = sphinx.quickstart:main - [distutils.commands] - build_sphinx = sphinx.setup_command:BuildDoc + C:\> py -m pip show --verbose sphinx + Name: Sphinx + Version: 1.4.5 + Summary: Python documentation generator + Home-page: http://sphinx-doc.org/ + Author: Georg Brandl + Author-email: georg@python.org + License: BSD + Location: /my/env/lib/python2.7/site-packages + Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six + Metadata-Version: 2.0 + Installer: + Classifiers: + Development Status :: 5 - Production/Stable + Environment :: Console + Environment :: Web Environment + Intended Audience :: Developers + Intended Audience :: Education + License :: OSI Approved :: BSD License + Operating System :: OS Independent + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 3 + Framework :: Sphinx + Framework :: Sphinx :: Extension + Framework :: Sphinx :: Theme + Topic :: Documentation + Topic :: Documentation :: Sphinx + Topic :: Text Processing + Topic :: Utilities + Entry-points: + [console_scripts] + sphinx-apidoc = sphinx.apidoc:main + sphinx-autogen = sphinx.ext.autosummary.generate:main + sphinx-build = sphinx:main + sphinx-quickstart = sphinx.quickstart:main + [distutils.commands] + build_sphinx = sphinx.setup_command:BuildDoc diff --git a/docs/html/reference/pip_uninstall.rst b/docs/html/reference/pip_uninstall.rst index ecc3b9210..8b31c5673 100644 --- a/docs/html/reference/pip_uninstall.rst +++ b/docs/html/reference/pip_uninstall.rst @@ -38,26 +38,26 @@ Examples #. Uninstall a package. - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: console + .. code-block:: console - $ python -m pip uninstall simplejson - Uninstalling simplejson: - /home/me/env/lib/python2.7/site-packages/simplejson - /home/me/env/lib/python2.7/site-packages/simplejson-2.2.1-py2.7.egg-info - Proceed (y/n)? y - Successfully uninstalled simplejson + $ python -m pip uninstall simplejson + Uninstalling simplejson: + /home/me/env/lib/python2.7/site-packages/simplejson + /home/me/env/lib/python2.7/site-packages/simplejson-2.2.1-py2.7.egg-info + Proceed (y/n)? y + Successfully uninstalled simplejson .. group-tab:: Windows - .. code-block:: console + .. code-block:: console - C:\> py -m pip uninstall simplejson - Uninstalling simplejson: - /home/me/env/lib/python2.7/site-packages/simplejson - /home/me/env/lib/python2.7/site-packages/simplejson-2.2.1-py2.7.egg-info - Proceed (y/n)? y - Successfully uninstalled simplejson + C:\> py -m pip uninstall simplejson + Uninstalling simplejson: + /home/me/env/lib/python2.7/site-packages/simplejson + /home/me/env/lib/python2.7/site-packages/simplejson-2.2.1-py2.7.egg-info + Proceed (y/n)? y + Successfully uninstalled simplejson diff --git a/docs/html/reference/pip_wheel.rst b/docs/html/reference/pip_wheel.rst index 5dd4b0328..c1bdf37f8 100644 --- a/docs/html/reference/pip_wheel.rst +++ b/docs/html/reference/pip_wheel.rst @@ -36,17 +36,17 @@ In order for pip to build a wheel, ``setup.py`` must implement the .. tabs:: - .. group-tab:: Unix/macOS + .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python setup.py bdist_wheel -d TARGET + python setup.py bdist_wheel -d TARGET - .. group-tab:: Windows + .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py setup.py bdist_wheel -d TARGET + py setup.py bdist_wheel -d TARGET This command must create a wheel compatible with the invoking Python @@ -64,17 +64,17 @@ example: .. tabs:: - .. group-tab:: Unix/macOS + .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell - python -m pip wheel --global-option bdist_ext --global-option -DFOO wheel + python -m pip wheel --global-option bdist_ext --global-option -DFOO wheel - .. group-tab:: Windows + .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell - py -m pip wheel --global-option bdist_ext --global-option -DFOO wheel + py -m pip wheel --global-option bdist_ext --global-option -DFOO wheel will result in a build command of @@ -103,34 +103,34 @@ Examples #. Build wheels for a requirement (and all its dependencies), and then install - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell python -m pip wheel --wheel-dir=/tmp/wheelhouse SomePackage python -m pip install --no-index --find-links=/tmp/wheelhouse SomePackage .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell py -m pip wheel --wheel-dir=/tmp/wheelhouse SomePackage py -m pip install --no-index --find-links=/tmp/wheelhouse SomePackage #. Build a wheel for a package from source - .. tabs:: + .. tabs:: .. group-tab:: Unix/macOS - .. code-block:: shell + .. code-block:: shell python -m pip wheel --no-binary SomePackage SomePackage .. group-tab:: Windows - .. code-block:: shell + .. code-block:: shell py -m pip wheel --no-binary SomePackage SomePackage From 294995bc47961b0d948e50521fc0a5ebfebab71b Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 14 Sep 2020 23:00:48 +0300 Subject: [PATCH 156/209] Fix typo --- docs/html/development/release-process.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/html/development/release-process.rst b/docs/html/development/release-process.rst index 66ff3ca7c..ebba13258 100644 --- a/docs/html/development/release-process.rst +++ b/docs/html/development/release-process.rst @@ -16,7 +16,7 @@ with version numbers. Our release months are January, April, July, October. The release date within that month will be up to the release manager for that release. If there are no changes, then that release month is skipped and the next release will be -3 month later. +3 months later. The release manager may, at their discretion, choose whether or not there will be a pre-release period for a release, and if there is may extend that From 0fc1044ff44a83b3c4f5ed340217c940def6ad37 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Wed, 16 Sep 2020 17:49:15 +0530 Subject: [PATCH 157/209] :newspaper: --- news/8861.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/8861.bugfix diff --git a/news/8861.bugfix b/news/8861.bugfix new file mode 100644 index 000000000..d623419fa --- /dev/null +++ b/news/8861.bugfix @@ -0,0 +1 @@ +Tweak the output during dependency resolution in the new resolver. From c947d00882ef6f12cbae1bfb71bc2677c4d2a82a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Tue, 25 Aug 2020 22:06:44 +0700 Subject: [PATCH 158/209] [fast-deps] Check download directory before making requests --- news/8804.feature | 3 +++ src/pip/_internal/operations/prepare.py | 29 +++++++++++++------------ 2 files changed, 18 insertions(+), 14 deletions(-) create mode 100644 news/8804.feature diff --git a/news/8804.feature b/news/8804.feature new file mode 100644 index 000000000..a29333342 --- /dev/null +++ b/news/8804.feature @@ -0,0 +1,3 @@ +Check the download directory for existing wheels to possibly avoid +fetching metadata when the ``fast-deps`` feature is used with +``pip wheel`` and ``pip download``. diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 0a2540526..09540edce 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -486,26 +486,27 @@ class RequirementPreparer(object): link = req.link self._log_preparing_link(req) with indent_log(): - wheel_dist = self._fetch_metadata_using_lazy_wheel(link) - if wheel_dist is not None: - req.needs_more_preparation = True - return wheel_dist + download_dir = self._get_download_dir(req.link) + if download_dir is not None and link.is_wheel: + hashes = self._get_linked_req_hashes(req) + file_path = _check_download_dir(req.link, download_dir, hashes) + if file_path is not None: + self._downloaded[req.link.url] = file_path, None + else: + file_path = None + + if file_path is None: + wheel_dist = self._fetch_metadata_using_lazy_wheel(link) + if wheel_dist is not None: + req.needs_more_preparation = True + return wheel_dist return self._prepare_linked_requirement(req, parallel_builds) def prepare_linked_requirements_more(self, reqs, parallel_builds=False): # type: (Iterable[InstallRequirement], bool) -> None """Prepare a linked requirement more, if needed.""" reqs = [req for req in reqs if req.needs_more_preparation] - links = [] # type: List[Link] - for req in reqs: - download_dir = self._get_download_dir(req.link) - if download_dir is not None: - hashes = self._get_linked_req_hashes(req) - file_path = _check_download_dir(req.link, download_dir, hashes) - if download_dir is None or file_path is None: - links.append(req.link) - else: - self._downloaded[req.link.url] = file_path, None + links = [req.link for req in reqs] # Let's download to a temporary directory. tmpdir = TempDirectory(kind="unpack", globally_managed=True).path From 2b07c5d28888c431437e770adff3d036b4b1e1f1 Mon Sep 17 00:00:00 2001 From: Vipul Kumar Date: Wed, 23 Sep 2020 04:27:53 +0000 Subject: [PATCH 159/209] End no-color's description with period punctuation It would be nice, if like description of other options, "--no-color"'s description also ends with a period. --- src/pip/_internal/cli/cmdoptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index ed42c5f5a..f94ddfabb 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -187,7 +187,7 @@ no_color = partial( dest='no_color', action='store_true', default=False, - help="Suppress colored output", + help="Suppress colored output.", ) # type: Callable[..., Option] version = partial( From 4c534e65d21d4b17a76e29794496400f72e40f6f Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Wed, 23 Sep 2020 18:27:39 +0530 Subject: [PATCH 160/209] Tweak message when -r is not passed on a requirements.txt --- src/pip/_internal/req/constructors.py | 2 +- tests/unit/test_req.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index 7a4641ef5..24fff26ec 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -159,7 +159,7 @@ def deduce_helpful_msg(req): """ msg = "" if os.path.exists(req): - msg = " It does exist." + msg = " The path does exist. " # Try to parse and check if it is a requirements file. try: with open(req, 'r') as fp: diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index a5a9d4bae..730a26a88 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -572,7 +572,7 @@ class TestInstallRequirement(object): install_req_from_line(req_file_path) err_msg = e.value.args[0] assert "Invalid requirement" in err_msg - assert "It looks like a path. It does exist." in err_msg + assert "It looks like a path. The path does exist." in err_msg assert "appears to be a requirements file." in err_msg assert "If that is the case, use the '-r' flag to install" in err_msg From e93c0ba6a326d39b3a9c52853707052f3c8e3e78 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Wed, 23 Sep 2020 19:50:53 +0530 Subject: [PATCH 161/209] Update linters None of these require any changes to the codebase. --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 04c72d8e3..f0e8adea4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ exclude: 'src/pip/_vendor/' repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.5.0 + rev: v3.2.0 hooks: - id: check-builtin-literals - id: check-added-large-files @@ -17,7 +17,7 @@ repos: exclude: .patch - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.1 + rev: 3.8.3 hooks: - id: flake8 additional_dependencies: [ @@ -56,6 +56,6 @@ repos: exclude: NEWS.rst - repo: https://github.com/mgedmin/check-manifest - rev: '0.42' + rev: '0.43' hooks: - id: check-manifest From 89dad17b87281a7cf8652d476edcc15c16f214a6 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Wed, 23 Sep 2020 19:51:42 +0530 Subject: [PATCH 162/209] Update linter: pygrep-hooks --- .pre-commit-config.yaml | 2 +- docs/html/user_guide.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f0e8adea4..6feaa1754 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,7 +44,7 @@ repos: args: ["--pretty", "-2"] - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.5.1 + rev: v1.6.0 hooks: - id: python-no-log-warn - id: python-no-eval diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index ce013c2a3..8e8b25259 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -1105,7 +1105,7 @@ The big change in this release is to the pip dependency resolver within pip. Computers need to know the right order to install pieces of software -("to install `x`, you need to install `y` first"). So, when Python +("to install ``x``, you need to install ``y`` first"). So, when Python programmers share software as packages, they have to precisely describe those installation prerequisites, and pip needs to navigate tricky situations where it's getting conflicting instructions. This new From 25ab172b55bba6213e4bfb68b5c6e1bc4bec857e Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Wed, 23 Sep 2020 18:38:01 +0530 Subject: [PATCH 163/209] Update linter: isort --- .pre-commit-config.yaml | 2 +- src/pip/_internal/__init__.py | 2 +- src/pip/_internal/build_env.py | 3 ++- src/pip/_internal/cache.py | 2 +- src/pip/_internal/cli/base_command.py | 5 ++--- src/pip/_internal/cli/cmdoptions.py | 3 ++- src/pip/_internal/cli/command_context.py | 2 +- src/pip/_internal/cli/main_parser.py | 2 +- src/pip/_internal/cli/spinners.py | 2 +- src/pip/_internal/commands/__init__.py | 1 + src/pip/_internal/commands/check.py | 2 +- src/pip/_internal/commands/completion.py | 2 +- src/pip/_internal/commands/configuration.py | 2 +- src/pip/_internal/commands/debug.py | 5 +++-- src/pip/_internal/commands/help.py | 6 ++++-- src/pip/_internal/commands/list.py | 5 +++-- src/pip/_internal/commands/search.py | 3 ++- src/pip/_internal/commands/show.py | 2 +- src/pip/_internal/configuration.py | 4 +--- src/pip/_internal/distributions/base.py | 3 ++- src/pip/_internal/distributions/installed.py | 1 + src/pip/_internal/distributions/sdist.py | 1 + src/pip/_internal/distributions/wheel.py | 1 + src/pip/_internal/exceptions.py | 4 ++-- src/pip/_internal/index/collector.py | 14 +++++++++++--- src/pip/_internal/index/package_finder.py | 9 ++++++++- src/pip/_internal/locations.py | 3 +-- src/pip/_internal/main.py | 2 +- src/pip/_internal/models/candidate.py | 1 + src/pip/_internal/models/direct_url.py | 4 +--- src/pip/_internal/models/format_control.py | 2 +- src/pip/_internal/models/link.py | 1 + src/pip/_internal/models/selection_prefs.py | 1 + src/pip/_internal/network/auth.py | 6 +++--- src/pip/_internal/network/cache.py | 2 +- src/pip/_internal/network/session.py | 4 +--- src/pip/_internal/network/xmlrpc.py | 1 + src/pip/_internal/operations/build/metadata.py | 3 ++- src/pip/_internal/operations/build/wheel.py | 1 + src/pip/_internal/operations/check.py | 5 ++--- src/pip/_internal/operations/freeze.py | 18 +++++++++++++----- src/pip/_internal/operations/install/wheel.py | 2 +- src/pip/_internal/pyproject.py | 2 +- src/pip/_internal/req/constructors.py | 5 ++--- src/pip/_internal/req/req_file.py | 10 +++++++++- src/pip/_internal/req/req_install.py | 12 ++++++------ src/pip/_internal/req/req_set.py | 1 + src/pip/_internal/req/req_tracker.py | 3 ++- src/pip/_internal/req/req_uninstall.py | 11 ++++++++++- src/pip/_internal/resolution/base.py | 1 + .../_internal/resolution/legacy/resolver.py | 1 + .../_internal/resolution/resolvelib/factory.py | 2 +- .../resolution/resolvelib/provider.py | 2 +- src/pip/_internal/utils/compat.py | 2 +- src/pip/_internal/utils/direct_url_helpers.py | 4 ++-- src/pip/_internal/utils/encoding.py | 2 +- src/pip/_internal/utils/entrypoints.py | 2 +- src/pip/_internal/utils/hashes.py | 5 ++--- src/pip/_internal/utils/misc.py | 15 +++++++++++++-- src/pip/_internal/utils/packaging.py | 3 ++- src/pip/_internal/utils/parallel.py | 2 +- src/pip/_internal/utils/subprocess.py | 9 ++++++++- src/pip/_internal/vcs/bazaar.py | 1 + src/pip/_internal/vcs/git.py | 1 + src/pip/_internal/vcs/subversion.py | 3 ++- src/pip/_internal/vcs/versioncontrol.py | 14 ++++++++++++-- src/pip/_internal/wheel_builder.py | 4 +--- tests/conftest.py | 3 ++- tests/functional/test_uninstall.py | 3 ++- tests/lib/__init__.py | 1 + tests/lib/server.py | 11 ++++++++++- tests/lib/wheel.py | 12 ++++++++++-- tests/unit/test_locations.py | 2 ++ 73 files changed, 198 insertions(+), 95 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6feaa1754..f1afdd473 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: exclude: tests/data - repo: https://github.com/timothycrosley/isort - rev: 4.3.21 + rev: 5.5.3 hooks: - id: isort files: \.py$ diff --git a/src/pip/_internal/__init__.py b/src/pip/_internal/__init__.py index 264c2cab8..a778e9948 100755 --- a/src/pip/_internal/__init__.py +++ b/src/pip/_internal/__init__.py @@ -2,7 +2,7 @@ import pip._internal.utils.inject_securetransport # noqa from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, List + from typing import List, Optional def main(args=None): diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 28d1ad689..a08e63cd0 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -19,7 +19,8 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from types import TracebackType - from typing import Tuple, Set, Iterable, Optional, List, Type + from typing import Iterable, List, Optional, Set, Tuple, Type + from pip._internal.index.package_finder import PackageFinder logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index 07db948b9..def8dd64a 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -17,7 +17,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.urls import path_to_url if MYPY_CHECK_RUNNING: - from typing import Optional, Set, List, Any, Dict + from typing import Any, Dict, List, Optional, Set from pip._vendor.packaging.tags import Tag diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py index 197400a72..8f14aa6b7 100644 --- a/src/pip/_internal/cli/base_command.py +++ b/src/pip/_internal/cli/base_command.py @@ -43,12 +43,11 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.virtualenv import running_under_virtualenv if MYPY_CHECK_RUNNING: - from typing import List, Optional, Tuple, Any from optparse import Values + from typing import Any, List, Optional, Tuple - from pip._internal.utils.temp_dir import ( + from pip._internal.utils.temp_dir import \ TempDirectoryTypeRegistry as TempDirRegistry - ) __all__ = ['Command'] diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index f94ddfabb..2f640b2cb 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -30,8 +30,9 @@ from pip._internal.utils.hashes import STRONG_HASHES from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any, Callable, Dict, Optional, Tuple from optparse import OptionParser, Values + from typing import Any, Callable, Dict, Optional, Tuple + from pip._internal.cli.parser import ConfigOptionParser diff --git a/src/pip/_internal/cli/command_context.py b/src/pip/_internal/cli/command_context.py index d1a64a776..669c77774 100644 --- a/src/pip/_internal/cli/command_context.py +++ b/src/pip/_internal/cli/command_context.py @@ -5,7 +5,7 @@ from pip._vendor.contextlib2 import ExitStack from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Iterator, ContextManager, TypeVar + from typing import ContextManager, Iterator, TypeVar _T = TypeVar('_T', covariant=True) diff --git a/src/pip/_internal/cli/main_parser.py b/src/pip/_internal/cli/main_parser.py index 08c82c1f7..6356d831d 100644 --- a/src/pip/_internal/cli/main_parser.py +++ b/src/pip/_internal/cli/main_parser.py @@ -15,7 +15,7 @@ from pip._internal.utils.misc import get_pip_version, get_prog from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Tuple, List + from typing import List, Tuple __all__ = ["create_main_parser", "parse_command"] diff --git a/src/pip/_internal/cli/spinners.py b/src/pip/_internal/cli/spinners.py index c6c4c5cd1..65c3c23d7 100644 --- a/src/pip/_internal/cli/spinners.py +++ b/src/pip/_internal/cli/spinners.py @@ -13,7 +13,7 @@ from pip._internal.utils.logging import get_indentation from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Iterator, IO + from typing import IO, Iterator logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/commands/__init__.py b/src/pip/_internal/commands/__init__.py index 6825fa6e2..4f0c4ba3a 100644 --- a/src/pip/_internal/commands/__init__.py +++ b/src/pip/_internal/commands/__init__.py @@ -18,6 +18,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import Any + from pip._internal.cli.base_command import Command diff --git a/src/pip/_internal/commands/check.py b/src/pip/_internal/commands/check.py index b557ca641..e066bb63c 100644 --- a/src/pip/_internal/commands/check.py +++ b/src/pip/_internal/commands/check.py @@ -12,8 +12,8 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING logger = logging.getLogger(__name__) if MYPY_CHECK_RUNNING: - from typing import List, Any from optparse import Values + from typing import Any, List class CheckCommand(Command): diff --git a/src/pip/_internal/commands/completion.py b/src/pip/_internal/commands/completion.py index 9b99f51f0..b19f1ed1a 100644 --- a/src/pip/_internal/commands/completion.py +++ b/src/pip/_internal/commands/completion.py @@ -9,8 +9,8 @@ from pip._internal.utils.misc import get_prog from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import List from optparse import Values + from typing import List BASE_COMPLETION = """ # pip {shell} completion start{script}# pip {shell} completion end diff --git a/src/pip/_internal/commands/configuration.py b/src/pip/_internal/commands/configuration.py index f9b3ab79d..2a6311acd 100644 --- a/src/pip/_internal/commands/configuration.py +++ b/src/pip/_internal/commands/configuration.py @@ -15,8 +15,8 @@ from pip._internal.utils.misc import get_prog, write_output from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import List, Any, Optional from optparse import Values + from typing import Any, List, Optional from pip._internal.configuration import Kind diff --git a/src/pip/_internal/commands/debug.py b/src/pip/_internal/commands/debug.py index ff369d7d9..1b65c4306 100644 --- a/src/pip/_internal/commands/debug.py +++ b/src/pip/_internal/commands/debug.py @@ -19,9 +19,10 @@ from pip._internal.utils.misc import get_pip_version from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from types import ModuleType - from typing import List, Optional, Dict from optparse import Values + from types import ModuleType + from typing import Dict, List, Optional + from pip._internal.configuration import Configuration logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/commands/help.py b/src/pip/_internal/commands/help.py index a2edc2989..2ab2b6d8f 100644 --- a/src/pip/_internal/commands/help.py +++ b/src/pip/_internal/commands/help.py @@ -6,8 +6,8 @@ from pip._internal.exceptions import CommandError from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import List from optparse import Values + from typing import List class HelpCommand(Command): @@ -20,7 +20,9 @@ class HelpCommand(Command): def run(self, options, args): # type: (Values, List[str]) -> int from pip._internal.commands import ( - commands_dict, create_command, get_similar_commands, + commands_dict, + create_command, + get_similar_commands, ) try: diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index 7c18a7d9f..a6dfa5fd5 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -24,10 +24,11 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from optparse import Values - from typing import List, Set, Tuple, Iterator + from typing import Iterator, List, Set, Tuple + + from pip._vendor.pkg_resources import Distribution from pip._internal.network.session import PipSession - from pip._vendor.pkg_resources import Distribution logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/commands/search.py b/src/pip/_internal/commands/search.py index ff0947202..ea4f47db5 100644 --- a/src/pip/_internal/commands/search.py +++ b/src/pip/_internal/commands/search.py @@ -24,7 +24,8 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from optparse import Values - from typing import List, Dict, Optional + from typing import Dict, List, Optional + from typing_extensions import TypedDict TransformedHit = TypedDict( 'TransformedHit', diff --git a/src/pip/_internal/commands/show.py b/src/pip/_internal/commands/show.py index 3892c5959..b0b3f3abd 100644 --- a/src/pip/_internal/commands/show.py +++ b/src/pip/_internal/commands/show.py @@ -14,7 +14,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from optparse import Values - from typing import List, Dict, Iterator + from typing import Dict, Iterator, List logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 13cab9230..23614fd2b 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -28,9 +28,7 @@ from pip._internal.utils.misc import ensure_dir, enum from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( - Any, Dict, Iterable, List, NewType, Optional, Tuple - ) + from typing import Any, Dict, Iterable, List, NewType, Optional, Tuple RawConfigParser = configparser.RawConfigParser # Shorthand Kind = NewType("Kind", str) diff --git a/src/pip/_internal/distributions/base.py b/src/pip/_internal/distributions/base.py index b836b98d1..3a789f804 100644 --- a/src/pip/_internal/distributions/base.py +++ b/src/pip/_internal/distributions/base.py @@ -8,8 +8,9 @@ if MYPY_CHECK_RUNNING: from typing import Optional from pip._vendor.pkg_resources import Distribution - from pip._internal.req import InstallRequirement + from pip._internal.index.package_finder import PackageFinder + from pip._internal.req import InstallRequirement @add_metaclass(abc.ABCMeta) diff --git a/src/pip/_internal/distributions/installed.py b/src/pip/_internal/distributions/installed.py index 0d15bf424..a813b211f 100644 --- a/src/pip/_internal/distributions/installed.py +++ b/src/pip/_internal/distributions/installed.py @@ -5,6 +5,7 @@ if MYPY_CHECK_RUNNING: from typing import Optional from pip._vendor.pkg_resources import Distribution + from pip._internal.index.package_finder import PackageFinder diff --git a/src/pip/_internal/distributions/sdist.py b/src/pip/_internal/distributions/sdist.py index be3d7d97a..06b9df09c 100644 --- a/src/pip/_internal/distributions/sdist.py +++ b/src/pip/_internal/distributions/sdist.py @@ -10,6 +10,7 @@ if MYPY_CHECK_RUNNING: from typing import Set, Tuple from pip._vendor.pkg_resources import Distribution + from pip._internal.index.package_finder import PackageFinder diff --git a/src/pip/_internal/distributions/wheel.py b/src/pip/_internal/distributions/wheel.py index bf3482b15..2adc22862 100644 --- a/src/pip/_internal/distributions/wheel.py +++ b/src/pip/_internal/distributions/wheel.py @@ -6,6 +6,7 @@ from pip._internal.utils.wheel import pkg_resources_distribution_for_wheel if MYPY_CHECK_RUNNING: from pip._vendor.pkg_resources import Distribution + from pip._internal.index.package_finder import PackageFinder diff --git a/src/pip/_internal/exceptions.py b/src/pip/_internal/exceptions.py index 3f26215d6..62bde1eed 100644 --- a/src/pip/_internal/exceptions.py +++ b/src/pip/_internal/exceptions.py @@ -9,10 +9,10 @@ from pip._vendor.six import iteritems from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any, Optional, List, Dict, Text + from typing import Any, Dict, List, Optional, Text from pip._vendor.pkg_resources import Distribution - from pip._vendor.requests.models import Response, Request + from pip._vendor.requests.models import Request, Response from pip._vendor.six import PY3 from pip._vendor.six.moves import configparser diff --git a/src/pip/_internal/index/collector.py b/src/pip/_internal/index/collector.py index 6c35fc660..e6230c767 100644 --- a/src/pip/_internal/index/collector.py +++ b/src/pip/_internal/index/collector.py @@ -28,12 +28,20 @@ from pip._internal.utils.urls import path_to_url, url_to_path from pip._internal.vcs import is_url, vcs if MYPY_CHECK_RUNNING: + import xml.etree.ElementTree from optparse import Values from typing import ( - Callable, Iterable, List, MutableMapping, Optional, - Protocol, Sequence, Tuple, TypeVar, Union, + Callable, + Iterable, + List, + MutableMapping, + Optional, + Protocol, + Sequence, + Tuple, + TypeVar, + Union, ) - import xml.etree.ElementTree from pip._vendor.requests import Response diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 84115783a..19a9c2c13 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -35,7 +35,14 @@ from pip._internal.utils.urls import url_to_path if MYPY_CHECK_RUNNING: from typing import ( - FrozenSet, Iterable, List, Optional, Set, Text, Tuple, Union, + FrozenSet, + Iterable, + List, + Optional, + Set, + Text, + Tuple, + Union, ) from pip._vendor.packaging.tags import Tag diff --git a/src/pip/_internal/locations.py b/src/pip/_internal/locations.py index 0c1235488..35a4512b4 100644 --- a/src/pip/_internal/locations.py +++ b/src/pip/_internal/locations.py @@ -22,9 +22,8 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING, cast from pip._internal.utils.virtualenv import running_under_virtualenv if MYPY_CHECK_RUNNING: - from typing import Dict, List, Optional, Union - from distutils.cmd import Command as DistutilsCommand + from typing import Dict, List, Optional, Union # Application Directories diff --git a/src/pip/_internal/main.py b/src/pip/_internal/main.py index 3208d5b88..1c99c49a1 100644 --- a/src/pip/_internal/main.py +++ b/src/pip/_internal/main.py @@ -1,7 +1,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, List + from typing import List, Optional def main(args=None): diff --git a/src/pip/_internal/models/candidate.py b/src/pip/_internal/models/candidate.py index 9149e0fc6..0d89a8c07 100644 --- a/src/pip/_internal/models/candidate.py +++ b/src/pip/_internal/models/candidate.py @@ -5,6 +5,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from pip._vendor.packaging.version import _BaseVersion + from pip._internal.models.link import Link diff --git a/src/pip/_internal/models/direct_url.py b/src/pip/_internal/models/direct_url.py index 87bd9fe4b..99aa68d12 100644 --- a/src/pip/_internal/models/direct_url.py +++ b/src/pip/_internal/models/direct_url.py @@ -8,9 +8,7 @@ from pip._vendor.six.moves.urllib import parse as urllib_parse from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( - Any, Dict, Iterable, Optional, Type, TypeVar, Union - ) + from typing import Any, Dict, Iterable, Optional, Type, TypeVar, Union T = TypeVar("T") diff --git a/src/pip/_internal/models/format_control.py b/src/pip/_internal/models/format_control.py index c6275e721..adcf61e28 100644 --- a/src/pip/_internal/models/format_control.py +++ b/src/pip/_internal/models/format_control.py @@ -4,7 +4,7 @@ from pip._internal.exceptions import CommandError from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, Set, FrozenSet + from typing import FrozenSet, Optional, Set class FormatControl(object): diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index c0d278ade..29ef402be 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -16,6 +16,7 @@ from pip._internal.utils.urls import path_to_url, url_to_path if MYPY_CHECK_RUNNING: from typing import Optional, Text, Tuple, Union + from pip._internal.index.collector import HTMLPage from pip._internal.utils.hashes import Hashes diff --git a/src/pip/_internal/models/selection_prefs.py b/src/pip/_internal/models/selection_prefs.py index 5db3ca91c..83110dd8f 100644 --- a/src/pip/_internal/models/selection_prefs.py +++ b/src/pip/_internal/models/selection_prefs.py @@ -2,6 +2,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import Optional + from pip._internal.models.format_control import FormatControl diff --git a/src/pip/_internal/network/auth.py b/src/pip/_internal/network/auth.py index c49deaaf1..357811a16 100644 --- a/src/pip/_internal/network/auth.py +++ b/src/pip/_internal/network/auth.py @@ -20,12 +20,12 @@ from pip._internal.utils.misc import ( from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Dict, Optional, Tuple, List, Any + from typing import Any, Dict, List, Optional, Tuple + + from pip._vendor.requests.models import Request, Response from pip._internal.vcs.versioncontrol import AuthInfo - from pip._vendor.requests.models import Response, Request - Credentials = Tuple[str, str, str] logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/network/cache.py b/src/pip/_internal/network/cache.py index a0d55b5e9..d2a1b7313 100644 --- a/src/pip/_internal/network/cache.py +++ b/src/pip/_internal/network/cache.py @@ -13,7 +13,7 @@ from pip._internal.utils.misc import ensure_dir from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, Iterator + from typing import Iterator, Optional def is_from_cache(response): diff --git a/src/pip/_internal/network/session.py b/src/pip/_internal/network/session.py index 68ef68db5..176f0fb68 100644 --- a/src/pip/_internal/network/session.py +++ b/src/pip/_internal/network/session.py @@ -37,9 +37,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.urls import url_to_path if MYPY_CHECK_RUNNING: - from typing import ( - Iterator, List, Optional, Tuple, Union, - ) + from typing import Iterator, List, Optional, Tuple, Union from pip._internal.models.link import Link diff --git a/src/pip/_internal/network/xmlrpc.py b/src/pip/_internal/network/xmlrpc.py index e61126241..504018f28 100644 --- a/src/pip/_internal/network/xmlrpc.py +++ b/src/pip/_internal/network/xmlrpc.py @@ -14,6 +14,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import Dict + from pip._internal.network.session import PipSession diff --git a/src/pip/_internal/operations/build/metadata.py b/src/pip/_internal/operations/build/metadata.py index cf52f8d8f..5709962b0 100644 --- a/src/pip/_internal/operations/build/metadata.py +++ b/src/pip/_internal/operations/build/metadata.py @@ -8,9 +8,10 @@ from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from pip._internal.build_env import BuildEnvironment from pip._vendor.pep517.wrappers import Pep517HookCaller + from pip._internal.build_env import BuildEnvironment + def generate_metadata(build_env, backend): # type: (BuildEnvironment, Pep517HookCaller) -> str diff --git a/src/pip/_internal/operations/build/wheel.py b/src/pip/_internal/operations/build/wheel.py index 0c28c4989..d16ee0966 100644 --- a/src/pip/_internal/operations/build/wheel.py +++ b/src/pip/_internal/operations/build/wheel.py @@ -6,6 +6,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import List, Optional + from pip._vendor.pep517.wrappers import Pep517HookCaller logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/operations/check.py b/src/pip/_internal/operations/check.py index 5714915bc..52e8116e1 100644 --- a/src/pip/_internal/operations/check.py +++ b/src/pip/_internal/operations/check.py @@ -16,10 +16,9 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING logger = logging.getLogger(__name__) if MYPY_CHECK_RUNNING: + from typing import Any, Callable, Dict, List, Optional, Set, Tuple + from pip._internal.req.req_install import InstallRequirement - from typing import ( - Any, Callable, Dict, Optional, Set, Tuple, List - ) # Shorthands PackageSet = Dict[str, 'PackageDetails'] diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py index ddb9cb232..b98b8cd79 100644 --- a/src/pip/_internal/operations/freeze.py +++ b/src/pip/_internal/operations/freeze.py @@ -26,12 +26,20 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import ( - Iterator, Optional, List, Container, Set, Dict, Tuple, Iterable, Union + Container, + Dict, + Iterable, + Iterator, + List, + Optional, + Set, + Tuple, + Union, ) + + from pip._vendor.pkg_resources import Distribution, Requirement + from pip._internal.cache import WheelCache - from pip._vendor.pkg_resources import ( - Distribution, Requirement - ) RequirementInfo = Tuple[Optional[Union[str, Requirement]], bool, List[str]] @@ -183,7 +191,7 @@ def get_requirement_info(dist): location = os.path.normcase(os.path.abspath(dist.location)) - from pip._internal.vcs import vcs, RemoteNotFoundError + from pip._internal.vcs import RemoteNotFoundError, vcs vcs_backend = vcs.get_backend_for_dir(location) if vcs_backend is None: diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py index e91b1b8d5..af6b39052 100644 --- a/src/pip/_internal/operations/install/wheel.py +++ b/src/pip/_internal/operations/install/wheel.py @@ -62,10 +62,10 @@ if not MYPY_CHECK_RUNNING: else: from email.message import Message from typing import ( + IO, Any, Callable, Dict, - IO, Iterable, Iterator, List, diff --git a/src/pip/_internal/pyproject.py b/src/pip/_internal/pyproject.py index 6b4faf7a7..4144a9ed6 100644 --- a/src/pip/_internal/pyproject.py +++ b/src/pip/_internal/pyproject.py @@ -12,7 +12,7 @@ from pip._internal.exceptions import InstallationError from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any, Optional, List + from typing import Any, List, Optional def _is_list_of_str(obj): diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index 7a4641ef5..b089c502d 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -31,9 +31,8 @@ from pip._internal.utils.urls import path_to_url from pip._internal.vcs import is_url, vcs if MYPY_CHECK_RUNNING: - from typing import ( - Any, Dict, Optional, Set, Tuple, Union, - ) + from typing import Any, Dict, Optional, Set, Tuple, Union + from pip._internal.req.req_file import ParsedRequirement diff --git a/src/pip/_internal/req/req_file.py b/src/pip/_internal/req/req_file.py index c8d7a0a5a..b070dc640 100644 --- a/src/pip/_internal/req/req_file.py +++ b/src/pip/_internal/req/req_file.py @@ -26,7 +26,15 @@ from pip._internal.utils.urls import get_url_scheme, url_to_path if MYPY_CHECK_RUNNING: from optparse import Values from typing import ( - Any, Callable, Dict, Iterator, List, NoReturn, Optional, Text, Tuple, + Any, + Callable, + Dict, + Iterator, + List, + NoReturn, + Optional, + Text, + Tuple, ) from pip._internal.index.package_finder import PackageFinder diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 72f792342..9a6763074 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -53,13 +53,13 @@ from pip._internal.utils.virtualenv import running_under_virtualenv from pip._internal.vcs import vcs if MYPY_CHECK_RUNNING: - from typing import ( - Any, Dict, Iterable, List, Optional, Sequence, Union, - ) - from pip._internal.build_env import BuildEnvironment - from pip._vendor.pkg_resources import Distribution - from pip._vendor.packaging.specifiers import SpecifierSet + from typing import Any, Dict, Iterable, List, Optional, Sequence, Union + from pip._vendor.packaging.markers import Marker + from pip._vendor.packaging.specifiers import SpecifierSet + from pip._vendor.pkg_resources import Distribution + + from pip._internal.build_env import BuildEnvironment logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/req/req_set.py b/src/pip/_internal/req/req_set.py index ab4b6f849..c9ea3be5d 100644 --- a/src/pip/_internal/req/req_set.py +++ b/src/pip/_internal/req/req_set.py @@ -12,6 +12,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import Dict, Iterable, List, Optional, Tuple + from pip._internal.req.req_install import InstallRequirement diff --git a/src/pip/_internal/req/req_tracker.py b/src/pip/_internal/req/req_tracker.py index 13fb24563..7379c307b 100644 --- a/src/pip/_internal/req/req_tracker.py +++ b/src/pip/_internal/req/req_tracker.py @@ -14,8 +14,9 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from types import TracebackType from typing import Dict, Iterator, Optional, Set, Type, Union - from pip._internal.req.req_install import InstallRequirement + from pip._internal.models.link import Link + from pip._internal.req.req_install import InstallRequirement logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/req/req_uninstall.py b/src/pip/_internal/req/req_uninstall.py index 69719d338..2e7dfcc73 100644 --- a/src/pip/_internal/req/req_uninstall.py +++ b/src/pip/_internal/req/req_uninstall.py @@ -29,8 +29,17 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import ( - Any, Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple, + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Optional, + Set, + Tuple, ) + from pip._vendor.pkg_resources import Distribution logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/resolution/base.py b/src/pip/_internal/resolution/base.py index 2fa118bd8..6d50555e5 100644 --- a/src/pip/_internal/resolution/base.py +++ b/src/pip/_internal/resolution/base.py @@ -2,6 +2,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import Callable, List + from pip._internal.req.req_install import InstallRequirement from pip._internal.req.req_set import RequirementSet diff --git a/src/pip/_internal/resolution/legacy/resolver.py b/src/pip/_internal/resolution/legacy/resolver.py index d2dafa77f..6ef00bba1 100644 --- a/src/pip/_internal/resolution/legacy/resolver.py +++ b/src/pip/_internal/resolution/legacy/resolver.py @@ -42,6 +42,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import DefaultDict, List, Optional, Set, Tuple + from pip._vendor.pkg_resources import Distribution from pip._internal.cache import WheelCache diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 97b0b2730..96e2d5331 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -38,8 +38,8 @@ from .requirements import ( if MYPY_CHECK_RUNNING: from typing import ( - FrozenSet, Dict, + FrozenSet, Iterable, List, Optional, diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index 80577a61c..99f99bfc2 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -16,7 +16,7 @@ if MYPY_CHECK_RUNNING: Union, ) - from .base import Requirement, Candidate + from .base import Candidate, Requirement from .factory import Factory # Notes on the relationship between the provider, the factory, and the diff --git a/src/pip/_internal/utils/compat.py b/src/pip/_internal/utils/compat.py index 89c5169af..cc6353678 100644 --- a/src/pip/_internal/utils/compat.py +++ b/src/pip/_internal/utils/compat.py @@ -246,8 +246,8 @@ else: def ioctl_GWINSZ(fd): try: import fcntl - import termios import struct + import termios cr = struct.unpack_from( 'hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '12345678') diff --git a/src/pip/_internal/utils/direct_url_helpers.py b/src/pip/_internal/utils/direct_url_helpers.py index f1fe209e9..a355a6c5e 100644 --- a/src/pip/_internal/utils/direct_url_helpers.py +++ b/src/pip/_internal/utils/direct_url_helpers.py @@ -20,10 +20,10 @@ except ImportError: if MYPY_CHECK_RUNNING: from typing import Optional - from pip._internal.models.link import Link - from pip._vendor.pkg_resources import Distribution + from pip._internal.models.link import Link + logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/utils/encoding.py b/src/pip/_internal/utils/encoding.py index 5b83d61bb..42a57535a 100644 --- a/src/pip/_internal/utils/encoding.py +++ b/src/pip/_internal/utils/encoding.py @@ -6,7 +6,7 @@ import sys from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import List, Tuple, Text + from typing import List, Text, Tuple BOMS = [ (codecs.BOM_UTF8, 'utf-8'), diff --git a/src/pip/_internal/utils/entrypoints.py b/src/pip/_internal/utils/entrypoints.py index befd01c89..64d1cb2bd 100644 --- a/src/pip/_internal/utils/entrypoints.py +++ b/src/pip/_internal/utils/entrypoints.py @@ -4,7 +4,7 @@ from pip._internal.cli.main import main from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, List + from typing import List, Optional def _wrapper(args=None): diff --git a/src/pip/_internal/utils/hashes.py b/src/pip/_internal/utils/hashes.py index d9f74a640..4d4e26b59 100644 --- a/src/pip/_internal/utils/hashes.py +++ b/src/pip/_internal/utils/hashes.py @@ -13,9 +13,8 @@ from pip._internal.utils.misc import read_chunks from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( - Dict, List, BinaryIO, NoReturn, Iterator - ) + from typing import BinaryIO, Dict, Iterator, List, NoReturn + from pip._vendor.six import PY3 if PY3: from hashlib import _Hash diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index 5629c60c1..289db0e39 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -54,9 +54,20 @@ else: if MYPY_CHECK_RUNNING: from typing import ( - Any, AnyStr, Callable, Container, Iterable, Iterator, List, Optional, - Text, Tuple, TypeVar, Union, + Any, + AnyStr, + Callable, + Container, + Iterable, + Iterator, + List, + Optional, + Text, + Tuple, + TypeVar, + Union, ) + from pip._vendor.pkg_resources import Distribution VersionInfo = Tuple[int, int, int] diff --git a/src/pip/_internal/utils/packaging.py b/src/pip/_internal/utils/packaging.py index 68aa86edb..27fd20423 100644 --- a/src/pip/_internal/utils/packaging.py +++ b/src/pip/_internal/utils/packaging.py @@ -11,8 +11,9 @@ from pip._internal.utils.misc import display_path from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, Tuple from email.message import Message + from typing import Optional, Tuple + from pip._vendor.pkg_resources import Distribution diff --git a/src/pip/_internal/utils/parallel.py b/src/pip/_internal/utils/parallel.py index 9fe1fe8b9..d4113bdc2 100644 --- a/src/pip/_internal/utils/parallel.py +++ b/src/pip/_internal/utils/parallel.py @@ -29,8 +29,8 @@ from pip._vendor.six.moves import map from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Callable, Iterable, Iterator, Union, TypeVar from multiprocessing import pool + from typing import Callable, Iterable, Iterator, TypeVar, Union Pool = Union[pool.Pool, pool.ThreadPool] S = TypeVar('S') diff --git a/src/pip/_internal/utils/subprocess.py b/src/pip/_internal/utils/subprocess.py index d398e68da..1dfe02fa0 100644 --- a/src/pip/_internal/utils/subprocess.py +++ b/src/pip/_internal/utils/subprocess.py @@ -15,7 +15,14 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import ( - Any, Callable, Iterable, List, Mapping, Optional, Text, Union, + Any, + Callable, + Iterable, + List, + Mapping, + Optional, + Text, + Union, ) CommandArgs = List[Union[str, HiddenText]] diff --git a/src/pip/_internal/vcs/bazaar.py b/src/pip/_internal/vcs/bazaar.py index 94408c52f..3180713f7 100644 --- a/src/pip/_internal/vcs/bazaar.py +++ b/src/pip/_internal/vcs/bazaar.py @@ -16,6 +16,7 @@ from pip._internal.vcs.versioncontrol import VersionControl, vcs if MYPY_CHECK_RUNNING: from typing import Optional, Tuple + from pip._internal.utils.misc import HiddenText from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index db8c72349..1831aede5 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -25,6 +25,7 @@ from pip._internal.vcs.versioncontrol import ( if MYPY_CHECK_RUNNING: from typing import Optional, Tuple + from pip._internal.utils.misc import HiddenText from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py index ab134970b..eae09c196 100644 --- a/src/pip/_internal/vcs/subversion.py +++ b/src/pip/_internal/vcs/subversion.py @@ -26,8 +26,9 @@ _svn_info_xml_url_re = re.compile(r'(.*)') if MYPY_CHECK_RUNNING: from typing import Optional, Tuple - from pip._internal.utils.subprocess import CommandArgs + from pip._internal.utils.misc import HiddenText + from pip._internal.utils.subprocess import CommandArgs from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py index 96f830f99..e4f61a99c 100644 --- a/src/pip/_internal/vcs/versioncontrol.py +++ b/src/pip/_internal/vcs/versioncontrol.py @@ -38,9 +38,19 @@ from pip._internal.utils.urls import get_url_scheme if MYPY_CHECK_RUNNING: from typing import ( - Dict, Iterable, Iterator, List, Optional, Text, Tuple, - Type, Union, Mapping, Any + Any, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Text, + Tuple, + Type, + Union, ) + from pip._internal.utils.misc import HiddenText from pip._internal.utils.subprocess import CommandArgs diff --git a/src/pip/_internal/wheel_builder.py b/src/pip/_internal/wheel_builder.py index fa08016bd..27fce66c2 100644 --- a/src/pip/_internal/wheel_builder.py +++ b/src/pip/_internal/wheel_builder.py @@ -19,9 +19,7 @@ from pip._internal.utils.urls import path_to_url from pip._internal.vcs import vcs if MYPY_CHECK_RUNNING: - from typing import ( - Any, Callable, Iterable, List, Optional, Tuple, - ) + from typing import Any, Callable, Iterable, List, Optional, Tuple from pip._internal.cache import WheelCache from pip._internal.req.req_install import InstallRequirement diff --git a/tests/conftest.py b/tests/conftest.py index 97a25f60e..ffd9f4215 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,7 +27,8 @@ from tests.lib.venv import VirtualEnvironment if MYPY_CHECK_RUNNING: from typing import Dict, Iterable - from tests.lib.server import MockServer as _MockServer, Responder + from tests.lib.server import MockServer as _MockServer + from tests.lib.server import Responder def pytest_addoption(parser): diff --git a/tests/functional/test_uninstall.py b/tests/functional/test_uninstall.py index 6e4aec0e5..5def65871 100644 --- a/tests/functional/test_uninstall.py +++ b/tests/functional/test_uninstall.py @@ -422,9 +422,10 @@ def test_uninstallpathset_no_paths(caplog): Test UninstallPathSet logs notification when there are no paths to uninstall """ - from pip._internal.req.req_uninstall import UninstallPathSet from pkg_resources import get_distribution + from pip._internal.req.req_uninstall import UninstallPathSet + caplog.set_level(logging.INFO) test_dist = get_distribution('pip') diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index ee243d0e6..07569d814 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -31,6 +31,7 @@ from tests.lib.wheel import make_wheel if MYPY_CHECK_RUNNING: from typing import List, Optional + from pip._internal.models.target_python import TargetPython diff --git a/tests/lib/server.py b/tests/lib/server.py index 6a8862ac5..ebbf120d3 100644 --- a/tests/lib/server.py +++ b/tests/lib/server.py @@ -15,7 +15,16 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from types import TracebackType from typing import ( - Any, Callable, Dict, Iterable, List, Optional, Text, Tuple, Type, Union + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Text, + Tuple, + Type, + Union, ) from werkzeug.serving import BaseWSGIServer diff --git a/tests/lib/wheel.py b/tests/lib/wheel.py index 8ea458658..d89a680a1 100644 --- a/tests/lib/wheel.py +++ b/tests/lib/wheel.py @@ -20,8 +20,16 @@ from tests.lib.path import Path if MYPY_CHECK_RUNNING: from typing import ( - AnyStr, Callable, Dict, List, Iterable, Optional, Tuple, Sequence, - TypeVar, Union, + AnyStr, + Callable, + Dict, + Iterable, + List, + Optional, + Sequence, + Tuple, + TypeVar, + Union, ) # path, digest, size diff --git a/tests/unit/test_locations.py b/tests/unit/test_locations.py index c9bbe7943..2ede236f9 100644 --- a/tests/unit/test_locations.py +++ b/tests/unit/test_locations.py @@ -100,6 +100,7 @@ class TestDistutilsScheme: f.parent.mkdir() f.write_text("[install]\ninstall-scripts=" + install_scripts) from distutils.dist import Distribution + # patch the function that returns what config files are present monkeypatch.setattr( Distribution, @@ -121,6 +122,7 @@ class TestDistutilsScheme: f.parent.mkdir() f.write_text("[install]\ninstall-lib=" + install_lib) from distutils.dist import Distribution + # patch the function that returns what config files are present monkeypatch.setattr( Distribution, From 3a8e8578a87458719ca0dbb9b31989fffba230a9 Mon Sep 17 00:00:00 2001 From: Elisha Hollander Date: Wed, 23 Sep 2020 17:42:29 +0300 Subject: [PATCH 164/209] Update LICENSE.txt --- LICENSE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.txt b/LICENSE.txt index 737fec5c5..75eb0fd80 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2008-2019 The pip developers (see AUTHORS.txt file) +Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the From a00f85d22bd86af2b8ecfff53efc732e95c20e7e Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Wed, 23 Sep 2020 19:59:10 +0530 Subject: [PATCH 165/209] Prepare flake8 for black --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index 45fd58a3e..6b034f0ee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,6 +16,7 @@ default_section = THIRDPARTY include_trailing_comma = true [flake8] +max-line-length = 88 exclude = ./build, .nox, @@ -28,6 +29,8 @@ ignore = G200, G202, # pycodestyle checks ignored in the default configuration E121, E123, E126, E133, E226, E241, E242, E704, W503, W504, W505, + # black adds spaces around ':' + E203, per-file-ignores = # G: The plugin logging-format treats every .log and .error as logging. noxfile.py: G From 58c594c06b46842e830faa7e1ce7a5df5819daef Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Wed, 23 Sep 2020 19:57:09 +0530 Subject: [PATCH 166/209] Prepare isort for black --- setup.cfg | 3 +-- src/pip/_internal/cli/base_command.py | 15 +++++-------- src/pip/_internal/cli/main_parser.py | 5 +---- src/pip/_internal/cli/req_command.py | 5 +---- src/pip/_internal/commands/configuration.py | 6 +----- src/pip/_internal/commands/search.py | 1 + src/pip/_internal/index/package_finder.py | 11 +--------- src/pip/_internal/models/target_python.py | 5 +---- src/pip/_internal/network/download.py | 12 ++--------- src/pip/_internal/network/lazy_wheel.py | 6 +----- src/pip/_internal/network/session.py | 1 + .../operations/build/wheel_legacy.py | 4 +--- src/pip/_internal/operations/check.py | 4 +--- src/pip/_internal/operations/freeze.py | 5 +---- src/pip/_internal/operations/install/wheel.py | 21 +++---------------- src/pip/_internal/operations/prepare.py | 11 ++-------- src/pip/_internal/req/req_file.py | 5 +---- src/pip/_internal/req/req_install.py | 10 +++++---- .../_internal/resolution/legacy/resolver.py | 5 +---- .../resolution/resolvelib/provider.py | 11 +--------- src/pip/_internal/self_outdated_check.py | 12 ++--------- src/pip/_internal/utils/hashes.py | 6 +----- src/pip/_internal/utils/misc.py | 14 +++---------- src/pip/_internal/utils/subprocess.py | 11 +--------- src/pip/_internal/vcs/versioncontrol.py | 6 +----- tests/functional/test_configuration.py | 5 +---- tests/functional/test_new_resolver_hashes.py | 5 +---- tests/functional/test_search.py | 6 +----- tests/functional/test_uninstall.py | 6 +----- tests/unit/resolution_resolvelib/conftest.py | 1 + tests/unit/test_finder.py | 5 +---- tests/unit/test_operations_prepare.py | 6 +----- tests/unit/test_req_file.py | 5 +---- tests/unit/test_resolution_legacy_resolver.py | 5 +---- tests/unit/test_utils.py | 6 +----- tests/unit/test_utils_unpacking.py | 6 +----- tests/unit/test_wheel.py | 4 +--- 37 files changed, 53 insertions(+), 202 deletions(-) diff --git a/setup.cfg b/setup.cfg index 6b034f0ee..450f953e4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,5 @@ [isort] +profile = black skip = ./build, .nox, @@ -6,14 +7,12 @@ skip = .scratch, _vendor, data -multi_line_output = 3 known_third_party = pip._vendor known_first_party = pip tests default_section = THIRDPARTY -include_trailing_comma = true [flake8] max-line-length = 88 diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py index 8f14aa6b7..e4b07e0ce 100644 --- a/src/pip/_internal/cli/base_command.py +++ b/src/pip/_internal/cli/base_command.py @@ -12,10 +12,7 @@ import traceback from pip._internal.cli import cmdoptions from pip._internal.cli.command_context import CommandContextMixIn -from pip._internal.cli.parser import ( - ConfigOptionParser, - UpdatingDefaultsHelpFormatter, -) +from pip._internal.cli.parser import ConfigOptionParser, UpdatingDefaultsHelpFormatter from pip._internal.cli.status_codes import ( ERROR, PREVIOUS_BUILD_DIR_ERROR, @@ -35,10 +32,7 @@ from pip._internal.utils.deprecation import deprecated from pip._internal.utils.filesystem import check_path_owner from pip._internal.utils.logging import BrokenStdoutLoggingError, setup_logging from pip._internal.utils.misc import get_prog, normalize_path -from pip._internal.utils.temp_dir import ( - global_tempdir_manager, - tempdir_registry, -) +from pip._internal.utils.temp_dir import global_tempdir_manager, tempdir_registry from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.virtualenv import running_under_virtualenv @@ -46,8 +40,9 @@ if MYPY_CHECK_RUNNING: from optparse import Values from typing import Any, List, Optional, Tuple - from pip._internal.utils.temp_dir import \ - TempDirectoryTypeRegistry as TempDirRegistry + from pip._internal.utils.temp_dir import ( + TempDirectoryTypeRegistry as TempDirRegistry, + ) __all__ = ['Command'] diff --git a/src/pip/_internal/cli/main_parser.py b/src/pip/_internal/cli/main_parser.py index 6356d831d..ba3cf68aa 100644 --- a/src/pip/_internal/cli/main_parser.py +++ b/src/pip/_internal/cli/main_parser.py @@ -5,10 +5,7 @@ import os import sys from pip._internal.cli import cmdoptions -from pip._internal.cli.parser import ( - ConfigOptionParser, - UpdatingDefaultsHelpFormatter, -) +from pip._internal.cli.parser import ConfigOptionParser, UpdatingDefaultsHelpFormatter from pip._internal.commands import commands_dict, get_similar_commands from pip._internal.exceptions import CommandError from pip._internal.utils.misc import get_pip_version, get_prog diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 76abce5ac..0757e34f6 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -38,10 +38,7 @@ if MYPY_CHECK_RUNNING: from pip._internal.req.req_install import InstallRequirement from pip._internal.req.req_tracker import RequirementTracker from pip._internal.resolution.base import BaseResolver - from pip._internal.utils.temp_dir import ( - TempDirectory, - TempDirectoryTypeRegistry, - ) + from pip._internal.utils.temp_dir import TempDirectory, TempDirectoryTypeRegistry logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/commands/configuration.py b/src/pip/_internal/commands/configuration.py index 2a6311acd..1ab90b47b 100644 --- a/src/pip/_internal/commands/configuration.py +++ b/src/pip/_internal/commands/configuration.py @@ -4,11 +4,7 @@ import subprocess from pip._internal.cli.base_command import Command from pip._internal.cli.status_codes import ERROR, SUCCESS -from pip._internal.configuration import ( - Configuration, - get_configuration_files, - kinds, -) +from pip._internal.configuration import Configuration, get_configuration_files, kinds from pip._internal.exceptions import PipError from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import get_prog, write_output diff --git a/src/pip/_internal/commands/search.py b/src/pip/_internal/commands/search.py index ea4f47db5..146d653e5 100644 --- a/src/pip/_internal/commands/search.py +++ b/src/pip/_internal/commands/search.py @@ -7,6 +7,7 @@ from collections import OrderedDict from pip._vendor import pkg_resources from pip._vendor.packaging.version import parse as parse_version + # NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is # why we ignore the type on this import from pip._vendor.six.moves import xmlrpc_client # type: ignore diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 19a9c2c13..5162a8191 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -34,16 +34,7 @@ from pip._internal.utils.unpacking import SUPPORTED_EXTENSIONS from pip._internal.utils.urls import url_to_path if MYPY_CHECK_RUNNING: - from typing import ( - FrozenSet, - Iterable, - List, - Optional, - Set, - Text, - Tuple, - Union, - ) + from typing import FrozenSet, Iterable, List, Optional, Set, Text, Tuple, Union from pip._vendor.packaging.tags import Tag from pip._vendor.packaging.version import _BaseVersion diff --git a/src/pip/_internal/models/target_python.py b/src/pip/_internal/models/target_python.py index 6d1ca7964..ad7e506a6 100644 --- a/src/pip/_internal/models/target_python.py +++ b/src/pip/_internal/models/target_python.py @@ -1,9 +1,6 @@ import sys -from pip._internal.utils.compatibility_tags import ( - get_supported, - version_info_to_nodot, -) +from pip._internal.utils.compatibility_tags import get_supported, version_info_to_nodot from pip._internal.utils.misc import normalize_version_info from pip._internal.utils.typing import MYPY_CHECK_RUNNING diff --git a/src/pip/_internal/network/download.py b/src/pip/_internal/network/download.py index 56feaabac..76896e899 100644 --- a/src/pip/_internal/network/download.py +++ b/src/pip/_internal/network/download.py @@ -11,16 +11,8 @@ from pip._internal.cli.progress_bars import DownloadProgressProvider from pip._internal.exceptions import NetworkConnectionError from pip._internal.models.index import PyPI from pip._internal.network.cache import is_from_cache -from pip._internal.network.utils import ( - HEADERS, - raise_for_status, - response_chunks, -) -from pip._internal.utils.misc import ( - format_size, - redact_auth_from_url, - splitext, -) +from pip._internal.network.utils import HEADERS, raise_for_status, response_chunks +from pip._internal.utils.misc import format_size, redact_auth_from_url, splitext from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: diff --git a/src/pip/_internal/network/lazy_wheel.py b/src/pip/_internal/network/lazy_wheel.py index a0f9e151d..608475aba 100644 --- a/src/pip/_internal/network/lazy_wheel.py +++ b/src/pip/_internal/network/lazy_wheel.py @@ -10,11 +10,7 @@ from zipfile import BadZipfile, ZipFile from pip._vendor.requests.models import CONTENT_CHUNK_SIZE from pip._vendor.six.moves import range -from pip._internal.network.utils import ( - HEADERS, - raise_for_status, - response_chunks, -) +from pip._internal.network.utils import HEADERS, raise_for_status, response_chunks from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.wheel import pkg_resources_distribution_for_wheel diff --git a/src/pip/_internal/network/session.py b/src/pip/_internal/network/session.py index 176f0fb68..454945d9a 100644 --- a/src/pip/_internal/network/session.py +++ b/src/pip/_internal/network/session.py @@ -25,6 +25,7 @@ from pip._vendor.urllib3.exceptions import InsecureRequestWarning from pip import __version__ from pip._internal.network.auth import MultiDomainBasicAuth from pip._internal.network.cache import SafeFileCache + # Import ssl from compat so the initial import occurs in only one place. from pip._internal.utils.compat import has_tls, ipaddress from pip._internal.utils.glibc import libc_ver diff --git a/src/pip/_internal/operations/build/wheel_legacy.py b/src/pip/_internal/operations/build/wheel_legacy.py index 37dc876ac..9da365e4d 100644 --- a/src/pip/_internal/operations/build/wheel_legacy.py +++ b/src/pip/_internal/operations/build/wheel_legacy.py @@ -2,9 +2,7 @@ import logging import os.path from pip._internal.cli.spinners import open_spinner -from pip._internal.utils.setuptools_build import ( - make_setuptools_bdist_wheel_args, -) +from pip._internal.utils.setuptools_build import make_setuptools_bdist_wheel_args from pip._internal.utils.subprocess import ( LOG_DIVIDER, call_subprocess, diff --git a/src/pip/_internal/operations/check.py b/src/pip/_internal/operations/check.py index 52e8116e1..bc44d4357 100644 --- a/src/pip/_internal/operations/check.py +++ b/src/pip/_internal/operations/check.py @@ -7,9 +7,7 @@ from collections import namedtuple from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.pkg_resources import RequirementParseError -from pip._internal.distributions import ( - make_distribution_for_install_requirement, -) +from pip._internal.distributions import make_distribution_for_install_requirement from pip._internal.utils.misc import get_installed_distributions from pip._internal.utils.typing import MYPY_CHECK_RUNNING diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py index b98b8cd79..d4f790cd4 100644 --- a/src/pip/_internal/operations/freeze.py +++ b/src/pip/_internal/operations/freeze.py @@ -18,10 +18,7 @@ from pip._internal.utils.direct_url_helpers import ( direct_url_as_pep440_direct_reference, dist_get_direct_url, ) -from pip._internal.utils.misc import ( - dist_is_editable, - get_installed_distributions, -) +from pip._internal.utils.misc import dist_is_editable, get_installed_distributions from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py index af6b39052..8b67ebb94 100644 --- a/src/pip/_internal/operations/install/wheel.py +++ b/src/pip/_internal/operations/install/wheel.py @@ -21,14 +21,7 @@ from zipfile import ZipFile from pip._vendor import pkg_resources from pip._vendor.distlib.scripts import ScriptMaker from pip._vendor.distlib.util import get_export_entry -from pip._vendor.six import ( - PY2, - ensure_str, - ensure_text, - itervalues, - reraise, - text_type, -) +from pip._vendor.six import PY2, ensure_str, ensure_text, itervalues, reraise, text_type from pip._vendor.six.moves import filterfalse, map from pip._internal.exceptions import InstallationError @@ -36,12 +29,7 @@ from pip._internal.locations import get_major_minor_version from pip._internal.models.direct_url import DIRECT_URL_METADATA_NAME, DirectUrl from pip._internal.models.scheme import SCHEME_KEYS from pip._internal.utils.filesystem import adjacent_tmp_file, replace -from pip._internal.utils.misc import ( - captured_stdout, - ensure_dir, - hash_file, - partition, -) +from pip._internal.utils.misc import captured_stdout, ensure_dir, hash_file, partition from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.unpacking import ( current_umask, @@ -49,10 +37,7 @@ from pip._internal.utils.unpacking import ( set_extracted_file_to_default_mode_plus_executable, zip_item_is_executable, ) -from pip._internal.utils.wheel import ( - parse_wheel, - pkg_resources_distribution_for_wheel, -) +from pip._internal.utils.wheel import parse_wheel, pkg_resources_distribution_for_wheel # Use the custom cast function at runtime to make cast work, # and import typing.cast when performing pre-commit and type diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index acff7ffa1..37f84f47e 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -12,9 +12,7 @@ import shutil from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.six import PY2 -from pip._internal.distributions import ( - make_distribution_for_install_requirement, -) +from pip._internal.distributions import make_distribution_for_install_requirement from pip._internal.distributions.installed import InstalledDistribution from pip._internal.exceptions import ( DirectoryUrlHashUnsupported, @@ -34,12 +32,7 @@ from pip._internal.network.lazy_wheel import ( from pip._internal.utils.filesystem import copy2_fixed from pip._internal.utils.hashes import MissingHashes from pip._internal.utils.logging import indent_log -from pip._internal.utils.misc import ( - display_path, - hide_url, - path_to_display, - rmtree, -) +from pip._internal.utils.misc import display_path, hide_url, path_to_display, rmtree from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.unpacking import unpack_file diff --git a/src/pip/_internal/req/req_file.py b/src/pip/_internal/req/req_file.py index b070dc640..c8c9165d3 100644 --- a/src/pip/_internal/req/req_file.py +++ b/src/pip/_internal/req/req_file.py @@ -13,10 +13,7 @@ import sys from pip._vendor.six.moves.urllib import parse as urllib_parse from pip._internal.cli import cmdoptions -from pip._internal.exceptions import ( - InstallationError, - RequirementsFileParseError, -) +from pip._internal.exceptions import InstallationError, RequirementsFileParseError from pip._internal.models.search_scope import SearchScope from pip._internal.network.utils import raise_for_status from pip._internal.utils.encoding import auto_decode diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 9a6763074..42999a59f 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -22,10 +22,12 @@ from pip._internal.exceptions import InstallationError from pip._internal.locations import get_scheme from pip._internal.models.link import Link from pip._internal.operations.build.metadata import generate_metadata -from pip._internal.operations.build.metadata_legacy import \ - generate_metadata as generate_metadata_legacy -from pip._internal.operations.install.editable_legacy import \ - install_editable as install_editable_legacy +from pip._internal.operations.build.metadata_legacy import ( + generate_metadata as generate_metadata_legacy, +) +from pip._internal.operations.install.editable_legacy import ( + install_editable as install_editable_legacy, +) from pip._internal.operations.install.legacy import LegacyInstallFailure from pip._internal.operations.install.legacy import install as install_legacy from pip._internal.operations.install.wheel import install_wheel diff --git a/src/pip/_internal/resolution/legacy/resolver.py b/src/pip/_internal/resolution/legacy/resolver.py index 6ef00bba1..d0fc1a7b3 100644 --- a/src/pip/_internal/resolution/legacy/resolver.py +++ b/src/pip/_internal/resolution/legacy/resolver.py @@ -34,10 +34,7 @@ from pip._internal.resolution.base import BaseResolver from pip._internal.utils.compatibility_tags import get_supported from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import dist_in_usersite, normalize_version_info -from pip._internal.utils.packaging import ( - check_requires_python, - get_requires_python, -) +from pip._internal.utils.packaging import check_requires_python, get_requires_python from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index 99f99bfc2..a1bab05a4 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -5,16 +5,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING from .base import Constraint if MYPY_CHECK_RUNNING: - from typing import ( - Any, - Dict, - Iterable, - Optional, - Sequence, - Set, - Tuple, - Union, - ) + from typing import Any, Dict, Iterable, Optional, Sequence, Set, Tuple, Union from .base import Candidate, Requirement from .factory import Factory diff --git a/src/pip/_internal/self_outdated_check.py b/src/pip/_internal/self_outdated_check.py index fbd9dfd48..c2d166b18 100644 --- a/src/pip/_internal/self_outdated_check.py +++ b/src/pip/_internal/self_outdated_check.py @@ -13,16 +13,8 @@ from pip._vendor.six import ensure_binary from pip._internal.index.collector import LinkCollector from pip._internal.index.package_finder import PackageFinder from pip._internal.models.selection_prefs import SelectionPreferences -from pip._internal.utils.filesystem import ( - adjacent_tmp_file, - check_path_owner, - replace, -) -from pip._internal.utils.misc import ( - ensure_dir, - get_distribution, - get_installed_version, -) +from pip._internal.utils.filesystem import adjacent_tmp_file, check_path_owner, replace +from pip._internal.utils.misc import ensure_dir, get_distribution, get_installed_version from pip._internal.utils.packaging import get_installer from pip._internal.utils.typing import MYPY_CHECK_RUNNING diff --git a/src/pip/_internal/utils/hashes.py b/src/pip/_internal/utils/hashes.py index 4d4e26b59..b306dafe7 100644 --- a/src/pip/_internal/utils/hashes.py +++ b/src/pip/_internal/utils/hashes.py @@ -4,11 +4,7 @@ import hashlib from pip._vendor.six import iteritems, iterkeys, itervalues -from pip._internal.exceptions import ( - HashMismatch, - HashMissing, - InstallationError, -) +from pip._internal.exceptions import HashMismatch, HashMissing, InstallationError from pip._internal.utils.misc import read_chunks from pip._internal.utils.typing import MYPY_CHECK_RUNNING diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index 289db0e39..c122beb32 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -20,6 +20,7 @@ from itertools import tee from pip._vendor import pkg_resources from pip._vendor.packaging.utils import canonicalize_name + # NOTE: retrying is not annotated in typeshed as on 2017-07-17, which is # why we ignore the type on this import. from pip._vendor.retrying import retry # type: ignore @@ -30,17 +31,8 @@ from pip._vendor.six.moves.urllib.parse import unquote as urllib_unquote from pip import __version__ from pip._internal.exceptions import CommandError -from pip._internal.locations import ( - get_major_minor_version, - site_packages, - user_site, -) -from pip._internal.utils.compat import ( - WINDOWS, - expanduser, - stdlib_pkgs, - str_to_display, -) +from pip._internal.locations import get_major_minor_version, site_packages, user_site +from pip._internal.utils.compat import WINDOWS, expanduser, stdlib_pkgs, str_to_display from pip._internal.utils.typing import MYPY_CHECK_RUNNING, cast from pip._internal.utils.virtualenv import ( running_under_virtualenv, diff --git a/src/pip/_internal/utils/subprocess.py b/src/pip/_internal/utils/subprocess.py index 1dfe02fa0..605e711e6 100644 --- a/src/pip/_internal/utils/subprocess.py +++ b/src/pip/_internal/utils/subprocess.py @@ -14,16 +14,7 @@ from pip._internal.utils.misc import HiddenText, path_to_display from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( - Any, - Callable, - Iterable, - List, - Mapping, - Optional, - Text, - Union, - ) + from typing import Any, Callable, Iterable, List, Mapping, Optional, Text, Union CommandArgs = List[Union[str, HiddenText]] diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py index e4f61a99c..219f79673 100644 --- a/src/pip/_internal/vcs/versioncontrol.py +++ b/src/pip/_internal/vcs/versioncontrol.py @@ -12,11 +12,7 @@ import sys from pip._vendor import pkg_resources from pip._vendor.six.moves.urllib import parse as urllib_parse -from pip._internal.exceptions import ( - BadCommand, - InstallationError, - SubProcessError, -) +from pip._internal.exceptions import BadCommand, InstallationError, SubProcessError from pip._internal.utils.compat import console_to_str, samefile from pip._internal.utils.logging import subprocess_logger from pip._internal.utils.misc import ( diff --git a/tests/functional/test_configuration.py b/tests/functional/test_configuration.py index 63b243f78..f820bdc19 100644 --- a/tests/functional/test_configuration.py +++ b/tests/functional/test_configuration.py @@ -7,10 +7,7 @@ import textwrap import pytest from pip._internal.cli.status_codes import ERROR -from pip._internal.configuration import ( - CONFIG_BASENAME, - get_configuration_files, -) +from pip._internal.configuration import CONFIG_BASENAME, get_configuration_files from tests.lib.configuration_helpers import ConfigurationMixin, kinds diff --git a/tests/functional/test_new_resolver_hashes.py b/tests/functional/test_new_resolver_hashes.py index 4b13ebc30..ad5e2f3d0 100644 --- a/tests/functional/test_new_resolver_hashes.py +++ b/tests/functional/test_new_resolver_hashes.py @@ -4,10 +4,7 @@ import hashlib import pytest from pip._internal.utils.urls import path_to_url -from tests.lib import ( - create_basic_sdist_for_package, - create_basic_wheel_for_package, -) +from tests.lib import create_basic_sdist_for_package, create_basic_wheel_for_package _FindLinks = collections.namedtuple( "_FindLinks", "index_html sdist_hash wheel_hash", diff --git a/tests/functional/test_search.py b/tests/functional/test_search.py index 5918b4f64..1892e26b5 100644 --- a/tests/functional/test_search.py +++ b/tests/functional/test_search.py @@ -5,11 +5,7 @@ import pytest from pip._internal.cli.status_codes import NO_MATCHES_FOUND, SUCCESS from pip._internal.commands import create_command -from pip._internal.commands.search import ( - highest_version, - print_results, - transform_hits, -) +from pip._internal.commands.search import highest_version, print_results, transform_hits from tests.lib import pyversion if pyversion >= '3': diff --git a/tests/functional/test_uninstall.py b/tests/functional/test_uninstall.py index 5def65871..6c687ada5 100644 --- a/tests/functional/test_uninstall.py +++ b/tests/functional/test_uninstall.py @@ -13,11 +13,7 @@ import pytest from pip._internal.req.constructors import install_req_from_line from pip._internal.utils.misc import rmtree -from tests.lib import ( - assert_all_changes, - create_test_package_with_setup, - need_svn, -) +from tests.lib import assert_all_changes, create_test_package_with_setup, need_svn from tests.lib.local_repos import local_checkout, local_repo diff --git a/tests/unit/resolution_resolvelib/conftest.py b/tests/unit/resolution_resolvelib/conftest.py index 87f5d129c..9c1c9e5c4 100644 --- a/tests/unit/resolution_resolvelib/conftest.py +++ b/tests/unit/resolution_resolvelib/conftest.py @@ -4,6 +4,7 @@ from pip._internal.cli.req_command import RequirementCommand from pip._internal.commands.install import InstallCommand from pip._internal.index.collector import LinkCollector from pip._internal.index.package_finder import PackageFinder + # from pip._internal.models.index import PyPI from pip._internal.models.search_scope import SearchScope from pip._internal.models.selection_prefs import SelectionPreferences diff --git a/tests/unit/test_finder.py b/tests/unit/test_finder.py index 853af723b..55fdab3b8 100644 --- a/tests/unit/test_finder.py +++ b/tests/unit/test_finder.py @@ -8,10 +8,7 @@ from pip._vendor.packaging.tags import Tag from pkg_resources import parse_version import pip._internal.utils.compatibility_tags -from pip._internal.exceptions import ( - BestVersionAlreadyInstalled, - DistributionNotFound, -) +from pip._internal.exceptions import BestVersionAlreadyInstalled, DistributionNotFound from pip._internal.index.package_finder import ( CandidateEvaluator, InstallationCandidate, diff --git a/tests/unit/test_operations_prepare.py b/tests/unit/test_operations_prepare.py index ab6aaf6aa..af3ce72a1 100644 --- a/tests/unit/test_operations_prepare.py +++ b/tests/unit/test_operations_prepare.py @@ -13,11 +13,7 @@ from pip._internal.network.session import PipSession from pip._internal.operations.prepare import _copy_source_tree, unpack_url from pip._internal.utils.hashes import Hashes from pip._internal.utils.urls import path_to_url -from tests.lib.filesystem import ( - get_filelist, - make_socket_file, - make_unreadable_file, -) +from tests.lib.filesystem import get_filelist, make_socket_file, make_unreadable_file from tests.lib.path import Path from tests.lib.requests_mocks import MockResponse diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index 69d93a0cd..f995d05a6 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -10,10 +10,7 @@ from pip._vendor.six import PY2 from pretend import stub import pip._internal.req.req_file # this will be monkeypatched -from pip._internal.exceptions import ( - InstallationError, - RequirementsFileParseError, -) +from pip._internal.exceptions import InstallationError, RequirementsFileParseError from pip._internal.models.format_control import FormatControl from pip._internal.network.session import PipSession from pip._internal.req.constructors import ( diff --git a/tests/unit/test_resolution_legacy_resolver.py b/tests/unit/test_resolution_legacy_resolver.py index 561313c00..0388b42be 100644 --- a/tests/unit/test_resolution_legacy_resolver.py +++ b/tests/unit/test_resolution_legacy_resolver.py @@ -4,10 +4,7 @@ import mock import pytest from pip._vendor import pkg_resources -from pip._internal.exceptions import ( - NoneMetadataError, - UnsupportedPythonVersion, -) +from pip._internal.exceptions import NoneMetadataError, UnsupportedPythonVersion from pip._internal.req.constructors import install_req_from_line from pip._internal.resolution.legacy.resolver import ( Resolver, diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 0a1c47cd7..14b4d7482 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -16,11 +16,7 @@ from io import BytesIO import pytest from mock import Mock, patch -from pip._internal.exceptions import ( - HashMismatch, - HashMissing, - InstallationError, -) +from pip._internal.exceptions import HashMismatch, HashMissing, InstallationError from pip._internal.utils.deprecation import PipDeprecationWarning, deprecated from pip._internal.utils.encoding import BOMS, auto_decode from pip._internal.utils.glibc import ( diff --git a/tests/unit/test_utils_unpacking.py b/tests/unit/test_utils_unpacking.py index d01ffb9cd..5c2be24d4 100644 --- a/tests/unit/test_utils_unpacking.py +++ b/tests/unit/test_utils_unpacking.py @@ -10,11 +10,7 @@ import zipfile import pytest from pip._internal.exceptions import InstallationError -from pip._internal.utils.unpacking import ( - is_within_directory, - untar_file, - unzip_file, -) +from pip._internal.utils.unpacking import is_within_directory, untar_file, unzip_file class TestUnpackArchives(object): diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index 6c9fe02b3..35916058a 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -19,9 +19,7 @@ from pip._internal.models.direct_url import ( DirectUrl, ) from pip._internal.models.scheme import Scheme -from pip._internal.operations.build.wheel_legacy import ( - get_legacy_build_wheel_path, -) +from pip._internal.operations.build.wheel_legacy import get_legacy_build_wheel_path from pip._internal.operations.install import wheel from pip._internal.utils.compat import WINDOWS from pip._internal.utils.misc import hash_file From f8fe3531b9177e89e680484bbf741c751e2e8172 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 23 Sep 2020 22:00:32 +0300 Subject: [PATCH 167/209] Remove unnecessary config for isort 5 --- setup.cfg | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 45fd58a3e..58342df49 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,10 +9,6 @@ skip = multi_line_output = 3 known_third_party = pip._vendor -known_first_party = - pip - tests -default_section = THIRDPARTY include_trailing_comma = true [flake8] From 342939ffb945d0009a25cf5d4d63cf83e1d23a4b Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 24 Sep 2020 02:11:32 +0530 Subject: [PATCH 168/209] Remove news entries for released changes --- news/8090.bugfix | 3 --- news/8695.bugfix | 3 --- news/8701.bugfix | 2 -- news/8716.bugfix | 2 -- news/8717.bugfix | 1 - news/8724.bugfix | 2 -- 6 files changed, 13 deletions(-) delete mode 100644 news/8090.bugfix delete mode 100644 news/8695.bugfix delete mode 100644 news/8701.bugfix delete mode 100644 news/8716.bugfix delete mode 100644 news/8717.bugfix delete mode 100644 news/8724.bugfix diff --git a/news/8090.bugfix b/news/8090.bugfix deleted file mode 100644 index e9f2b7cbb..000000000 --- a/news/8090.bugfix +++ /dev/null @@ -1,3 +0,0 @@ -Only attempt to use the keyring once and if it fails, don't try again. -This prevents spamming users with several keyring unlock prompts when they -cannot unlock or don't want to do so. diff --git a/news/8695.bugfix b/news/8695.bugfix deleted file mode 100644 index 668e4672e..000000000 --- a/news/8695.bugfix +++ /dev/null @@ -1,3 +0,0 @@ -Fix regression that distributions in system site-packages are not correctly -found when a virtual environment is configured with ``system-site-packages`` -on. diff --git a/news/8701.bugfix b/news/8701.bugfix deleted file mode 100644 index 086a8f2eb..000000000 --- a/news/8701.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Disable caching for range requests, which causes corrupted wheels -when pip tries to obtain metadata using the feature ``fast-deps``. diff --git a/news/8716.bugfix b/news/8716.bugfix deleted file mode 100644 index 086a8f2eb..000000000 --- a/news/8716.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Disable caching for range requests, which causes corrupted wheels -when pip tries to obtain metadata using the feature ``fast-deps``. diff --git a/news/8717.bugfix b/news/8717.bugfix deleted file mode 100644 index e8c8533c4..000000000 --- a/news/8717.bugfix +++ /dev/null @@ -1 +0,0 @@ -Always use UTF-8 to read ``pyvenv.cfg`` to match the built-in ``venv``. diff --git a/news/8724.bugfix b/news/8724.bugfix deleted file mode 100644 index 8641098dd..000000000 --- a/news/8724.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -2020 Resolver: Correctly handle marker evaluation in constraints and exclude -them if their markers do not match the current environment. From d45ba65c37108499d17a7ffd2cbcb07fdade2ae4 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 24 Sep 2020 01:33:56 +0300 Subject: [PATCH 169/209] Include http directory in 'pip cache info' and 'pip cache purge' --- news/8892.feature | 1 + src/pip/_internal/commands/cache.py | 41 +++++++++---- tests/functional/test_cache.py | 90 +++++++++++++++++++++++++---- 3 files changed, 112 insertions(+), 20 deletions(-) create mode 100644 news/8892.feature diff --git a/news/8892.feature b/news/8892.feature new file mode 100644 index 000000000..96c99bf8c --- /dev/null +++ b/news/8892.feature @@ -0,0 +1 @@ +Include http subdirectory in ``pip cache info`` and ``pip cache purge`` commands. diff --git a/src/pip/_internal/commands/cache.py b/src/pip/_internal/commands/cache.py index b9d3ed410..efba24f1a 100644 --- a/src/pip/_internal/commands/cache.py +++ b/src/pip/_internal/commands/cache.py @@ -102,19 +102,30 @@ class CacheCommand(Command): if args: raise CommandError('Too many arguments') + num_http_files = len(self._find_http_files(options)) num_packages = len(self._find_wheels(options, '*')) - cache_location = self._wheels_cache_dir(options) - cache_size = filesystem.format_directory_size(cache_location) + http_cache_location = self._cache_dir(options, 'http') + wheels_cache_location = self._cache_dir(options, 'wheels') + http_cache_size = filesystem.format_directory_size(http_cache_location) + wheels_cache_size = filesystem.format_directory_size( + wheels_cache_location + ) message = textwrap.dedent(""" - Location: {location} - Size: {size} + HTTP files location: {http_cache_location} + HTTP files size: {http_cache_size} + Number of HTTP files: {num_http_files} + Wheels location: {wheels_cache_location} + Wheels size: {wheels_cache_size} Number of wheels: {package_count} """).format( - location=cache_location, + http_cache_location=http_cache_location, + http_cache_size=http_cache_size, + num_http_files=num_http_files, + wheels_cache_location=wheels_cache_location, package_count=num_packages, - size=cache_size, + wheels_cache_size=wheels_cache_size, ).strip() logger.info(message) @@ -169,6 +180,11 @@ class CacheCommand(Command): raise CommandError('Please provide a pattern') files = self._find_wheels(options, args[0]) + + # Only fetch http files if no specific pattern given + if args[0] == '*': + files += self._find_http_files(options) + if not files: raise CommandError('No matching packages') @@ -184,13 +200,18 @@ class CacheCommand(Command): return self.remove_cache_items(options, ['*']) - def _wheels_cache_dir(self, options): - # type: (Values) -> str - return os.path.join(options.cache_dir, 'wheels') + def _cache_dir(self, options, subdir): + # type: (Values, str) -> str + return os.path.join(options.cache_dir, subdir) + + def _find_http_files(self, options): + # type: (Values) -> List[str] + http_dir = self._cache_dir(options, 'http') + return filesystem.find_files(http_dir, '*') def _find_wheels(self, options, pattern): # type: (Values, str) -> List[str] - wheel_dir = self._wheels_cache_dir(options) + wheel_dir = self._cache_dir(options, 'wheels') # The wheel filename format, as specified in PEP 427, is: # {distribution}-{version}(-{build})?-{python}-{abi}-{platform}.whl diff --git a/tests/functional/test_cache.py b/tests/functional/test_cache.py index 603e11b5b..a1ffd9090 100644 --- a/tests/functional/test_cache.py +++ b/tests/functional/test_cache.py @@ -15,11 +15,30 @@ def cache_dir(script): return result.stdout.strip() +@pytest.fixture +def http_cache_dir(cache_dir): + return os.path.normcase(os.path.join(cache_dir, 'http')) + + @pytest.fixture def wheel_cache_dir(cache_dir): return os.path.normcase(os.path.join(cache_dir, 'wheels')) +@pytest.fixture +def http_cache_files(http_cache_dir): + destination = os.path.join(http_cache_dir, 'arbitrary', 'pathname') + + if not os.path.exists(destination): + return [] + + filenames = glob(os.path.join(destination, '*')) + files = [] + for filename in filenames: + files.append(os.path.join(destination, filename)) + return files + + @pytest.fixture def wheel_cache_files(wheel_cache_dir): destination = os.path.join(wheel_cache_dir, 'arbitrary', 'pathname') @@ -34,6 +53,24 @@ def wheel_cache_files(wheel_cache_dir): return files +@pytest.fixture +def populate_http_cache(http_cache_dir): + destination = os.path.join(http_cache_dir, 'arbitrary', 'pathname') + os.makedirs(destination) + + files = [ + ('aaaaaaaaa', os.path.join(destination, 'aaaaaaaaa')), + ('bbbbbbbbb', os.path.join(destination, 'bbbbbbbbb')), + ('ccccccccc', os.path.join(destination, 'ccccccccc')), + ] + + for _name, filename in files: + with open(filename, 'w'): + pass + + return files + + @pytest.fixture def populate_wheel_cache(wheel_cache_dir): destination = os.path.join(wheel_cache_dir, 'arbitrary', 'pathname') @@ -83,6 +120,29 @@ def list_matches_wheel_abspath(wheel_name, result): and os.path.exists(l), lines)) +@pytest.fixture +def remove_matches_http(http_cache_dir): + """Returns True if any line in `result`, which should be the output of + a `pip cache purge` call, matches `http_filename`. + + E.g., If http_filename is `aaaaaaaaa`, it searches for a line equal to + `Removed /arbitrary/pathname/aaaaaaaaa`. + """ + + def _remove_matches_http(http_filename, result): + lines = result.stdout.splitlines() + + # The "/arbitrary/pathname/" bit is an implementation detail of how + # the `populate_http_cache` fixture is implemented. + path = os.path.join( + http_cache_dir, 'arbitrary', 'pathname', http_filename, + ) + expected = 'Removed {}'.format(path) + return expected in lines + + return _remove_matches_http + + @pytest.fixture def remove_matches_wheel(wheel_cache_dir): """Returns True if any line in `result`, which should be the output of @@ -124,11 +184,14 @@ def test_cache_dir_too_many_args(script, cache_dir): assert 'ERROR: Too many arguments' in result.stderr.splitlines() -@pytest.mark.usefixtures("populate_wheel_cache") -def test_cache_info(script, wheel_cache_dir, wheel_cache_files): +@pytest.mark.usefixtures("populate_http_cache", "populate_wheel_cache") +def test_cache_info( + script, http_cache_dir, wheel_cache_dir, wheel_cache_files +): result = script.pip('cache', 'info') - assert 'Location: {}'.format(wheel_cache_dir) in result.stdout + assert 'HTTP files location: {}'.format(http_cache_dir) in result.stdout + assert 'Wheels location: {}'.format(wheel_cache_dir) in result.stdout num_wheels = len(wheel_cache_files) assert 'Number of wheels: {}'.format(num_wheels) in result.stdout @@ -265,21 +328,28 @@ def test_cache_remove_name_and_version_match(script, remove_matches_wheel): assert not remove_matches_wheel('zzz-7.8.9', result) -@pytest.mark.usefixtures("populate_wheel_cache") -def test_cache_purge(script, remove_matches_wheel): - """Running `pip cache purge` should remove all cached wheels.""" +@pytest.mark.usefixtures("populate_http_cache", "populate_wheel_cache") +def test_cache_purge(script, remove_matches_http, remove_matches_wheel): + """Running `pip cache purge` should remove all cached http files and + wheels.""" result = script.pip('cache', 'purge', '--verbose') + assert remove_matches_http('aaaaaaaaa', result) + assert remove_matches_http('bbbbbbbbb', result) + assert remove_matches_http('ccccccccc', result) + assert remove_matches_wheel('yyy-1.2.3', result) assert remove_matches_wheel('zzz-4.5.6', result) assert remove_matches_wheel('zzz-4.5.7', result) assert remove_matches_wheel('zzz-7.8.9', result) -@pytest.mark.usefixtures("populate_wheel_cache") -def test_cache_purge_too_many_args(script, wheel_cache_files): +@pytest.mark.usefixtures("populate_http_cache", "populate_wheel_cache") +def test_cache_purge_too_many_args( + script, http_cache_files, wheel_cache_files +): """Running `pip cache purge aaa` should raise an error and remove no - cached wheels.""" + cached http files or wheels.""" result = script.pip('cache', 'purge', 'aaa', '--verbose', expect_error=True) assert result.stdout == '' @@ -289,7 +359,7 @@ def test_cache_purge_too_many_args(script, wheel_cache_files): assert 'ERROR: Too many arguments' in result.stderr.splitlines() # Make sure nothing was deleted. - for filename in wheel_cache_files: + for filename in http_cache_files + wheel_cache_files: assert os.path.exists(filename) From e5deff909c8f4eed0f6e60e769570e4e2c08c6b7 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 24 Sep 2020 00:36:02 +0530 Subject: [PATCH 170/209] Add black to pre-commit, ignoring all files Each of the lines in this "exclude" pattern will get a dedicated PR to remove it and would blacken that part of the codebase. --- .pre-commit-config.yaml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f1afdd473..8a62a90f6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,6 +16,42 @@ repos: - id: trailing-whitespace exclude: .patch +- repo: https://github.com/psf/black + rev: 20.8b1 + hooks: + - id: black + exclude: | + (?x) + ^docs/| + ^src/pip/_internal/cli| + ^src/pip/_internal/commands| + ^src/pip/_internal/distributions| + ^src/pip/_internal/index| + ^src/pip/_internal/models| + ^src/pip/_internal/network| + ^src/pip/_internal/operations| + ^src/pip/_internal/req| + ^src/pip/_internal/resolution| + ^src/pip/_internal/utils| + ^src/pip/_internal/vcs| + ^src/pip/_internal/\w+\.py$| + ^src/pip/__main__.py$| + ^tools/| + # Tests + ^tests/conftest.py| + ^tests/yaml| + ^tests/lib| + ^tests/data| + ^tests/unit| + ^tests/functional/(?!test_install)| + ^tests/functional/test_install| + # Files in the root of the repository + ^setup.py| + ^noxfile.py| + # A blank ignore, to avoid merge conflicts later. + ^$ + + - repo: https://gitlab.com/pycqa/flake8 rev: 3.8.3 hooks: From 8fc985eccba7f9cc0c6f53ff8ea08d6897895776 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 24 Sep 2020 14:35:53 +0300 Subject: [PATCH 171/209] Flake8: use extend-ignore to avoid needing to redefine defaults --- setup.cfg | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 450f953e4..59716a080 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,10 +24,8 @@ exclude = _vendor, data enable-extensions = G -ignore = +extend-ignore = G200, G202, - # pycodestyle checks ignored in the default configuration - E121, E123, E126, E133, E226, E241, E242, E704, W503, W504, W505, # black adds spaces around ':' E203, per-file-ignores = From 67bff6199ff7c8b080ce839d9d0e748b8d39b161 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sat, 5 Sep 2020 23:55:06 +0200 Subject: [PATCH 172/209] Add a change fragment about issue #8783 / PR #8848 --- news/8783.doc | 1 + news/8848.doc | 1 + 2 files changed, 2 insertions(+) create mode 100644 news/8783.doc create mode 120000 news/8848.doc diff --git a/news/8783.doc b/news/8783.doc new file mode 100644 index 000000000..6d2bb8762 --- /dev/null +++ b/news/8783.doc @@ -0,0 +1 @@ +Added initial UX feedback widgets to docs. diff --git a/news/8848.doc b/news/8848.doc new file mode 120000 index 000000000..a318abd14 --- /dev/null +++ b/news/8848.doc @@ -0,0 +1 @@ +8783.doc \ No newline at end of file From 3301284810f905bdb25c67233952892f7852df2d Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sat, 5 Sep 2020 21:01:37 +0200 Subject: [PATCH 173/209] =?UTF-8?q?=F0=9F=93=9D=20Add=20initial=20sphinx?= =?UTF-8?q?=20ext=20for=20per-doc=20feedbacks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves #8783 --- docs/docs_feedback_sphinxext.py | 165 ++++++++++++++++++++++++++++++++ docs/html/conf.py | 22 ++++- 2 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 docs/docs_feedback_sphinxext.py diff --git a/docs/docs_feedback_sphinxext.py b/docs/docs_feedback_sphinxext.py new file mode 100644 index 000000000..90f2ddd74 --- /dev/null +++ b/docs/docs_feedback_sphinxext.py @@ -0,0 +1,165 @@ +"""A sphinx extension for collecting per doc feedback.""" + +from __future__ import annotations + +from itertools import chain +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Dict, List, Union + + from sphinx.application import Sphinx + + +DEFAULT_DOC_LINES_THRESHOLD = 250 +RST_INDENT = 4 +EMAIL_INDENT = 6 + + +def _modify_rst_document_source_on_read( + app: Sphinx, + docname: str, + source: List[str], +) -> None: + """Add info block to top and bottom of each document source. + + This function modifies RST source in-place by adding an admonition + block at the top and the bottom of each document right after it's + been read from disk preserving :orphan: at top, if present. + """ + admonition_type = app.config.docs_feedback_admonition_type + big_doc_lines = app.config.docs_feedback_big_doc_lines + escaped_email = app.config.docs_feedback_email.replace(' ', r'\ ') + excluded_documents = set(app.config.docs_feedback_excluded_documents) + questions_list = app.config.docs_feedback_questions_list + + valid_admonitions = { + 'attention', 'caution', 'danger', 'error', 'hint', + 'important', 'note', 'tip', 'warning', 'admonition', + } + + if admonition_type not in valid_admonitions: + raise ValueError( + 'Expected `docs_feedback_admonition_type` to be one of ' + f'{valid_admonitions} but got {admonition_type}.' + ) + + if not questions_list: + raise ValueError( + 'Expected `docs_feedback_questions_list` to list questions ' + 'but got none.' + ) + + if docname in excluded_documents: + # NOTE: Completely ignore any document + # NOTE: listed in 'docs_feedback_excluded_documents'. + return + + is_doc_big = source[0].count('\n') >= big_doc_lines + + questions_list_rst = '\n'.join( + f'{" " * RST_INDENT}{number!s}. {question}' + for number, question in enumerate(questions_list, 1) + ) + questions_list_urlencoded = ( + '\n'.join( + f'\n{" " * RST_INDENT}{number!s}. {question} ' + for number, question in enumerate( + chain( + (f'Document: {docname}. Page URL: https://', ), + questions_list, + ), + ) + ). + rstrip('\r\n\t '). + replace('\r', '%0D'). + replace('\n', '%0A'). + replace(' ', '%20') + ) + + admonition_msg = rf""" + **Did this article help?** + + We are currently doing research to improve pip's documentation + and would love your feedback. + Please `email us`_ and let us know{{let_us_know_ending}} + +{{questions_list_rst}} + + .. _email us: + mailto:{escaped_email}\ + ?subject=[Doc:\ {docname}]\ Pip\ docs\ feedback\ \ + (URL\:\ https\://)\ + &body={questions_list_urlencoded} + """ + let_us_know_ending = ':' + + info_block_bottom = ( + f'.. {admonition_type}::\n\t\t{admonition_msg.format_map(locals())}\n' + ) + + questions_list_rst = '' + let_us_know_ending = ( + ' why you came to this page and what on it helped ' + 'you and what did not. ' + '(:issue:`Read more about this research <8517>`)' + ) + info_block_top = '' if is_doc_big else ( + f'.. {admonition_type}::\n\t\t{admonition_msg.format_map(locals())}\n' + ) + + orphan_mark = ':orphan:' + is_orphan = orphan_mark in source[0] + if is_orphan: + source[0].replace(orphan_mark, '') + else: + orphan_mark = '' + + source[0] = '\n\n'.join(( + orphan_mark, info_block_top, source[0], info_block_bottom, + )) + + +def setup(app: Sphinx) -> Dict[str, Union[bool, str]]: + """Initialize the Sphinx extension. + + This function adds a callback for modifying the document sources + in-place on read. + + It also declares the extension settings changable via :file:`conf.py`. + """ + rebuild_trigger = 'html' # rebuild full html on settings change + app.add_config_value( + 'docs_feedback_admonition_type', + default='important', + rebuild=rebuild_trigger, + ) + app.add_config_value( + 'docs_feedback_big_doc_lines', + default=DEFAULT_DOC_LINES_THRESHOLD, + rebuild=rebuild_trigger, + ) + app.add_config_value( + 'docs_feedback_email', + default='Docs UX Team ', + rebuild=rebuild_trigger, + ) + app.add_config_value( + 'docs_feedback_excluded_documents', + default=set(), + rebuild=rebuild_trigger, + ) + app.add_config_value( + 'docs_feedback_questions_list', + default=(), + rebuild=rebuild_trigger, + ) + + app.add_css_file('important-admonition.css') + app.connect('source-read', _modify_rst_document_source_on_read) + + return { + 'parallel_read_safe': True, + 'parallel_write_safe': True, + 'version': 'builtin', + } diff --git a/docs/html/conf.py b/docs/html/conf.py index a88ac33e2..b859e9902 100644 --- a/docs/html/conf.py +++ b/docs/html/conf.py @@ -31,10 +31,14 @@ sys.path.insert(0, docs_dir) # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. # extensions = ['sphinx.ext.autodoc'] extensions = [ + # native: 'sphinx.ext.extlinks', - 'pip_sphinxext', 'sphinx.ext.intersphinx', + # third-party: 'sphinx_tabs.tabs', + # in-tree: + 'docs_feedback_sphinxext', + 'pip_sphinxext', ] # intersphinx @@ -308,3 +312,19 @@ for fname in raw_subcommands: ) man_pages.append((fname_base, outname, description, u'pip developers', 1)) + +# -- Options for docs_feedback_sphinxext -------------------------------------- + +# NOTE: Must be one of 'attention', 'caution', 'danger', 'error', 'hint', +# NOTE: 'important', 'note', 'tip', 'warning' or 'admonition'. +docs_feedback_admonition_type = 'important' +docs_feedback_big_doc_lines = 50 # bigger docs will have a banner on top +docs_feedback_email = 'Docs UX Team ' +docs_feedback_excluded_documents = { # these won't have any banners + 'news', +} +docs_feedback_questions_list = ( + 'What problem were you trying to solve when you came to this page?', + 'What content was useful?', + 'What content was not useful?', +) From 13962cd6adcf233c8e9b5a2875e706a19019cd24 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sat, 5 Sep 2020 23:51:11 +0200 Subject: [PATCH 174/209] =?UTF-8?q?=F0=9F=8E=A8=20Colorize=20the=20"import?= =?UTF-8?q?ant"=20admonition=20blocks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- MANIFEST.in | 2 +- docs/html/_static/important-admonition.css | 8 ++++++++ docs/html/conf.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 docs/html/_static/important-admonition.css diff --git a/MANIFEST.in b/MANIFEST.in index aa6a1d0e7..24d455378 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -22,7 +22,7 @@ exclude noxfile.py recursive-include src/pip/_vendor *.pem recursive-include src/pip/_vendor py.typed -recursive-include docs Makefile *.rst *.py *.bat +recursive-include docs *.css *.rst *.py exclude src/pip/_vendor/six exclude src/pip/_vendor/six/moves diff --git a/docs/html/_static/important-admonition.css b/docs/html/_static/important-admonition.css new file mode 100644 index 000000000..a73ae2e4d --- /dev/null +++ b/docs/html/_static/important-admonition.css @@ -0,0 +1,8 @@ +.admonition.important { + background-color: rgb(219, 250, 244); + border: 1px solid rgb(26, 188, 156); +} + +.admonition.important>.admonition-title { + color: rgb(26, 188, 156); +} diff --git a/docs/html/conf.py b/docs/html/conf.py index b859e9902..444d15a81 100644 --- a/docs/html/conf.py +++ b/docs/html/conf.py @@ -181,7 +181,7 @@ html_theme_options = { # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = [] +html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. From 1135ac041d253122cf9791a0978332737f449195 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 24 Sep 2020 15:51:44 +0800 Subject: [PATCH 175/209] Move lru_cache to utils for reuse --- src/pip/_internal/index/collector.py | 26 ++------------------------ src/pip/_internal/utils/compat.py | 24 +++++++++++++++++++++++- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/pip/_internal/index/collector.py b/src/pip/_internal/index/collector.py index e6230c767..7b9abbf69 100644 --- a/src/pip/_internal/index/collector.py +++ b/src/pip/_internal/index/collector.py @@ -21,6 +21,7 @@ from pip._internal.exceptions import NetworkConnectionError from pip._internal.models.link import Link from pip._internal.models.search_scope import SearchScope from pip._internal.network.utils import raise_for_status +from pip._internal.utils.compat import lru_cache from pip._internal.utils.filetypes import ARCHIVE_EXTENSIONS from pip._internal.utils.misc import pairwise, redact_auth_from_url from pip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -36,10 +37,8 @@ if MYPY_CHECK_RUNNING: List, MutableMapping, Optional, - Protocol, Sequence, Tuple, - TypeVar, Union, ) @@ -50,31 +49,10 @@ if MYPY_CHECK_RUNNING: HTMLElement = xml.etree.ElementTree.Element ResponseHeaders = MutableMapping[str, str] - # Used in the @lru_cache polyfill. - F = TypeVar('F') - - class LruCache(Protocol): - def __call__(self, maxsize=None): - # type: (Optional[int]) -> Callable[[F], F] - raise NotImplementedError - logger = logging.getLogger(__name__) -# Fallback to noop_lru_cache in Python 2 -# TODO: this can be removed when python 2 support is dropped! -def noop_lru_cache(maxsize=None): - # type: (Optional[int]) -> Callable[[F], F] - def _wrapper(f): - # type: (F) -> F - return f - return _wrapper - - -_lru_cache = getattr(functools, "lru_cache", noop_lru_cache) # type: LruCache - - def _match_vcs_scheme(url): # type: (str) -> Optional[str] """Look for VCS schemes in the URL. @@ -344,7 +322,7 @@ def with_cached_html_pages( `page` has `page.cache_link_parsing == False`. """ - @_lru_cache(maxsize=None) + @lru_cache(maxsize=None) def wrapper(cacheable_page): # type: (CacheablePageContent) -> List[Link] return list(fn(cacheable_page.page)) diff --git a/src/pip/_internal/utils/compat.py b/src/pip/_internal/utils/compat.py index cc6353678..2196e6e0a 100644 --- a/src/pip/_internal/utils/compat.py +++ b/src/pip/_internal/utils/compat.py @@ -7,6 +7,7 @@ distributions.""" from __future__ import absolute_import, division import codecs +import functools import locale import logging import os @@ -18,7 +19,15 @@ from pip._vendor.six import PY2, text_type from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, Text, Tuple, Union + from typing import Callable, Optional, Protocol, Text, Tuple, TypeVar, Union + + # Used in the @lru_cache polyfill. + F = TypeVar('F') + + class LruCache(Protocol): + def __call__(self, maxsize=None): + # type: (Optional[int]) -> Callable[[F], F] + raise NotImplementedError try: import ipaddress @@ -269,3 +278,16 @@ else: if not cr: cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) return int(cr[1]), int(cr[0]) + + +# Fallback to noop_lru_cache in Python 2 +# TODO: this can be removed when python 2 support is dropped! +def noop_lru_cache(maxsize=None): + # type: (Optional[int]) -> Callable[[F], F] + def _wrapper(f): + # type: (F) -> F + return f + return _wrapper + + +lru_cache = getattr(functools, "lru_cache", noop_lru_cache) # type: LruCache From 1dd6d562789b06dc1508212505e32f36bd71d310 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 24 Sep 2020 15:52:52 +0800 Subject: [PATCH 176/209] Cache PackageFinder.find_all_candidates() --- src/pip/_internal/index/package_finder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 5162a8191..b361e194d 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -25,6 +25,7 @@ from pip._internal.models.link import Link from pip._internal.models.selection_prefs import SelectionPreferences from pip._internal.models.target_python import TargetPython from pip._internal.models.wheel import Wheel +from pip._internal.utils.compat import lru_cache from pip._internal.utils.filetypes import WHEEL_EXTENSION from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import build_netloc @@ -799,6 +800,7 @@ class PackageFinder(object): return package_links + @lru_cache(maxsize=None) def find_all_candidates(self, project_name): # type: (str) -> List[InstallationCandidate] """Find all available InstallationCandidate for project_name From 22eeff47707605b2537c0ee7a7ad23662649fb0a Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 26 Sep 2020 15:08:41 +0800 Subject: [PATCH 177/209] News --- news/8905.feature.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 news/8905.feature.rst diff --git a/news/8905.feature.rst b/news/8905.feature.rst new file mode 100644 index 000000000..5d27d40c2 --- /dev/null +++ b/news/8905.feature.rst @@ -0,0 +1,3 @@ +Cache package listings on index packages so they are guarenteed to stay stable +during a pip command session. This also improves performance when a index page +is accessed multiple times during the command session. From dedecf0735c34e03906e225a95231bf10d6dde79 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 26 Sep 2020 19:55:00 +0800 Subject: [PATCH 178/209] Remove the .rst suffix from news fragment --- news/{8905.feature.rst => 8905.feature} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename news/{8905.feature.rst => 8905.feature} (100%) diff --git a/news/8905.feature.rst b/news/8905.feature similarity index 100% rename from news/8905.feature.rst rename to news/8905.feature From b215120b5ab1315c963ee409b720753ac690c7f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Sat, 26 Sep 2020 14:41:42 +0100 Subject: [PATCH 179/209] Revert "Merge pull request #8391 from VikramJayanthi17/error-swallow-fix" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 7a60395dbd646401192252c4c2834607ff05fa96, reversing changes made to d3ce025e8dac297fe550e2acfa730288d715b6a7. It fixes devendored pip. See #8916. Signed-off-by: Filipe Laíns --- src/pip/_vendor/__init__.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/pip/_vendor/__init__.py b/src/pip/_vendor/__init__.py index 581db54c8..c3db83ff6 100644 --- a/src/pip/_vendor/__init__.py +++ b/src/pip/_vendor/__init__.py @@ -32,11 +32,15 @@ def vendored(modulename): try: __import__(modulename, globals(), locals(), level=0) except ImportError: - # This error used to be silenced in earlier variants of this file, to instead - # raise the error when pip actually tries to use the missing module. - # Based on inputs in #5354, this was changed to explicitly raise the error. - # Re-raising the exception without modifying it is an intentional choice. - raise + # We can just silently allow import failures to pass here. If we + # got to this point it means that ``import pip._vendor.whatever`` + # failed and so did ``import whatever``. Since we're importing this + # upfront in an attempt to alias imports, not erroring here will + # just mean we get a regular import error whenever pip *actually* + # tries to import one of these modules to use it, which actually + # gives us a better error message than we would have otherwise + # gotten. + pass else: sys.modules[vendored_name] = sys.modules[modulename] base, head = vendored_name.rsplit(".", 1) From d4686b57935baf28fa709195af57975a88238a9d Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 27 Sep 2020 21:11:35 +0300 Subject: [PATCH 180/209] Document Python support policy --- docs/html/development/release-process.rst | 8 +++++++- news/8927.removal | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 news/8927.removal diff --git a/docs/html/development/release-process.rst b/docs/html/development/release-process.rst index ebba13258..1c6889ef0 100644 --- a/docs/html/development/release-process.rst +++ b/docs/html/development/release-process.rst @@ -65,7 +65,7 @@ their merits. ``pip._internal.utils.deprecation.deprecated``. The function is not a part of pip's public API. -Python 2 support +Python 2 Support ---------------- pip will continue to ensure that it runs on Python 2.7 after the CPython 2.7 @@ -79,6 +79,12 @@ only bugs will be considered, and merged (subject to normal review processes). Note that there may be delays due to the lack of developer resources for reviewing such pull requests. +Python Support Policy +--------------------- + +In general, a given Python version is supported until its usage on PyPI falls below 5%. +This is at the maintainers' discretion, in case extraordinary circumstances arise. + .. _`Feature Flags`: Feature Flags diff --git a/news/8927.removal b/news/8927.removal new file mode 100644 index 000000000..0032fa5f2 --- /dev/null +++ b/news/8927.removal @@ -0,0 +1 @@ +Document that Python versions are generally supported until PyPI usage falls below 5%. From 806c112ed098426f9aaf5e4812d52c8f6ac10ecc Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 28 Sep 2020 00:05:08 +0800 Subject: [PATCH 181/209] Don't crash on 'check' when METADATA is missing --- news/8676.feature | 2 ++ src/pip/_internal/operations/check.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 news/8676.feature diff --git a/news/8676.feature b/news/8676.feature new file mode 100644 index 000000000..f8da963f6 --- /dev/null +++ b/news/8676.feature @@ -0,0 +1,2 @@ +Improve error message friendliness when an environment has packages with +corrupted metadata. diff --git a/src/pip/_internal/operations/check.py b/src/pip/_internal/operations/check.py index bc44d4357..5dee6bcb4 100644 --- a/src/pip/_internal/operations/check.py +++ b/src/pip/_internal/operations/check.py @@ -45,8 +45,8 @@ def create_package_set_from_installed(**kwargs): name = canonicalize_name(dist.project_name) try: package_set[name] = PackageDetails(dist.version, dist.requires()) - except RequirementParseError as e: - # Don't crash on broken metadata + except (OSError, RequirementParseError) as e: + # Don't crash on unreadable or broken metadata logger.warning("Error parsing requirements for %s: %s", name, e) problems = True return package_set, problems From fbf710ea5dc6e8aa3a84a80691df1daded5a424f Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 28 Sep 2020 09:52:15 +0300 Subject: [PATCH 182/209] Pin pytest whilst pytest-rerunfailures is incompatible with pytest 6.1 --- tools/requirements/tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/requirements/tests.txt b/tools/requirements/tests.txt index ef87225d6..6ae8c5d43 100644 --- a/tools/requirements/tests.txt +++ b/tools/requirements/tests.txt @@ -5,7 +5,7 @@ enum34; python_version < '3.4' freezegun mock pretend -pytest +pytest<6.1.0 # Until https://github.com/pytest-dev/pytest-rerunfailures/issues/128 is fixed pytest-cov pytest-rerunfailures pytest-timeout From ebc13756128b8f85ce1ea76cd5f507e7d0c928d4 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 26 Sep 2020 16:06:44 +0800 Subject: [PATCH 183/209] Make private attribute looks private --- src/pip/_internal/resolution/resolvelib/provider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index a1bab05a4..8ad1c65d3 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -43,7 +43,7 @@ class PipProvider(AbstractProvider): self._constraints = constraints self._ignore_dependencies = ignore_dependencies self._upgrade_strategy = upgrade_strategy - self.user_requested = user_requested + self._user_requested = user_requested def _sort_matches(self, matches): # type: (Iterable[Candidate]) -> Sequence[Candidate] @@ -84,7 +84,7 @@ class PipProvider(AbstractProvider): if self._upgrade_strategy == "eager": return True elif self._upgrade_strategy == "only-if-needed": - return (name in self.user_requested) + return (name in self._user_requested) return False def sort_key(c): From c2de8974d422596909c2700636dff07dd3bf1225 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sun, 27 Sep 2020 23:16:25 +0800 Subject: [PATCH 184/209] Resolve user-requested requirements first --- src/pip/_internal/resolution/resolvelib/provider.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index 8ad1c65d3..8264b471c 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -115,11 +115,18 @@ class PipProvider(AbstractProvider): self, resolution, # type: Optional[Candidate] candidates, # type: Sequence[Candidate] - information # type: Sequence[Tuple[Requirement, Candidate]] + information # type: Sequence[Tuple[Requirement, Optional[Candidate]]] ): # type: (...) -> Any - # Use the "usual" value for now - return len(candidates) + """Return a sort key to determine what dependency to look next. + + A smaller value makes a dependency higher priority. We put direct + (user-requested) dependencies first since they may contain useful + user-specified version ranges. Users tend to expect us to catch + problems in them early as well. + """ + transitive = all(parent is not None for _, parent in information) + return (transitive, len(candidates)) def find_matches(self, requirements): # type: (Sequence[Requirement]) -> Iterable[Candidate] From df6d3c701a02e7bcea71ae932f2f02917894664a Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 28 Sep 2020 22:22:58 +0800 Subject: [PATCH 185/209] News fragment --- news/8924.feature | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 news/8924.feature diff --git a/news/8924.feature b/news/8924.feature new file mode 100644 index 000000000..c607aa0d0 --- /dev/null +++ b/news/8924.feature @@ -0,0 +1,2 @@ +New resolver: Tweak resolution logic to improve user experience when +user-supplied requirements conflict. From 6138184e2c84842472ac24d17777618f79c0aea1 Mon Sep 17 00:00:00 2001 From: Sumana Harihareswara Date: Mon, 28 Sep 2020 15:19:17 -0400 Subject: [PATCH 186/209] Update docs index page about resolver and UX work --- docs/html/index.rst | 3 ++- news/a2fa2e68-01bf-11eb-a0b1-4fe8cb1f9dcf.trivial | 0 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 news/a2fa2e68-01bf-11eb-a0b1-4fe8cb1f9dcf.trivial diff --git a/docs/html/index.rst b/docs/html/index.rst index 860abcac9..64fc34c9d 100644 --- a/docs/html/index.rst +++ b/docs/html/index.rst @@ -17,7 +17,7 @@ Please take a look at our documentation for how to install and use pip: ux_research_design news -In 2020, we're working on improvements to the heart of pip. Please `learn more and take our survey`_ to help us do it right. +In 2020, we're working on improvements to the heart of pip: :ref:`Resolver changes 2020`. Please `learn more and take our survey`_ to help us do it right, and `join our user experience surveys pool`_. If you find bugs, need help, or want to talk to the developers, please use our mailing lists or chat rooms: @@ -40,6 +40,7 @@ rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_. .. _package installer: https://packaging.python.org/guides/tool-recommendations/ .. _Python Package Index: https://pypi.org +.. _join our user experience surveys pool: ux_research_design .. _learn more and take our survey: https://pyfound.blogspot.com/2020/03/new-pip-resolver-to-roll-out-this-year.html .. _Installation: https://pip.pypa.io/en/stable/installing.html .. _Documentation: https://pip.pypa.io/en/stable/ diff --git a/news/a2fa2e68-01bf-11eb-a0b1-4fe8cb1f9dcf.trivial b/news/a2fa2e68-01bf-11eb-a0b1-4fe8cb1f9dcf.trivial new file mode 100644 index 000000000..e69de29bb From 4fc4333abf9270b33acaf305eb1e1c5d38db177f Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Tue, 29 Sep 2020 09:57:46 +0300 Subject: [PATCH 187/209] Revert "Pin pytest whilst pytest-rerunfailures is incompatible with pytest 6.1" This reverts commit fbf710ea5dc6e8aa3a84a80691df1daded5a424f. --- tools/requirements/tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/requirements/tests.txt b/tools/requirements/tests.txt index 6ae8c5d43..ef87225d6 100644 --- a/tools/requirements/tests.txt +++ b/tools/requirements/tests.txt @@ -5,7 +5,7 @@ enum34; python_version < '3.4' freezegun mock pretend -pytest<6.1.0 # Until https://github.com/pytest-dev/pytest-rerunfailures/issues/128 is fixed +pytest pytest-cov pytest-rerunfailures pytest-timeout From 3b44d70b5e85e0b71d9f1e4a888497352d7412e5 Mon Sep 17 00:00:00 2001 From: Sumana Harihareswara Date: Wed, 30 Sep 2020 07:10:09 -0400 Subject: [PATCH 188/209] docs: Add how to default to new resolver Fixes #8661. --- docs/html/user_guide.rst | 7 +++++++ news/bc7f9ea0-030d-11eb-92cb-6b2b625d02fc.trivial | 0 2 files changed, 7 insertions(+) create mode 100644 news/bc7f9ea0-030d-11eb-92cb-6b2b625d02fc.trivial diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 06f85cba8..a2d13c433 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -1545,6 +1545,12 @@ How to test - If you have a build pipeline that depends on pip installing your dependencies for you, check that the new resolver does what you need. + + - If you'd like pip to default to using the new resolver, run ``pip + config set global.use-feature 2020-resolver`` (for more on that + and the alternate ``PIP_USE_FEATURE`` environment variable + option, see `issue 8661`_). + - Run your project’s CI (test suite, build process, etc.) using the new resolver, and let us know of any issues. - If you have encountered resolver issues with pip in the past, @@ -1671,6 +1677,7 @@ announcements on the `low-traffic packaging announcements list`_ and .. _freeze: https://pip.pypa.io/en/latest/reference/pip_freeze/ .. _resolver testing survey: https://tools.simplysecure.org/survey/index.php?r=survey/index&sid=989272&lang=en +.. _issue 8661: https://github.com/pypa/pip/issues/8661 .. _our announcement on the PSF blog: http://pyfound.blogspot.com/2020/03/new-pip-resolver-to-roll-out-this-year.html .. _tensorflow: https://pypi.org/project/tensorflow/ .. _low-traffic packaging announcements list: https://mail.python.org/mailman3/lists/pypi-announce.python.org/ diff --git a/news/bc7f9ea0-030d-11eb-92cb-6b2b625d02fc.trivial b/news/bc7f9ea0-030d-11eb-92cb-6b2b625d02fc.trivial new file mode 100644 index 000000000..e69de29bb From 2ef8040495711c7e7e695f80a35e208f7404f879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Thu, 1 Oct 2020 20:56:01 +0700 Subject: [PATCH 189/209] Comment and rework conditionals in download dir check Co-authored-by: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com> --- src/pip/_internal/operations/prepare.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 09540edce..fd5ff7817 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -486,20 +486,25 @@ class RequirementPreparer(object): link = req.link self._log_preparing_link(req) with indent_log(): + # Check if the relevant file is already available + # in the download directory + file_path = None download_dir = self._get_download_dir(req.link) if download_dir is not None and link.is_wheel: hashes = self._get_linked_req_hashes(req) file_path = _check_download_dir(req.link, download_dir, hashes) - if file_path is not None: - self._downloaded[req.link.url] = file_path, None - else: - file_path = None - if file_path is None: + if file_path is not None: + # The file is already available, so mark it as downloaded + self._downloaded[req.link.url] = file_path, None + else: + # The file is not available, attempt to fetch only metadata wheel_dist = self._fetch_metadata_using_lazy_wheel(link) if wheel_dist is not None: req.needs_more_preparation = True return wheel_dist + + # None of the optimizations worked, fully prepare the requirement return self._prepare_linked_requirement(req, parallel_builds) def prepare_linked_requirements_more(self, reqs, parallel_builds=False): From 0652a2f016b959a4adca67806b3aa30991196ab7 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Fri, 2 Oct 2020 09:59:38 +0300 Subject: [PATCH 190/209] Rename to 'Package index page cache location' --- src/pip/_internal/commands/cache.py | 4 ++-- tests/functional/test_cache.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/commands/cache.py b/src/pip/_internal/commands/cache.py index efba24f1a..ec21be68f 100644 --- a/src/pip/_internal/commands/cache.py +++ b/src/pip/_internal/commands/cache.py @@ -113,8 +113,8 @@ class CacheCommand(Command): ) message = textwrap.dedent(""" - HTTP files location: {http_cache_location} - HTTP files size: {http_cache_size} + Package index page cache location: {http_cache_location} + Package index page cache size: {http_cache_size} Number of HTTP files: {num_http_files} Wheels location: {wheels_cache_location} Wheels size: {wheels_cache_size} diff --git a/tests/functional/test_cache.py b/tests/functional/test_cache.py index a1ffd9090..872f55982 100644 --- a/tests/functional/test_cache.py +++ b/tests/functional/test_cache.py @@ -190,7 +190,10 @@ def test_cache_info( ): result = script.pip('cache', 'info') - assert 'HTTP files location: {}'.format(http_cache_dir) in result.stdout + assert ( + 'Package index page cache location: {}'.format(http_cache_dir) + in result.stdout + ) assert 'Wheels location: {}'.format(wheel_cache_dir) in result.stdout num_wheels = len(wheel_cache_files) assert 'Number of wheels: {}'.format(num_wheels) in result.stdout From c5744d5643a2a4fbbaaaa5352af77ad6dce19e16 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 26 Apr 2020 20:40:25 +0530 Subject: [PATCH 191/209] Enforce news/*.{correct-kind}.rst naming --- .pre-commit-config.yaml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a62a90f6..13b3abc62 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -85,11 +85,18 @@ repos: - id: python-no-log-warn - id: python-no-eval - id: rst-backticks - # Validate existing ReST files and NEWS fragments. - files: .*\.rst$|^news/.* + files: .*\.rst$ types: [file] - # The errors flagged in NEWS.rst are old. - exclude: NEWS.rst + exclude: NEWS.rst # The errors flagged in NEWS.rst are old. + +- repo: local + hooks: + - id: news-fragment-filenames + name: NEWS fragment + language: fail + entry: NEWS fragment files must be named *.(process|removal|feature|bugfix|vendor|doc|trivial).rst + exclude: ^news/(.gitignore|.*\.(process|removal|feature|bugfix|vendor|doc|trivial).rst) + files: ^news/ - repo: https://github.com/mgedmin/check-manifest rev: '0.43' From 603b2fa4ca886358f776c63596fc5682817213bc Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 24 Sep 2020 02:06:05 +0530 Subject: [PATCH 192/209] Rework the NEWS entries section --- docs/html/development/contributing.rst | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/html/development/contributing.rst b/docs/html/development/contributing.rst index 15690dae4..3d2d5ead0 100644 --- a/docs/html/development/contributing.rst +++ b/docs/html/development/contributing.rst @@ -70,15 +70,17 @@ such, but it is preferred to have a dedicated issue (for example, in case the PR ends up rejected due to code quality reasons). Once you have an issue or pull request, you take the number and you create a -file inside of the ``news/`` directory named after that issue number with an -extension of ``removal``, ``feature``, ``bugfix``, or ``doc``. Thus if your -issue or PR number is ``1234`` and this change is fixing a bug, then you would -create a file ``news/1234.bugfix``. PRs can span multiple categories by creating -multiple files (for instance, if you added a feature and deprecated/removed the -old feature at the same time, you would create ``news/NNNN.feature`` and -``news/NNNN.removal``). Likewise if a PR touches multiple issues/PRs you may -create a file for each of them with the exact same contents and Towncrier will -deduplicate them. +file inside of the ``news/`` directory, named after that issue number with a +"type" of ``removal``, ``feature``, ``bugfix``, or ``doc`` associated with it. + +If your issue or PR number is ``1234`` and this change is fixing a bug, +then you would create a file ``news/1234.bugfix.rst``. PRs can span multiple +categories by creating multiple files (for instance, if you added a feature and +deprecated/removed the old feature at the same time, you would create +``news/NNNN.feature.rst`` and ``news/NNNN.removal.rst``). + +If a PR touches multiple issues/PRs, you may create a file for each of them +with the exact same contents and Towncrier will deduplicate them. Contents of a NEWS entry ------------------------ From 8e8b6a6c7d024c76262e16e58719e479a0feec7d Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 3 Oct 2020 16:45:39 +0530 Subject: [PATCH 193/209] Rename the NEWS fragments --- ...5.trivial => 093f7456-cc25-4df9-9518-4732b1e07fe5.trivial.rst} | 0 ...5.trivial => 0e494986-202e-4275-b7ec-d6f046c0aa05.trivial.rst} | 0 ...1.trivial => 2a1e2773-eae6-43a4-b075-55f49e713fb1.trivial.rst} | 0 ...b.trivial => 4c82ffb4-cde3-4dd5-8f37-6f4ef53e028b.trivial.rst} | 0 ...2.trivial => 50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial.rst} | 0 ...e.trivial => 559bb022-21ae-498c-a2ce-2c354d880f5e.trivial.rst} | 0 ...0.trivial => 5e8c60c2-d540-4a25-af03-100d848acbc0.trivial.rst} | 0 ...1.trivial => 629892ca-55da-4ca9-9cff-c15373e97ad1.trivial.rst} | 0 ...f.trivial => 6f8eff8d-9886-4e00-b431-5c809500e6bf.trivial.rst} | 0 news/{7231.doc => 7231.doc.rst} | 0 news/{7311.doc => 7311.doc.rst} | 0 news/{8103.bugfix => 8103.bugfix.rst} | 0 news/{8181.removal => 8181.removal.rst} | 0 news/{8355.feature => 8355.feature.rst} | 0 news/{8417.removal => 8417.removal.rst} | 0 news/{8578.bugfix => 8578.bugfix.rst} | 0 news/{8636.doc => 8636.doc.rst} | 0 news/{8676.feature => 8676.feature.rst} | 0 news/{8696.bugfix => 8696.bugfix.rst} | 0 news/{8752.feature => 8752.feature.rst} | 0 news/{8758.bugfix => 8758.bugfix.rst} | 0 news/{8781.trivial => 8781.trivial.rst} | 0 news/{8783.doc => 8783.doc.rst} | 0 news/{8792.bugfix => 8792.bugfix.rst} | 0 news/{8804.feature => 8804.feature.rst} | 0 news/{8807.doc => 8807.doc.rst} | 0 news/{8815.feature => 8815.feature.rst} | 0 news/{8839.bugfix => 8839.bugfix.rst} | 0 news/{8848.doc => 8848.doc.rst} | 0 news/{8861.bugfix => 8861.bugfix.rst} | 0 news/{8892.feature => 8892.feature.rst} | 0 news/{8905.feature => 8905.feature.rst} | 0 news/{8924.feature => 8924.feature.rst} | 0 news/{8927.removal => 8927.removal.rst} | 0 ...0.trivial => 946beace-6164-4d1a-a05d-e9bebf43ccd0.trivial.rst} | 0 ...9.trivial => 9f8da1d9-dd18-47e9-b334-5eb862054409.trivial.rst} | 0 ...f.trivial => a2fa2e68-01bf-11eb-a0b1-4fe8cb1f9dcf.trivial.rst} | 0 ...3.trivial => a3a2b1b7-744e-4533-b3ff-6e7a1843d573.trivial.rst} | 0 ...d.trivial => c6182139-edb4-4bf6-bc3f-2d37cb5759ad.trivial.rst} | 0 ...5.trivial => d53425e4-767e-4d73-bce5-88644b781855.trivial.rst} | 0 ...f.trivial => d90a40c1-15b7-46b9-9162-335bb346b53f.trivial.rst} | 0 41 files changed, 0 insertions(+), 0 deletions(-) rename news/{093f7456-cc25-4df9-9518-4732b1e07fe5.trivial => 093f7456-cc25-4df9-9518-4732b1e07fe5.trivial.rst} (100%) rename news/{0e494986-202e-4275-b7ec-d6f046c0aa05.trivial => 0e494986-202e-4275-b7ec-d6f046c0aa05.trivial.rst} (100%) rename news/{2a1e2773-eae6-43a4-b075-55f49e713fb1.trivial => 2a1e2773-eae6-43a4-b075-55f49e713fb1.trivial.rst} (100%) rename news/{4c82ffb4-cde3-4dd5-8f37-6f4ef53e028b.trivial => 4c82ffb4-cde3-4dd5-8f37-6f4ef53e028b.trivial.rst} (100%) rename news/{50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial => 50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial.rst} (100%) rename news/{559bb022-21ae-498c-a2ce-2c354d880f5e.trivial => 559bb022-21ae-498c-a2ce-2c354d880f5e.trivial.rst} (100%) rename news/{5e8c60c2-d540-4a25-af03-100d848acbc0.trivial => 5e8c60c2-d540-4a25-af03-100d848acbc0.trivial.rst} (100%) rename news/{629892ca-55da-4ca9-9cff-c15373e97ad1.trivial => 629892ca-55da-4ca9-9cff-c15373e97ad1.trivial.rst} (100%) rename news/{6f8eff8d-9886-4e00-b431-5c809500e6bf.trivial => 6f8eff8d-9886-4e00-b431-5c809500e6bf.trivial.rst} (100%) rename news/{7231.doc => 7231.doc.rst} (100%) rename news/{7311.doc => 7311.doc.rst} (100%) rename news/{8103.bugfix => 8103.bugfix.rst} (100%) rename news/{8181.removal => 8181.removal.rst} (100%) rename news/{8355.feature => 8355.feature.rst} (100%) rename news/{8417.removal => 8417.removal.rst} (100%) rename news/{8578.bugfix => 8578.bugfix.rst} (100%) rename news/{8636.doc => 8636.doc.rst} (100%) rename news/{8676.feature => 8676.feature.rst} (100%) rename news/{8696.bugfix => 8696.bugfix.rst} (100%) rename news/{8752.feature => 8752.feature.rst} (100%) rename news/{8758.bugfix => 8758.bugfix.rst} (100%) rename news/{8781.trivial => 8781.trivial.rst} (100%) rename news/{8783.doc => 8783.doc.rst} (100%) rename news/{8792.bugfix => 8792.bugfix.rst} (100%) rename news/{8804.feature => 8804.feature.rst} (100%) rename news/{8807.doc => 8807.doc.rst} (100%) rename news/{8815.feature => 8815.feature.rst} (100%) rename news/{8839.bugfix => 8839.bugfix.rst} (100%) rename news/{8848.doc => 8848.doc.rst} (100%) rename news/{8861.bugfix => 8861.bugfix.rst} (100%) rename news/{8892.feature => 8892.feature.rst} (100%) rename news/{8905.feature => 8905.feature.rst} (100%) rename news/{8924.feature => 8924.feature.rst} (100%) rename news/{8927.removal => 8927.removal.rst} (100%) rename news/{946beace-6164-4d1a-a05d-e9bebf43ccd0.trivial => 946beace-6164-4d1a-a05d-e9bebf43ccd0.trivial.rst} (100%) rename news/{9f8da1d9-dd18-47e9-b334-5eb862054409.trivial => 9f8da1d9-dd18-47e9-b334-5eb862054409.trivial.rst} (100%) rename news/{a2fa2e68-01bf-11eb-a0b1-4fe8cb1f9dcf.trivial => a2fa2e68-01bf-11eb-a0b1-4fe8cb1f9dcf.trivial.rst} (100%) rename news/{a3a2b1b7-744e-4533-b3ff-6e7a1843d573.trivial => a3a2b1b7-744e-4533-b3ff-6e7a1843d573.trivial.rst} (100%) rename news/{c6182139-edb4-4bf6-bc3f-2d37cb5759ad.trivial => c6182139-edb4-4bf6-bc3f-2d37cb5759ad.trivial.rst} (100%) rename news/{d53425e4-767e-4d73-bce5-88644b781855.trivial => d53425e4-767e-4d73-bce5-88644b781855.trivial.rst} (100%) rename news/{d90a40c1-15b7-46b9-9162-335bb346b53f.trivial => d90a40c1-15b7-46b9-9162-335bb346b53f.trivial.rst} (100%) diff --git a/news/093f7456-cc25-4df9-9518-4732b1e07fe5.trivial b/news/093f7456-cc25-4df9-9518-4732b1e07fe5.trivial.rst similarity index 100% rename from news/093f7456-cc25-4df9-9518-4732b1e07fe5.trivial rename to news/093f7456-cc25-4df9-9518-4732b1e07fe5.trivial.rst diff --git a/news/0e494986-202e-4275-b7ec-d6f046c0aa05.trivial b/news/0e494986-202e-4275-b7ec-d6f046c0aa05.trivial.rst similarity index 100% rename from news/0e494986-202e-4275-b7ec-d6f046c0aa05.trivial rename to news/0e494986-202e-4275-b7ec-d6f046c0aa05.trivial.rst diff --git a/news/2a1e2773-eae6-43a4-b075-55f49e713fb1.trivial b/news/2a1e2773-eae6-43a4-b075-55f49e713fb1.trivial.rst similarity index 100% rename from news/2a1e2773-eae6-43a4-b075-55f49e713fb1.trivial rename to news/2a1e2773-eae6-43a4-b075-55f49e713fb1.trivial.rst diff --git a/news/4c82ffb4-cde3-4dd5-8f37-6f4ef53e028b.trivial b/news/4c82ffb4-cde3-4dd5-8f37-6f4ef53e028b.trivial.rst similarity index 100% rename from news/4c82ffb4-cde3-4dd5-8f37-6f4ef53e028b.trivial rename to news/4c82ffb4-cde3-4dd5-8f37-6f4ef53e028b.trivial.rst diff --git a/news/50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial b/news/50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial.rst similarity index 100% rename from news/50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial rename to news/50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial.rst diff --git a/news/559bb022-21ae-498c-a2ce-2c354d880f5e.trivial b/news/559bb022-21ae-498c-a2ce-2c354d880f5e.trivial.rst similarity index 100% rename from news/559bb022-21ae-498c-a2ce-2c354d880f5e.trivial rename to news/559bb022-21ae-498c-a2ce-2c354d880f5e.trivial.rst diff --git a/news/5e8c60c2-d540-4a25-af03-100d848acbc0.trivial b/news/5e8c60c2-d540-4a25-af03-100d848acbc0.trivial.rst similarity index 100% rename from news/5e8c60c2-d540-4a25-af03-100d848acbc0.trivial rename to news/5e8c60c2-d540-4a25-af03-100d848acbc0.trivial.rst diff --git a/news/629892ca-55da-4ca9-9cff-c15373e97ad1.trivial b/news/629892ca-55da-4ca9-9cff-c15373e97ad1.trivial.rst similarity index 100% rename from news/629892ca-55da-4ca9-9cff-c15373e97ad1.trivial rename to news/629892ca-55da-4ca9-9cff-c15373e97ad1.trivial.rst diff --git a/news/6f8eff8d-9886-4e00-b431-5c809500e6bf.trivial b/news/6f8eff8d-9886-4e00-b431-5c809500e6bf.trivial.rst similarity index 100% rename from news/6f8eff8d-9886-4e00-b431-5c809500e6bf.trivial rename to news/6f8eff8d-9886-4e00-b431-5c809500e6bf.trivial.rst diff --git a/news/7231.doc b/news/7231.doc.rst similarity index 100% rename from news/7231.doc rename to news/7231.doc.rst diff --git a/news/7311.doc b/news/7311.doc.rst similarity index 100% rename from news/7311.doc rename to news/7311.doc.rst diff --git a/news/8103.bugfix b/news/8103.bugfix.rst similarity index 100% rename from news/8103.bugfix rename to news/8103.bugfix.rst diff --git a/news/8181.removal b/news/8181.removal.rst similarity index 100% rename from news/8181.removal rename to news/8181.removal.rst diff --git a/news/8355.feature b/news/8355.feature.rst similarity index 100% rename from news/8355.feature rename to news/8355.feature.rst diff --git a/news/8417.removal b/news/8417.removal.rst similarity index 100% rename from news/8417.removal rename to news/8417.removal.rst diff --git a/news/8578.bugfix b/news/8578.bugfix.rst similarity index 100% rename from news/8578.bugfix rename to news/8578.bugfix.rst diff --git a/news/8636.doc b/news/8636.doc.rst similarity index 100% rename from news/8636.doc rename to news/8636.doc.rst diff --git a/news/8676.feature b/news/8676.feature.rst similarity index 100% rename from news/8676.feature rename to news/8676.feature.rst diff --git a/news/8696.bugfix b/news/8696.bugfix.rst similarity index 100% rename from news/8696.bugfix rename to news/8696.bugfix.rst diff --git a/news/8752.feature b/news/8752.feature.rst similarity index 100% rename from news/8752.feature rename to news/8752.feature.rst diff --git a/news/8758.bugfix b/news/8758.bugfix.rst similarity index 100% rename from news/8758.bugfix rename to news/8758.bugfix.rst diff --git a/news/8781.trivial b/news/8781.trivial.rst similarity index 100% rename from news/8781.trivial rename to news/8781.trivial.rst diff --git a/news/8783.doc b/news/8783.doc.rst similarity index 100% rename from news/8783.doc rename to news/8783.doc.rst diff --git a/news/8792.bugfix b/news/8792.bugfix.rst similarity index 100% rename from news/8792.bugfix rename to news/8792.bugfix.rst diff --git a/news/8804.feature b/news/8804.feature.rst similarity index 100% rename from news/8804.feature rename to news/8804.feature.rst diff --git a/news/8807.doc b/news/8807.doc.rst similarity index 100% rename from news/8807.doc rename to news/8807.doc.rst diff --git a/news/8815.feature b/news/8815.feature.rst similarity index 100% rename from news/8815.feature rename to news/8815.feature.rst diff --git a/news/8839.bugfix b/news/8839.bugfix.rst similarity index 100% rename from news/8839.bugfix rename to news/8839.bugfix.rst diff --git a/news/8848.doc b/news/8848.doc.rst similarity index 100% rename from news/8848.doc rename to news/8848.doc.rst diff --git a/news/8861.bugfix b/news/8861.bugfix.rst similarity index 100% rename from news/8861.bugfix rename to news/8861.bugfix.rst diff --git a/news/8892.feature b/news/8892.feature.rst similarity index 100% rename from news/8892.feature rename to news/8892.feature.rst diff --git a/news/8905.feature b/news/8905.feature.rst similarity index 100% rename from news/8905.feature rename to news/8905.feature.rst diff --git a/news/8924.feature b/news/8924.feature.rst similarity index 100% rename from news/8924.feature rename to news/8924.feature.rst diff --git a/news/8927.removal b/news/8927.removal.rst similarity index 100% rename from news/8927.removal rename to news/8927.removal.rst diff --git a/news/946beace-6164-4d1a-a05d-e9bebf43ccd0.trivial b/news/946beace-6164-4d1a-a05d-e9bebf43ccd0.trivial.rst similarity index 100% rename from news/946beace-6164-4d1a-a05d-e9bebf43ccd0.trivial rename to news/946beace-6164-4d1a-a05d-e9bebf43ccd0.trivial.rst diff --git a/news/9f8da1d9-dd18-47e9-b334-5eb862054409.trivial b/news/9f8da1d9-dd18-47e9-b334-5eb862054409.trivial.rst similarity index 100% rename from news/9f8da1d9-dd18-47e9-b334-5eb862054409.trivial rename to news/9f8da1d9-dd18-47e9-b334-5eb862054409.trivial.rst diff --git a/news/a2fa2e68-01bf-11eb-a0b1-4fe8cb1f9dcf.trivial b/news/a2fa2e68-01bf-11eb-a0b1-4fe8cb1f9dcf.trivial.rst similarity index 100% rename from news/a2fa2e68-01bf-11eb-a0b1-4fe8cb1f9dcf.trivial rename to news/a2fa2e68-01bf-11eb-a0b1-4fe8cb1f9dcf.trivial.rst diff --git a/news/a3a2b1b7-744e-4533-b3ff-6e7a1843d573.trivial b/news/a3a2b1b7-744e-4533-b3ff-6e7a1843d573.trivial.rst similarity index 100% rename from news/a3a2b1b7-744e-4533-b3ff-6e7a1843d573.trivial rename to news/a3a2b1b7-744e-4533-b3ff-6e7a1843d573.trivial.rst diff --git a/news/c6182139-edb4-4bf6-bc3f-2d37cb5759ad.trivial b/news/c6182139-edb4-4bf6-bc3f-2d37cb5759ad.trivial.rst similarity index 100% rename from news/c6182139-edb4-4bf6-bc3f-2d37cb5759ad.trivial rename to news/c6182139-edb4-4bf6-bc3f-2d37cb5759ad.trivial.rst diff --git a/news/d53425e4-767e-4d73-bce5-88644b781855.trivial b/news/d53425e4-767e-4d73-bce5-88644b781855.trivial.rst similarity index 100% rename from news/d53425e4-767e-4d73-bce5-88644b781855.trivial rename to news/d53425e4-767e-4d73-bce5-88644b781855.trivial.rst diff --git a/news/d90a40c1-15b7-46b9-9162-335bb346b53f.trivial b/news/d90a40c1-15b7-46b9-9162-335bb346b53f.trivial.rst similarity index 100% rename from news/d90a40c1-15b7-46b9-9162-335bb346b53f.trivial rename to news/d90a40c1-15b7-46b9-9162-335bb346b53f.trivial.rst From 717f54a931d96936cc001b9afe49ddbb510b7e11 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 4 Oct 2020 23:14:46 +0530 Subject: [PATCH 194/209] Change the symlink to be a proper file instead. --- news/8848.doc.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 120000 => 100644 news/8848.doc.rst diff --git a/news/8848.doc.rst b/news/8848.doc.rst deleted file mode 120000 index a318abd14..000000000 --- a/news/8848.doc.rst +++ /dev/null @@ -1 +0,0 @@ -8783.doc \ No newline at end of file diff --git a/news/8848.doc.rst b/news/8848.doc.rst new file mode 100644 index 000000000..6d2bb8762 --- /dev/null +++ b/news/8848.doc.rst @@ -0,0 +1 @@ +Added initial UX feedback widgets to docs. From 234bcf6c1176b80f4d44dba6d669acdb4673e3b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Tue, 6 Oct 2020 21:33:35 +0700 Subject: [PATCH 195/209] Enforce news/*.rst --- ...c.trivial => 24715eb2-c118-473b-9d99-4f6ce8bbfa83.trivial.rst} | 0 news/bc7f9ea0-030d-11eb-92cb-6b2b625d02fc.trivial.rst | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename news/{bc7f9ea0-030d-11eb-92cb-6b2b625d02fc.trivial => 24715eb2-c118-473b-9d99-4f6ce8bbfa83.trivial.rst} (100%) create mode 100644 news/bc7f9ea0-030d-11eb-92cb-6b2b625d02fc.trivial.rst diff --git a/news/bc7f9ea0-030d-11eb-92cb-6b2b625d02fc.trivial b/news/24715eb2-c118-473b-9d99-4f6ce8bbfa83.trivial.rst similarity index 100% rename from news/bc7f9ea0-030d-11eb-92cb-6b2b625d02fc.trivial rename to news/24715eb2-c118-473b-9d99-4f6ce8bbfa83.trivial.rst diff --git a/news/bc7f9ea0-030d-11eb-92cb-6b2b625d02fc.trivial.rst b/news/bc7f9ea0-030d-11eb-92cb-6b2b625d02fc.trivial.rst new file mode 100644 index 000000000..e69de29bb From 78b294e7461c0df200b894a54362681c7e994815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Fri, 4 Sep 2020 16:20:13 +0700 Subject: [PATCH 196/209] Remove download_dir exist check Both pip download and wheel call endure_dir on the directory. --- src/pip/_internal/commands/download.py | 1 - src/pip/_internal/operations/prepare.py | 18 ++---------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/src/pip/_internal/commands/download.py b/src/pip/_internal/commands/download.py index 0861d9e67..31eebd962 100644 --- a/src/pip/_internal/commands/download.py +++ b/src/pip/_internal/commands/download.py @@ -87,7 +87,6 @@ class DownloadCommand(RequirementCommand): cmdoptions.check_dist_restriction(options) options.download_dir = normalize_path(options.download_dir) - ensure_dir(options.download_dir) session = self.get_default_session(options) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 3060bafa1..c48bcca63 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -353,20 +353,6 @@ class RequirementPreparer(object): # Previous "header" printed for a link-based InstallRequirement self._previous_requirement_header = ("", "") - @property - def _download_should_save(self): - # type: () -> bool - if not self.download_dir: - return False - - if os.path.exists(self.download_dir): - return True - - logger.critical('Could not find download directory') - raise InstallationError( - "Could not find or access download directory '{}'" - .format(self.download_dir)) - def _log_preparing_link(self, req): # type: (InstallRequirement) -> None """Provide context for the requirement being prepared.""" @@ -568,9 +554,9 @@ class RequirementPreparer(object): download_path = display_path(download_location) logger.info('Saved %s', download_path) - if self._download_should_save: + if link.is_vcs: # Make a .zip of the source_dir we already created. - if link.is_vcs: + if self.download_dir is not None: req.archive(self.download_dir) return dist From 6887b0795b3567ac2a85578e5509dd8cfcc2b284 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Sat, 12 Sep 2020 20:57:12 +0700 Subject: [PATCH 197/209] Merge usage of download_dir and wheel_download_dir In every cases, at least one of them is None. By doing this, it is also possible to simplify wrapper codes around download_dir. --- src/pip/_internal/cli/req_command.py | 2 -- src/pip/_internal/commands/wheel.py | 2 +- src/pip/_internal/operations/prepare.py | 46 +++++++------------------ src/pip/_internal/req/req_install.py | 4 ++- tests/unit/test_req.py | 1 - 5 files changed, 17 insertions(+), 38 deletions(-) diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 0757e34f6..03cc52f69 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -203,7 +203,6 @@ class RequirementCommand(IndexGroupCommand): finder, # type: PackageFinder use_user_site, # type: bool download_dir=None, # type: str - wheel_download_dir=None, # type: str ): # type: (...) -> RequirementPreparer """ @@ -229,7 +228,6 @@ class RequirementCommand(IndexGroupCommand): build_dir=temp_build_dir_path, src_dir=options.src_dir, download_dir=download_dir, - wheel_download_dir=wheel_download_dir, build_isolation=options.build_isolation, req_tracker=req_tracker, session=session, diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index 0f718566b..38a9b197f 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -137,7 +137,7 @@ class WheelCommand(RequirementCommand): req_tracker=req_tracker, session=session, finder=finder, - wheel_download_dir=options.wheel_dir, + download_dir=options.wheel_dir, use_user_site=False, ) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index c48bcca63..8767115b8 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -300,7 +300,6 @@ class RequirementPreparer(object): build_dir, # type: str download_dir, # type: Optional[str] src_dir, # type: str - wheel_download_dir, # type: Optional[str] build_isolation, # type: bool req_tracker, # type: RequirementTracker session, # type: PipSession @@ -325,16 +324,6 @@ class RequirementPreparer(object): # not saved, and are deleted immediately after unpacking. self.download_dir = download_dir - # Where still-packed .whl files should be written to. If None, they are - # written to the download_dir parameter. Separate to download_dir to - # permit only keeping wheel archives for pip wheel. - self.wheel_download_dir = wheel_download_dir - - # NOTE - # download_dir and wheel_download_dir overlap semantically and may - # be combined if we're willing to have non-wheel archives present in - # the wheelhouse output by 'pip wheel'. - # Is build isolation allowed? self.build_isolation = build_isolation @@ -371,15 +360,8 @@ class RequirementPreparer(object): with indent_log(): logger.info("Using cached %s", req.link.filename) - def _get_download_dir(self, link): - # type: (Link) -> Optional[str] - if link.is_wheel and self.wheel_download_dir: - # Download wheels to a dedicated dir when doing `pip wheel`. - return self.wheel_download_dir - return self.download_dir - - def _ensure_link_req_src_dir(self, req, download_dir, parallel_builds): - # type: (InstallRequirement, Optional[str], bool) -> None + def _ensure_link_req_src_dir(self, req, parallel_builds): + # type: (InstallRequirement, bool) -> None """Ensure source_dir of a linked InstallRequirement.""" # Since source_dir is only set for editable requirements. if req.link.is_wheel: @@ -480,10 +462,9 @@ class RequirementPreparer(object): # Check if the relevant file is already available # in the download directory file_path = None - download_dir = self._get_download_dir(req.link) - if download_dir is not None and link.is_wheel: + if self.download_dir is not None and link.is_wheel: hashes = self._get_linked_req_hashes(req) - file_path = _check_download_dir(req.link, download_dir, hashes) + file_path = _check_download_dir(req.link, self.download_dir, hashes) if file_path is not None: # The file is already available, so mark it as downloaded @@ -514,15 +495,14 @@ class RequirementPreparer(object): # type: (InstallRequirement, bool) -> Distribution assert req.link link = req.link - download_dir = self._get_download_dir(link) - self._ensure_link_req_src_dir(req, download_dir, parallel_builds) + self._ensure_link_req_src_dir(req, parallel_builds) hashes = self._get_linked_req_hashes(req) if link.url not in self._downloaded: try: local_file = unpack_url( link, req.source_dir, self._download, - download_dir, hashes, + self.download_dir, hashes, ) except NetworkConnectionError as exc: raise InstallationError( @@ -544,11 +524,13 @@ class RequirementPreparer(object): req, self.req_tracker, self.finder, self.build_isolation, ) - if download_dir: + if self.download_dir is not None: if link.is_existing_dir(): logger.info('Link is a directory, ignoring download_dir') elif local_file: - download_location = os.path.join(download_dir, link.filename) + download_location = os.path.join( + self.download_dir, link.filename + ) if not os.path.exists(download_location): shutil.copy(local_file.path, download_location) download_path = display_path(download_location) @@ -556,8 +538,7 @@ class RequirementPreparer(object): if link.is_vcs: # Make a .zip of the source_dir we already created. - if self.download_dir is not None: - req.archive(self.download_dir) + req.archive(self.download_dir) return dist def prepare_editable_requirement( @@ -579,14 +560,13 @@ class RequirementPreparer(object): 'hash.'.format(req) ) req.ensure_has_source_dir(self.src_dir) - req.update_editable(not self._download_should_save) + req.update_editable(self.download_dir is None) dist = _get_prepared_distribution( req, self.req_tracker, self.finder, self.build_isolation, ) - if self._download_should_save: - req.archive(self.download_dir) + req.archive(self.download_dir) req.check_if_exists(self.use_user_site) return dist diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 42999a59f..8ce299503 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -700,12 +700,14 @@ class InstallRequirement(object): return self.name + '/' + name def archive(self, build_dir): - # type: (str) -> None + # type: (Optional[str]) -> None """Saves archive to provided build_dir. Used for saving downloaded VCS requirements as part of `pip download`. """ assert self.source_dir + if build_dir is None: + return create_archive = True archive_name = '{}-{}.zip'.format(self.name, self.metadata["version"]) diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index 730a26a88..083d2c2c6 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -82,7 +82,6 @@ class TestRequirementSet(object): build_dir=os.path.join(self.tempdir, 'build'), src_dir=os.path.join(self.tempdir, 'src'), download_dir=None, - wheel_download_dir=None, build_isolation=True, req_tracker=tracker, session=session, From b28e2c4928cc62d90b738a4613886fb1e2ad6a81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Sat, 12 Sep 2020 21:08:49 +0700 Subject: [PATCH 198/209] New resolver: Avoid polluting dest dir Previously, during dependency resolution for `pip download -d ` or `pip wheel -w `, distributions downloaded are always saved to , even for those are only used in backtracking and are not part of the returned requirement set. --- news/8827.bugfix.rst | 2 ++ src/pip/_internal/commands/download.py | 1 + src/pip/_internal/commands/wheel.py | 11 +++++--- src/pip/_internal/operations/prepare.py | 36 ++++++++++++++++--------- 4 files changed, 33 insertions(+), 17 deletions(-) create mode 100644 news/8827.bugfix.rst diff --git a/news/8827.bugfix.rst b/news/8827.bugfix.rst new file mode 100644 index 000000000..608cd3d5c --- /dev/null +++ b/news/8827.bugfix.rst @@ -0,0 +1,2 @@ +Avoid polluting the destination directory by resolution artifacts +when the new resolver is used for ``pip download`` or ``pip wheel``. diff --git a/src/pip/_internal/commands/download.py b/src/pip/_internal/commands/download.py index 31eebd962..2f151e049 100644 --- a/src/pip/_internal/commands/download.py +++ b/src/pip/_internal/commands/download.py @@ -137,6 +137,7 @@ class DownloadCommand(RequirementCommand): for req in requirement_set.requirements.values(): if not req.editable and req.satisfied_by is None: assert req.name is not None + preparer.save_linked_requirement(req) downloaded.append(req.name) if downloaded: write_output('Successfully downloaded %s', ' '.join(downloaded)) diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index 38a9b197f..8f5783c35 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -21,6 +21,7 @@ if MYPY_CHECK_RUNNING: from optparse import Values from typing import List + from pip._internal.req.req_install import InstallRequirement logger = logging.getLogger(__name__) @@ -156,10 +157,12 @@ class WheelCommand(RequirementCommand): reqs, check_supported_wheels=True ) - reqs_to_build = [ - r for r in requirement_set.requirements.values() - if should_build_for_wheel_command(r) - ] + reqs_to_build = [] # type: List[InstallRequirement] + for req in requirement_set.requirements.values(): + if req.is_wheel: + preparer.save_linked_requirement(req) + elif should_build_for_wheel_command(req): + reqs_to_build.append(req) # build wheels build_successes, build_failures = build( diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 8767115b8..de017504a 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -523,23 +523,33 @@ class RequirementPreparer(object): dist = _get_prepared_distribution( req, self.req_tracker, self.finder, self.build_isolation, ) + return dist - if self.download_dir is not None: - if link.is_existing_dir(): - logger.info('Link is a directory, ignoring download_dir') - elif local_file: - download_location = os.path.join( - self.download_dir, link.filename - ) - if not os.path.exists(download_location): - shutil.copy(local_file.path, download_location) - download_path = display_path(download_location) - logger.info('Saved %s', download_path) - + def save_linked_requirement(self, req): + # type: (InstallRequirement) -> None + assert self.download_dir is not None + assert req.link is not None + link = req.link if link.is_vcs: # Make a .zip of the source_dir we already created. req.archive(self.download_dir) - return dist + return + + if link.is_existing_dir(): + logger.debug( + 'Not copying link to destination directory ' + 'since it is a directory: %s', link, + ) + return + if req.local_file_path is None: + # No distribution was downloaded for this requirement. + return + + download_location = os.path.join(self.download_dir, link.filename) + if not os.path.exists(download_location): + shutil.copy(req.local_file_path, download_location) + download_path = display_path(download_location) + logger.info('Saved %s', download_path) def prepare_editable_requirement( self, From 960dca9949cf742b2766f9fc20277321f44cf09c Mon Sep 17 00:00:00 2001 From: Noah Gorny Date: Fri, 9 Oct 2020 16:32:09 +0300 Subject: [PATCH 199/209] resolvelib: factory: Get installed distributions correctly --- src/pip/_internal/resolution/resolvelib/factory.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 96e2d5331..3300cc8c5 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -98,9 +98,15 @@ class Factory(object): self._editable_candidate_cache = {} # type: Cache[EditableCandidate] if not ignore_installed: + packages = get_installed_distributions( + local_only=False, + include_editables=True, + editables_only=False, + user_only=False, + paths=None, + ) self._installed_dists = { - canonicalize_name(dist.project_name): dist - for dist in get_installed_distributions() + canonicalize_name(p.key): p for p in packages } else: self._installed_dists = {} From 7e02958a1e03fffe0bb3b79c9d8a4f8f78d9ac03 Mon Sep 17 00:00:00 2001 From: Noah Gorny Date: Fri, 9 Oct 2020 16:44:09 +0300 Subject: [PATCH 200/209] Add news fragment for 8963 --- news/8963.bugfix.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 news/8963.bugfix.rst diff --git a/news/8963.bugfix.rst b/news/8963.bugfix.rst new file mode 100644 index 000000000..62c01b464 --- /dev/null +++ b/news/8963.bugfix.rst @@ -0,0 +1,2 @@ +Correctly search for installed distributions in new resolver logic in order +to not miss packages (virtualenv packages from system-wide-packages for example) From 8326148149e1993928bd63cbd1e3a39d79867ce4 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 28 Sep 2020 16:36:04 +0800 Subject: [PATCH 201/209] Implement "lazy sequence" to avoid Internet find_matches() is modified to return a special type that implements the sequence protocol (instead of a plain list). This special sequence type tries to use the installed candidate as the first element if possible, and only access indexes when the installed candidate is considered unsatisfactory. --- .../resolution/resolvelib/factory.py | 67 +++++----- .../resolution/resolvelib/found_candidates.py | 122 ++++++++++++++++++ .../resolution/resolvelib/provider.py | 91 +++---------- .../resolution_resolvelib/test_requirement.py | 9 +- 4 files changed, 179 insertions(+), 110 deletions(-) create mode 100644 src/pip/_internal/resolution/resolvelib/found_candidates.py diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 3300cc8c5..0ac8d1af9 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -1,7 +1,5 @@ -import collections import logging -from pip._vendor import six from pip._vendor.packaging.utils import canonicalize_name from pip._internal.exceptions import ( @@ -30,6 +28,7 @@ from .candidates import ( LinkCandidate, RequiresPythonCandidate, ) +from .found_candidates import FoundCandidates from .requirements import ( ExplicitRequirement, RequiresPythonRequirement, @@ -41,6 +40,7 @@ if MYPY_CHECK_RUNNING: Dict, FrozenSet, Iterable, + Iterator, List, Optional, Sequence, @@ -98,15 +98,9 @@ class Factory(object): self._editable_candidate_cache = {} # type: Cache[EditableCandidate] if not ignore_installed: - packages = get_installed_distributions( - local_only=False, - include_editables=True, - editables_only=False, - user_only=False, - paths=None, - ) self._installed_dists = { - canonicalize_name(p.key): p for p in packages + canonicalize_name(dist.project_name): dist + for dist in get_installed_distributions(local_only=False) } else: self._installed_dists = {} @@ -160,6 +154,7 @@ class Factory(object): ireqs, # type: Sequence[InstallRequirement] specifier, # type: SpecifierSet hashes, # type: Hashes + prefers_installed, # type: bool ): # type: (...) -> Iterable[Candidate] if not ireqs: @@ -178,54 +173,49 @@ class Factory(object): hashes &= ireq.hashes(trust_internet=False) extras |= frozenset(ireq.extras) - # We use this to ensure that we only yield a single candidate for - # each version (the finder's preferred one for that version). The - # requirement needs to return only one candidate per version, so we - # implement that logic here so that requirements using this helper - # don't all have to do the same thing later. - candidates = collections.OrderedDict() # type: VersionCandidates - # Get the installed version, if it matches, unless the user # specified `--force-reinstall`, when we want the version from # the index instead. - installed_version = None installed_candidate = None if not self._force_reinstall and name in self._installed_dists: installed_dist = self._installed_dists[name] - installed_version = installed_dist.parsed_version - if specifier.contains(installed_version, prereleases=True): + if specifier.contains(installed_dist.version, prereleases=True): installed_candidate = self._make_candidate_from_dist( dist=installed_dist, extras=extras, template=template, ) - found = self._finder.find_best_candidate( - project_name=name, - specifier=specifier, - hashes=hashes, - ) - for ican in found.iter_applicable(): - if ican.version == installed_version and installed_candidate: - candidate = installed_candidate - else: - candidate = self._make_candidate_from_link( + def iter_index_candidates(): + # type: () -> Iterator[Candidate] + result = self._finder.find_best_candidate( + project_name=name, + specifier=specifier, + hashes=hashes, + ) + # PackageFinder returns earlier versions first, so we reverse. + for ican in reversed(list(result.iter_applicable())): + yield self._make_candidate_from_link( link=ican.link, extras=extras, template=template, name=name, version=ican.version, ) - candidates[ican.version] = candidate - # Yield the installed version even if it is not found on the index. - if installed_version and installed_candidate: - candidates[installed_version] = installed_candidate + return FoundCandidates( + iter_index_candidates, + installed_candidate, + prefers_installed, + ) - return six.itervalues(candidates) - - def find_candidates(self, requirements, constraint): - # type: (Sequence[Requirement], Constraint) -> Iterable[Candidate] + def find_candidates( + self, + requirements, # type: Sequence[Requirement] + constraint, # type: Constraint + prefers_installed, # type: bool + ): + # type: (...) -> Iterable[Candidate] explicit_candidates = set() # type: Set[Candidate] ireqs = [] # type: List[InstallRequirement] for req in requirements: @@ -242,6 +232,7 @@ class Factory(object): ireqs, constraint.specifier, constraint.hashes, + prefers_installed, ) if constraint: diff --git a/src/pip/_internal/resolution/resolvelib/found_candidates.py b/src/pip/_internal/resolution/resolvelib/found_candidates.py new file mode 100644 index 000000000..0ddcb1fcc --- /dev/null +++ b/src/pip/_internal/resolution/resolvelib/found_candidates.py @@ -0,0 +1,122 @@ +from pip._vendor.six.moves import collections_abc + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Callable, Iterator, Optional, Set + + from pip._vendor.packaging.version import _BaseVersion + + from .base import Candidate + + +class _InstalledFirstCandidatesIterator(collections_abc.Iterator): + """Iterator for ``FoundCandidates``. + + This iterator is used when the resolver prefers to keep the version of an + already-installed package. The already-installed candidate is always + returned first. Candidates from index are accessed only when the resolver + wants them, and the already-installed version is excluded from them. + """ + def __init__( + self, + get_others, # type: Callable[[], Iterator[Candidate]] + installed, # type: Optional[Candidate] + ): + self._installed = installed + self._get_others = get_others + self._others = None # type: Optional[Iterator[Candidate]] + self._returned = set() # type: Set[_BaseVersion] + + def __next__(self): + # type: () -> Candidate + if self._installed and self._installed.version not in self._returned: + self._returned.add(self._installed.version) + return self._installed + if self._others is None: + self._others = self._get_others() + cand = next(self._others) + while cand.version in self._returned: + cand = next(self._others) + self._returned.add(cand.version) + return cand + + next = __next__ # XXX: Python 2. + + +class _InstalledReplacesCandidatesIterator(collections_abc.Iterator): + """Iterator for ``FoundCandidates``. + + This iterator is used when the resolver prefers to upgrade an + already-installed package. Candidates from index are returned in their + normal ordering, except replaced when the version is already installed. + """ + def __init__( + self, + get_others, # type: Callable[[], Iterator[Candidate]] + installed, # type: Optional[Candidate] + ): + self._installed = installed + self._get_others = get_others + self._others = None # type: Optional[Iterator[Candidate]] + self._returned = set() # type: Set[_BaseVersion] + + def __next__(self): + # type: () -> Candidate + if self._others is None: + self._others = self._get_others() + cand = next(self._others) + while cand.version in self._returned: + cand = next(self._others) + if self._installed and cand.version == self._installed.version: + cand = self._installed + self._returned.add(cand.version) + return cand + + next = __next__ # XXX: Python 2. + + +class FoundCandidates(collections_abc.Sequence): + """A lazy sequence to provide candidates to the resolver. + + The intended usage is to return this from `find_matches()` so the resolver + can iterate through the sequence multiple times, but only access the index + page when remote packages are actually needed. This improve performances + when suitable candidates are already installed on disk. + """ + def __init__( + self, + get_others, # type: Callable[[], Iterator[Candidate]] + installed, # type: Optional[Candidate] + prefers_installed, # type: bool + ): + self._get_others = get_others + self._installed = installed + self._prefers_installed = prefers_installed + + def __getitem__(self, index): + # type: (int) -> Candidate + for i, value in enumerate(self): + if index == i: + return value + raise IndexError(index) + + def __iter__(self): + # type: () -> Iterator[Candidate] + if self._prefers_installed: + klass = _InstalledFirstCandidatesIterator + else: + klass = _InstalledReplacesCandidatesIterator + return klass(self._get_others, self._installed) + + def __len__(self): + # type: () -> int + return sum(1 for _ in self) + + def __bool__(self): + # type: () -> bool + if self._prefers_installed and self._installed: + return True + return any(self) + + __nonzero__ = __bool__ # XXX: Python 2. diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index 8264b471c..7f7d0e154 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -45,30 +45,26 @@ class PipProvider(AbstractProvider): self._upgrade_strategy = upgrade_strategy self._user_requested = user_requested - def _sort_matches(self, matches): - # type: (Iterable[Candidate]) -> Sequence[Candidate] + def identify(self, dependency): + # type: (Union[Requirement, Candidate]) -> str + return dependency.name - # The requirement is responsible for returning a sequence of potential - # candidates, one per version. The provider handles the logic of - # deciding the order in which these candidates should be passed to - # the resolver. + def get_preference( + self, + resolution, # type: Optional[Candidate] + candidates, # type: Sequence[Candidate] + information # type: Sequence[Tuple[Requirement, Candidate]] + ): + # type: (...) -> Any + transitive = all(parent is not None for _, parent in information) + return (transitive, bool(candidates)) - # The `matches` argument is a sequence of candidates, one per version, - # which are potential options to be installed. The requirement will - # have already sorted out whether to give us an already-installed - # candidate or a version from PyPI (i.e., it will deal with options - # like --force-reinstall and --ignore-installed). + def find_matches(self, requirements): + # type: (Sequence[Requirement]) -> Iterable[Candidate] + if not requirements: + return [] + name = requirements[0].name - # We now work out the correct order. - # - # 1. If no other considerations apply, later versions take priority. - # 2. An already installed distribution is preferred over any other, - # unless the user has requested an upgrade. - # Upgrades are allowed when: - # * The --upgrade flag is set, and - # - The project was specified on the command line, or - # - The project is a dependency and the "eager" upgrade strategy - # was requested. def _eligible_for_upgrade(name): # type: (str) -> bool """Are upgrades allowed for this project? @@ -87,56 +83,11 @@ class PipProvider(AbstractProvider): return (name in self._user_requested) return False - def sort_key(c): - # type: (Candidate) -> int - """Return a sort key for the matches. - - The highest priority should be given to installed candidates that - are not eligible for upgrade. We use the integer value in the first - part of the key to sort these before other candidates. - - We only pull the installed candidate to the bottom (i.e. most - preferred), but otherwise keep the ordering returned by the - requirement. The requirement is responsible for returning a list - otherwise sorted for the resolver, taking account for versions - and binary preferences as specified by the user. - """ - if c.is_installed and not _eligible_for_upgrade(c.name): - return 1 - return 0 - - return sorted(matches, key=sort_key) - - def identify(self, dependency): - # type: (Union[Requirement, Candidate]) -> str - return dependency.name - - def get_preference( - self, - resolution, # type: Optional[Candidate] - candidates, # type: Sequence[Candidate] - information # type: Sequence[Tuple[Requirement, Optional[Candidate]]] - ): - # type: (...) -> Any - """Return a sort key to determine what dependency to look next. - - A smaller value makes a dependency higher priority. We put direct - (user-requested) dependencies first since they may contain useful - user-specified version ranges. Users tend to expect us to catch - problems in them early as well. - """ - transitive = all(parent is not None for _, parent in information) - return (transitive, len(candidates)) - - def find_matches(self, requirements): - # type: (Sequence[Requirement]) -> Iterable[Candidate] - if not requirements: - return [] - constraint = self._constraints.get( - requirements[0].name, Constraint.empty(), + return self._factory.find_candidates( + requirements, + constraint=self._constraints.get(name, Constraint.empty()), + prefers_installed=(not _eligible_for_upgrade(name)), ) - candidates = self._factory.find_candidates(requirements, constraint) - return reversed(self._sort_matches(candidates)) def is_satisfied_by(self, requirement, candidate): # type: (Requirement, Candidate) -> bool diff --git a/tests/unit/resolution_resolvelib/test_requirement.py b/tests/unit/resolution_resolvelib/test_requirement.py index a03edb6f7..48c5f7347 100644 --- a/tests/unit/resolution_resolvelib/test_requirement.py +++ b/tests/unit/resolution_resolvelib/test_requirement.py @@ -58,7 +58,9 @@ def test_new_resolver_correct_number_of_matches(test_cases, factory): """Requirements should return the correct number of candidates""" for spec, _, match_count in test_cases: req = factory.make_requirement_from_spec(spec, comes_from=None) - matches = factory.find_candidates([req], Constraint.empty()) + matches = factory.find_candidates( + [req], Constraint.empty(), prefers_installed=False, + ) assert len(list(matches)) == match_count @@ -67,7 +69,10 @@ def test_new_resolver_candidates_match_requirement(test_cases, factory): """ for spec, _, _ in test_cases: req = factory.make_requirement_from_spec(spec, comes_from=None) - for c in factory.find_candidates([req], Constraint.empty()): + candidates = factory.find_candidates( + [req], Constraint.empty(), prefers_installed=False, + ) + for c in candidates: assert isinstance(c, Candidate) assert req.is_satisfied_by(c) From a270ca561695164e817eebe11c8129bebaf11f3e Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 29 Sep 2020 22:24:39 +0800 Subject: [PATCH 202/209] Mypy is wrong --- src/pip/_internal/resolution/resolvelib/found_candidates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/resolution/resolvelib/found_candidates.py b/src/pip/_internal/resolution/resolvelib/found_candidates.py index 0ddcb1fcc..be50f52d1 100644 --- a/src/pip/_internal/resolution/resolvelib/found_candidates.py +++ b/src/pip/_internal/resolution/resolvelib/found_candidates.py @@ -1,4 +1,4 @@ -from pip._vendor.six.moves import collections_abc +from pip._vendor.six.moves import collections_abc # type: ignore from pip._internal.utils.typing import MYPY_CHECK_RUNNING From 01c9b6cf25cab66422c8182e66d0749cf5d7ac64 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 30 Sep 2020 15:09:59 +0800 Subject: [PATCH 203/209] Cache results and remove unused implementation --- .../resolution/resolvelib/found_candidates.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/found_candidates.py b/src/pip/_internal/resolution/resolvelib/found_candidates.py index be50f52d1..43591967f 100644 --- a/src/pip/_internal/resolution/resolvelib/found_candidates.py +++ b/src/pip/_internal/resolution/resolvelib/found_candidates.py @@ -1,5 +1,6 @@ from pip._vendor.six.moves import collections_abc # type: ignore +from pip._internal.utils.compat import lru_cache from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: @@ -96,10 +97,10 @@ class FoundCandidates(collections_abc.Sequence): def __getitem__(self, index): # type: (int) -> Candidate - for i, value in enumerate(self): - if index == i: - return value - raise IndexError(index) + # Implemented to satisfy the ABC check, This is not needed by the + # resolver, and should not be used by the provider either (for + # performance reasons). + raise NotImplementedError("don't do this") def __iter__(self): # type: () -> Iterator[Candidate] @@ -109,10 +110,12 @@ class FoundCandidates(collections_abc.Sequence): klass = _InstalledReplacesCandidatesIterator return klass(self._get_others, self._installed) + @lru_cache(maxsize=1) def __len__(self): # type: () -> int return sum(1 for _ in self) + @lru_cache(maxsize=1) def __bool__(self): # type: () -> bool if self._prefers_installed and self._installed: From 270e183718ec17598e345bd56027fa691dd13b5a Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 30 Sep 2020 15:55:49 +0800 Subject: [PATCH 204/209] News --- news/8023.feature.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 news/8023.feature.rst diff --git a/news/8023.feature.rst b/news/8023.feature.rst new file mode 100644 index 000000000..c886e9a66 --- /dev/null +++ b/news/8023.feature.rst @@ -0,0 +1,2 @@ +New resolver: Avoid accessing indexes when the installed candidate is preferred +and considered good enough. From 6e3d56897b159212813136b0126f82737105b924 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 30 Sep 2020 21:36:15 +0800 Subject: [PATCH 205/209] Always return the installed version --- .../resolution/resolvelib/found_candidates.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/found_candidates.py b/src/pip/_internal/resolution/resolvelib/found_candidates.py index 43591967f..c9b21727a 100644 --- a/src/pip/_internal/resolution/resolvelib/found_candidates.py +++ b/src/pip/_internal/resolution/resolvelib/found_candidates.py @@ -66,10 +66,19 @@ class _InstalledReplacesCandidatesIterator(collections_abc.Iterator): # type: () -> Candidate if self._others is None: self._others = self._get_others() - cand = next(self._others) - while cand.version in self._returned: + try: cand = next(self._others) - if self._installed and cand.version == self._installed.version: + while cand.version in self._returned: + cand = next(self._others) + if self._installed and cand.version == self._installed.version: + cand = self._installed + except StopIteration: + # Return the already-installed candidate as the last item if its + # version does not exist on the index. + if not self._installed: + raise + if self._installed.version in self._returned: + raise cand = self._installed self._returned.add(cand.version) return cand From d22775819bb70f6f321e839b4e3f3677c72f41fb Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 2 Oct 2020 00:12:28 +0800 Subject: [PATCH 206/209] Test for candidate ordering --- tests/functional/test_new_resolver.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/functional/test_new_resolver.py b/tests/functional/test_new_resolver.py index 1dab8d470..1718ab8a8 100644 --- a/tests/functional/test_new_resolver.py +++ b/tests/functional/test_new_resolver.py @@ -1020,3 +1020,29 @@ def test_new_resolver_no_deps_checks_requires_python(script): "{}.{}.{} not in '<2'".format(*sys.version_info[:3]) ) assert message in result.stderr + + +def test_new_resolver_prefers_installed_in_upgrade_if_latest(script): + create_basic_wheel_for_package(script, "pkg", "1") + local_pkg = create_test_package_with_setup(script, name="pkg", version="2") + + # Install the version that's not on the index. + script.pip( + "install", + "--use-feature=2020-resolver", + "--no-cache-dir", + "--no-index", + local_pkg, + ) + + # Now --upgrade should still pick the local version because it's "better". + script.pip( + "install", + "--use-feature=2020-resolver", + "--no-cache-dir", + "--no-index", + "--find-links", script.scratch_path, + "--upgrade", + "pkg", + ) + assert_installed(script, pkg="2") From 17d0086ea2b1dbff45ae8e2be88225fd366d67dd Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 1 Oct 2020 23:41:03 +0800 Subject: [PATCH 207/209] Do this all over again --- .../resolution/resolvelib/found_candidates.py | 109 ++++++------------ 1 file changed, 38 insertions(+), 71 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/found_candidates.py b/src/pip/_internal/resolution/resolvelib/found_candidates.py index c9b21727a..491290466 100644 --- a/src/pip/_internal/resolution/resolvelib/found_candidates.py +++ b/src/pip/_internal/resolution/resolvelib/found_candidates.py @@ -1,89 +1,51 @@ +import functools +import itertools + from pip._vendor.six.moves import collections_abc # type: ignore from pip._internal.utils.compat import lru_cache from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Callable, Iterator, Optional, Set + from typing import Any, Callable, Iterator, Optional, Set from pip._vendor.packaging.version import _BaseVersion from .base import Candidate -class _InstalledFirstCandidatesIterator(collections_abc.Iterator): - """Iterator for ``FoundCandidates``. - - This iterator is used when the resolver prefers to keep the version of an - already-installed package. The already-installed candidate is always - returned first. Candidates from index are accessed only when the resolver - wants them, and the already-installed version is excluded from them. - """ - def __init__( - self, - get_others, # type: Callable[[], Iterator[Candidate]] - installed, # type: Optional[Candidate] - ): - self._installed = installed - self._get_others = get_others - self._others = None # type: Optional[Iterator[Candidate]] - self._returned = set() # type: Set[_BaseVersion] - - def __next__(self): - # type: () -> Candidate - if self._installed and self._installed.version not in self._returned: - self._returned.add(self._installed.version) - return self._installed - if self._others is None: - self._others = self._get_others() - cand = next(self._others) - while cand.version in self._returned: - cand = next(self._others) - self._returned.add(cand.version) - return cand - - next = __next__ # XXX: Python 2. +def _deduplicated_by_version(candidates): + # type: (Iterator[Candidate]) -> Iterator[Candidate] + returned = set() # type: Set[_BaseVersion] + for candidate in candidates: + if candidate.version in returned: + continue + returned.add(candidate.version) + yield candidate -class _InstalledReplacesCandidatesIterator(collections_abc.Iterator): +def _replaces_sort_key(installed, candidate): + # type: (Candidate, Candidate) -> Any + return (candidate.version, candidate is installed) + + +def _insert_installed(installed, others): + # type: (Candidate, Iterator[Candidate]) -> Iterator[Candidate] """Iterator for ``FoundCandidates``. This iterator is used when the resolver prefers to upgrade an already-installed package. Candidates from index are returned in their normal ordering, except replaced when the version is already installed. + + The sort key prefers the installed candidate over candidates of the same + version from the index, so it is chosen on de-duplication. """ - def __init__( - self, - get_others, # type: Callable[[], Iterator[Candidate]] - installed, # type: Optional[Candidate] - ): - self._installed = installed - self._get_others = get_others - self._others = None # type: Optional[Iterator[Candidate]] - self._returned = set() # type: Set[_BaseVersion] - - def __next__(self): - # type: () -> Candidate - if self._others is None: - self._others = self._get_others() - try: - cand = next(self._others) - while cand.version in self._returned: - cand = next(self._others) - if self._installed and cand.version == self._installed.version: - cand = self._installed - except StopIteration: - # Return the already-installed candidate as the last item if its - # version does not exist on the index. - if not self._installed: - raise - if self._installed.version in self._returned: - raise - cand = self._installed - self._returned.add(cand.version) - return cand - - next = __next__ # XXX: Python 2. + candidates = sorted( + itertools.chain(others, [installed]), + key=functools.partial(_replaces_sort_key, installed), + reverse=True, + ) + return iter(candidates) class FoundCandidates(collections_abc.Sequence): @@ -106,22 +68,27 @@ class FoundCandidates(collections_abc.Sequence): def __getitem__(self, index): # type: (int) -> Candidate - # Implemented to satisfy the ABC check, This is not needed by the + # Implemented to satisfy the ABC check. This is not needed by the # resolver, and should not be used by the provider either (for # performance reasons). raise NotImplementedError("don't do this") def __iter__(self): # type: () -> Iterator[Candidate] - if self._prefers_installed: - klass = _InstalledFirstCandidatesIterator + if not self._installed: + candidates = self._get_others() + elif self._prefers_installed: + candidates = itertools.chain([self._installed], self._get_others()) else: - klass = _InstalledReplacesCandidatesIterator - return klass(self._get_others, self._installed) + candidates = _insert_installed(self._installed, self._get_others()) + return _deduplicated_by_version(candidates) @lru_cache(maxsize=1) def __len__(self): # type: () -> int + # Implement to satisfy the ABC check and used in tests. This is not + # needed by the resolver, and should not be used by the provider either + # (for performance reasons). return sum(1 for _ in self) @lru_cache(maxsize=1) From 761433cee8ad0146bf38b2736a22c91d1dd63615 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 2 Oct 2020 19:55:05 +0800 Subject: [PATCH 208/209] Eliminate len() usage in tests --- .../_internal/resolution/resolvelib/found_candidates.py | 9 ++++----- tests/unit/resolution_resolvelib/test_requirement.py | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/found_candidates.py b/src/pip/_internal/resolution/resolvelib/found_candidates.py index 491290466..db8232240 100644 --- a/src/pip/_internal/resolution/resolvelib/found_candidates.py +++ b/src/pip/_internal/resolution/resolvelib/found_candidates.py @@ -83,13 +83,12 @@ class FoundCandidates(collections_abc.Sequence): candidates = _insert_installed(self._installed, self._get_others()) return _deduplicated_by_version(candidates) - @lru_cache(maxsize=1) def __len__(self): # type: () -> int - # Implement to satisfy the ABC check and used in tests. This is not - # needed by the resolver, and should not be used by the provider either - # (for performance reasons). - return sum(1 for _ in self) + # Implemented to satisfy the ABC check. This is not needed by the + # resolver, and should not be used by the provider either (for + # performance reasons). + raise NotImplementedError("don't do this") @lru_cache(maxsize=1) def __bool__(self): diff --git a/tests/unit/resolution_resolvelib/test_requirement.py b/tests/unit/resolution_resolvelib/test_requirement.py index 48c5f7347..6149fd1ae 100644 --- a/tests/unit/resolution_resolvelib/test_requirement.py +++ b/tests/unit/resolution_resolvelib/test_requirement.py @@ -61,7 +61,7 @@ def test_new_resolver_correct_number_of_matches(test_cases, factory): matches = factory.find_candidates( [req], Constraint.empty(), prefers_installed=False, ) - assert len(list(matches)) == match_count + assert sum(1 for _ in matches) == match_count def test_new_resolver_candidates_match_requirement(test_cases, factory): From b921db84bdace474139477089ef1865a0217825f Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 2 Oct 2020 19:55:14 +0800 Subject: [PATCH 209/209] Improve sorting logic --- .../resolution/resolvelib/found_candidates.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/found_candidates.py b/src/pip/_internal/resolution/resolvelib/found_candidates.py index db8232240..a669e8936 100644 --- a/src/pip/_internal/resolution/resolvelib/found_candidates.py +++ b/src/pip/_internal/resolution/resolvelib/found_candidates.py @@ -1,5 +1,5 @@ -import functools import itertools +import operator from pip._vendor.six.moves import collections_abc # type: ignore @@ -7,7 +7,7 @@ from pip._internal.utils.compat import lru_cache from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any, Callable, Iterator, Optional, Set + from typing import Callable, Iterator, Optional, Set from pip._vendor.packaging.version import _BaseVersion @@ -24,11 +24,6 @@ def _deduplicated_by_version(candidates): yield candidate -def _replaces_sort_key(installed, candidate): - # type: (Candidate, Candidate) -> Any - return (candidate.version, candidate is installed) - - def _insert_installed(installed, others): # type: (Candidate, Iterator[Candidate]) -> Iterator[Candidate] """Iterator for ``FoundCandidates``. @@ -37,12 +32,15 @@ def _insert_installed(installed, others): already-installed package. Candidates from index are returned in their normal ordering, except replaced when the version is already installed. - The sort key prefers the installed candidate over candidates of the same - version from the index, so it is chosen on de-duplication. + Since candidates from index are already sorted by reverse version order, + `sorted()` here would keep the ordering mostly intact, only shuffling the + already-installed candidate into the correct position. We put the already- + installed candidate in front of those from the index, so it's put in front + after sorting due to Python sorting's stableness guarentee. """ candidates = sorted( - itertools.chain(others, [installed]), - key=functools.partial(_replaces_sort_key, installed), + itertools.chain([installed], others), + key=operator.attrgetter("version"), reverse=True, ) return iter(candidates)