From c7b477777a0fc5f2f8b71c2e6b6eb542a7c65126 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com> Date: Tue, 11 Aug 2020 15:41:35 +0530 Subject: [PATCH 1/8] Merge pull request #8730 from McSinyx/news-8701-8716 Add news for disabling range response caching --- news/8701.bugfix | 2 ++ news/8716.bugfix | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 news/8701.bugfix create mode 100644 news/8716.bugfix diff --git a/news/8701.bugfix b/news/8701.bugfix new file mode 100644 index 000000000..086a8f2eb --- /dev/null +++ b/news/8701.bugfix @@ -0,0 +1,2 @@ +Disable caching for range requests, which causes corrupted wheels +when pip tries to obtain metadata using the feature ``fast-deps``. diff --git a/news/8716.bugfix b/news/8716.bugfix new file mode 100644 index 000000000..086a8f2eb --- /dev/null +++ b/news/8716.bugfix @@ -0,0 +1,2 @@ +Disable caching for range requests, which causes corrupted wheels +when pip tries to obtain metadata using the feature ``fast-deps``. From 626d6316829e45ee2e9fafc67b8ae6d4708a905d Mon Sep 17 00:00:00 2001 From: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com> Date: Tue, 11 Aug 2020 15:41:02 +0530 Subject: [PATCH 2/8] Merge pull request #8744 from hroncok/keyring_global_nope When one keyring attempt fails, don't bother with more --- news/8090.bugfix | 3 +++ src/pip/_internal/network/auth.py | 2 ++ tests/unit/test_network_auth.py | 26 ++++++++++++++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 news/8090.bugfix diff --git a/news/8090.bugfix b/news/8090.bugfix new file mode 100644 index 000000000..e9f2b7cbb --- /dev/null +++ b/news/8090.bugfix @@ -0,0 +1,3 @@ +Only attempt to use the keyring once and if it fails, don't try again. +This prevents spamming users with several keyring unlock prompts when they +cannot unlock or don't want to do so. diff --git a/src/pip/_internal/network/auth.py b/src/pip/_internal/network/auth.py index ca729fcdf..c49deaaf1 100644 --- a/src/pip/_internal/network/auth.py +++ b/src/pip/_internal/network/auth.py @@ -44,6 +44,7 @@ except Exception as exc: def get_keyring_auth(url, username): # type: (str, str) -> Optional[AuthInfo] """Return the tuple auth for a given url from keyring.""" + global keyring if not url or not keyring: return None @@ -69,6 +70,7 @@ def get_keyring_auth(url, username): logger.warning( "Keyring is skipped due to an exception: %s", str(exc), ) + keyring = None return None diff --git a/tests/unit/test_network_auth.py b/tests/unit/test_network_auth.py index 08320cfa1..8116b627f 100644 --- a/tests/unit/test_network_auth.py +++ b/tests/unit/test_network_auth.py @@ -242,3 +242,29 @@ def test_keyring_get_credential(monkeypatch, url, expect): assert auth._get_new_credentials( url, allow_netrc=False, allow_keyring=True ) == expect + + +class KeyringModuleBroken(object): + """Represents the current supported API of keyring, but broken""" + + def __init__(self): + self._call_count = 0 + + def get_credential(self, system, username): + self._call_count += 1 + raise Exception("This keyring is broken!") + + +def test_broken_keyring_disables_keyring(monkeypatch): + keyring_broken = KeyringModuleBroken() + monkeypatch.setattr(pip._internal.network.auth, 'keyring', keyring_broken) + + auth = MultiDomainBasicAuth(index_urls=["http://example.com/"]) + + assert keyring_broken._call_count == 0 + for i in range(5): + url = "http://example.com/path" + str(i) + assert auth._get_new_credentials( + url, allow_netrc=False, allow_keyring=True + ) == (None, None) + assert keyring_broken._call_count == 1 From e04cd89f6fcc37f7fab00829dcb349838c8837a8 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com> Date: Mon, 10 Aug 2020 20:30:55 +0530 Subject: [PATCH 3/8] Merge pull request #8702 from uranusjr/get-distribution-looks-for-all --- news/8695.bugfix | 3 ++ src/pip/_internal/commands/search.py | 1 + src/pip/_internal/utils/misc.py | 27 +++++++++-- tests/unit/test_utils.py | 70 +++++++++++++++++++++++----- 4 files changed, 84 insertions(+), 17 deletions(-) create mode 100644 news/8695.bugfix diff --git a/news/8695.bugfix b/news/8695.bugfix new file mode 100644 index 000000000..668e4672e --- /dev/null +++ b/news/8695.bugfix @@ -0,0 +1,3 @@ +Fix regression that distributions in system site-packages are not correctly +found when a virtual environment is configured with ``system-site-packages`` +on. diff --git a/src/pip/_internal/commands/search.py b/src/pip/_internal/commands/search.py index e906ce766..ff0947202 100644 --- a/src/pip/_internal/commands/search.py +++ b/src/pip/_internal/commands/search.py @@ -140,6 +140,7 @@ def print_results(hits, name_column_width=None, terminal_width=None): write_output(line) if name in installed_packages: dist = get_distribution(name) + assert dist is not None with indent_log(): if dist.version == latest: write_output('INSTALLED: %s (latest)', dist.version) diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index 24a745562..5629c60c1 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -483,22 +483,39 @@ def get_installed_distributions( ] -def search_distribution(req_name): +def _search_distribution(req_name): + # type: (str) -> Optional[Distribution] + """Find a distribution matching the ``req_name`` in the environment. + This searches from *all* distributions available in the environment, to + match the behavior of ``pkg_resources.get_distribution()``. + """ # Canonicalize the name before searching in the list of # installed distributions and also while creating the package # dictionary to get the Distribution object req_name = canonicalize_name(req_name) - packages = get_installed_distributions(skip=()) + packages = get_installed_distributions( + local_only=False, + skip=(), + include_editables=True, + editables_only=False, + user_only=False, + paths=None, + ) pkg_dict = {canonicalize_name(p.key): p for p in packages} return pkg_dict.get(req_name) def get_distribution(req_name): - """Given a requirement name, return the installed Distribution object""" + # type: (str) -> Optional[Distribution] + """Given a requirement name, return the installed Distribution object. + + This searches from *all* distributions available in the environment, to + match the behavior of ``pkg_resources.get_distribution()``. + """ # Search the distribution by looking through the working set - dist = search_distribution(req_name) + dist = _search_distribution(req_name) # If distribution could not be found, call working_set.require # to update the working set, and try to find the distribution @@ -514,7 +531,7 @@ def get_distribution(req_name): pkg_resources.working_set.require(req_name) except pkg_resources.DistributionNotFound: return None - return search_distribution(req_name) + return _search_distribution(req_name) def egg_link_path(dist): diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ebabd29e2..0a1c47cd7 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -5,6 +5,7 @@ util tests """ import codecs +import itertools import os import shutil import stat @@ -34,6 +35,7 @@ from pip._internal.utils.misc import ( build_url_from_netloc, egg_link_path, format_size, + get_distribution, get_installed_distributions, get_prog, hide_url, @@ -192,26 +194,30 @@ class Tests_EgglinkPath: @patch('pip._internal.utils.misc.dist_in_usersite') @patch('pip._internal.utils.misc.dist_is_local') @patch('pip._internal.utils.misc.dist_is_editable') -class Tests_get_installed_distributions: - """test util.get_installed_distributions""" +class TestsGetDistributions(object): + """Test get_installed_distributions() and get_distribution(). + """ + class MockWorkingSet(list): + def require(self, name): + pass - workingset = [ - Mock(test_name="global"), - Mock(test_name="editable"), - Mock(test_name="normal"), - Mock(test_name="user"), - ] + workingset = MockWorkingSet(( + Mock(test_name="global", key="global"), + Mock(test_name="editable", key="editable"), + Mock(test_name="normal", key="normal"), + Mock(test_name="user", key="user"), + )) - workingset_stdlib = [ + workingset_stdlib = MockWorkingSet(( Mock(test_name='normal', key='argparse'), Mock(test_name='normal', key='wsgiref') - ] + )) - workingset_freeze = [ + workingset_freeze = MockWorkingSet(( Mock(test_name='normal', key='pip'), Mock(test_name='normal', key='setuptools'), Mock(test_name='normal', key='distribute') - ] + )) def dist_is_editable(self, dist): return dist.test_name == "editable" @@ -287,6 +293,46 @@ class Tests_get_installed_distributions: skip=('setuptools', 'pip', 'distribute')) assert len(dists) == 0 + @pytest.mark.parametrize( + "working_set, req_name", + itertools.chain( + itertools.product([workingset], (d.key for d in workingset)), + itertools.product( + [workingset_stdlib], (d.key for d in workingset_stdlib), + ), + ), + ) + def test_get_distribution( + self, + mock_dist_is_editable, + mock_dist_is_local, + mock_dist_in_usersite, + working_set, + req_name, + ): + """Ensure get_distribution() finds all kinds of distributions. + """ + mock_dist_is_editable.side_effect = self.dist_is_editable + mock_dist_is_local.side_effect = self.dist_is_local + mock_dist_in_usersite.side_effect = self.dist_in_usersite + with patch("pip._vendor.pkg_resources.working_set", working_set): + dist = get_distribution(req_name) + assert dist is not None + assert dist.key == req_name + + @patch('pip._vendor.pkg_resources.working_set', workingset) + def test_get_distribution_nonexist( + self, + mock_dist_is_editable, + mock_dist_is_local, + mock_dist_in_usersite, + ): + mock_dist_is_editable.side_effect = self.dist_is_editable + mock_dist_is_local.side_effect = self.dist_is_local + mock_dist_in_usersite.side_effect = self.dist_in_usersite + dist = get_distribution("non-exist") + assert dist is None + def test_rmtree_errorhandler_nonexistent_directory(tmpdir): """ From 516c7431bcab438721931c8eeba0b4c33740288a Mon Sep 17 00:00:00 2001 From: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com> Date: Fri, 7 Aug 2020 15:11:32 +0530 Subject: [PATCH 4/8] Merge pull request #8718 from uranusjr/pyvenv-cfg-encoding Always use UTF-8 to read pyvenv.cfg --- news/8717.bugfix | 1 + src/pip/_internal/utils/virtualenv.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 news/8717.bugfix diff --git a/news/8717.bugfix b/news/8717.bugfix new file mode 100644 index 000000000..e8c8533c4 --- /dev/null +++ b/news/8717.bugfix @@ -0,0 +1 @@ +Always use UTF-8 to read ``pyvenv.cfg`` to match the built-in ``venv``. diff --git a/src/pip/_internal/utils/virtualenv.py b/src/pip/_internal/utils/virtualenv.py index 596a69a7d..4a7812873 100644 --- a/src/pip/_internal/utils/virtualenv.py +++ b/src/pip/_internal/utils/virtualenv.py @@ -1,5 +1,6 @@ from __future__ import absolute_import +import io import logging import os import re @@ -51,7 +52,9 @@ def _get_pyvenv_cfg_lines(): """ pyvenv_cfg_file = os.path.join(sys.prefix, 'pyvenv.cfg') try: - with open(pyvenv_cfg_file) as f: + # Although PEP 405 does not specify, the built-in venv module always + # writes with UTF-8. (pypa/pip#8717) + with io.open(pyvenv_cfg_file, encoding='utf-8') as f: return f.read().splitlines() # avoids trailing newlines except IOError: return None From 0ebe453140ccf2a8c3c537b4800fd2d7b97715e5 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam <3275593+pradyunsg@users.noreply.github.com> Date: Fri, 7 Aug 2020 14:41:46 +0530 Subject: [PATCH 5/8] Merge pull request #8727 from uranusjr/new-resolver-constraint-markers --- news/8724.bugfix | 2 ++ .../resolution/resolvelib/resolver.py | 3 ++- tests/functional/test_new_resolver.py | 25 +++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 news/8724.bugfix diff --git a/news/8724.bugfix b/news/8724.bugfix new file mode 100644 index 000000000..8641098dd --- /dev/null +++ b/news/8724.bugfix @@ -0,0 +1,2 @@ +2020 Resolver: Correctly handle marker evaluation in constraints and exclude +them if their markers do not match the current environment. diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py index 43ea24863..aecddb113 100644 --- a/src/pip/_internal/resolution/resolvelib/resolver.py +++ b/src/pip/_internal/resolution/resolvelib/resolver.py @@ -90,7 +90,8 @@ class Resolver(BaseResolver): problem = check_invalid_constraint_type(req) if problem: raise InstallationError(problem) - + if not req.match_markers(): + continue name = canonicalize_name(req.name) if name in constraints: constraints[name] = constraints[name] & req.specifier diff --git a/tests/functional/test_new_resolver.py b/tests/functional/test_new_resolver.py index 932918033..46e32ddd3 100644 --- a/tests/functional/test_new_resolver.py +++ b/tests/functional/test_new_resolver.py @@ -1,6 +1,7 @@ import json import os import sys +import textwrap import pytest from pip._vendor.packaging.utils import canonicalize_name @@ -729,6 +730,30 @@ def test_new_resolver_constraint_on_path(script): assert msg in result.stderr, str(result) +def test_new_resolver_constraint_only_marker_match(script): + create_basic_wheel_for_package(script, "pkg", "1.0") + create_basic_wheel_for_package(script, "pkg", "2.0") + create_basic_wheel_for_package(script, "pkg", "3.0") + + constrants_content = textwrap.dedent( + """ + pkg==1.0; python_version == "{ver[0]}.{ver[1]}" # Always satisfies. + pkg==2.0; python_version < "0" # Never satisfies. + """ + ).format(ver=sys.version_info) + constraints_txt = script.scratch_path / "constraints.txt" + constraints_txt.write_text(constrants_content) + + script.pip( + "install", "--use-feature=2020-resolver", + "--no-cache-dir", "--no-index", + "-c", constraints_txt, + "--find-links", script.scratch_path, + "pkg", + ) + assert_installed(script, pkg="1.0") + + def test_new_resolver_upgrade_needs_option(script): # Install pkg 1.0.0 create_basic_wheel_for_package(script, "pkg", "1.0.0") From b9e403b1730e8555018794f758e7a9fc871e3833 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Wed, 5 Aug 2020 19:58:22 -0400 Subject: [PATCH 6/8] Merge pull request #8716 from McSinyx/fix-range-request-cache Disable caching for range requests --- news/50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial | 0 src/pip/_internal/network/lazy_wheel.py | 6 ++++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 news/50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial diff --git a/news/50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial b/news/50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/network/lazy_wheel.py b/src/pip/_internal/network/lazy_wheel.py index 16be0d297..a0f9e151d 100644 --- a/src/pip/_internal/network/lazy_wheel.py +++ b/src/pip/_internal/network/lazy_wheel.py @@ -194,8 +194,10 @@ class LazyZipOverHTTP(object): def _stream_response(self, start, end, base_headers=HEADERS): # type: (int, int, Dict[str, str]) -> Response """Return HTTP response to a range request from start to end.""" - headers = {'Range': 'bytes={}-{}'.format(start, end)} - headers.update(base_headers) + headers = base_headers.copy() + headers['Range'] = 'bytes={}-{}'.format(start, end) + # TODO: Get range requests to be correctly cached + headers['Cache-Control'] = 'no-cache' return self._session.get(self._url, headers=headers, stream=True) def _merge(self, start, end, left, right): From e16ebf1b7fd77ad96eeb5ec07ae5326bea02268a Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 11 Aug 2020 16:56:41 +0530 Subject: [PATCH 7/8] Bump for release --- NEWS.rst | 19 +++++++++++++++++++ ...cf024d-0a74-44c8-b3e9-483dd826fff2.trivial | 0 news/8090.bugfix | 3 --- news/8695.bugfix | 3 --- news/8701.bugfix | 2 -- news/8716.bugfix | 2 -- news/8717.bugfix | 1 - news/8724.bugfix | 2 -- src/pip/__init__.py | 2 +- 9 files changed, 20 insertions(+), 14 deletions(-) delete mode 100644 news/50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial delete mode 100644 news/8090.bugfix delete mode 100644 news/8695.bugfix delete mode 100644 news/8701.bugfix delete mode 100644 news/8716.bugfix delete mode 100644 news/8717.bugfix delete mode 100644 news/8724.bugfix diff --git a/NEWS.rst b/NEWS.rst index aaacfffb4..a0376a06f 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -9,6 +9,25 @@ .. towncrier release notes start +20.2.2 (2020-08-11) +=================== + +Bug Fixes +--------- + +- Only attempt to use the keyring once and if it fails, don't try again. + This prevents spamming users with several keyring unlock prompts when they + cannot unlock or don't want to do so. (`#8090 `_) +- Fix regression that distributions in system site-packages are not correctly + found when a virtual environment is configured with ``system-site-packages`` + on. (`#8695 `_) +- Disable caching for range requests, which causes corrupted wheels + when pip tries to obtain metadata using the feature ``fast-deps``. (`#8701 `_, `#8716 `_) +- Always use UTF-8 to read ``pyvenv.cfg`` to match the built-in ``venv``. (`#8717 `_) +- 2020 Resolver: Correctly handle marker evaluation in constraints and exclude + them if their markers do not match the current environment. (`#8724 `_) + + 20.2.1 (2020-08-04) =================== diff --git a/news/50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial b/news/50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/8090.bugfix b/news/8090.bugfix deleted file mode 100644 index e9f2b7cbb..000000000 --- a/news/8090.bugfix +++ /dev/null @@ -1,3 +0,0 @@ -Only attempt to use the keyring once and if it fails, don't try again. -This prevents spamming users with several keyring unlock prompts when they -cannot unlock or don't want to do so. diff --git a/news/8695.bugfix b/news/8695.bugfix deleted file mode 100644 index 668e4672e..000000000 --- a/news/8695.bugfix +++ /dev/null @@ -1,3 +0,0 @@ -Fix regression that distributions in system site-packages are not correctly -found when a virtual environment is configured with ``system-site-packages`` -on. diff --git a/news/8701.bugfix b/news/8701.bugfix deleted file mode 100644 index 086a8f2eb..000000000 --- a/news/8701.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Disable caching for range requests, which causes corrupted wheels -when pip tries to obtain metadata using the feature ``fast-deps``. diff --git a/news/8716.bugfix b/news/8716.bugfix deleted file mode 100644 index 086a8f2eb..000000000 --- a/news/8716.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Disable caching for range requests, which causes corrupted wheels -when pip tries to obtain metadata using the feature ``fast-deps``. diff --git a/news/8717.bugfix b/news/8717.bugfix deleted file mode 100644 index e8c8533c4..000000000 --- a/news/8717.bugfix +++ /dev/null @@ -1 +0,0 @@ -Always use UTF-8 to read ``pyvenv.cfg`` to match the built-in ``venv``. diff --git a/news/8724.bugfix b/news/8724.bugfix deleted file mode 100644 index 8641098dd..000000000 --- a/news/8724.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -2020 Resolver: Correctly handle marker evaluation in constraints and exclude -them if their markers do not match the current environment. diff --git a/src/pip/__init__.py b/src/pip/__init__.py index 6a730519c..611753fed 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -4,7 +4,7 @@ if MYPY_CHECK_RUNNING: from typing import List, Optional -__version__ = "20.2.1" +__version__ = "20.2.2" def main(args=None): From cef8abd268d7516dc98a6dc5fe3e61b3fbd6944f Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Tue, 11 Aug 2020 16:56:42 +0530 Subject: [PATCH 8/8] Bump for development --- src/pip/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/__init__.py b/src/pip/__init__.py index 611753fed..5a2f3c317 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -4,7 +4,7 @@ if MYPY_CHECK_RUNNING: from typing import List, Optional -__version__ = "20.2.2" +__version__ = "20.3.dev0" def main(args=None):