diff --git a/news/8905.feature b/news/8905.feature new file mode 100644 index 000000000..5d27d40c2 --- /dev/null +++ b/news/8905.feature @@ -0,0 +1,3 @@ +Cache package listings on index packages so they are guarenteed to stay stable +during a pip command session. This also improves performance when a index page +is accessed multiple times during the command session. diff --git a/src/pip/_internal/index/collector.py b/src/pip/_internal/index/collector.py index 6c35fc660..ef2100f89 100644 --- a/src/pip/_internal/index/collector.py +++ b/src/pip/_internal/index/collector.py @@ -21,6 +21,7 @@ from pip._internal.exceptions import NetworkConnectionError from pip._internal.models.link import Link from pip._internal.models.search_scope import SearchScope from pip._internal.network.utils import raise_for_status +from pip._internal.utils.compat import lru_cache from pip._internal.utils.filetypes import ARCHIVE_EXTENSIONS from pip._internal.utils.misc import pairwise, redact_auth_from_url from pip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -30,8 +31,14 @@ from pip._internal.vcs import is_url, vcs if MYPY_CHECK_RUNNING: from optparse import Values from typing import ( - Callable, Iterable, List, MutableMapping, Optional, - Protocol, Sequence, Tuple, TypeVar, Union, + Callable, + Iterable, + List, + MutableMapping, + Optional, + Sequence, + Tuple, + Union, ) import xml.etree.ElementTree @@ -42,31 +49,10 @@ if MYPY_CHECK_RUNNING: HTMLElement = xml.etree.ElementTree.Element ResponseHeaders = MutableMapping[str, str] - # Used in the @lru_cache polyfill. - F = TypeVar('F') - - class LruCache(Protocol): - def __call__(self, maxsize=None): - # type: (Optional[int]) -> Callable[[F], F] - raise NotImplementedError - logger = logging.getLogger(__name__) -# Fallback to noop_lru_cache in Python 2 -# TODO: this can be removed when python 2 support is dropped! -def noop_lru_cache(maxsize=None): - # type: (Optional[int]) -> Callable[[F], F] - def _wrapper(f): - # type: (F) -> F - return f - return _wrapper - - -_lru_cache = getattr(functools, "lru_cache", noop_lru_cache) # type: LruCache - - def _match_vcs_scheme(url): # type: (str) -> Optional[str] """Look for VCS schemes in the URL. @@ -336,7 +322,7 @@ def with_cached_html_pages( `page` has `page.cache_link_parsing == False`. """ - @_lru_cache(maxsize=None) + @lru_cache(maxsize=None) def wrapper(cacheable_page): # type: (CacheablePageContent) -> List[Link] return list(fn(cacheable_page.page)) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 84115783a..8ceccee6c 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -25,6 +25,7 @@ from pip._internal.models.link import Link from pip._internal.models.selection_prefs import SelectionPreferences from pip._internal.models.target_python import TargetPython from pip._internal.models.wheel import Wheel +from pip._internal.utils.compat import lru_cache from pip._internal.utils.filetypes import WHEEL_EXTENSION from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import build_netloc @@ -801,6 +802,7 @@ class PackageFinder(object): return package_links + @lru_cache(maxsize=None) def find_all_candidates(self, project_name): # type: (str) -> List[InstallationCandidate] """Find all available InstallationCandidate for project_name diff --git a/src/pip/_internal/utils/compat.py b/src/pip/_internal/utils/compat.py index 89c5169af..9a2bb7800 100644 --- a/src/pip/_internal/utils/compat.py +++ b/src/pip/_internal/utils/compat.py @@ -7,6 +7,7 @@ distributions.""" from __future__ import absolute_import, division import codecs +import functools import locale import logging import os @@ -18,7 +19,15 @@ from pip._vendor.six import PY2, text_type from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, Text, Tuple, Union + from typing import Callable, Optional, Protocol, Text, Tuple, TypeVar, Union + + # Used in the @lru_cache polyfill. + F = TypeVar('F') + + class LruCache(Protocol): + def __call__(self, maxsize=None): + # type: (Optional[int]) -> Callable[[F], F] + raise NotImplementedError try: import ipaddress @@ -269,3 +278,16 @@ else: if not cr: cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) return int(cr[1]), int(cr[0]) + + +# Fallback to noop_lru_cache in Python 2 +# TODO: this can be removed when python 2 support is dropped! +def noop_lru_cache(maxsize=None): + # type: (Optional[int]) -> Callable[[F], F] + def _wrapper(f): + # type: (F) -> F + return f + return _wrapper + + +lru_cache = getattr(functools, "lru_cache", noop_lru_cache) # type: LruCache