McSinyx
/
pip
mirror of https://github.com/pypa/pip
1
1
Fork 0
Code Issues Releases Activity

Merge pull request #6624 from cjerdonek/add-search-scope 

Add SearchScope class for --find-links and --index-url related options
This commit is contained in:
Chris Jerdonek 2019-06-21 07:30:55 -07:00 committed by GitHub
parent 26cdedd618 7d08bb37a5
commit c3c61b5c51
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 201 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/pip/_internal/build_env.py
View File

@ -177,14 +177,17 @@ class BuildEnvironment(object):
formats = getattr(finder.format_control, format_control)
args.extend(('--' + format_control.replace('_', '-'),
','.join(sorted(formats or {':none:'}))))
if finder.index_urls:
args.extend(['-i', finder.index_urls[0]])
for extra_index in finder.index_urls[1:]:
index_urls = finder.index_urls
if index_urls:
args.extend(['-i', index_urls[0]])
for extra_index in index_urls[1:]:
args.extend(['--extra-index-url', extra_index])
else:
args.append('--no-index')
for link in finder.find_links:
args.extend(['--find-links', link])
for host in finder.trusted_hosts:
args.extend(['--trusted-host', host])
if finder.allow_all_prereleases:

101
src/pip/_internal/index.py
View File

@ -6,7 +6,6 @@ import itertools
import logging
import mimetypes
import os
import posixpath
import re
from collections import namedtuple
@ -19,21 +18,21 @@ from pip._vendor.requests.exceptions import HTTPError, RetryError, SSLError
from pip._vendor.six.moves.urllib import parse as urllib_parse
from pip._vendor.six.moves.urllib import request as urllib_request
from pip._internal.download import HAS_TLS, is_url, url_to_path
from pip._internal.download import is_url, url_to_path
from pip._internal.exceptions import (
BestVersionAlreadyInstalled, DistributionNotFound, InvalidWheelFilename,
UnsupportedWheel,
)
from pip._internal.models.candidate import InstallationCandidate
from pip._internal.models.format_control import FormatControl
from pip._internal.models.index import PyPI
from pip._internal.models.link import Link
from pip._internal.models.search_scope import SearchScope
from pip._internal.models.target_python import TargetPython
from pip._internal.utils.compat import ipaddress
from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import (
ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, WHEEL_EXTENSION, normalize_path,
path_to_url, redact_password_from_url,
ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, WHEEL_EXTENSION, path_to_url,
redact_password_from_url,
)
from pip._internal.utils.packaging import check_requires_python
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
@ -560,8 +559,7 @@ class PackageFinder(object):
def __init__(
self,
candidate_evaluator, # type: CandidateEvaluator
find_links, # type: List[str]
index_urls, # type: List[str]
search_scope, # type: SearchScope
session, # type: PipSession
format_control=None, # type: Optional[FormatControl]
trusted_hosts=None, # type: Optional[List[str]]
@ -583,8 +581,7 @@ class PackageFinder(object):
format_control = format_control or FormatControl(set(), set())
self.candidate_evaluator = candidate_evaluator
self.find_links = find_links
self.index_urls = index_urls
self.search_scope = search_scope
self.session = session
self.format_control = format_control
self.trusted_hosts = trusted_hosts
@ -626,47 +623,36 @@ class PackageFinder(object):
"'session'"
)
# Build find_links. If an argument starts with ~, it may be
# a local file relative to a home directory. So try normalizing
# it and if it exists, use the normalized version.
# This is deliberately conservative - it might be fine just to
# blindly normalize anything starting with a ~...
built_find_links = [] # type: List[str]
for link in find_links:
if link.startswith('~'):
new_link = normalize_path(link)
if os.path.exists(new_link):
link = new_link
built_find_links.append(link)
search_scope = SearchScope.create(
find_links=find_links,
index_urls=index_urls,
)
candidate_evaluator = CandidateEvaluator(
target_python=target_python, prefer_binary=prefer_binary,
target_python=target_python,
prefer_binary=prefer_binary,
allow_all_prereleases=allow_all_prereleases,
ignore_requires_python=ignore_requires_python,
)
# If we don't have TLS enabled, then WARN if anyplace we're looking
# relies on TLS.
if not HAS_TLS:
for link in itertools.chain(index_urls, built_find_links):
parsed = urllib_parse.urlparse(link)
if parsed.scheme == "https":
logger.warning(
"pip is configured with locations that require "
"TLS/SSL, however the ssl module in Python is not "
"available."
)
break
return cls(
candidate_evaluator=candidate_evaluator,
find_links=built_find_links,
index_urls=index_urls,
search_scope=search_scope,
session=session,
format_control=format_control,
trusted_hosts=trusted_hosts,
)
@property
def find_links(self):
# type: () -> List[str]
return self.search_scope.find_links
@property
def index_urls(self):
# type: () -> List[str]
return self.search_scope.index_urls
@property
def allow_all_prereleases(self):
# type: () -> bool
@ -701,21 +687,6 @@ class PackageFinder(object):
for host in self.trusted_hosts:
yield ('*', host, '*')
def get_formatted_locations(self):
# type: () -> str
lines = []
if self.index_urls and self.index_urls != [PyPI.simple_url]:
lines.append(
"Looking in indexes: {}".format(", ".join(
redact_password_from_url(url) for url in self.index_urls))
)
if self.find_links:
lines.append(
"Looking in links: {}".format(", ".join(
redact_password_from_url(url) for url in self.find_links))
)
return "\n".join(lines)
@staticmethod
def _sort_locations(locations, expand_dir=False):
# type: (Sequence[str], bool) -> Tuple[List[str], List[str]]
@ -848,29 +819,6 @@ class PackageFinder(object):
return False
def _get_index_urls_locations(self, project_name):
# type: (str) -> List[str]
"""Returns the locations found via self.index_urls
Checks the url_name on the main (first in the list) index and
use this url_name to produce all locations
"""
def mkurl_pypi_url(url):
loc = posixpath.join(
url,
urllib_parse.quote(canonicalize_name(project_name)))
# For maximum compatibility with easy_install, ensure the path
# ends in a trailing slash. Although this isn't in the spec
# (and PyPI can handle it without the slash) some other index
# implementations might break if they relied on easy_install's
# behavior.
if not loc.endswith('/'):
loc = loc + '/'
return loc
return [mkurl_pypi_url(url) for url in self.index_urls]
def find_all_candidates(self, project_name):
# type: (str) -> List[InstallationCandidate]
"""Find all available InstallationCandidate for project_name
@ -881,7 +829,8 @@ class PackageFinder(object):
See CandidateEvaluator.evaluate_link() for details on which files
are accepted.
"""
index_locations = self._get_index_urls_locations(project_name)
search_scope = self.search_scope
index_locations = search_scope.get_index_urls_locations(project_name)
index_file_loc, index_url_loc = self._sort_locations(index_locations)
fl_file_loc, fl_url_loc = self._sort_locations(
self.find_links, expand_dir=True,

3
src/pip/_internal/legacy_resolve.py
View File

@ -179,7 +179,8 @@ class Resolver(object):
)
# Display where finder is looking for packages
locations = self.finder.get_formatted_locations()
search_scope = self.finder.search_scope
locations = search_scope.get_formatted_locations()
if locations:
logger.info(locations)

113
src/pip/_internal/models/search_scope.py Normal file
View File

@ -0,0 +1,113 @@
import itertools
import logging
import os
import posixpath
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.six.moves.urllib import parse as urllib_parse
from pip._internal.download import HAS_TLS
from pip._internal.models.index import PyPI
from pip._internal.utils.misc import normalize_path, redact_password_from_url
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
if MYPY_CHECK_RUNNING:
from typing import List
logger = logging.getLogger(__name__)
class SearchScope(object):
"""
Encapsulates the locations that pip is configured to search.
"""
@classmethod
def create(
cls,
find_links, # type: List[str]
index_urls, # type: List[str]
):
# type: (...) -> SearchScope
"""
Create a SearchScope object after normalizing the `find_links`.
"""
# Build find_links. If an argument starts with ~, it may be
# a local file relative to a home directory. So try normalizing
# it and if it exists, use the normalized version.
# This is deliberately conservative - it might be fine just to
# blindly normalize anything starting with a ~...
built_find_links = [] # type: List[str]
for link in find_links:
if link.startswith('~'):
new_link = normalize_path(link)
if os.path.exists(new_link):
link = new_link
built_find_links.append(link)
# If we don't have TLS enabled, then WARN if anyplace we're looking
# relies on TLS.
if not HAS_TLS:
for link in itertools.chain(index_urls, built_find_links):
parsed = urllib_parse.urlparse(link)
if parsed.scheme == 'https':
logger.warning(
'pip is configured with locations that require '
'TLS/SSL, however the ssl module in Python is not '
'available.'
)
break
return cls(
find_links=built_find_links,
index_urls=index_urls,
)
def __init__(
self,
find_links, # type: List[str]
index_urls, # type: List[str]
):
# type: (...) -> None
self.find_links = find_links
self.index_urls = index_urls
def get_formatted_locations(self):
# type: () -> str
lines = []
if self.index_urls and self.index_urls != [PyPI.simple_url]:
lines.append(
'Looking in indexes: {}'.format(', '.join(
redact_password_from_url(url) for url in self.index_urls))
)
if self.find_links:
lines.append(
'Looking in links: {}'.format(', '.join(
redact_password_from_url(url) for url in self.find_links))
)
return '\n'.join(lines)
def get_index_urls_locations(self, project_name):
# type: (str) -> List[str]
"""Returns the locations found via self.index_urls
Checks the url_name on the main (first in the list) index and
use this url_name to produce all locations
"""
def mkurl_pypi_url(url):
loc = posixpath.join(
url,
urllib_parse.quote(canonicalize_name(project_name)))
# For maximum compatibility with easy_install, ensure the path
# ends in a trailing slash. Although this isn't in the spec
# (and PyPI can handle it without the slash) some other index
# implementations might break if they relied on easy_install's
# behavior.
if not loc.endswith('/'):
loc = loc + '/'
return loc
return [mkurl_pypi_url(url) for url in self.index_urls]

18
src/pip/_internal/req/req_file.py
View File

@ -16,6 +16,7 @@ from pip._vendor.six.moves.urllib import parse as urllib_parse
from pip._internal.cli import cmdoptions
from pip._internal.download import get_file_content
from pip._internal.exceptions import RequirementsFileParseError
from pip._internal.models.search_scope import SearchScope
from pip._internal.req.constructors import (
install_req_from_editable, install_req_from_line,
)
@ -238,12 +239,14 @@ def process_line(
# set finder options
elif finder:
find_links = finder.find_links
index_urls = finder.index_urls
if opts.index_url:
finder.index_urls = [opts.index_url]
index_urls = [opts.index_url]
if opts.no_index is True:
finder.index_urls = []
index_urls = []
if opts.extra_index_urls:
finder.index_urls.extend(opts.extra_index_urls)
index_urls.extend(opts.extra_index_urls)
if opts.find_links:
# FIXME: it would be nice to keep track of the source
# of the find_links: support a find-links local path
@ -253,7 +256,14 @@ def process_line(
relative_to_reqs_file = os.path.join(req_dir, value)
if os.path.exists(relative_to_reqs_file):
value = relative_to_reqs_file
finder.find_links.append(value)
find_links.append(value)
search_scope = SearchScope(
find_links=find_links,
index_urls=index_urls,
)
finder.search_scope = search_scope
if opts.pre:
finder.set_allow_all_prereleases()
for host in opts.trusted_hosts or []:

9
tests/unit/test_finder.py
View File

@ -463,15 +463,6 @@ class TestCandidateEvaluator(object):
assert actual == (False, expected_msg)
def test_get_index_urls_locations():
"""Check that the canonical name is on all indexes"""
finder = make_test_finder(index_urls=['file://index1/', 'file://index2'])
locations = finder._get_index_urls_locations(
install_req_from_line('Complex_Name').name)
assert locations == ['file://index1/complex-name/',
'file://index2/complex-name/']
def test_find_all_candidates_nothing():
"""Find nothing without anything"""
finder = make_test_finder()

21
tests/unit/test_index.py
View File

@ -385,27 +385,6 @@ def test_secure_origin(location, trusted, expected):
assert logger.called == expected
def test_get_formatted_locations_basic_auth():
"""
Test that basic authentication credentials defined in URL
is not included in formatted output.
"""
index_urls = [
'https://pypi.org/simple',
'https://repo-user:repo-pass@repo.domain.com',
]
find_links = [
'https://links-user:links-pass@page.domain.com'
]
finder = make_test_finder(find_links=find_links, index_urls=index_urls)
result = finder.get_formatted_locations()
assert 'repo-user:****@repo.domain.com' in result
assert 'repo-pass' not in result
assert 'links-user:****@page.domain.com' in result
assert 'links-pass' not in result
@pytest.mark.parametrize(
("egg_info", "canonical_name", "expected"),
[

41
tests/unit/test_search_scope.py Normal file
View File

@ -0,0 +1,41 @@
from pip._internal.models.search_scope import SearchScope
from pip._internal.req.constructors import install_req_from_line
class TestSearchScope:
def test_get_formatted_locations_basic_auth(self):
"""
Test that basic authentication credentials defined in URL
is not included in formatted output.
"""
index_urls = [
'https://pypi.org/simple',
'https://repo-user:repo-pass@repo.domain.com',
]
find_links = [
'https://links-user:links-pass@page.domain.com'
]
search_scope = SearchScope(
find_links=find_links, index_urls=index_urls,
)
result = search_scope.get_formatted_locations()
assert 'repo-user:****@repo.domain.com' in result
assert 'repo-pass' not in result
assert 'links-user:****@page.domain.com' in result
assert 'links-pass' not in result
def test_get_index_urls_locations(self):
"""Check that the canonical name is on all indexes"""
search_scope = SearchScope(
find_links=[],
index_urls=['file://index1/', 'file://index2'],
)
actual = search_scope.get_index_urls_locations(
install_req_from_line('Complex_Name').name
)
assert actual == [
'file://index1/complex-name/',
'file://index2/complex-name/',
]