Add type annotations for pip._internal.req (#6063)

This commit is contained in:
Maxim Kurnikov 2018-12-17 14:13:00 +03:00 committed by Pradyun Gedam
parent 410072bc8b
commit 05eb7d8e92
7 changed files with 314 additions and 102 deletions

View File

@ -6,7 +6,10 @@ from .req_install import InstallRequirement
from .req_set import RequirementSet
from .req_file import parse_requirements
from pip._internal.utils.logging import indent_log
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
if MYPY_CHECK_RUNNING:
from typing import List, Sequence # noqa: F401
__all__ = [
"RequirementSet", "InstallRequirement",
@ -16,8 +19,13 @@ __all__ = [
logger = logging.getLogger(__name__)
def install_given_reqs(to_install, install_options, global_options=(),
*args, **kwargs):
def install_given_reqs(
to_install, # type: List[InstallRequirement]
install_options, # type: List[str]
global_options=(), # type: Sequence[str]
*args, **kwargs
):
# type: (...) -> List[InstallRequirement]
"""
Install everything in the given list.

View File

@ -25,9 +25,17 @@ from pip._internal.models.index import PyPI, TestPyPI
from pip._internal.models.link import Link
from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils.misc import is_installable_dir
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
from pip._internal.vcs import vcs
from pip._internal.wheel import Wheel
if MYPY_CHECK_RUNNING:
from typing import ( # noqa: F401
Optional, Tuple, Set, Any, Mapping, Union, Text
)
from pip._internal.cache import WheelCache # noqa: F401
__all__ = [
"install_req_from_editable", "install_req_from_line",
"parse_editable"
@ -38,6 +46,7 @@ operators = Specifier._operators.keys()
def _strip_extras(path):
# type: (str) -> Tuple[str, Optional[str]]
m = re.match(r'^(.+)(\[[^\]]+\])$', path)
extras = None
if m:
@ -50,6 +59,7 @@ def _strip_extras(path):
def parse_editable(editable_req):
# type: (str) -> Tuple[Optional[str], str, Optional[Set[str]]]
"""Parses an editable requirement into:
- a requirement name
- an URL
@ -115,6 +125,7 @@ def parse_editable(editable_req):
def deduce_helpful_msg(req):
# type: (str) -> str
"""Returns helpful msg in case requirements file does not exist,
or cannot be parsed.
@ -135,7 +146,7 @@ def deduce_helpful_msg(req):
" the packages specified within it."
except RequirementParseError:
logger.debug("Cannot parse '%s' as requirements \
file" % (req), exc_info=1)
file" % (req), exc_info=True)
else:
msg += " File '%s' does not exist." % (req)
return msg
@ -145,9 +156,15 @@ def deduce_helpful_msg(req):
def install_req_from_editable(
editable_req, comes_from=None, use_pep517=None, isolated=False,
options=None, wheel_cache=None, constraint=False
editable_req, # type: str
comes_from=None, # type: Optional[str]
use_pep517=None, # type: Optional[bool]
isolated=False, # type: bool
options=None, # type: Optional[Mapping[Text, Any]]
wheel_cache=None, # type: Optional[WheelCache]
constraint=False # type: bool
):
# type: (...) -> InstallRequirement
name, url, extras_override = parse_editable(editable_req)
if url.startswith('file:'):
source_dir = url_to_path(url)
@ -175,9 +192,15 @@ def install_req_from_editable(
def install_req_from_line(
name, comes_from=None, use_pep517=None, isolated=False, options=None,
wheel_cache=None, constraint=False
name, # type: str
comes_from=None, # type: Optional[Union[str, InstallRequirement]]
use_pep517=None, # type: Optional[bool]
isolated=False, # type: bool
options=None, # type: Optional[Mapping[Text, Any]]
wheel_cache=None, # type: Optional[WheelCache]
constraint=False # type: bool
):
# type: (...) -> InstallRequirement
"""Creates an InstallRequirement from a name, which might be a
requirement, directory containing 'setup.py', filename, or URL.
"""
@ -186,24 +209,24 @@ def install_req_from_line(
else:
marker_sep = ';'
if marker_sep in name:
name, markers = name.split(marker_sep, 1)
markers = markers.strip()
if not markers:
name, markers_as_string = name.split(marker_sep, 1)
markers_as_string = markers_as_string.strip()
if not markers_as_string:
markers = None
else:
markers = Marker(markers)
markers = Marker(markers_as_string)
else:
markers = None
name = name.strip()
req = None
req_as_string = None
path = os.path.normpath(os.path.abspath(name))
link = None
extras = None
extras_as_string = None
if is_url(name):
link = Link(name)
else:
p, extras = _strip_extras(path)
p, extras_as_string = _strip_extras(path)
looks_like_dir = os.path.isdir(p) and (
os.path.sep in name or
(os.path.altsep is not None and os.path.altsep in name) or
@ -234,34 +257,37 @@ def install_req_from_line(
# wheel file
if link.is_wheel:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
req = "%s==%s" % (wheel.name, wheel.version)
req_as_string = "%s==%s" % (wheel.name, wheel.version)
else:
# set the req to the egg fragment. when it's not there, this
# will become an 'unnamed' requirement
req = link.egg_fragment
req_as_string = link.egg_fragment
# a requirement specifier
else:
req = name
req_as_string = name
if extras:
extras = Requirement("placeholder" + extras.lower()).extras
if extras_as_string:
extras = Requirement("placeholder" + extras_as_string.lower()).extras
else:
extras = ()
if req is not None:
if req_as_string is not None:
try:
req = Requirement(req)
req = Requirement(req_as_string)
except InvalidRequirement:
if os.path.sep in req:
if os.path.sep in req_as_string:
add_msg = "It looks like a path."
add_msg += deduce_helpful_msg(req)
elif '=' in req and not any(op in req for op in operators):
add_msg += deduce_helpful_msg(req_as_string)
elif ('=' in req_as_string and
not any(op in req_as_string for op in operators)):
add_msg = "= is not a valid operator. Did you mean == ?"
else:
add_msg = ""
raise InstallationError(
"Invalid requirement: '%s'\n%s" % (req, add_msg)
"Invalid requirement: '%s'\n%s" % (req_as_string, add_msg)
)
else:
req = None
return InstallRequirement(
req, comes_from, link=link, markers=markers,
@ -273,12 +299,16 @@ def install_req_from_line(
)
def install_req_from_req(
req, comes_from=None, isolated=False, wheel_cache=None,
use_pep517=None
def install_req_from_req_string(
req_string, # type: str
comes_from=None, # type: Optional[InstallRequirement]
isolated=False, # type: bool
wheel_cache=None, # type: Optional[WheelCache]
use_pep517=None # type: Optional[bool]
):
# type: (...) -> InstallRequirement
try:
req = Requirement(req)
req = Requirement(req_string)
except InvalidRequirement:
raise InstallationError("Invalid requirement: '%s'" % req)

View File

@ -19,6 +19,18 @@ from pip._internal.exceptions import RequirementsFileParseError
from pip._internal.req.constructors import (
install_req_from_editable, install_req_from_line,
)
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
if MYPY_CHECK_RUNNING:
from typing import ( # noqa: F401
Iterator, Tuple, Optional, List, Callable, Text
)
from pip._internal.req import InstallRequirement # noqa: F401
from pip._internal.cache import WheelCache # noqa: F401
from pip._internal.index import PackageFinder # noqa: F401
from pip._internal.download import PipSession # noqa: F401
ReqFileLines = Iterator[Tuple[int, Text]]
__all__ = ['parse_requirements']
@ -46,22 +58,30 @@ SUPPORTED_OPTIONS = [
cmdoptions.process_dependency_links,
cmdoptions.trusted_host,
cmdoptions.require_hashes,
]
] # type: List[Callable[..., optparse.Option]]
# options to be passed to requirements
SUPPORTED_OPTIONS_REQ = [
cmdoptions.install_options,
cmdoptions.global_options,
cmdoptions.hash,
]
] # type: List[Callable[..., optparse.Option]]
# the 'dest' string values
SUPPORTED_OPTIONS_REQ_DEST = [o().dest for o in SUPPORTED_OPTIONS_REQ]
def parse_requirements(filename, finder=None, comes_from=None, options=None,
session=None, constraint=False, wheel_cache=None,
use_pep517=None):
def parse_requirements(
filename, # type: str
finder=None, # type: Optional[PackageFinder]
comes_from=None, # type: Optional[str]
options=None, # type: Optional[optparse.Values]
session=None, # type: Optional[PipSession]
constraint=False, # type: bool
wheel_cache=None, # type: Optional[WheelCache]
use_pep517=None # type: Optional[bool]
):
# type: (...) -> Iterator[InstallRequirement]
"""Parse a requirements file and yield InstallRequirement instances.
:param filename: Path or url of requirements file.
@ -95,12 +115,13 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None,
def preprocess(content, options):
# type: (Text, Optional[optparse.Values]) -> ReqFileLines
"""Split, filter, and join lines, and return a line iterator
:param content: the content of the requirements file
:param options: cli options
"""
lines_enum = enumerate(content.splitlines(), start=1)
lines_enum = enumerate(content.splitlines(), start=1) # type: ReqFileLines
lines_enum = join_lines(lines_enum)
lines_enum = ignore_comments(lines_enum)
lines_enum = skip_regex(lines_enum, options)
@ -108,9 +129,19 @@ def preprocess(content, options):
return lines_enum
def process_line(line, filename, line_number, finder=None, comes_from=None,
options=None, session=None, wheel_cache=None,
use_pep517=None, constraint=False):
def process_line(
line, # type: Text
filename, # type: str
line_number, # type: int
finder=None, # type: Optional[PackageFinder]
comes_from=None, # type: Optional[str]
options=None, # type: Optional[optparse.Values]
session=None, # type: Optional[PipSession]
wheel_cache=None, # type: Optional[WheelCache]
use_pep517=None, # type: Optional[bool]
constraint=False # type: bool
):
# type: (...) -> Iterator[InstallRequirement]
"""Process a single requirements line; This can result in creating/yielding
requirements, or updating the finder.
@ -130,15 +161,20 @@ def process_line(line, filename, line_number, finder=None, comes_from=None,
"""
parser = build_parser(line)
defaults = parser.get_default_values()
defaults.index_url = None
# fixed in mypy==0.650
defaults.index_url = None # type: ignore
if finder:
# `finder.format_control` will be updated during parsing
defaults.format_control = finder.format_control
# fixed in mypy==0.650
defaults.format_control = finder.format_control # type: ignore
args_str, options_str = break_args_options(line)
if sys.version_info < (2, 7, 3):
# Prior to 2.7.3, shlex cannot deal with unicode entries
options_str = options_str.encode('utf8')
opts, _ = parser.parse_args(shlex.split(options_str), defaults)
# https://github.com/python/mypy/issues/1174
options_str = options_str.encode('utf8') # type: ignore
# https://github.com/python/mypy/issues/1174
opts, _ = parser.parse_args(shlex.split(options_str), # type: ignore
defaults)
# preserve for the nested code path
line_comes_from = '%s %s (line %s)' % (
@ -187,16 +223,17 @@ def process_line(line, filename, line_number, finder=None, comes_from=None,
# do a join so relative paths work
req_path = os.path.join(os.path.dirname(filename), req_path)
# TODO: Why not use `comes_from='-r {} (line {})'` here as well?
parser = parse_requirements(
parsed_reqs = parse_requirements(
req_path, finder, comes_from, options, session,
constraint=nested_constraint, wheel_cache=wheel_cache
)
for req in parser:
for req in parsed_reqs:
yield req
# percolate hash-checking option upward
elif opts.require_hashes:
options.require_hashes = opts.require_hashes
# fixed in mypy==0.650
options.require_hashes = opts.require_hashes # type: ignore
# set finder options
elif finder:
@ -226,6 +263,7 @@ def process_line(line, filename, line_number, finder=None, comes_from=None,
def break_args_options(line):
# type: (Text) -> Tuple[str, Text]
"""Break up the line into an args and options string. We only want to shlex
(and then optparse) the options, not the args. args can contain markers
which are corrupted by shlex.
@ -239,10 +277,11 @@ def break_args_options(line):
else:
args.append(token)
options.pop(0)
return ' '.join(args), ' '.join(options)
return ' '.join(args), ' '.join(options) # type: ignore
def build_parser(line):
# type: (Text) -> optparse.OptionParser
"""
Return a parser for parsing requirement lines
"""
@ -259,20 +298,25 @@ def build_parser(line):
# add offending line
msg = 'Invalid requirement: %s\n%s' % (line, msg)
raise RequirementsFileParseError(msg)
parser.exit = parser_exit
# ignore type, because mypy disallows assigning to a method,
# see https://github.com/python/mypy/issues/2427
parser.exit = parser_exit # type: ignore
return parser
def join_lines(lines_enum):
# type: (ReqFileLines) -> ReqFileLines
"""Joins a line ending in '\' with the previous line (except when following
comments). The joined line takes on the index of the first line.
"""
primary_line_number = None
new_line = []
new_line = [] # type: List[Text]
for line_number, line in lines_enum:
if not line.endswith('\\') or COMMENT_RE.match(line):
if COMMENT_RE.match(line):
# fixed in mypy==0.641
if not line.endswith('\\') or COMMENT_RE.match(line): # type: ignore
# fixed in mypy==0.641
if COMMENT_RE.match(line): # type: ignore
# this ensures comments are always matched later
line = ' ' + line
if new_line:
@ -294,17 +338,20 @@ def join_lines(lines_enum):
def ignore_comments(lines_enum):
# type: (ReqFileLines) -> ReqFileLines
"""
Strips comments and filter empty lines.
"""
for line_number, line in lines_enum:
line = COMMENT_RE.sub('', line)
# fixed in mypy==0.641
line = COMMENT_RE.sub('', line) # type: ignore
line = line.strip()
if line:
yield line_number, line
def skip_regex(lines_enum, options):
# type: (ReqFileLines, Optional[optparse.Values]) -> ReqFileLines
"""
Skip lines that match '--skip-requirements-regex' pattern
@ -318,6 +365,7 @@ def skip_regex(lines_enum, options):
def expand_env_variables(lines_enum):
# type: (ReqFileLines) -> ReqFileLines
"""Replace all environment variables that can be retrieved via `os.getenv`.
The only allowed format for environment variables defined in the
@ -334,7 +382,8 @@ def expand_env_variables(lines_enum):
to uppercase letter, digits and the `_` (underscore).
"""
for line_number, line in lines_enum:
for env_var, var_name in ENV_VAR_RE.findall(line):
# fixed in mypy==0.641
for env_var, var_name in ENV_VAR_RE.findall(line): # type: ignore
value = os.getenv(var_name)
if not value:
continue

View File

@ -35,10 +35,22 @@ from pip._internal.utils.misc import (
from pip._internal.utils.packaging import get_metadata
from pip._internal.utils.setuptools_build import SETUPTOOLS_SHIM
from pip._internal.utils.temp_dir import TempDirectory
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
from pip._internal.utils.ui import open_spinner
from pip._internal.vcs import vcs
from pip._internal.wheel import move_wheel_files
if MYPY_CHECK_RUNNING:
from typing import ( # noqa: F401
Optional, Iterable, List, Union, Any, Mapping, Text, Sequence
)
from pip._vendor.pkg_resources import Distribution # noqa: F401
from pip._internal.index import PackageFinder # noqa: F401
from pip._internal.cache import WheelCache # noqa: F401
from pip._vendor.packaging.specifiers import SpecifierSet # noqa: F401
from pip._vendor.packaging.markers import Marker # noqa: F401
logger = logging.getLogger(__name__)
@ -49,10 +61,23 @@ class InstallRequirement(object):
installing the said requirement.
"""
def __init__(self, req, comes_from, source_dir=None, editable=False,
link=None, update=True, markers=None, use_pep517=None,
isolated=False, options=None, wheel_cache=None,
constraint=False, extras=()):
def __init__(
self,
req, # type: Optional[Requirement]
comes_from, # type: Optional[Union[str, InstallRequirement]]
source_dir=None, # type: Optional[str]
editable=False, # type: bool
link=None, # type: Optional[Link]
update=True, # type: bool
markers=None, # type: Optional[Marker]
use_pep517=None, # type: Optional[bool]
isolated=False, # type: bool
options=None, # type: Optional[Mapping[Text, Any]]
wheel_cache=None, # type: Optional[WheelCache]
constraint=False, # type: bool
extras=() # type: Iterable[str]
):
# type: (...) -> None
assert req is None or isinstance(req, Requirement), req
self.req = req
self.comes_from = comes_from
@ -64,10 +89,10 @@ class InstallRequirement(object):
self.editable = editable
self._wheel_cache = wheel_cache
if link is not None:
self.link = self.original_link = link
else:
self.link = self.original_link = req and req.url and Link(req.url)
if link is None and req and req.url:
# PEP 508 URL requirement
link = Link(req.url)
self.link = self.original_link = link
if extras:
self.extras = extras
@ -77,11 +102,11 @@ class InstallRequirement(object):
}
else:
self.extras = set()
if markers is not None:
self.markers = markers
else:
self.markers = req and req.marker
self._egg_info_path = None
if markers is None and req:
markers = req.marker
self.markers = markers
self._egg_info_path = None # type: Optional[str]
# This holds the pkg_resources.Distribution object if this requirement
# is already available:
self.satisfied_by = None
@ -92,11 +117,11 @@ class InstallRequirement(object):
self._temp_build_dir = TempDirectory(kind="req-build")
# Used to store the global directory where the _temp_build_dir should
# have been created. Cf _correct_build_location method.
self._ideal_build_dir = None
self._ideal_build_dir = None # type: Optional[str]
# True if the editable should be updated:
self.update = update
# Set to True after successful installation
self.install_succeeded = None
self.install_succeeded = None # type: Optional[bool]
# UninstallPathSet of uninstalled distribution (for possible rollback)
self.uninstalled_pathset = None
self.options = options if options else {}
@ -111,16 +136,16 @@ class InstallRequirement(object):
# gets stored. We need this to pass to build_wheel, so the backend
# can ensure that the wheel matches the metadata (see the PEP for
# details).
self.metadata_directory = None
self.metadata_directory = None # type: Optional[str]
# The static build requirements (from pyproject.toml)
self.pyproject_requires = None
self.pyproject_requires = None # type: Optional[List[str]]
# Build requirements that we will check are available
self.requirements_to_check = []
self.requirements_to_check = [] # type: List[str]
# The PEP 517 backend we should use to build the project
self.pep517_backend = None
self.pep517_backend = None # type: Optional[Pep517HookCaller]
# Are we using PEP 517 for this requirement?
# After pyproject.toml has been loaded, the only valid values are True
@ -154,6 +179,7 @@ class InstallRequirement(object):
self.__class__.__name__, str(self), self.editable)
def populate_link(self, finder, upgrade, require_hashes):
# type: (PackageFinder, bool, bool) -> None
"""Ensure that if a link can be found for this, that it is found.
Note that self.link may still be None - if Upgrade is False and the
@ -176,16 +202,19 @@ class InstallRequirement(object):
# Things that are valid for all kinds of requirements?
@property
def name(self):
# type: () -> Optional[str]
if self.req is None:
return None
return native_str(pkg_resources.safe_name(self.req.name))
@property
def specifier(self):
# type: () -> SpecifierSet
return self.req.specifier
@property
def is_pinned(self):
# type: () -> bool
"""Return whether I am pinned to an exact version.
For example, some-package==1.2 is pinned; some-package>1.2 is not.
@ -199,6 +228,7 @@ class InstallRequirement(object):
return get_installed_version(self.name)
def match_markers(self, extras_requested=None):
# type: (Optional[Iterable[str]]) -> bool
if not extras_requested:
# Provide an extra to safely evaluate the markers
# without matching any extra
@ -212,6 +242,7 @@ class InstallRequirement(object):
@property
def has_hash_options(self):
# type: () -> bool
"""Return whether any known-good hashes are specified as options.
These activate --require-hashes mode; hashes specified as part of a
@ -221,6 +252,7 @@ class InstallRequirement(object):
return bool(self.options.get('hashes', {}))
def hashes(self, trust_internet=True):
# type: (bool) -> Hashes
"""Return a hash-comparer that considers my option- and URL-based
hashes to be known-good.
@ -242,6 +274,7 @@ class InstallRequirement(object):
return Hashes(good_hashes)
def from_path(self):
# type: () -> Optional[str]
"""Format a nice indicator to show where this "comes from"
"""
if self.req is None:
@ -257,6 +290,7 @@ class InstallRequirement(object):
return s
def build_location(self, build_dir):
# type: (str) -> Optional[str]
assert build_dir is not None
if self._temp_build_dir.path is not None:
return self._temp_build_dir.path
@ -284,6 +318,7 @@ class InstallRequirement(object):
return os.path.join(build_dir, name)
def _correct_build_location(self):
# type: () -> None
"""Move self._temp_build_dir to self._ideal_build_dir/self.req.name
For some requirements (e.g. a path to a directory), the name of the
@ -297,7 +332,8 @@ class InstallRequirement(object):
return
assert self.req is not None
assert self._temp_build_dir.path
assert self._ideal_build_dir.path
assert (self._ideal_build_dir is not None and
self._ideal_build_dir.path) # type: ignore
old_location = self._temp_build_dir.path
self._temp_build_dir.path = None
@ -325,6 +361,7 @@ class InstallRequirement(object):
self.metadata_directory = new_meta
def remove_temporary_source(self):
# type: () -> None
"""Remove the source files from this requirement, if they are marked
for deletion"""
if self.source_dir and os.path.exists(
@ -336,6 +373,7 @@ class InstallRequirement(object):
self.build_env.cleanup()
def check_if_exists(self, use_user_site):
# type: (bool) -> bool
"""Find an installed distribution that satisfies or conflicts
with this requirement, and set self.satisfied_by or
self.conflicts_with appropriately.
@ -379,11 +417,22 @@ class InstallRequirement(object):
# Things valid for wheels
@property
def is_wheel(self):
return self.link and self.link.is_wheel
# type: () -> bool
if not self.link:
return False
return self.link.is_wheel
def move_wheel_files(self, wheeldir, root=None, home=None, prefix=None,
warn_script_location=True, use_user_site=False,
pycompile=True):
def move_wheel_files(
self,
wheeldir, # type: str
root=None, # type: Optional[str]
home=None, # type: Optional[str]
prefix=None, # type: Optional[str]
warn_script_location=True, # type: bool
use_user_site=False, # type: bool
pycompile=True # type: bool
):
# type: (...) -> None
move_wheel_files(
self.name, self.req, wheeldir,
user=use_user_site,
@ -398,12 +447,14 @@ class InstallRequirement(object):
# Things valid for sdists
@property
def setup_py_dir(self):
# type: () -> str
return os.path.join(
self.source_dir,
self.link and self.link.subdirectory_fragment or '')
@property
def setup_py(self):
# type: () -> str
assert self.source_dir, "No source dir for %s" % self
setup_py = os.path.join(self.setup_py_dir, 'setup.py')
@ -416,6 +467,7 @@ class InstallRequirement(object):
@property
def pyproject_toml(self):
# type: () -> str
assert self.source_dir, "No source dir for %s" % self
pp_toml = os.path.join(self.setup_py_dir, 'pyproject.toml')
@ -427,6 +479,7 @@ class InstallRequirement(object):
return pp_toml
def load_pyproject_toml(self):
# type: () -> None
"""Load the pyproject.toml file.
After calling this routine, all of the attributes related to PEP 517
@ -467,6 +520,7 @@ class InstallRequirement(object):
self.pep517_backend._subprocess_runner = runner
def prepare_metadata(self):
# type: () -> None
"""Ensure that project metadata is available.
Under PEP 517, call the backend hook to prepare the metadata.
@ -505,6 +559,7 @@ class InstallRequirement(object):
self.req = Requirement(metadata_name)
def prepare_pep517_metadata(self):
# type: () -> None
assert self.pep517_backend is not None
metadata_dir = os.path.join(
@ -526,6 +581,7 @@ class InstallRequirement(object):
self.metadata_directory = os.path.join(metadata_dir, distinfo_dir)
def run_egg_info(self):
# type: () -> None
if self.name:
logger.debug(
'Running setup.py (path:%s) egg_info for package %s',
@ -545,7 +601,7 @@ class InstallRequirement(object):
# source code will be mistaken for an installed egg, causing
# problems
if self.editable:
egg_base_option = []
egg_base_option = [] # type: List[str]
else:
egg_info_dir = os.path.join(self.setup_py_dir, 'pip-egg-info')
ensure_dir(egg_info_dir)
@ -559,6 +615,7 @@ class InstallRequirement(object):
@property
def egg_info_path(self):
# type: () -> str
if self._egg_info_path is None:
if self.editable:
base = self.source_dir
@ -617,6 +674,7 @@ class InstallRequirement(object):
return self._metadata
def get_dist(self):
# type: () -> Distribution
"""Return a pkg_resources.Distribution for this requirement"""
if self.metadata_directory:
base_dir, distinfo = os.path.split(self.metadata_directory)
@ -630,7 +688,8 @@ class InstallRequirement(object):
base_dir = os.path.dirname(egg_info)
metadata = pkg_resources.PathMetadata(base_dir, egg_info)
dist_name = os.path.splitext(os.path.basename(egg_info))[0]
typ = pkg_resources.Distribution
# https://github.com/python/mypy/issues/1174
typ = pkg_resources.Distribution # type: ignore
return typ(
base_dir,
@ -639,6 +698,7 @@ class InstallRequirement(object):
)
def assert_source_matches_version(self):
# type: () -> None
assert self.source_dir
version = self.metadata['version']
if self.req.specifier and version not in self.req.specifier:
@ -657,6 +717,7 @@ class InstallRequirement(object):
# For both source distributions and editables
def ensure_has_source_dir(self, parent_dir):
# type: (str) -> str
"""Ensure that a source_dir is set.
This will create a temporary build dir if the name of the requirement
@ -671,8 +732,13 @@ class InstallRequirement(object):
return self.source_dir
# For editable installations
def install_editable(self, install_options,
global_options=(), prefix=None):
def install_editable(
self,
install_options, # type: List[str]
global_options=(), # type: Sequence[str]
prefix=None # type: Optional[str]
):
# type: (...) -> None
logger.info('Running setup.py develop for %s', self.name)
if self.isolated:
@ -702,6 +768,7 @@ class InstallRequirement(object):
self.install_succeeded = True
def update_editable(self, obtain=True):
# type: (bool) -> None
if not self.link:
logger.debug(
"Cannot update repository at %s; repository location is "
@ -733,6 +800,7 @@ class InstallRequirement(object):
# Top-level Actions
def uninstall(self, auto_confirm=False, verbose=False,
use_user_site=False):
# type: (bool, bool, bool) -> Optional[UninstallPathSet]
"""
Uninstall the distribution currently satisfying this requirement.
@ -747,7 +815,7 @@ class InstallRequirement(object):
"""
if not self.check_if_exists(use_user_site):
logger.warning("Skipping %s as it is not installed.", self.name)
return
return None
dist = self.satisfied_by or self.conflicts_with
uninstalled_pathset = UninstallPathSet.from_dist(dist)
@ -762,9 +830,16 @@ class InstallRequirement(object):
name = name.replace(os.path.sep, '/')
return name
def _get_archive_name(self, path, parentdir, rootdir):
# type: (str, str, str) -> str
path = os.path.join(parentdir, path)
name = self._clean_zip_name(path, rootdir)
return self.name + '/' + name
# TODO: Investigate if this should be kept in InstallRequirement
# Seems to be used only when VCS + downloads
def archive(self, build_dir):
# type: (str) -> None
assert self.source_dir
create_archive = True
archive_name = '%s-%s.zip' % (self.name, self.metadata["version"])
@ -798,23 +873,37 @@ class InstallRequirement(object):
if 'pip-egg-info' in dirnames:
dirnames.remove('pip-egg-info')
for dirname in dirnames:
dirname = os.path.join(dirpath, dirname)
name = self._clean_zip_name(dirname, dir)
zipdir = zipfile.ZipInfo(self.name + '/' + name + '/')
dir_arcname = self._get_archive_name(dirname,
parentdir=dirpath,
rootdir=dir)
# should be fixed in mypy==0.650
# see https://github.com/python/typeshed/pull/2628
zipdir = zipfile.ZipInfo(dir_arcname + '/') # type: ignore
zipdir.external_attr = 0x1ED << 16 # 0o755
zip.writestr(zipdir, '')
for filename in filenames:
if filename == PIP_DELETE_MARKER_FILENAME:
continue
file_arcname = self._get_archive_name(filename,
parentdir=dirpath,
rootdir=dir)
filename = os.path.join(dirpath, filename)
name = self._clean_zip_name(filename, dir)
zip.write(filename, self.name + '/' + name)
zip.write(filename, file_arcname)
zip.close()
logger.info('Saved %s', display_path(archive_path))
def install(self, install_options, global_options=None, root=None,
home=None, prefix=None, warn_script_location=True,
use_user_site=False, pycompile=True):
def install(
self,
install_options, # type: List[str]
global_options=None, # type: Optional[Sequence[str]]
root=None, # type: Optional[str]
home=None, # type: Optional[str]
prefix=None, # type: Optional[str]
warn_script_location=True, # type: bool
use_user_site=False, # type: bool
pycompile=True # type: bool
):
# type: (...) -> None
global_options = global_options if global_options is not None else []
if self.editable:
self.install_editable(
@ -844,7 +933,8 @@ class InstallRequirement(object):
self.options.get('install_options', [])
if self.isolated:
global_options = global_options + ["--no-user-cfg"]
# https://github.com/python/mypy/issues/1174
global_options = global_options + ["--no-user-cfg"] # type: ignore
with TempDirectory(kind="record") as temp_dir:
record_filename = os.path.join(temp_dir.path, 'install-record.txt')
@ -903,8 +993,15 @@ class InstallRequirement(object):
with open(inst_files_path, 'w') as f:
f.write('\n'.join(new_lines) + '\n')
def get_install_args(self, global_options, record_filename, root, prefix,
pycompile):
def get_install_args(
self,
global_options, # type: Sequence[str]
record_filename, # type: str
root, # type: Optional[str]
prefix, # type: Optional[str]
pycompile # type: bool
):
# type: (...) -> List[str]
install_args = [sys.executable, "-u"]
install_args.append('-c')
install_args.append(SETUPTOOLS_SHIM % self.setup_py)

View File

@ -5,26 +5,33 @@ from collections import OrderedDict
from pip._internal.exceptions import InstallationError
from pip._internal.utils.logging import indent_log
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
from pip._internal.wheel import Wheel
if MYPY_CHECK_RUNNING:
from typing import Optional, List, Tuple, Dict, Iterable # noqa: F401
from pip._internal.req.req_install import InstallRequirement # noqa: F401
logger = logging.getLogger(__name__)
class RequirementSet(object):
def __init__(self, require_hashes=False, check_supported_wheels=True):
# type: (bool, bool) -> None
"""Create a RequirementSet.
"""
self.requirements = OrderedDict()
self.requirements = OrderedDict() # type: Dict[str, InstallRequirement] # noqa: E501
self.require_hashes = require_hashes
self.check_supported_wheels = check_supported_wheels
# Mapping of alias: real_name
self.requirement_aliases = {}
self.unnamed_requirements = []
self.successfully_downloaded = []
self.reqs_to_cleanup = []
self.requirement_aliases = {} # type: Dict[str, str]
self.unnamed_requirements = [] # type: List[InstallRequirement]
self.successfully_downloaded = [] # type: List[InstallRequirement]
self.reqs_to_cleanup = [] # type: List[InstallRequirement]
def __str__(self):
reqs = [req for req in self.requirements.values()
@ -39,8 +46,13 @@ class RequirementSet(object):
return ('<%s object; %d requirement(s): %s>'
% (self.__class__.__name__, len(reqs), reqs_str))
def add_requirement(self, install_req, parent_req_name=None,
extras_requested=None):
def add_requirement(
self,
install_req, # type: InstallRequirement
parent_req_name=None, # type: Optional[str]
extras_requested=None # type: Optional[Iterable[str]]
):
# type: (...) -> Tuple[List[InstallRequirement], Optional[InstallRequirement]] # noqa: E501
"""Add install_req as a requirement to install.
:param parent_req_name: The name of the requirement that needed this
@ -152,6 +164,7 @@ class RequirementSet(object):
return [existing_req], existing_req
def has_requirement(self, project_name):
# type: (str) -> bool
name = project_name.lower()
if (name in self.requirements and
not self.requirements[name].constraint or
@ -162,10 +175,12 @@ class RequirementSet(object):
@property
def has_requirements(self):
# type: () -> List[InstallRequirement]
return list(req for req in self.requirements.values() if not
req.constraint) or self.unnamed_requirements
def get_requirement(self, project_name):
# type: (str) -> InstallRequirement
for name in project_name, project_name.lower():
if name in self.requirements:
return self.requirements[name]
@ -174,6 +189,7 @@ class RequirementSet(object):
raise KeyError("No project with the name %r" % project_name)
def cleanup_files(self):
# type: () -> None
"""Clean up files, remove builds."""
logger.debug('Cleaning up...')
with indent_log():

View File

@ -7,6 +7,12 @@ import logging
import os
from pip._internal.utils.temp_dir import TempDirectory
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
if MYPY_CHECK_RUNNING:
from typing import Set, Iterator # noqa: F401
from pip._internal.req.req_install import InstallRequirement # noqa: F401
from pip._internal.models.link import Link # noqa: F401
logger = logging.getLogger(__name__)
@ -14,6 +20,7 @@ logger = logging.getLogger(__name__)
class RequirementTracker(object):
def __init__(self):
# type: () -> None
self._root = os.environ.get('PIP_REQ_TRACKER')
if self._root is None:
self._temp_dir = TempDirectory(delete=False, kind='req-tracker')
@ -23,7 +30,7 @@ class RequirementTracker(object):
else:
self._temp_dir = None
logger.debug('Re-using requirements tracker %r', self._root)
self._entries = set()
self._entries = set() # type: Set[InstallRequirement]
def __enter__(self):
return self
@ -32,10 +39,12 @@ class RequirementTracker(object):
self.cleanup()
def _entry_path(self, link):
# type: (Link) -> str
hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest()
return os.path.join(self._root, hashed)
def add(self, req):
# type: (InstallRequirement) -> None
link = req.link
info = str(req)
entry_path = self._entry_path(link)
@ -54,12 +63,14 @@ class RequirementTracker(object):
logger.debug('Added %s to build tracker %r', req, self._root)
def remove(self, req):
# type: (InstallRequirement) -> None
link = req.link
self._entries.remove(req)
os.unlink(self._entry_path(link))
logger.debug('Removed %s from build tracker %r', req, self._root)
def cleanup(self):
# type: () -> None
for req in set(self._entries):
self.remove(req)
remove = self._temp_dir is not None
@ -71,6 +82,7 @@ class RequirementTracker(object):
@contextlib.contextmanager
def track(self, req):
# type: (InstallRequirement) -> Iterator[None]
self.add(req)
yield
self.remove(req)

View File

@ -18,7 +18,7 @@ from pip._internal.exceptions import (
BestVersionAlreadyInstalled, DistributionNotFound, HashError, HashErrors,
UnsupportedPythonVersion,
)
from pip._internal.req.constructors import install_req_from_req
from pip._internal.req.constructors import install_req_from_req_string
from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import dist_in_usersite, ensure_dir
from pip._internal.utils.packaging import check_dist_requires_python
@ -269,7 +269,7 @@ class Resolver(object):
more_reqs = []
def add_req(subreq, extras_requested):
sub_install_req = install_req_from_req(
sub_install_req = install_req_from_req_string(
str(subreq),
req_to_install,
isolated=self.isolated,