From 05eb7d8e92605d93205dbc4958cbed81e6f11542 Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Mon, 17 Dec 2018 14:13:00 +0300 Subject: [PATCH] Add type annotations for pip._internal.req (#6063) --- src/pip/_internal/req/__init__.py | 12 +- src/pip/_internal/req/constructors.py | 84 ++++++++---- src/pip/_internal/req/req_file.py | 95 ++++++++++---- src/pip/_internal/req/req_install.py | 177 ++++++++++++++++++++------ src/pip/_internal/req/req_set.py | 30 ++++- src/pip/_internal/req/req_tracker.py | 14 +- src/pip/_internal/resolve.py | 4 +- 7 files changed, 314 insertions(+), 102 deletions(-) diff --git a/src/pip/_internal/req/__init__.py b/src/pip/_internal/req/__init__.py index b270498e2..5e4eb92f0 100644 --- a/src/pip/_internal/req/__init__.py +++ b/src/pip/_internal/req/__init__.py @@ -6,7 +6,10 @@ from .req_install import InstallRequirement from .req_set import RequirementSet from .req_file import parse_requirements from pip._internal.utils.logging import indent_log +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +if MYPY_CHECK_RUNNING: + from typing import List, Sequence # noqa: F401 __all__ = [ "RequirementSet", "InstallRequirement", @@ -16,8 +19,13 @@ __all__ = [ logger = logging.getLogger(__name__) -def install_given_reqs(to_install, install_options, global_options=(), - *args, **kwargs): +def install_given_reqs( + to_install, # type: List[InstallRequirement] + install_options, # type: List[str] + global_options=(), # type: Sequence[str] + *args, **kwargs +): + # type: (...) -> List[InstallRequirement] """ Install everything in the given list. diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index 640efd453..c26c1ed28 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -25,9 +25,17 @@ from pip._internal.models.index import PyPI, TestPyPI from pip._internal.models.link import Link from pip._internal.req.req_install import InstallRequirement from pip._internal.utils.misc import is_installable_dir +from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.vcs import vcs from pip._internal.wheel import Wheel +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Optional, Tuple, Set, Any, Mapping, Union, Text + ) + from pip._internal.cache import WheelCache # noqa: F401 + + __all__ = [ "install_req_from_editable", "install_req_from_line", "parse_editable" @@ -38,6 +46,7 @@ operators = Specifier._operators.keys() def _strip_extras(path): + # type: (str) -> Tuple[str, Optional[str]] m = re.match(r'^(.+)(\[[^\]]+\])$', path) extras = None if m: @@ -50,6 +59,7 @@ def _strip_extras(path): def parse_editable(editable_req): + # type: (str) -> Tuple[Optional[str], str, Optional[Set[str]]] """Parses an editable requirement into: - a requirement name - an URL @@ -115,6 +125,7 @@ def parse_editable(editable_req): def deduce_helpful_msg(req): + # type: (str) -> str """Returns helpful msg in case requirements file does not exist, or cannot be parsed. @@ -135,7 +146,7 @@ def deduce_helpful_msg(req): " the packages specified within it." except RequirementParseError: logger.debug("Cannot parse '%s' as requirements \ - file" % (req), exc_info=1) + file" % (req), exc_info=True) else: msg += " File '%s' does not exist." % (req) return msg @@ -145,9 +156,15 @@ def deduce_helpful_msg(req): def install_req_from_editable( - editable_req, comes_from=None, use_pep517=None, isolated=False, - options=None, wheel_cache=None, constraint=False + editable_req, # type: str + comes_from=None, # type: Optional[str] + use_pep517=None, # type: Optional[bool] + isolated=False, # type: bool + options=None, # type: Optional[Mapping[Text, Any]] + wheel_cache=None, # type: Optional[WheelCache] + constraint=False # type: bool ): + # type: (...) -> InstallRequirement name, url, extras_override = parse_editable(editable_req) if url.startswith('file:'): source_dir = url_to_path(url) @@ -175,9 +192,15 @@ def install_req_from_editable( def install_req_from_line( - name, comes_from=None, use_pep517=None, isolated=False, options=None, - wheel_cache=None, constraint=False + name, # type: str + comes_from=None, # type: Optional[Union[str, InstallRequirement]] + use_pep517=None, # type: Optional[bool] + isolated=False, # type: bool + options=None, # type: Optional[Mapping[Text, Any]] + wheel_cache=None, # type: Optional[WheelCache] + constraint=False # type: bool ): + # type: (...) -> InstallRequirement """Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. """ @@ -186,24 +209,24 @@ def install_req_from_line( else: marker_sep = ';' if marker_sep in name: - name, markers = name.split(marker_sep, 1) - markers = markers.strip() - if not markers: + name, markers_as_string = name.split(marker_sep, 1) + markers_as_string = markers_as_string.strip() + if not markers_as_string: markers = None else: - markers = Marker(markers) + markers = Marker(markers_as_string) else: markers = None name = name.strip() - req = None + req_as_string = None path = os.path.normpath(os.path.abspath(name)) link = None - extras = None + extras_as_string = None if is_url(name): link = Link(name) else: - p, extras = _strip_extras(path) + p, extras_as_string = _strip_extras(path) looks_like_dir = os.path.isdir(p) and ( os.path.sep in name or (os.path.altsep is not None and os.path.altsep in name) or @@ -234,34 +257,37 @@ def install_req_from_line( # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename - req = "%s==%s" % (wheel.name, wheel.version) + req_as_string = "%s==%s" % (wheel.name, wheel.version) else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement - req = link.egg_fragment + req_as_string = link.egg_fragment # a requirement specifier else: - req = name + req_as_string = name - if extras: - extras = Requirement("placeholder" + extras.lower()).extras + if extras_as_string: + extras = Requirement("placeholder" + extras_as_string.lower()).extras else: extras = () - if req is not None: + if req_as_string is not None: try: - req = Requirement(req) + req = Requirement(req_as_string) except InvalidRequirement: - if os.path.sep in req: + if os.path.sep in req_as_string: add_msg = "It looks like a path." - add_msg += deduce_helpful_msg(req) - elif '=' in req and not any(op in req for op in operators): + add_msg += deduce_helpful_msg(req_as_string) + elif ('=' in req_as_string and + not any(op in req_as_string for op in operators)): add_msg = "= is not a valid operator. Did you mean == ?" else: add_msg = "" raise InstallationError( - "Invalid requirement: '%s'\n%s" % (req, add_msg) + "Invalid requirement: '%s'\n%s" % (req_as_string, add_msg) ) + else: + req = None return InstallRequirement( req, comes_from, link=link, markers=markers, @@ -273,12 +299,16 @@ def install_req_from_line( ) -def install_req_from_req( - req, comes_from=None, isolated=False, wheel_cache=None, - use_pep517=None +def install_req_from_req_string( + req_string, # type: str + comes_from=None, # type: Optional[InstallRequirement] + isolated=False, # type: bool + wheel_cache=None, # type: Optional[WheelCache] + use_pep517=None # type: Optional[bool] ): + # type: (...) -> InstallRequirement try: - req = Requirement(req) + req = Requirement(req_string) except InvalidRequirement: raise InstallationError("Invalid requirement: '%s'" % req) diff --git a/src/pip/_internal/req/req_file.py b/src/pip/_internal/req/req_file.py index b332f6853..50976e1c5 100644 --- a/src/pip/_internal/req/req_file.py +++ b/src/pip/_internal/req/req_file.py @@ -19,6 +19,18 @@ from pip._internal.exceptions import RequirementsFileParseError from pip._internal.req.constructors import ( install_req_from_editable, install_req_from_line, ) +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Iterator, Tuple, Optional, List, Callable, Text + ) + from pip._internal.req import InstallRequirement # noqa: F401 + from pip._internal.cache import WheelCache # noqa: F401 + from pip._internal.index import PackageFinder # noqa: F401 + from pip._internal.download import PipSession # noqa: F401 + + ReqFileLines = Iterator[Tuple[int, Text]] __all__ = ['parse_requirements'] @@ -46,22 +58,30 @@ SUPPORTED_OPTIONS = [ cmdoptions.process_dependency_links, cmdoptions.trusted_host, cmdoptions.require_hashes, -] +] # type: List[Callable[..., optparse.Option]] # options to be passed to requirements SUPPORTED_OPTIONS_REQ = [ cmdoptions.install_options, cmdoptions.global_options, cmdoptions.hash, -] +] # type: List[Callable[..., optparse.Option]] # the 'dest' string values SUPPORTED_OPTIONS_REQ_DEST = [o().dest for o in SUPPORTED_OPTIONS_REQ] -def parse_requirements(filename, finder=None, comes_from=None, options=None, - session=None, constraint=False, wheel_cache=None, - use_pep517=None): +def parse_requirements( + filename, # type: str + finder=None, # type: Optional[PackageFinder] + comes_from=None, # type: Optional[str] + options=None, # type: Optional[optparse.Values] + session=None, # type: Optional[PipSession] + constraint=False, # type: bool + wheel_cache=None, # type: Optional[WheelCache] + use_pep517=None # type: Optional[bool] +): + # type: (...) -> Iterator[InstallRequirement] """Parse a requirements file and yield InstallRequirement instances. :param filename: Path or url of requirements file. @@ -95,12 +115,13 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None, def preprocess(content, options): + # type: (Text, Optional[optparse.Values]) -> ReqFileLines """Split, filter, and join lines, and return a line iterator :param content: the content of the requirements file :param options: cli options """ - lines_enum = enumerate(content.splitlines(), start=1) + lines_enum = enumerate(content.splitlines(), start=1) # type: ReqFileLines lines_enum = join_lines(lines_enum) lines_enum = ignore_comments(lines_enum) lines_enum = skip_regex(lines_enum, options) @@ -108,9 +129,19 @@ def preprocess(content, options): return lines_enum -def process_line(line, filename, line_number, finder=None, comes_from=None, - options=None, session=None, wheel_cache=None, - use_pep517=None, constraint=False): +def process_line( + line, # type: Text + filename, # type: str + line_number, # type: int + finder=None, # type: Optional[PackageFinder] + comes_from=None, # type: Optional[str] + options=None, # type: Optional[optparse.Values] + session=None, # type: Optional[PipSession] + wheel_cache=None, # type: Optional[WheelCache] + use_pep517=None, # type: Optional[bool] + constraint=False # type: bool +): + # type: (...) -> Iterator[InstallRequirement] """Process a single requirements line; This can result in creating/yielding requirements, or updating the finder. @@ -130,15 +161,20 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, """ parser = build_parser(line) defaults = parser.get_default_values() - defaults.index_url = None + # fixed in mypy==0.650 + defaults.index_url = None # type: ignore if finder: # `finder.format_control` will be updated during parsing - defaults.format_control = finder.format_control + # fixed in mypy==0.650 + defaults.format_control = finder.format_control # type: ignore args_str, options_str = break_args_options(line) if sys.version_info < (2, 7, 3): # Prior to 2.7.3, shlex cannot deal with unicode entries - options_str = options_str.encode('utf8') - opts, _ = parser.parse_args(shlex.split(options_str), defaults) + # https://github.com/python/mypy/issues/1174 + options_str = options_str.encode('utf8') # type: ignore + # https://github.com/python/mypy/issues/1174 + opts, _ = parser.parse_args(shlex.split(options_str), # type: ignore + defaults) # preserve for the nested code path line_comes_from = '%s %s (line %s)' % ( @@ -187,16 +223,17 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, # do a join so relative paths work req_path = os.path.join(os.path.dirname(filename), req_path) # TODO: Why not use `comes_from='-r {} (line {})'` here as well? - parser = parse_requirements( + parsed_reqs = parse_requirements( req_path, finder, comes_from, options, session, constraint=nested_constraint, wheel_cache=wheel_cache ) - for req in parser: + for req in parsed_reqs: yield req # percolate hash-checking option upward elif opts.require_hashes: - options.require_hashes = opts.require_hashes + # fixed in mypy==0.650 + options.require_hashes = opts.require_hashes # type: ignore # set finder options elif finder: @@ -226,6 +263,7 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, def break_args_options(line): + # type: (Text) -> Tuple[str, Text] """Break up the line into an args and options string. We only want to shlex (and then optparse) the options, not the args. args can contain markers which are corrupted by shlex. @@ -239,10 +277,11 @@ def break_args_options(line): else: args.append(token) options.pop(0) - return ' '.join(args), ' '.join(options) + return ' '.join(args), ' '.join(options) # type: ignore def build_parser(line): + # type: (Text) -> optparse.OptionParser """ Return a parser for parsing requirement lines """ @@ -259,20 +298,25 @@ def build_parser(line): # add offending line msg = 'Invalid requirement: %s\n%s' % (line, msg) raise RequirementsFileParseError(msg) - parser.exit = parser_exit + # ignore type, because mypy disallows assigning to a method, + # see https://github.com/python/mypy/issues/2427 + parser.exit = parser_exit # type: ignore return parser def join_lines(lines_enum): + # type: (ReqFileLines) -> ReqFileLines """Joins a line ending in '\' with the previous line (except when following comments). The joined line takes on the index of the first line. """ primary_line_number = None - new_line = [] + new_line = [] # type: List[Text] for line_number, line in lines_enum: - if not line.endswith('\\') or COMMENT_RE.match(line): - if COMMENT_RE.match(line): + # fixed in mypy==0.641 + if not line.endswith('\\') or COMMENT_RE.match(line): # type: ignore + # fixed in mypy==0.641 + if COMMENT_RE.match(line): # type: ignore # this ensures comments are always matched later line = ' ' + line if new_line: @@ -294,17 +338,20 @@ def join_lines(lines_enum): def ignore_comments(lines_enum): + # type: (ReqFileLines) -> ReqFileLines """ Strips comments and filter empty lines. """ for line_number, line in lines_enum: - line = COMMENT_RE.sub('', line) + # fixed in mypy==0.641 + line = COMMENT_RE.sub('', line) # type: ignore line = line.strip() if line: yield line_number, line def skip_regex(lines_enum, options): + # type: (ReqFileLines, Optional[optparse.Values]) -> ReqFileLines """ Skip lines that match '--skip-requirements-regex' pattern @@ -318,6 +365,7 @@ def skip_regex(lines_enum, options): def expand_env_variables(lines_enum): + # type: (ReqFileLines) -> ReqFileLines """Replace all environment variables that can be retrieved via `os.getenv`. The only allowed format for environment variables defined in the @@ -334,7 +382,8 @@ def expand_env_variables(lines_enum): to uppercase letter, digits and the `_` (underscore). """ for line_number, line in lines_enum: - for env_var, var_name in ENV_VAR_RE.findall(line): + # fixed in mypy==0.641 + for env_var, var_name in ENV_VAR_RE.findall(line): # type: ignore value = os.getenv(var_name) if not value: continue diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 075f86e9b..3be06f91d 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -35,10 +35,22 @@ from pip._internal.utils.misc import ( from pip._internal.utils.packaging import get_metadata from pip._internal.utils.setuptools_build import SETUPTOOLS_SHIM from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.ui import open_spinner from pip._internal.vcs import vcs from pip._internal.wheel import move_wheel_files +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Optional, Iterable, List, Union, Any, Mapping, Text, Sequence + ) + from pip._vendor.pkg_resources import Distribution # noqa: F401 + from pip._internal.index import PackageFinder # noqa: F401 + from pip._internal.cache import WheelCache # noqa: F401 + from pip._vendor.packaging.specifiers import SpecifierSet # noqa: F401 + from pip._vendor.packaging.markers import Marker # noqa: F401 + + logger = logging.getLogger(__name__) @@ -49,10 +61,23 @@ class InstallRequirement(object): installing the said requirement. """ - def __init__(self, req, comes_from, source_dir=None, editable=False, - link=None, update=True, markers=None, use_pep517=None, - isolated=False, options=None, wheel_cache=None, - constraint=False, extras=()): + def __init__( + self, + req, # type: Optional[Requirement] + comes_from, # type: Optional[Union[str, InstallRequirement]] + source_dir=None, # type: Optional[str] + editable=False, # type: bool + link=None, # type: Optional[Link] + update=True, # type: bool + markers=None, # type: Optional[Marker] + use_pep517=None, # type: Optional[bool] + isolated=False, # type: bool + options=None, # type: Optional[Mapping[Text, Any]] + wheel_cache=None, # type: Optional[WheelCache] + constraint=False, # type: bool + extras=() # type: Iterable[str] + ): + # type: (...) -> None assert req is None or isinstance(req, Requirement), req self.req = req self.comes_from = comes_from @@ -64,10 +89,10 @@ class InstallRequirement(object): self.editable = editable self._wheel_cache = wheel_cache - if link is not None: - self.link = self.original_link = link - else: - self.link = self.original_link = req and req.url and Link(req.url) + if link is None and req and req.url: + # PEP 508 URL requirement + link = Link(req.url) + self.link = self.original_link = link if extras: self.extras = extras @@ -77,11 +102,11 @@ class InstallRequirement(object): } else: self.extras = set() - if markers is not None: - self.markers = markers - else: - self.markers = req and req.marker - self._egg_info_path = None + if markers is None and req: + markers = req.marker + self.markers = markers + + self._egg_info_path = None # type: Optional[str] # This holds the pkg_resources.Distribution object if this requirement # is already available: self.satisfied_by = None @@ -92,11 +117,11 @@ class InstallRequirement(object): self._temp_build_dir = TempDirectory(kind="req-build") # Used to store the global directory where the _temp_build_dir should # have been created. Cf _correct_build_location method. - self._ideal_build_dir = None + self._ideal_build_dir = None # type: Optional[str] # True if the editable should be updated: self.update = update # Set to True after successful installation - self.install_succeeded = None + self.install_succeeded = None # type: Optional[bool] # UninstallPathSet of uninstalled distribution (for possible rollback) self.uninstalled_pathset = None self.options = options if options else {} @@ -111,16 +136,16 @@ class InstallRequirement(object): # gets stored. We need this to pass to build_wheel, so the backend # can ensure that the wheel matches the metadata (see the PEP for # details). - self.metadata_directory = None + self.metadata_directory = None # type: Optional[str] # The static build requirements (from pyproject.toml) - self.pyproject_requires = None + self.pyproject_requires = None # type: Optional[List[str]] # Build requirements that we will check are available - self.requirements_to_check = [] + self.requirements_to_check = [] # type: List[str] # The PEP 517 backend we should use to build the project - self.pep517_backend = None + self.pep517_backend = None # type: Optional[Pep517HookCaller] # Are we using PEP 517 for this requirement? # After pyproject.toml has been loaded, the only valid values are True @@ -154,6 +179,7 @@ class InstallRequirement(object): self.__class__.__name__, str(self), self.editable) def populate_link(self, finder, upgrade, require_hashes): + # type: (PackageFinder, bool, bool) -> None """Ensure that if a link can be found for this, that it is found. Note that self.link may still be None - if Upgrade is False and the @@ -176,16 +202,19 @@ class InstallRequirement(object): # Things that are valid for all kinds of requirements? @property def name(self): + # type: () -> Optional[str] if self.req is None: return None return native_str(pkg_resources.safe_name(self.req.name)) @property def specifier(self): + # type: () -> SpecifierSet return self.req.specifier @property def is_pinned(self): + # type: () -> bool """Return whether I am pinned to an exact version. For example, some-package==1.2 is pinned; some-package>1.2 is not. @@ -199,6 +228,7 @@ class InstallRequirement(object): return get_installed_version(self.name) def match_markers(self, extras_requested=None): + # type: (Optional[Iterable[str]]) -> bool if not extras_requested: # Provide an extra to safely evaluate the markers # without matching any extra @@ -212,6 +242,7 @@ class InstallRequirement(object): @property def has_hash_options(self): + # type: () -> bool """Return whether any known-good hashes are specified as options. These activate --require-hashes mode; hashes specified as part of a @@ -221,6 +252,7 @@ class InstallRequirement(object): return bool(self.options.get('hashes', {})) def hashes(self, trust_internet=True): + # type: (bool) -> Hashes """Return a hash-comparer that considers my option- and URL-based hashes to be known-good. @@ -242,6 +274,7 @@ class InstallRequirement(object): return Hashes(good_hashes) def from_path(self): + # type: () -> Optional[str] """Format a nice indicator to show where this "comes from" """ if self.req is None: @@ -257,6 +290,7 @@ class InstallRequirement(object): return s def build_location(self, build_dir): + # type: (str) -> Optional[str] assert build_dir is not None if self._temp_build_dir.path is not None: return self._temp_build_dir.path @@ -284,6 +318,7 @@ class InstallRequirement(object): return os.path.join(build_dir, name) def _correct_build_location(self): + # type: () -> None """Move self._temp_build_dir to self._ideal_build_dir/self.req.name For some requirements (e.g. a path to a directory), the name of the @@ -297,7 +332,8 @@ class InstallRequirement(object): return assert self.req is not None assert self._temp_build_dir.path - assert self._ideal_build_dir.path + assert (self._ideal_build_dir is not None and + self._ideal_build_dir.path) # type: ignore old_location = self._temp_build_dir.path self._temp_build_dir.path = None @@ -325,6 +361,7 @@ class InstallRequirement(object): self.metadata_directory = new_meta def remove_temporary_source(self): + # type: () -> None """Remove the source files from this requirement, if they are marked for deletion""" if self.source_dir and os.path.exists( @@ -336,6 +373,7 @@ class InstallRequirement(object): self.build_env.cleanup() def check_if_exists(self, use_user_site): + # type: (bool) -> bool """Find an installed distribution that satisfies or conflicts with this requirement, and set self.satisfied_by or self.conflicts_with appropriately. @@ -379,11 +417,22 @@ class InstallRequirement(object): # Things valid for wheels @property def is_wheel(self): - return self.link and self.link.is_wheel + # type: () -> bool + if not self.link: + return False + return self.link.is_wheel - def move_wheel_files(self, wheeldir, root=None, home=None, prefix=None, - warn_script_location=True, use_user_site=False, - pycompile=True): + def move_wheel_files( + self, + wheeldir, # type: str + root=None, # type: Optional[str] + home=None, # type: Optional[str] + prefix=None, # type: Optional[str] + warn_script_location=True, # type: bool + use_user_site=False, # type: bool + pycompile=True # type: bool + ): + # type: (...) -> None move_wheel_files( self.name, self.req, wheeldir, user=use_user_site, @@ -398,12 +447,14 @@ class InstallRequirement(object): # Things valid for sdists @property def setup_py_dir(self): + # type: () -> str return os.path.join( self.source_dir, self.link and self.link.subdirectory_fragment or '') @property def setup_py(self): + # type: () -> str assert self.source_dir, "No source dir for %s" % self setup_py = os.path.join(self.setup_py_dir, 'setup.py') @@ -416,6 +467,7 @@ class InstallRequirement(object): @property def pyproject_toml(self): + # type: () -> str assert self.source_dir, "No source dir for %s" % self pp_toml = os.path.join(self.setup_py_dir, 'pyproject.toml') @@ -427,6 +479,7 @@ class InstallRequirement(object): return pp_toml def load_pyproject_toml(self): + # type: () -> None """Load the pyproject.toml file. After calling this routine, all of the attributes related to PEP 517 @@ -467,6 +520,7 @@ class InstallRequirement(object): self.pep517_backend._subprocess_runner = runner def prepare_metadata(self): + # type: () -> None """Ensure that project metadata is available. Under PEP 517, call the backend hook to prepare the metadata. @@ -505,6 +559,7 @@ class InstallRequirement(object): self.req = Requirement(metadata_name) def prepare_pep517_metadata(self): + # type: () -> None assert self.pep517_backend is not None metadata_dir = os.path.join( @@ -526,6 +581,7 @@ class InstallRequirement(object): self.metadata_directory = os.path.join(metadata_dir, distinfo_dir) def run_egg_info(self): + # type: () -> None if self.name: logger.debug( 'Running setup.py (path:%s) egg_info for package %s', @@ -545,7 +601,7 @@ class InstallRequirement(object): # source code will be mistaken for an installed egg, causing # problems if self.editable: - egg_base_option = [] + egg_base_option = [] # type: List[str] else: egg_info_dir = os.path.join(self.setup_py_dir, 'pip-egg-info') ensure_dir(egg_info_dir) @@ -559,6 +615,7 @@ class InstallRequirement(object): @property def egg_info_path(self): + # type: () -> str if self._egg_info_path is None: if self.editable: base = self.source_dir @@ -617,6 +674,7 @@ class InstallRequirement(object): return self._metadata def get_dist(self): + # type: () -> Distribution """Return a pkg_resources.Distribution for this requirement""" if self.metadata_directory: base_dir, distinfo = os.path.split(self.metadata_directory) @@ -630,7 +688,8 @@ class InstallRequirement(object): base_dir = os.path.dirname(egg_info) metadata = pkg_resources.PathMetadata(base_dir, egg_info) dist_name = os.path.splitext(os.path.basename(egg_info))[0] - typ = pkg_resources.Distribution + # https://github.com/python/mypy/issues/1174 + typ = pkg_resources.Distribution # type: ignore return typ( base_dir, @@ -639,6 +698,7 @@ class InstallRequirement(object): ) def assert_source_matches_version(self): + # type: () -> None assert self.source_dir version = self.metadata['version'] if self.req.specifier and version not in self.req.specifier: @@ -657,6 +717,7 @@ class InstallRequirement(object): # For both source distributions and editables def ensure_has_source_dir(self, parent_dir): + # type: (str) -> str """Ensure that a source_dir is set. This will create a temporary build dir if the name of the requirement @@ -671,8 +732,13 @@ class InstallRequirement(object): return self.source_dir # For editable installations - def install_editable(self, install_options, - global_options=(), prefix=None): + def install_editable( + self, + install_options, # type: List[str] + global_options=(), # type: Sequence[str] + prefix=None # type: Optional[str] + ): + # type: (...) -> None logger.info('Running setup.py develop for %s', self.name) if self.isolated: @@ -702,6 +768,7 @@ class InstallRequirement(object): self.install_succeeded = True def update_editable(self, obtain=True): + # type: (bool) -> None if not self.link: logger.debug( "Cannot update repository at %s; repository location is " @@ -733,6 +800,7 @@ class InstallRequirement(object): # Top-level Actions def uninstall(self, auto_confirm=False, verbose=False, use_user_site=False): + # type: (bool, bool, bool) -> Optional[UninstallPathSet] """ Uninstall the distribution currently satisfying this requirement. @@ -747,7 +815,7 @@ class InstallRequirement(object): """ if not self.check_if_exists(use_user_site): logger.warning("Skipping %s as it is not installed.", self.name) - return + return None dist = self.satisfied_by or self.conflicts_with uninstalled_pathset = UninstallPathSet.from_dist(dist) @@ -762,9 +830,16 @@ class InstallRequirement(object): name = name.replace(os.path.sep, '/') return name + def _get_archive_name(self, path, parentdir, rootdir): + # type: (str, str, str) -> str + path = os.path.join(parentdir, path) + name = self._clean_zip_name(path, rootdir) + return self.name + '/' + name + # TODO: Investigate if this should be kept in InstallRequirement # Seems to be used only when VCS + downloads def archive(self, build_dir): + # type: (str) -> None assert self.source_dir create_archive = True archive_name = '%s-%s.zip' % (self.name, self.metadata["version"]) @@ -798,23 +873,37 @@ class InstallRequirement(object): if 'pip-egg-info' in dirnames: dirnames.remove('pip-egg-info') for dirname in dirnames: - dirname = os.path.join(dirpath, dirname) - name = self._clean_zip_name(dirname, dir) - zipdir = zipfile.ZipInfo(self.name + '/' + name + '/') + dir_arcname = self._get_archive_name(dirname, + parentdir=dirpath, + rootdir=dir) + # should be fixed in mypy==0.650 + # see https://github.com/python/typeshed/pull/2628 + zipdir = zipfile.ZipInfo(dir_arcname + '/') # type: ignore zipdir.external_attr = 0x1ED << 16 # 0o755 zip.writestr(zipdir, '') for filename in filenames: if filename == PIP_DELETE_MARKER_FILENAME: continue + file_arcname = self._get_archive_name(filename, + parentdir=dirpath, + rootdir=dir) filename = os.path.join(dirpath, filename) - name = self._clean_zip_name(filename, dir) - zip.write(filename, self.name + '/' + name) + zip.write(filename, file_arcname) zip.close() logger.info('Saved %s', display_path(archive_path)) - def install(self, install_options, global_options=None, root=None, - home=None, prefix=None, warn_script_location=True, - use_user_site=False, pycompile=True): + def install( + self, + install_options, # type: List[str] + global_options=None, # type: Optional[Sequence[str]] + root=None, # type: Optional[str] + home=None, # type: Optional[str] + prefix=None, # type: Optional[str] + warn_script_location=True, # type: bool + use_user_site=False, # type: bool + pycompile=True # type: bool + ): + # type: (...) -> None global_options = global_options if global_options is not None else [] if self.editable: self.install_editable( @@ -844,7 +933,8 @@ class InstallRequirement(object): self.options.get('install_options', []) if self.isolated: - global_options = global_options + ["--no-user-cfg"] + # https://github.com/python/mypy/issues/1174 + global_options = global_options + ["--no-user-cfg"] # type: ignore with TempDirectory(kind="record") as temp_dir: record_filename = os.path.join(temp_dir.path, 'install-record.txt') @@ -903,8 +993,15 @@ class InstallRequirement(object): with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n') - def get_install_args(self, global_options, record_filename, root, prefix, - pycompile): + def get_install_args( + self, + global_options, # type: Sequence[str] + record_filename, # type: str + root, # type: Optional[str] + prefix, # type: Optional[str] + pycompile # type: bool + ): + # type: (...) -> List[str] install_args = [sys.executable, "-u"] install_args.append('-c') install_args.append(SETUPTOOLS_SHIM % self.setup_py) diff --git a/src/pip/_internal/req/req_set.py b/src/pip/_internal/req/req_set.py index b1983171d..d1410e935 100644 --- a/src/pip/_internal/req/req_set.py +++ b/src/pip/_internal/req/req_set.py @@ -5,26 +5,33 @@ from collections import OrderedDict from pip._internal.exceptions import InstallationError from pip._internal.utils.logging import indent_log +from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.wheel import Wheel +if MYPY_CHECK_RUNNING: + from typing import Optional, List, Tuple, Dict, Iterable # noqa: F401 + from pip._internal.req.req_install import InstallRequirement # noqa: F401 + + logger = logging.getLogger(__name__) class RequirementSet(object): def __init__(self, require_hashes=False, check_supported_wheels=True): + # type: (bool, bool) -> None """Create a RequirementSet. """ - self.requirements = OrderedDict() + self.requirements = OrderedDict() # type: Dict[str, InstallRequirement] # noqa: E501 self.require_hashes = require_hashes self.check_supported_wheels = check_supported_wheels # Mapping of alias: real_name - self.requirement_aliases = {} - self.unnamed_requirements = [] - self.successfully_downloaded = [] - self.reqs_to_cleanup = [] + self.requirement_aliases = {} # type: Dict[str, str] + self.unnamed_requirements = [] # type: List[InstallRequirement] + self.successfully_downloaded = [] # type: List[InstallRequirement] + self.reqs_to_cleanup = [] # type: List[InstallRequirement] def __str__(self): reqs = [req for req in self.requirements.values() @@ -39,8 +46,13 @@ class RequirementSet(object): return ('<%s object; %d requirement(s): %s>' % (self.__class__.__name__, len(reqs), reqs_str)) - def add_requirement(self, install_req, parent_req_name=None, - extras_requested=None): + def add_requirement( + self, + install_req, # type: InstallRequirement + parent_req_name=None, # type: Optional[str] + extras_requested=None # type: Optional[Iterable[str]] + ): + # type: (...) -> Tuple[List[InstallRequirement], Optional[InstallRequirement]] # noqa: E501 """Add install_req as a requirement to install. :param parent_req_name: The name of the requirement that needed this @@ -152,6 +164,7 @@ class RequirementSet(object): return [existing_req], existing_req def has_requirement(self, project_name): + # type: (str) -> bool name = project_name.lower() if (name in self.requirements and not self.requirements[name].constraint or @@ -162,10 +175,12 @@ class RequirementSet(object): @property def has_requirements(self): + # type: () -> List[InstallRequirement] return list(req for req in self.requirements.values() if not req.constraint) or self.unnamed_requirements def get_requirement(self, project_name): + # type: (str) -> InstallRequirement for name in project_name, project_name.lower(): if name in self.requirements: return self.requirements[name] @@ -174,6 +189,7 @@ class RequirementSet(object): raise KeyError("No project with the name %r" % project_name) def cleanup_files(self): + # type: () -> None """Clean up files, remove builds.""" logger.debug('Cleaning up...') with indent_log(): diff --git a/src/pip/_internal/req/req_tracker.py b/src/pip/_internal/req/req_tracker.py index 0a86f4cd3..82e084a4c 100644 --- a/src/pip/_internal/req/req_tracker.py +++ b/src/pip/_internal/req/req_tracker.py @@ -7,6 +7,12 @@ import logging import os from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Set, Iterator # noqa: F401 + from pip._internal.req.req_install import InstallRequirement # noqa: F401 + from pip._internal.models.link import Link # noqa: F401 logger = logging.getLogger(__name__) @@ -14,6 +20,7 @@ logger = logging.getLogger(__name__) class RequirementTracker(object): def __init__(self): + # type: () -> None self._root = os.environ.get('PIP_REQ_TRACKER') if self._root is None: self._temp_dir = TempDirectory(delete=False, kind='req-tracker') @@ -23,7 +30,7 @@ class RequirementTracker(object): else: self._temp_dir = None logger.debug('Re-using requirements tracker %r', self._root) - self._entries = set() + self._entries = set() # type: Set[InstallRequirement] def __enter__(self): return self @@ -32,10 +39,12 @@ class RequirementTracker(object): self.cleanup() def _entry_path(self, link): + # type: (Link) -> str hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest() return os.path.join(self._root, hashed) def add(self, req): + # type: (InstallRequirement) -> None link = req.link info = str(req) entry_path = self._entry_path(link) @@ -54,12 +63,14 @@ class RequirementTracker(object): logger.debug('Added %s to build tracker %r', req, self._root) def remove(self, req): + # type: (InstallRequirement) -> None link = req.link self._entries.remove(req) os.unlink(self._entry_path(link)) logger.debug('Removed %s from build tracker %r', req, self._root) def cleanup(self): + # type: () -> None for req in set(self._entries): self.remove(req) remove = self._temp_dir is not None @@ -71,6 +82,7 @@ class RequirementTracker(object): @contextlib.contextmanager def track(self, req): + # type: (InstallRequirement) -> Iterator[None] self.add(req) yield self.remove(req) diff --git a/src/pip/_internal/resolve.py b/src/pip/_internal/resolve.py index a911a348b..794070561 100644 --- a/src/pip/_internal/resolve.py +++ b/src/pip/_internal/resolve.py @@ -18,7 +18,7 @@ from pip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, HashError, HashErrors, UnsupportedPythonVersion, ) -from pip._internal.req.constructors import install_req_from_req +from pip._internal.req.constructors import install_req_from_req_string from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import dist_in_usersite, ensure_dir from pip._internal.utils.packaging import check_dist_requires_python @@ -269,7 +269,7 @@ class Resolver(object): more_reqs = [] def add_req(subreq, extras_requested): - sub_install_req = install_req_from_req( + sub_install_req = install_req_from_req_string( str(subreq), req_to_install, isolated=self.isolated,