From 963b31aa3d9adaf8427aefe0e19d24e7ca423c2b Mon Sep 17 00:00:00 2001 From: "Pradyun S. Gedam" Date: Sun, 23 Jul 2017 21:46:20 +0530 Subject: [PATCH] Breakup method in pip.operations.prepare (#4602) --- pip/operations/prepare.py | 390 ++++++++++++++++++++------------------ 1 file changed, 207 insertions(+), 183 deletions(-) diff --git a/pip/operations/prepare.py b/pip/operations/prepare.py index e5a4b3a87..04db19d02 100644 --- a/pip/operations/prepare.py +++ b/pip/operations/prepare.py @@ -149,195 +149,219 @@ class RequirementPreparer(object): return False def prepare_requirement(self, req, resolver): - # TODO: Breakup into smaller functions - # TODO: Add a nice docstring + """Prepare a requirement for installation + + Returns an AbstractDist that can be used to install the package + """ + # TODO: Remove circular dependency on resolver + assert resolver.require_hashes is not None, ( + "require_hashes should have been set in Resolver.resolve()" + ) + if req.editable: - logger.info('Obtaining %s', req) + return self._prepare_editable_requirement(req, resolver) + + # satisfied_by is only evaluated by calling _check_skip_installed, + # so it must be None here. + assert req.satisfied_by is None + if not resolver.ignore_installed: + skip_reason = resolver._check_skip_installed(req) + + if req.satisfied_by: + return self._prepare_installed_requirement( + req, resolver, skip_reason + ) + + return self._prepare_linked_requirement(req, resolver) + + def _prepare_linked_requirement(self, req, resolver): + """Prepare a requirement that would be obtained from req.link + """ + # TODO: Breakup into smaller functions + if req.link and req.link.scheme == 'file': + path = url_to_path(req.link.url) + logger.info('Processing %s', display_path(path)) else: - # satisfied_by is only evaluated by calling _check_skip_installed, - # so it must be None here. - assert req.satisfied_by is None - if not resolver.ignore_installed: - skip_reason = resolver._check_skip_installed(req) - - if req.satisfied_by: - assert skip_reason is not None, ( - '_check_skip_installed returned None but ' - 'req.satisfied_by is set to %r' - % (req.satisfied_by,)) - logger.info( - 'Requirement %s: %s (%s)', skip_reason, - req, - req.satisfied_by.version) - else: - if (req.link and - req.link.scheme == 'file'): - path = url_to_path(req.link.url) - logger.info('Processing %s', display_path(path)) - else: - logger.info('Collecting %s', req) - - assert resolver.require_hashes is not None, \ - "This should have been set in resolve()" + logger.info('Collecting %s', req) with indent_log(): - # ################################ # - # # vcs update or unpack archive # # - # ################################ # - if req.editable: - if resolver.require_hashes: - raise InstallationError( - 'The editable requirement %s cannot be installed when ' - 'requiring hashes, because there is no single file to ' - 'hash.' % req) - req.ensure_has_source_dir(self.src_dir) - req.update_editable(not self._download_should_save) - abstract_dist = make_abstract_dist(req) - abstract_dist.prep_for_dist() - if self._download_should_save: - req.archive(self.download_dir) - req.check_if_exists() - elif req.satisfied_by: - if resolver.require_hashes: - logger.debug( - 'Since it is already installed, we are trusting this ' - 'package without checking its hash. To ensure a ' - 'completely repeatable environment, install into an ' - 'empty virtualenv.') - abstract_dist = Installed(req) - else: - # @@ if filesystem packages are not marked - # editable in a req, a non deterministic error - # occurs when the script attempts to unpack the - # build directory - req.ensure_has_source_dir(self.build_dir) - # If a checkout exists, it's unwise to keep going. version - # inconsistencies are logged later, but do not fail the - # installation. - # FIXME: this won't upgrade when there's an existing - # package unpacked in `req.source_dir` - # package unpacked in `req.source_dir` - if os.path.exists( - os.path.join(req.source_dir, 'setup.py')): - raise PreviousBuildDirError( - "pip can't proceed with requirements '%s' due to a" - " pre-existing build directory (%s). This is " - "likely due to a previous installation that failed" - ". pip is being responsible and not assuming it " - "can delete this. Please delete it and try again." - % (req, req.source_dir) - ) - req.populate_link( - resolver.finder, - resolver._is_upgrade_allowed(req), - resolver.require_hashes + # @@ if filesystem packages are not marked + # editable in a req, a non deterministic error + # occurs when the script attempts to unpack the + # build directory + req.ensure_has_source_dir(self.build_dir) + # If a checkout exists, it's unwise to keep going. version + # inconsistencies are logged later, but do not fail the + # installation. + # FIXME: this won't upgrade when there's an existing + # package unpacked in `req.source_dir` + # package unpacked in `req.source_dir` + if os.path.exists(os.path.join(req.source_dir, 'setup.py')): + raise PreviousBuildDirError( + "pip can't proceed with requirements '%s' due to a" + " pre-existing build directory (%s). This is " + "likely due to a previous installation that failed" + ". pip is being responsible and not assuming it " + "can delete this. Please delete it and try again." + % (req, req.source_dir) ) - # We can't hit this spot and have populate_link return None. - # req.satisfied_by is None here (because we're - # guarded) and upgrade has no impact except when satisfied_by - # is not None. - # Then inside find_requirement existing_applicable -> False - # If no new versions are found, DistributionNotFound is raised, - # otherwise a result is guaranteed. - assert req.link - link = req.link + req.populate_link( + resolver.finder, + resolver._is_upgrade_allowed(req), + resolver.require_hashes + ) + # We can't hit this spot and have populate_link return None. + # req.satisfied_by is None here (because we're + # guarded) and upgrade has no impact except when satisfied_by + # is not None. + # Then inside find_requirement existing_applicable -> False + # If no new versions are found, DistributionNotFound is raised, + # otherwise a result is guaranteed. + assert req.link + link = req.link - # Now that we have the real link, we can tell what kind of - # requirements we have and raise some more informative errors - # than otherwise. (For example, we can raise VcsHashUnsupported - # for a VCS URL rather than HashMissing.) - if resolver.require_hashes: - # We could check these first 2 conditions inside - # unpack_url and save repetition of conditions, but then - # we would report less-useful error messages for - # unhashable requirements, complaining that there's no - # hash provided. - if is_vcs_url(link): - raise VcsHashUnsupported() - elif is_file_url(link) and is_dir_url(link): - raise DirectoryUrlHashUnsupported() - if (not req.original_link and - not req.is_pinned): - # Unpinned packages are asking for trouble when a new - # version is uploaded. This isn't a security check, but - # it saves users a surprising hash mismatch in the - # future. - # - # file:/// URLs aren't pinnable, so don't complain - # about them not being pinned. - raise HashUnpinned() - hashes = req.hashes( - trust_internet=not resolver.require_hashes) - if resolver.require_hashes and not hashes: - # Known-good hashes are missing for this requirement, so - # shim it with a facade object that will provoke hash - # computation and then raise a HashMissing exception - # showing the user what the hash should be. - hashes = MissingHashes() + # Now that we have the real link, we can tell what kind of + # requirements we have and raise some more informative errors + # than otherwise. (For example, we can raise VcsHashUnsupported + # for a VCS URL rather than HashMissing.) + if resolver.require_hashes: + # We could check these first 2 conditions inside + # unpack_url and save repetition of conditions, but then + # we would report less-useful error messages for + # unhashable requirements, complaining that there's no + # hash provided. + if is_vcs_url(link): + raise VcsHashUnsupported() + elif is_file_url(link) and is_dir_url(link): + raise DirectoryUrlHashUnsupported() + if not req.original_link and not req.is_pinned: + # Unpinned packages are asking for trouble when a new + # version is uploaded. This isn't a security check, but + # it saves users a surprising hash mismatch in the + # future. + # + # file:/// URLs aren't pinnable, so don't complain + # about them not being pinned. + raise HashUnpinned() + hashes = req.hashes(trust_internet=not resolver.require_hashes) + if resolver.require_hashes and not hashes: + # Known-good hashes are missing for this requirement, so + # shim it with a facade object that will provoke hash + # computation and then raise a HashMissing exception + # showing the user what the hash should be. + hashes = MissingHashes() - try: - download_dir = self.download_dir - # We always delete unpacked sdists after pip ran. - autodelete_unpacked = True - if req.link.is_wheel \ - and self.wheel_download_dir: - # when doing 'pip wheel` we download wheels to a - # dedicated dir. - download_dir = self.wheel_download_dir - if req.link.is_wheel: - if download_dir: - # When downloading, we only unpack wheels to get - # metadata. - autodelete_unpacked = True - else: - # When installing a wheel, we use the unpacked - # wheel. - autodelete_unpacked = False - unpack_url( - req.link, req.source_dir, - download_dir, autodelete_unpacked, - session=resolver.session, hashes=hashes, - progress_bar=self.progress_bar) - except requests.HTTPError as exc: - logger.critical( - 'Could not install requirement %s because ' - 'of error %s', - req, - exc, - ) - raise InstallationError( - 'Could not install requirement %s because ' - 'of HTTP error %s for URL %s' % - (req, exc, req.link) - ) - abstract_dist = make_abstract_dist(req) - abstract_dist.prep_for_dist() - if self._download_should_save: - # Make a .zip of the source_dir we already created. - if req.link.scheme in vcs.all_schemes: - req.archive(self.download_dir) - # req.req is only avail after unpack for URL - # pkgs repeat check_if_exists to uninstall-on-upgrade - # (#14) - if not resolver.ignore_installed: - req.check_if_exists() - if req.satisfied_by: - should_modify = ( - resolver.upgrade_strategy != "to-satisfy-only" or - resolver.ignore_installed - ) - if should_modify: - # don't uninstall conflict if user install and - # conflict is not user install - if not (resolver.use_user_site and not - dist_in_usersite(req.satisfied_by)): - req.conflicts_with = \ - req.satisfied_by - req.satisfied_by = None + try: + download_dir = self.download_dir + # We always delete unpacked sdists after pip ran. + autodelete_unpacked = True + if req.link.is_wheel and self.wheel_download_dir: + # when doing 'pip wheel` we download wheels to a + # dedicated dir. + download_dir = self.wheel_download_dir + if req.link.is_wheel: + if download_dir: + # When downloading, we only unpack wheels to get + # metadata. + autodelete_unpacked = True else: - logger.info( - 'Requirement already satisfied (use ' - '--upgrade to upgrade): %s', - req, - ) + # When installing a wheel, we use the unpacked + # wheel. + autodelete_unpacked = False + unpack_url( + req.link, req.source_dir, + download_dir, autodelete_unpacked, + session=resolver.session, hashes=hashes, + progress_bar=self.progress_bar + ) + except requests.HTTPError as exc: + logger.critical( + 'Could not install requirement %s because of error %s', + req, + exc, + ) + raise InstallationError( + 'Could not install requirement %s because of HTTP ' + 'error %s for URL %s' % + (req, exc, req.link) + ) + abstract_dist = make_abstract_dist(req) + abstract_dist.prep_for_dist() + if self._download_should_save: + # Make a .zip of the source_dir we already created. + if req.link.scheme in vcs.all_schemes: + req.archive(self.download_dir) + # req.req is only avail after unpack for URL + # pkgs repeat check_if_exists to uninstall-on-upgrade + # (#14) + if not resolver.ignore_installed: + req.check_if_exists() + if req.satisfied_by: + should_modify = ( + resolver.upgrade_strategy != "to-satisfy-only" or + resolver.ignore_installed + ) + if should_modify: + # don't uninstall conflict if user install and + # conflict is not user install + if not (resolver.use_user_site and + not dist_in_usersite(req.satisfied_by)): + req.conflicts_with = req.satisfied_by + req.satisfied_by = None + else: + logger.info( + 'Requirement already satisfied (use ' + '--upgrade to upgrade): %s', + req, + ) + return abstract_dist + + def _prepare_editable_requirement(self, req, resolver): + """Prepare an editable requirement + """ + assert req.editable, "cannot prepare a non-editable req as editable" + + logger.info('Obtaining %s', req) + + with indent_log(): + if resolver.require_hashes: + raise InstallationError( + 'The editable requirement %s cannot be installed when ' + 'requiring hashes, because there is no single file to ' + 'hash.' % req + ) + req.ensure_has_source_dir(self.src_dir) + req.update_editable(not self._download_should_save) + + abstract_dist = make_abstract_dist(req) + abstract_dist.prep_for_dist() + + if self._download_should_save: + req.archive(self.download_dir) + req.check_if_exists() + + return abstract_dist + + def _prepare_installed_requirement(self, req, resolver, skip_reason): + """Prepare an already-installed requirement + """ + assert req.satisfied_by, "req should have been satisfied but isn't" + assert skip_reason is not None, ( + "did not get skip reason skipped but req.satisfied_by " + "is set to %r" % (req.satisfied_by,) + ) + logger.info( + 'Requirement %s: %s (%s)', + skip_reason, req, req.satisfied_by.version + ) + with indent_log(): + if resolver.require_hashes: + logger.debug( + 'Since it is already installed, we are trusting this ' + 'package without checking its hash. To ensure a ' + 'completely repeatable environment, install into an ' + 'empty virtualenv.' + ) + abstract_dist = Installed(req) + return abstract_dist