From 0e1da584f418ae0088b43d01248572e2ff53d3a1 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 2 Jun 2013 13:03:56 -0400 Subject: [PATCH] Differentiate between internal and external links where possible * By default ignore external links * Add the ``--allow-external`` flag that enables external links globally * Fallback to allowing all links if we cannot determine the API version of the parsed page * Inform the user of ``--allow-external`` if nothing was found to install --- pip/cmdoptions.py | 11 +++++++- pip/commands/install.py | 4 ++- pip/index.py | 59 ++++++++++++++++++++++++++++++++++++++--- pip/req.py | 2 ++ 4 files changed, 71 insertions(+), 5 deletions(-) diff --git a/pip/cmdoptions.py b/pip/cmdoptions.py index ecfcbeb71..3f8f6f1b1 100644 --- a/pip/cmdoptions.py +++ b/pip/cmdoptions.py @@ -63,6 +63,14 @@ mirrors = make_option( default=[], help='Specific mirror URLs to query when --use-mirrors is used.') +allow_external = make_option( + "--allow-external", + dest="allow_external", + action="store_true", + default=False, + help="Allow the installation of externally hosted files", +) + requirements = make_option( '-r', '--requirement', dest='requirements', @@ -138,6 +146,7 @@ index_group = { no_index, find_links, use_mirrors, - mirrors + mirrors, + allow_external, ] } diff --git a/pip/commands/install.py b/pip/commands/install.py index ba65b45d2..cfc0f858e 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -162,7 +162,9 @@ class InstallCommand(Command): index_urls=index_urls, use_mirrors=options.use_mirrors, mirrors=options.mirrors, - use_wheel=options.use_wheel) + use_wheel=options.use_wheel, + allow_external=options.allow_external, + ) def run(self, options, args): if options.download_dir: diff --git a/pip/index.py b/pip/index.py index 717da182c..57e79bf32 100644 --- a/pip/index.py +++ b/pip/index.py @@ -48,7 +48,7 @@ class PackageFinder(object): def __init__(self, find_links, index_urls, use_mirrors=False, mirrors=None, main_mirror_url=None, - use_wheel=False): + use_wheel=False, allow_external=False): self.find_links = find_links self.index_urls = index_urls self.dependency_links = [] @@ -62,6 +62,12 @@ class PackageFinder(object): self.mirror_urls = [] self.use_wheel = use_wheel + self.allow_external = allow_external + + # Stores if we ignored any external links so that we can instruct + # end users how to install them if no distributions are available + self.need_warn_external = False + @property def use_wheel(self): return self._use_wheel @@ -219,6 +225,11 @@ class PackageFinder(object): [Link(url) for url in file_locations], req.name.lower())) if not found_versions and not page_versions and not dependency_versions and not file_versions: logger.fatal('Could not find any downloads that satisfy the requirement %s' % req) + + if self.need_warn_external: + logger.warn("Some externally hosted files were ignored (use " + "--allow-external to allow).") + raise DistributionNotFound('No distributions at all found for %s' % req) installed_version = [] if req.satisfied_by is not None: @@ -251,6 +262,11 @@ class PackageFinder(object): if not applicable_versions: logger.fatal('Could not find a version that satisfies the requirement %s (from versions: %s)' % (req, ', '.join([version for parsed_version, link, version in all_versions]))) + + if self.need_warn_external: + logger.warn("Some externally hosted files were ignored (use " + "--allow-external to allow).") + raise DistributionNotFound('No distributions matching the version for %s' % req) if applicable_versions[0][1] is InfLink: # We have an existing version, and its the best version @@ -388,6 +404,16 @@ class PackageFinder(object): if version is None: logger.debug('Skipping link %s; wrong project name (not %s)' % (link, search_name)) return [] + + if (link.internal is not None + and not link.internal + and not self.allow_external): + # We have a link that we are sure is external, so we should skip + # it unless we are allowing externals + logger.debug("Skipping %s because it is externally hosted." % link) + self.need_warn_external = True + return [] + match = self._py_version_re.search(version) if match: version = version[:match.start()] @@ -613,6 +639,21 @@ class HTMLPage(object): finally: resp.close() + @property + def api_version(self): + if not hasattr(self, "_api_version"): + _api_version = None + + metas = [x for x in self.parsed.findall(".//meta") + if x.get("name", "").lower() == "api-version"] + if metas: + try: + _api_version = int(metas[0].get("value", None)) + except (TypeError, ValueError): + _api_version = None + self._api_version = _api_version + return self._api_version + @property def base_url(self): if not hasattr(self, "_base_url"): @@ -630,7 +671,18 @@ class HTMLPage(object): if anchor.get("href"): href = anchor.get("href") url = self.clean_link(urlparse.urljoin(self.base_url, href)) - yield Link(url, self) + + # Determine if this link is internal. If that distinction + # doesn't make sense in this context, then we don't make + # any distinction. + internal = None + if self.api_version and self.api_version >= 2: + # Only api_versions >= 2 have a distinction between + # external and internal links + internal = bool(anchor.get("rel") + and "internal" in anchor.get("rel").split()) + + yield Link(url, self, internal=internal) def rel_links(self): for url in self.explicit_rel_links(): @@ -679,9 +731,10 @@ class HTMLPage(object): class Link(object): - def __init__(self, url, comes_from=None): + def __init__(self, url, comes_from=None, internal=None): self.url = url self.comes_from = comes_from + self.internal = internal # Set whether it's a wheel self.wheel = None diff --git a/pip/req.py b/pip/req.py index 7a6068256..9f3e690e8 100644 --- a/pip/req.py +++ b/pip/req.py @@ -1395,6 +1395,8 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None): finder.use_wheel = True elif line.startswith('--no-index'): finder.index_urls = [] + elif line.startswith("--allow-external"): + finder.allow_external = True else: comes_from = '-r %s (line %s)' % (filename, line_number) if line.startswith('-e') or line.startswith('--editable'):