Update pkg resources (#3630)

* Drop markers inside InstallRequirement.check_if_exists

Any requirement that are checked if they are installed may specify
an environment marker. Since the marker may include information that
we no longer have context for (such as which extra requires that
requirement), and we have already decided that we are going to install
the requirement, drop the marker when we check if it's installed.

* Update pkg_resources from setuptools 20.7.0

pip/_vendor/README.rst

@@ -95,7 +95,7 @@ such as OS packages.
 pkg_resources
 -------------
 
-pkg_resources has been pulled in from setuptools 20.4
+pkg_resources has been pulled in from setuptools 20.7.0
 
 
 Modifications
@@ -104,7 +104,7 @@ Modifications
 * html5lib has been modified to import six from pip._vendor
 * pkg_resources has been modified to import its externs from pip._vendor
 * CacheControl has been modified to import its dependencies from pip._vendor
-* packaging has been modified to import its dependencies from pip._vendor.
+* packaging has been modified to import its dependencies from pip._vendor
 
 
 Debundling

pip/_vendor/pkg_resources/__init__.py

@@ -28,8 +28,6 @@ import warnings
 import stat
 import functools
 import pkgutil
-import token
-import symbol
 import operator
 import platform
 import collections
@@ -67,11 +65,6 @@ try:
 except ImportError:
     importlib_machinery = None
 
-try:
-    import parser
-except ImportError:
-    pass
-
 from pip._vendor import packaging
 __import__('pip._vendor.packaging.version')
 __import__('pip._vendor.packaging.specifiers')
@@ -799,6 +792,8 @@ class WorkingSet(object):
         best = {}
         to_activate = []
 
+        req_extras = _ReqExtras()
+
         # Mapping of requirement to set of distributions that required it;
         # useful for reporting info about conflicts.
         required_by = collections.defaultdict(set)
@@ -809,6 +804,10 @@ class WorkingSet(object):
             if req in processed:
                 # Ignore cyclic or redundant dependencies
                 continue
+
+            if not req_extras.markers_pass(req):
+                continue
+
             dist = best.get(req.key)
             if dist is None:
                 # Find the best distribution and add it to the map
@@ -841,6 +840,7 @@ class WorkingSet(object):
             # Register the new requirements needed by req
             for new_requirement in new_requirements:
                 required_by[new_requirement].add(req.project_name)
+                req_extras[new_requirement] = req.extras
 
             processed[req] = True
 
@@ -973,6 +973,26 @@ class WorkingSet(object):
         self.callbacks = callbacks[:]
 
 
+class _ReqExtras(dict):
+    """
+    Map each requirement to the extras that demanded it.
+    """
+
+    def markers_pass(self, req):
+        """
+        Evaluate markers for req against each extra that
+        demanded it.
+
+        Return False if the req has a marker and fails
+        evaluation. Otherwise, return True.
+        """
+        evals = (
+            req.marker.evaluate({'extra': extra})
+            for extra in self.get(req) or ['']
+        )
+        return not req.marker or any(evals)
+
+
 class Environment(object):
     """Searchable snapshot of distributions on a search path"""
 
@@ -1839,7 +1859,13 @@ class FileMetadata(EmptyProvider):
     def get_metadata(self, name):
         if name=='PKG-INFO':
             with io.open(self.path, encoding='utf-8') as f:
-                metadata = f.read()
+                try:
+                    metadata = f.read()
+                except UnicodeDecodeError as exc:
+                    # add path context to error message
+                    tmpl = " in {self.path}"
+                    exc.reason += tmpl.format(self=self)
+                    raise
             return metadata
         raise KeyError("No metadata except PKG-INFO is available")
 

pip/req/req_install.py

@@ -990,7 +990,13 @@ class InstallRequirement(object):
         if self.req is None:
             return False
         try:
-            self.satisfied_by = pkg_resources.get_distribution(str(self.req))
+            # get_distribution() will resolve the entire list of requirements
+            # anyway, and we've already determined that we need the requirement
+            # in question, so strip the marker so that we don't try to
+            # evaluate it.
+            no_marker = Requirement(str(self.req))
+            no_marker.marker = None
+            self.satisfied_by = pkg_resources.get_distribution(str(no_marker))
         except pkg_resources.DistributionNotFound:
             return False
         except pkg_resources.VersionConflict: