Explicitly request the identity encoding as insurance against http://bugs.python.org/issue8732

pip/index.py

@@ -6,6 +6,7 @@ import re
 import threading
 import posixpath
 import pkg_resources
+import urllib
 import urllib2
 import urlparse
 import httplib
@@ -14,7 +15,7 @@ from Queue import Queue
 from Queue import Empty as QueueEmpty
 from pip.log import logger
 from pip.util import Inf, path_to_url2, url_to_path, geturl
-from pip.util import normalize_name, splitext
+from pip.util import normalize_name, splitext, urlopen
 from pip.exceptions import DistributionNotFound
 
 __all__ = ['PackageFinder']
@@ -395,10 +396,9 @@ class HTMLPage(object):
                             return None
             logger.debug('Getting page %s' % url)
             try:
-                resp = urllib2.urlopen(url)
+                resp = urlopen(url)
             except IOError:
-                import urllib
-                resp = urllib2.urlopen(urllib.basejoin(url,'index.html'))
+                resp = urlopen(urllib.basejoin(url,'index.html'))
 
             real_url = geturl(resp)
             headers = resp.info()

pip/util.py

@@ -433,3 +433,10 @@ def get_terminal_size():
     if not cr:
         cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80))
     return int(cr[1]), int(cr[0])
+
+# Insurance against "creative" interpretation of the RFC:
+# http://bugs.python.org/issue8732
+def urlopen(url):
+    if isinstance(url, basestring):
+        url = urllib2.Request(url, headers={'Accept-encoding':'identity'})
+    return urllib2.urlopen(url)