Respect explicit encoding declarations in setup.py files.

In Python 2, the exec statement handles encoding for us, but in
Python 3 the encoding must be specified when opening the file
(if it's not specified it uses the system locale encoding, so
previously this would work only if your locale environment variables
specified the same encoding as the setup.py file).

On Python 3.2+ the tokenize.open function is available to interpret
the encoding declaration; fixing this for python 3.0 and 3.1 is more
difficult.

pip/req.py

@@ -299,6 +299,7 @@ __file__ = __SETUP_PY__
 from setuptools.command import egg_info
 import pkg_resources
 import os
+import tokenize
 def replacement_run(self):
     self.mkpath(self.egg_info)
     installer = self.distribution.fetch_build_egg
@@ -309,7 +310,7 @@ def replacement_run(self):
             writer(self, ep.name, os.path.join(self.egg_info,ep.name))
     self.find_sources()
 egg_info.egg_info.run = replacement_run
-exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))
+exec(compile(getattr(tokenize, 'open', open)(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))
 """
 
     def egg_info_data(self, filename):
@@ -638,8 +639,8 @@ exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))
             install_args = [sys.executable]
             install_args.append('-c')
             install_args.append(
-            "import setuptools;__file__=%r;"\
-            "exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))" % self.setup_py)
+            "import setuptools, tokenize;__file__=%r;"\
+            "exec(compile(getattr(tokenize, 'open', open)(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))" % self.setup_py)
             install_args += list(global_options) + ['install','--record', record_filename]
 
             if not self.as_egg:
@@ -728,7 +729,7 @@ exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))
             ## FIXME: should we do --install-headers here too?
             call_subprocess(
                 [sys.executable, '-c',
-                 "import setuptools; __file__=%r; exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))" % self.setup_py]
+                 "import setuptools, tokenize; __file__=%r; exec(compile(getattr(tokenize, 'open', open)(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))" % self.setup_py]
                 + list(global_options) + ['develop', '--no-deps'] + list(install_options),
 
                 cwd=self.source_dir, filter_stdout=self._filter_install,

tests/data/packages/SetupPyLatin1/setup.py

@@ -0,0 +1,7 @@
+# -*- coding: latin-1 -*-
+
+from distutils.core import setup
+
+setup(name="SetupPyUTF8",
+      author="Saúl Ibarra Corretgé",
+      )

tests/data/packages/SetupPyUTF8/setup.py

@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+
+from distutils.core import setup
+
+setup(name="SetupPyUTF8",
+      author="Saúl Ibarra Corretgé",
+      )

tests/functional/test_install.py

@@ -440,6 +440,15 @@ def test_install_package_that_emits_unicode(script, data):
     assert 'FakeError: this package designed to fail on install' in result.stdout
     assert 'UnicodeDecodeError' not in result.stdout
 
+def test_install_package_with_utf8_setup(script, data):
+    """Install a package with a setup.py that declares a utf-8 encoding."""
+    to_install = data.packages.join("SetupPyUTF8")
+    script.pip('install', to_install)
+
+def test_install_package_with_latin1_setup(script, data):
+    """Install a package with a setup.py that declares a latin-1 encoding."""
+    to_install = data.packages.join("SetupPyLatin1")
+    script.pip('install', to_install)
 
 def test_url_req_case_mismatch(script, data):
     """