Merge pull request #8541 from chrahunt/refactor/compile-file-next-steps

Byte-compile files after installation

src/pip/_internal/operations/install/wheel.py

@@ -7,6 +7,7 @@ import collections
 import compileall
 import contextlib
 import csv
+import importlib
 import logging
 import os.path
 import re
@@ -451,14 +452,6 @@ def install_unpacked_wheel(
     changed = set()  # type: Set[RecordPath]
     generated = []  # type: List[str]
 
-    # Compile all of the pyc files that we're going to be installing
-    if pycompile:
-        with captured_stdout() as stdout:
-            with warnings.catch_warnings():
-                warnings.filterwarnings('ignore')
-                compileall.compile_dir(source, force=True, quiet=True)
-        logger.debug(stdout.getvalue())
-
     def record_installed(srcfile, destfile, modified=False):
         # type: (text_type, text_type, bool) -> None
         """Map archive RECORD paths to installation RECORD paths."""
@@ -583,6 +576,52 @@ def install_unpacked_wheel(
                 filter=filter,
             )
 
+    def pyc_source_file_paths():
+        # type: () -> Iterator[text_type]
+        # We de-duplicate installation paths, since there can be overlap (e.g.
+        # file in .data maps to same location as file in wheel root).
+        # Sorting installation paths makes it easier to reproduce and debug
+        # issues related to permissions on existing files.
+        for installed_path in sorted(set(installed.values())):
+            full_installed_path = os.path.join(lib_dir, installed_path)
+            if not os.path.isfile(full_installed_path):
+                continue
+            if not full_installed_path.endswith('.py'):
+                continue
+            yield full_installed_path
+
+    def pyc_output_path(path):
+        # type: (text_type) -> text_type
+        """Return the path the pyc file would have been written to.
+        """
+        if PY2:
+            if sys.flags.optimize:
+                return path + 'o'
+            else:
+                return path + 'c'
+        else:
+            return importlib.util.cache_from_source(path)
+
+    # Compile all of the pyc files for the installed files
+    if pycompile:
+        with captured_stdout() as stdout:
+            with warnings.catch_warnings():
+                warnings.filterwarnings('ignore')
+                for path in pyc_source_file_paths():
+                    # Python 2's `compileall.compile_file` requires a str in
+                    # error cases, so we must convert to the native type.
+                    path_arg = ensure_str(
+                        path, encoding=sys.getfilesystemencoding()
+                    )
+                    success = compileall.compile_file(
+                        path_arg, force=True, quiet=True
+                    )
+                    if success:
+                        pyc_path = pyc_output_path(path)
+                        assert os.path.exists(pyc_path)
+                        record_installed(pyc_path, pyc_path)
+        logger.debug(stdout.getvalue())
+
     maker = PipScriptMaker(None, scheme.scripts)
 
     # Ensure old scripts are overwritten.

tests/unit/test_wheel.py

@@ -258,9 +258,15 @@ class TestInstallUnpackedWheel(object):
     """
 
     def prep(self, data, tmpdir):
+        # Since Path implements __add__, os.path.join returns a Path object.
+        # Passing Path objects to interfaces expecting str (like
+        # `compileall.compile_file`) can cause failures, so we normalize it
+        # to a string here.
+        tmpdir = str(tmpdir)
         self.name = 'sample'
-        self.wheelpath = data.packages.joinpath(
-            'sample-1.2.0-py2.py3-none-any.whl')
+        self.wheelpath = os.path.join(
+            str(data.packages), 'sample-1.2.0-py2.py3-none-any.whl'
+        )
         self.req = Requirement('sample')
         self.src = os.path.join(tmpdir, 'src')
         self.dest = os.path.join(tmpdir, 'dest')