merge master branch into pull request.

news/2732.bugfix

@@ -0,0 +1 @@
+Interactive setup.py files will no longer hang indefinitely.

news/4501.feature

@@ -0,0 +1,3 @@
+Installing from a local directory or a VCS URL now builds a wheel to install,
+rather than running ``setup.py install``. Wheels from these sources are not
+cached.

news/4976.bugfix

@@ -0,0 +1 @@
+Abort if reading configuration causes encoding errors.

news/4982.bugfix

@@ -0,0 +1 @@
+Interactive setup.py files will no longer hang indefinitely.

src/pip/_internal/baseparser.py

@@ -9,7 +9,7 @@ from distutils.util import strtobool
 
 from pip._vendor.six import string_types
 
-from pip._internal.configuration import Configuration
+from pip._internal.configuration import Configuration, ConfigurationError
 from pip._internal.utils.compat import get_terminal_size
 
 logger = logging.getLogger(__name__)
@@ -177,9 +177,6 @@ class ConfigOptionParser(CustomOptionParser):
         the environ. Does a little special handling for certain types of
         options (lists)."""
 
-        # Load the configuration
-        self.config.load()
-
         # Accumulate complex default state.
         self.values = optparse.Values(self.defaults)
         late_eval = set()
@@ -224,6 +221,12 @@ class ConfigOptionParser(CustomOptionParser):
             # Old, pre-Optik 1.5 behaviour.
             return optparse.Values(self.defaults)
 
+        # Load the configuration, or error out in case of an error
+        try:
+            self.config.load()
+        except ConfigurationError as err:
+            self.exit(2, err.args[0])
+
         defaults = self._update_defaults(self.defaults.copy())  # ours
         for option in self._get_all_options():
             default = defaults.get(option.dest)

src/pip/_internal/cache.py

@@ -11,6 +11,7 @@ from pip._vendor.packaging.utils import canonicalize_name
 from pip._internal import index
 from pip._internal.compat import expanduser
 from pip._internal.download import path_to_url
+from pip._internal.utils.temp_dir import TempDirectory
 from pip._internal.wheel import InvalidWheelFilename, Wheel
 
 logger = logging.getLogger(__name__)
@@ -102,13 +103,18 @@ class Cache(object):
 
         return index.Link(path_to_url(path))
 
+    def cleanup(self):
+        pass
 
-class WheelCache(Cache):
+
+class SimpleWheelCache(Cache):
     """A cache of wheels for future installs.
     """
 
     def __init__(self, cache_dir, format_control):
-        super(WheelCache, self).__init__(cache_dir, format_control, {"binary"})
+        super(SimpleWheelCache, self).__init__(
+            cache_dir, format_control, {"binary"}
+        )
 
     def get_path_for_link(self, link):
         """Return a directory to store cached wheels for link
@@ -127,8 +133,7 @@ class WheelCache(Cache):
         """
         parts = self._get_cache_path_parts(link)
 
-        # Inside of the base location for cached wheels, expand our parts and
-        # join them all together.
+        # Store wheels within the root cache_dir
         return os.path.join(self.cache_dir, "wheels", *parts)
 
     def get(self, link, package_name):
@@ -148,3 +153,50 @@ class WheelCache(Cache):
             return link
 
         return self._link_for_candidate(link, min(candidates)[1])
+
+
+class EphemWheelCache(SimpleWheelCache):
+    """A SimpleWheelCache that creates it's own temporary cache directory
+    """
+
+    def __init__(self, format_control):
+        self._temp_dir = TempDirectory(kind="ephem-wheel-cache")
+        self._temp_dir.create()
+
+        super(EphemWheelCache, self).__init__(
+            self._temp_dir.path, format_control
+        )
+
+    def cleanup(self):
+        self._temp_dir.cleanup()
+
+
+class WheelCache(Cache):
+    """Wraps EphemWheelCache and SimpleWheelCache into a single Cache
+
+    This Cache allows for gracefully degradation, using the ephem wheel cache
+    when a certain link is not found in the simple wheel cache first.
+    """
+
+    def __init__(self, cache_dir, format_control):
+        super(WheelCache, self).__init__(
+            cache_dir, format_control, {'binary'}
+        )
+        self._wheel_cache = SimpleWheelCache(cache_dir, format_control)
+        self._ephem_cache = EphemWheelCache(format_control)
+
+    def get_path_for_link(self, link):
+        return self._wheel_cache.get_path_for_link(link)
+
+    def get_ephem_path_for_link(self, link):
+        return self._ephem_cache.get_path_for_link(link)
+
+    def get(self, link, package_name):
+        retval = self._wheel_cache.get(link, package_name)
+        if retval is link:
+            retval = self._ephem_cache.get(link, package_name)
+        return retval
+
+    def cleanup(self):
+        self._wheel_cache.cleanup()
+        self._ephem_cache.cleanup()

src/pip/_internal/commands/freeze.py

@@ -89,5 +89,8 @@ class FreezeCommand(Command):
             exclude_editable=options.exclude_editable,
         )
 
-        for line in freeze(**freeze_kwargs):
-            sys.stdout.write(line + '\n')
+        try:
+            for line in freeze(**freeze_kwargs):
+                sys.stdout.write(line + '\n')
+        finally:
+            wheel_cache.cleanup()

src/pip/_internal/commands/install.py

@@ -224,10 +224,10 @@ class InstallCommand(RequirementCommand):
         global_options = options.global_options or []
 
         with self._build_session(options) as session:
-
             finder = self._build_package_finder(options, session)
             build_delete = (not (options.no_clean or options.build_dir))
             wheel_cache = WheelCache(options.cache_dir, options.format_control)
+
             if options.cache_dir and not check_path_owner(options.cache_dir):
                 logger.warning(
                     "The directory '%s' or its parent directory is not owned "
@@ -249,18 +249,19 @@ class InstallCommand(RequirementCommand):
                     use_user_site=options.use_user_site,
                 )
 
-                self.populate_requirement_set(
-                    requirement_set, args, options, finder, session, self.name,
-                    wheel_cache
-                )
-                preparer = RequirementPreparer(
-                    build_dir=directory.path,
-                    src_dir=options.src_dir,
-                    download_dir=None,
-                    wheel_download_dir=None,
-                    progress_bar=options.progress_bar,
-                )
                 try:
+                    self.populate_requirement_set(
+                        requirement_set, args, options, finder, session,
+                        self.name, wheel_cache
+                    )
+                    preparer = RequirementPreparer(
+                        build_dir=directory.path,
+                        src_dir=options.src_dir,
+                        download_dir=None,
+                        wheel_download_dir=None,
+                        progress_bar=options.progress_bar,
+                    )
+
                     resolver = Resolver(
                         preparer=preparer,
                         finder=finder,
@@ -349,6 +350,7 @@ class InstallCommand(RequirementCommand):
                     # Clean up
                     if not options.no_clean:
                         requirement_set.cleanup_files()
+                        wheel_cache.cleanup()
 
         if options.target_dir:
             self._handle_target_dir(

src/pip/_internal/commands/wheel.py

@@ -146,35 +146,35 @@ class WheelCommand(RequirementCommand):
                     require_hashes=options.require_hashes,
                 )
 
-                self.populate_requirement_set(
-                    requirement_set, args, options, finder, session, self.name,
-                    wheel_cache
-                )
-
-                preparer = RequirementPreparer(
-                    build_dir=directory.path,
-                    src_dir=options.src_dir,
-                    download_dir=None,
-                    wheel_download_dir=options.wheel_dir,
-                    progress_bar=options.progress_bar,
-                )
-
-                resolver = Resolver(
-                    preparer=preparer,
-                    finder=finder,
-                    session=session,
-                    wheel_cache=wheel_cache,
-                    use_user_site=False,
-                    upgrade_strategy="to-satisfy-only",
-                    force_reinstall=False,
-                    ignore_dependencies=options.ignore_dependencies,
-                    ignore_requires_python=options.ignore_requires_python,
-                    ignore_installed=True,
-                    isolated=options.isolated_mode,
-                )
-                resolver.resolve(requirement_set)
-
                 try:
+                    self.populate_requirement_set(
+                        requirement_set, args, options, finder, session,
+                        self.name, wheel_cache
+                    )
+
+                    preparer = RequirementPreparer(
+                        build_dir=directory.path,
+                        src_dir=options.src_dir,
+                        download_dir=None,
+                        wheel_download_dir=options.wheel_dir,
+                        progress_bar=options.progress_bar,
+                    )
+
+                    resolver = Resolver(
+                        preparer=preparer,
+                        finder=finder,
+                        session=session,
+                        wheel_cache=wheel_cache,
+                        use_user_site=False,
+                        upgrade_strategy="to-satisfy-only",
+                        force_reinstall=False,
+                        ignore_dependencies=options.ignore_dependencies,
+                        ignore_requires_python=options.ignore_requires_python,
+                        ignore_installed=True,
+                        isolated=options.isolated_mode,
+                    )
+                    resolver.resolve(requirement_set)
+
                     # build wheels
                     wb = WheelBuilder(
                         finder, preparer, wheel_cache,
@@ -195,3 +195,4 @@ class WheelCommand(RequirementCommand):
                 finally:
                     if not options.no_clean:
                         requirement_set.cleanup_files()
+                        wheel_cache.cleanup()

src/pip/_internal/configuration.py

@@ -11,6 +11,7 @@ Some terminology:
   A single word describing where the configuration key-value pair came from
 """
 
+import locale
 import logging
 import os
 
@@ -283,8 +284,14 @@ class Configuration(object):
         # Doing this is useful when modifying and saving files, where we don't
         # need to construct a parser.
         if os.path.exists(fname):
-            parser.read(fname)
-
+            try:
+                parser.read(fname)
+            except UnicodeDecodeError:
+                raise ConfigurationError((
+                    "ERROR: "
+                    "Configuration file contains invalid %s characters.\n"
+                    "Please fix your configuration, located at %s\n"
+                ) % (locale.getpreferredencoding(False), fname))
         return parser
 
     def _load_environment_vars(self):

src/pip/_internal/resolve.py

@@ -220,7 +220,9 @@ class Resolver(object):
         if req.satisfied_by:
             should_modify = (
                 self.upgrade_strategy != "to-satisfy-only" or
-                self.force_reinstall or self.ignore_installed
+                self.force_reinstall or
+                self.ignore_installed or
+                req.link.scheme == 'file'
             )
             if should_modify:
                 self._set_req_to_reinstall(req)

src/pip/_internal/utils/misc.py

@@ -648,9 +648,10 @@ def call_subprocess(cmd, show_stdout=True, cwd=None,
         env.pop(name, None)
     try:
         proc = subprocess.Popen(
-            cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout,
-            cwd=cwd, env=env,
+            cmd, stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
+            stdout=stdout, cwd=cwd, env=env,
         )
+        proc.stdin.close()
     except Exception as exc:
         logger.critical(
             "Error %s while executing command %s", exc, command_desc,

src/pip/_internal/wheel.py

@@ -806,6 +806,7 @@ class WheelBuilder(object):
 
         buildset = []
         for req in requirements:
+            ephem_cache = False
             if req.constraint:
                 continue
             if req.is_wheel:
@@ -816,7 +817,8 @@ class WheelBuilder(object):
             elif autobuilding and req.editable:
                 pass
             elif autobuilding and req.link and not req.link.is_artifact:
-                pass
+                # VCS checkout. Build wheel just for this run.
+                ephem_cache = True
             elif autobuilding and not req.source_dir:
                 pass
             else:
@@ -824,9 +826,8 @@ class WheelBuilder(object):
                     link = req.link
                     base, ext = link.splitext()
                     if index.egg_info_matches(base, None, link) is None:
-                        # Doesn't look like a package - don't autobuild a wheel
-                        # because we'll have no way to lookup the result sanely
-                        continue
+                        # E.g. local directory. Build wheel just for this run.
+                        ephem_cache = True
                     if "binary" not in index.fmt_ctl_formats(
                             self.finder.format_control,
                             canonicalize_name(req.name)):
@@ -835,7 +836,7 @@ class WheelBuilder(object):
                             "being disabled for it.", req.name,
                         )
                         continue
-                buildset.append(req)
+                buildset.append((req, ephem_cache))
 
         if not buildset:
             return True
@@ -843,15 +844,19 @@ class WheelBuilder(object):
         # Build the wheels.
         logger.info(
             'Building wheels for collected packages: %s',
-            ', '.join([req.name for req in buildset]),
+            ', '.join([req.name for (req, _) in buildset]),
         )
+        _cache = self.wheel_cache  # shorter name
         with indent_log():
             build_success, build_failure = [], []
-            for req in buildset:
+            for req, ephem in buildset:
                 python_tag = None
                 if autobuilding:
                     python_tag = pep425tags.implementation_tag
-                    output_dir = self.wheel_cache.get_path_for_link(req.link)
+                    if ephem:
+                        output_dir = _cache.get_ephem_path_for_link(req.link)
+                    else:
+                        output_dir = _cache.get_path_for_link(req.link)
                     try:
                         ensure_dir(output_dir)
                     except OSError as e:

src/pip/_vendor/appdirs.py

@@ -557,18 +557,14 @@ def _get_win_folder_with_jna(csidl_name):
 
 if system == "win32":
     try:
-        import win32com.shell
-        _get_win_folder = _get_win_folder_with_pywin32
+        from ctypes import windll
+        _get_win_folder = _get_win_folder_with_ctypes
     except ImportError:
         try:
-            from ctypes import windll
-            _get_win_folder = _get_win_folder_with_ctypes
+            import com.sun.jna
+            _get_win_folder = _get_win_folder_with_jna
         except ImportError:
-            try:
-                import com.sun.jna
-                _get_win_folder = _get_win_folder_with_jna
-            except ImportError:
-                _get_win_folder = _get_win_folder_from_registry
+            _get_win_folder = _get_win_folder_from_registry
 
 
 #---- self test code

src/pip/_vendor/appdirs.pyi

@@ -1 +0,0 @@
-from appdirs import *

tasks/vendoring/__init__.py

@@ -144,9 +144,11 @@ def update_stubs(ctx):
 
     print("[vendoring.update_stubs] Add mypy stubs")
 
-    # Some projects need stubs other than a simple <name>.pyi
     extra_stubs_needed = {
-        "six": ["six.__init__", "six.moves"]
+        # Some projects need stubs other than a simple <name>.pyi
+        "six": ["six.__init__", "six.moves"],
+        # Some projects should not have stubs coz they're single file modules
+        "appdirs": [],
     }
 
     for lib in vendored_libs:

tasks/vendoring/patches/appdirs.patch

@@ -0,0 +1,28 @@
+diff --git a/src/pip/_vendor/appdirs.py b/src/pip/_vendor/appdirs.py
+index ae67001a..2bd39110 100644
+--- a/src/pip/_vendor/appdirs.py
++++ b/src/pip/_vendor/appdirs.py
+@@ -557,18 +557,14 @@ def _get_win_folder_with_jna(csidl_name):
+
+ if system == "win32":
+     try:
+-        import win32com.shell
+-        _get_win_folder = _get_win_folder_with_pywin32
++        from ctypes import windll
++        _get_win_folder = _get_win_folder_with_ctypes
+     except ImportError:
+         try:
+-            from ctypes import windll
+-            _get_win_folder = _get_win_folder_with_ctypes
++            import com.sun.jna
++            _get_win_folder = _get_win_folder_with_jna
+         except ImportError:
+-            try:
+-                import com.sun.jna
+-                _get_win_folder = _get_win_folder_with_jna
+-            except ImportError:
+-                _get_win_folder = _get_win_folder_from_registry
++            _get_win_folder = _get_win_folder_from_registry
+
+
+ #---- self test code

tests/functional/test_install.py

@@ -994,16 +994,15 @@ def test_install_builds_wheels(script, data, common_wheels):
     # and built wheels for upper and wheelbroken
     assert "Running setup.py bdist_wheel for upper" in str(res), str(res)
     assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res)
-    # But not requires_wheel... which is a local dir and thus uncachable.
-    assert "Running setup.py bdist_wheel for requir" not in str(res), str(res)
+    # Wheels are built for local directories, but not cached.
+    assert "Running setup.py bdist_wheel for requir" in str(res), str(res)
     # wheelbroken has to run install
     # into the cache
     assert wheels != [], str(res)
     # and installed from the wheel
     assert "Running setup.py install for upper" not in str(res), str(res)
-    # the local tree can't build a wheel (because we can't assume that every
-    # build will have a suitable unique key to cache on).
-    assert "Running setup.py install for requires-wheel" in str(res), str(res)
+    # Wheels are built for local directories, but not cached.
+    assert "Running setup.py install for requir" not in str(res), str(res)
     # wheelbroken has to run install
     assert "Running setup.py install for wheelb" in str(res), str(res)
     # We want to make sure we used the correct implementation tag
@@ -1027,13 +1026,12 @@ def test_install_no_binary_disables_building_wheels(
     assert expected in str(res), str(res)
     # and built wheels for wheelbroken only
     assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res)
-    # But not requires_wheel... which is a local dir and thus uncachable.
-    assert "Running setup.py bdist_wheel for requir" not in str(res), str(res)
-    # Nor upper, which was blacklisted
+    # Wheels are built for local directories, but not cached across runs
+    assert "Running setup.py bdist_wheel for requir" in str(res), str(res)
+    # Don't build wheel for upper which was blacklisted
     assert "Running setup.py bdist_wheel for upper" not in str(res), str(res)
-    # the local tree can't build a wheel (because we can't assume that every
-    # build will have a suitable unique key to cache on).
-    assert "Running setup.py install for requires-wheel" in str(res), str(res)
+    # Wheels are built for local directories, but not cached across runs
+    assert "Running setup.py install for requir" not in str(res), str(res)
     # And these two fell back to sdist based installed.
     assert "Running setup.py install for wheelb" in str(res), str(res)
     assert "Running setup.py install for upper" in str(res), str(res)

tests/functional/test_install_cleanup.py

@@ -31,7 +31,7 @@ def test_no_clean_option_blocks_cleaning_after_install(script, data):
     build = script.base_path / 'pip-build'
     script.pip(
         'install', '--no-clean', '--no-index', '--build', build,
-        '--find-links=%s' % data.find_links, 'simple',
+        '--find-links=%s' % data.find_links, 'simple', expect_temp=True,
     )
     assert exists(build)
 
@@ -134,7 +134,7 @@ def test_cleanup_prevented_upon_build_dir_exception(script, data):
     result = script.pip(
         'install', '-f', data.find_links, '--no-index', 'simple',
         '--build', build,
-        expect_error=True,
+        expect_error=True, expect_temp=True,
     )
 
     assert result.returncode == PREVIOUS_BUILD_DIR_ERROR

tests/functional/test_install_user.py

@@ -68,24 +68,29 @@ class Tests_UserSite:
         )
         result.assert_installed('INITools', use_user_site=True)
 
-    def test_install_curdir_usersite(self, script, virtualenv, data):
+    @pytest.mark.network
+    def test_install_from_current_directory_into_usersite(
+            self, script, virtualenv, data, common_wheels):
         """
         Test installing current directory ('.') into usersite
         """
         virtualenv.system_site_packages = True
+        script.pip("install", "wheel", '--no-index', '-f', common_wheels)
+
         run_from = data.packages.join("FSPkg")
         result = script.pip(
             'install', '-vvv', '--user', curdir,
             cwd=run_from,
             expect_error=False,
         )
+
         fspkg_folder = script.user_site / 'fspkg'
-        egg_info_folder = (
-            script.user_site / 'FSPkg-0.1.dev0-py%s.egg-info' % pyversion
-        )
         assert fspkg_folder in result.files_created, result.stdout
 
-        assert egg_info_folder in result.files_created
+        dist_info_folder = (
+            script.user_site / 'FSPkg-0.1.dev0.dist-info'
+        )
+        assert dist_info_folder in result.files_created
 
     def test_install_user_venv_nositepkgs_fails(self, script, data):
         """

tests/functional/test_wheel.py

@@ -193,7 +193,7 @@ def test_pip_wheel_fail_cause_of_previous_build_dir(
     result = script.pip(
         'wheel', '--no-index', '--find-links=%s' % data.find_links,
         '--build', script.venv_path / 'build',
-        'simple==3.0', expect_error=True,
+        'simple==3.0', expect_error=True, expect_temp=True,
     )
 
     # Then I see that the error code is the right one

tests/unit/test_utils.py

@@ -23,8 +23,8 @@ from pip._internal.utils.encoding import auto_decode
 from pip._internal.utils.glibc import check_glibc_version
 from pip._internal.utils.hashes import Hashes, MissingHashes
 from pip._internal.utils.misc import (
-    egg_link_path, ensure_dir, get_installed_distributions, get_prog,
-    normalize_path, rmtree, untar_file, unzip_file,
+    call_subprocess, egg_link_path, ensure_dir, get_installed_distributions,
+    get_prog, normalize_path, rmtree, untar_file, unzip_file,
 )
 from pip._internal.utils.packaging import check_dist_requires_python
 from pip._internal.utils.temp_dir import TempDirectory
@@ -612,3 +612,15 @@ class TestGetProg(object):
             executable
         )
         assert get_prog() == expected
+
+
+def test_call_subprocess_works_okay_when_just_given_nothing():
+    try:
+        call_subprocess([sys.executable, '-c', 'print("Hello")'])
+    except Exception:
+        assert False, "Expected subprocess call to succeed"
+
+
+def test_call_subprocess_closes_stdin():
+    with pytest.raises(InstallationError):
+        call_subprocess([sys.executable, '-c', 'input()'])