Use a secure randomized build directory when possible

CHANGES.txt

@@ -98,6 +98,9 @@
   looking for any file located inside of it instead of relying on the record
   file listing a directory. (:pull:`2076`)
 
+* Fixed :issue:`1964`, :issue:`1935`, :issue:`676`, Use a randomized and secure
+  default build directory when possible. (:pull:`2122`)
+
 
 **1.5.6 (2014-05-16)**
 

pip/cmdoptions.py

@@ -11,9 +11,7 @@ from __future__ import absolute_import
 
 import copy
 from optparse import OptionGroup, SUPPRESS_HELP, Option
-from pip.locations import (
+from pip.locations import CA_BUNDLE_PATH, USER_CACHE_DIR, src_prefix
-    CA_BUNDLE_PATH, USER_CACHE_DIR, build_prefix, src_prefix,
-)
 
 
 def make_option_group(group, parser):
@@ -360,10 +358,8 @@ build_dir = OptionMaker(
     '-b', '--build', '--build-dir', '--build-directory',
     dest='build_dir',
     metavar='dir',
-    default=build_prefix,
+    help='Directory to unpack packages into and build in.'
-    help='Directory to unpack packages into and build in. '
+)
-    'The default in a virtualenv is "<venv path>/build". '
-    'The default for global installs is "<OS temp dir>/pip_build_<username>".')
 
 install_options = OptionMaker(
     '--install-option',

pip/commands/install.py

@@ -7,13 +7,14 @@ import shutil
 import warnings
 
 from pip.req import InstallRequirement, RequirementSet, parse_requirements
-from pip.locations import virtualenv_no_global, distutils_scheme
+from pip.locations import build_prefix, virtualenv_no_global, distutils_scheme
 from pip.basecommand import Command
 from pip.index import PackageFinder
 from pip.exceptions import (
     InstallationError, CommandError, PreviousBuildDirError,
 )
 from pip import cmdoptions
+from pip.utils.build import BuildDirectory
 from pip.utils.deprecation import RemovedInPip7Warning, RemovedInPip8Warning
 
 
@@ -209,7 +210,16 @@ class InstallCommand(Command):
         if options.download_dir:
             options.no_install = True
             options.ignore_installed = True
-        options.build_dir = os.path.abspath(options.build_dir)
+
+        # If we have --no-install or --no-download and no --build we use the
+        # legacy static build dir
+        if (options.build_dir is None
+                and (options.no_install or options.no_download)):
+            options.build_dir = build_prefix
+
+        if options.build_dir:
+            options.build_dir = os.path.abspath(options.build_dir)
+
         options.src_dir = os.path.abspath(options.src_dir)
         install_options = options.install_options or []
         if options.use_user_site:
@@ -268,113 +278,125 @@ class InstallCommand(Command):
 
             finder = self._build_package_finder(options, index_urls, session)
 
-            requirement_set = RequirementSet(
+            build_delete = (not (options.no_clean or options.build_dir))
-                build_dir=options.build_dir,
+            with BuildDirectory(options.build_dir,
-                src_dir=options.src_dir,
+                                delete=build_delete) as build_dir:
-                download_dir=options.download_dir,
+                requirement_set = RequirementSet(
-                upgrade=options.upgrade,
+                    build_dir=build_dir,
-                as_egg=options.as_egg,
+                    src_dir=options.src_dir,
-                ignore_installed=options.ignore_installed,
+                    download_dir=options.download_dir,
-                ignore_dependencies=options.ignore_dependencies,
+                    upgrade=options.upgrade,
-                force_reinstall=options.force_reinstall,
+                    as_egg=options.as_egg,
-                use_user_site=options.use_user_site,
+                    ignore_installed=options.ignore_installed,
-                target_dir=temp_target_dir,
+                    ignore_dependencies=options.ignore_dependencies,
-                session=session,
+                    force_reinstall=options.force_reinstall,
-                pycompile=options.compile,
+                    use_user_site=options.use_user_site,
-            )
+                    target_dir=temp_target_dir,
-            for name in args:
+                    session=session,
-                requirement_set.add_requirement(
+                    pycompile=options.compile,
-                    InstallRequirement.from_line(name, None))
-            for name in options.editables:
-                requirement_set.add_requirement(
-                    InstallRequirement.from_editable(
-                        name,
-                        default_vcs=options.default_vcs
-                    )
                 )
-            for filename in options.requirements:
-                for req in parse_requirements(
-                        filename,
-                        finder=finder, options=options, session=session):
-                    requirement_set.add_requirement(req)
-            if not requirement_set.has_requirements:
-                opts = {'name': self.name}
-                if options.find_links:
-                    msg = ('You must give at least one requirement to %(name)s'
-                           ' (maybe you meant "pip %(name)s %(links)s"?)' %
-                           dict(opts, links=' '.join(options.find_links)))
-                else:
-                    msg = ('You must give at least one requirement '
-                           'to %(name)s (see "pip help %(name)s")' % opts)
-                logger.warning(msg)
-                return
 
-            try:
+                for name in args:
-                if not options.no_download:
+                    requirement_set.add_requirement(
-                    requirement_set.prepare_files(finder)
+                        InstallRequirement.from_line(name, None))
-                else:
-                    requirement_set.locate_files()
 
-                if not options.no_install:
+                for name in options.editables:
-                    requirement_set.install(
+                    requirement_set.add_requirement(
-                        install_options,
+                        InstallRequirement.from_editable(
-                        global_options,
+                            name,
-                        root=options.root_path,
+                            default_vcs=options.default_vcs
+                        )
                     )
-                    installed = ' '.join([
-                        req.name for req in
-                        requirement_set.successfully_installed
-                    ])
-                    if installed:
-                        logger.info('Successfully installed %s', installed)
-                else:
-                    downloaded = ' '.join([
-                        req.name
-                        for req in requirement_set.successfully_downloaded
-                    ])
-                    if downloaded:
-                        logger.info('Successfully downloaded %s', downloaded)
-            except PreviousBuildDirError:
-                options.no_clean = True
-                raise
-            finally:
-                # Clean up
-                if ((not options.no_clean)
-                        and ((not options.no_install)
-                             or options.download_dir)):
-                    requirement_set.cleanup_files()
 
-            if options.target_dir:
+                for filename in options.requirements:
-                if not os.path.exists(options.target_dir):
+                    for req in parse_requirements(
-                    os.makedirs(options.target_dir)
+                            filename,
-                lib_dir = distutils_scheme('', home=temp_target_dir)['purelib']
+                            finder=finder, options=options, session=session):
-                for item in os.listdir(lib_dir):
+                        requirement_set.add_requirement(req)
-                    target_item_dir = os.path.join(options.target_dir, item)
-                    if os.path.exists(target_item_dir):
-                        if not options.upgrade:
-                            logger.warning(
-                                'Target directory %s already exists. Specify '
-                                '--upgrade to force replacement.',
-                                target_item_dir
-                            )
-                            continue
-                        if os.path.islink(target_item_dir):
-                            logger.warning(
-                                'Target directory %s already exists and is '
-                                'a link. Pip will not automatically replace '
-                                'links, please remove if replacement is '
-                                'desired.',
-                                target_item_dir
-                            )
-                            continue
-                        if os.path.isdir(target_item_dir):
-                            shutil.rmtree(target_item_dir)
-                        else:
-                            os.remove(target_item_dir)
 
-                    shutil.move(
+                if not requirement_set.has_requirements:
-                        os.path.join(lib_dir, item),
+                    opts = {'name': self.name}
-                        target_item_dir
+                    if options.find_links:
-                    )
+                        msg = ('You must give at least one requirement to '
-                shutil.rmtree(temp_target_dir)
+                               '%(name)s (maybe you meant "pip %(name)s '
-            return requirement_set
+                               '%(links)s"?)' %
+                               dict(opts, links=' '.join(options.find_links)))
+                    else:
+                        msg = ('You must give at least one requirement '
+                               'to %(name)s (see "pip help %(name)s")' % opts)
+                    logger.warning(msg)
+                    return
+
+                try:
+                    if not options.no_download:
+                        requirement_set.prepare_files(finder)
+                    else:
+                        requirement_set.locate_files()
+
+                    if not options.no_install:
+                        requirement_set.install(
+                            install_options,
+                            global_options,
+                            root=options.root_path,
+                        )
+                        installed = ' '.join([
+                            req.name for req in
+                            requirement_set.successfully_installed
+                        ])
+                        if installed:
+                            logger.info('Successfully installed %s', installed)
+                    else:
+                        downloaded = ' '.join([
+                            req.name
+                            for req in requirement_set.successfully_downloaded
+                        ])
+                        if downloaded:
+                            logger.info(
+                                'Successfully downloaded %s', downloaded
+                            )
+                except PreviousBuildDirError:
+                    options.no_clean = True
+                    raise
+                finally:
+                    # Clean up
+                    if ((not options.no_clean)
+                            and ((not options.no_install)
+                                 or options.download_dir)):
+                        requirement_set.cleanup_files()
+
+        if options.target_dir:
+            if not os.path.exists(options.target_dir):
+                os.makedirs(options.target_dir)
+
+            lib_dir = distutils_scheme('', home=temp_target_dir)['purelib']
+
+            for item in os.listdir(lib_dir):
+                target_item_dir = os.path.join(options.target_dir, item)
+                if os.path.exists(target_item_dir):
+                    if not options.upgrade:
+                        logger.warning(
+                            'Target directory %s already exists. Specify '
+                            '--upgrade to force replacement.',
+                            target_item_dir
+                        )
+                        continue
+                    if os.path.islink(target_item_dir):
+                        logger.warning(
+                            'Target directory %s already exists and is '
+                            'a link. Pip will not automatically replace '
+                            'links, please remove if replacement is '
+                            'desired.',
+                            target_item_dir
+                        )
+                        continue
+                    if os.path.isdir(target_item_dir):
+                        shutil.rmtree(target_item_dir)
+                    else:
+                        os.remove(target_item_dir)
+
+                shutil.move(
+                    os.path.join(lib_dir, item),
+                    target_item_dir
+                )
+            shutil.rmtree(temp_target_dir)
+        return requirement_set

pip/commands/wheel.py

@@ -10,6 +10,7 @@ from pip.index import PackageFinder
 from pip.exceptions import CommandError, PreviousBuildDirError
 from pip.req import InstallRequirement, RequirementSet, parse_requirements
 from pip.utils import normalize_path
+from pip.utils.build import BuildDirectory
 from pip.utils.deprecation import RemovedInPip7Warning, RemovedInPip8Warning
 from pip.wheel import WheelBuilder
 from pip import cmdoptions
@@ -157,6 +158,9 @@ class WheelCommand(Command):
                 RemovedInPip8Warning,
             )
 
+        if options.build_dir:
+            options.build_dir = os.path.abspath(options.build_dir)
+
         with self._build_session(options) as session:
 
             finder = PackageFinder(
@@ -171,63 +175,67 @@ class WheelCommand(Command):
                 session=session,
             )
 
-            options.build_dir = os.path.abspath(options.build_dir)
+            build_delete = (not (options.no_clean or options.build_dir))
-            requirement_set = RequirementSet(
+            with BuildDirectory(options.build_dir,
-                build_dir=options.build_dir,
+                                delete=build_delete) as build_dir:
-                src_dir=options.src_dir,
+                requirement_set = RequirementSet(
-                download_dir=None,
+                    build_dir=build_dir,
-                ignore_dependencies=options.ignore_dependencies,
+                    src_dir=options.src_dir,
-                ignore_installed=True,
+                    download_dir=None,
-                session=session,
+                    ignore_dependencies=options.ignore_dependencies,
-                wheel_download_dir=options.wheel_dir
+                    ignore_installed=True,
-            )
+                    session=session,
+                    wheel_download_dir=options.wheel_dir
+                )
 
-            # make the wheelhouse
+                # make the wheelhouse
-            if not os.path.exists(options.wheel_dir):
+                if not os.path.exists(options.wheel_dir):
-                os.makedirs(options.wheel_dir)
+                    os.makedirs(options.wheel_dir)
 
-            # parse args and/or requirements files
+                # parse args and/or requirements files
-            for name in args:
+                for name in args:
-                requirement_set.add_requirement(
+                    requirement_set.add_requirement(
-                    InstallRequirement.from_line(name, None))
+                        InstallRequirement.from_line(name, None))
-            for name in options.editables:
+                for name in options.editables:
-                requirement_set.add_requirement(
+                    requirement_set.add_requirement(
-                    InstallRequirement.from_editable(
+                        InstallRequirement.from_editable(
-                        name,
+                            name,
-                        default_vcs=options.default_vcs
+                            default_vcs=options.default_vcs
+                        )
                     )
-                )
+                for filename in options.requirements:
-            for filename in options.requirements:
+                    for req in parse_requirements(
-                for req in parse_requirements(
+                            filename,
-                        filename,
+                            finder=finder,
-                        finder=finder,
+                            options=options,
-                        options=options,
+                            session=session):
-                        session=session):
+                        requirement_set.add_requirement(req)
-                    requirement_set.add_requirement(req)
 
-            # fail if no requirements
+                # fail if no requirements
-            if not requirement_set.has_requirements:
+                if not requirement_set.has_requirements:
-                logger.error(
+                    logger.error(
-                    "You must give at least one requirement to %s "
+                        "You must give at least one requirement to %s "
-                    "(see \"pip help %s\")",
+                        "(see \"pip help %s\")",
-                    self.name,
+                        self.name,
-                )
+                    )
-                return
+                    return
 
-            try:
+                try:
-                # build wheels
+                    # build wheels
-                wb = WheelBuilder(
+                    wb = WheelBuilder(
-                    requirement_set,
+                        requirement_set,
-                    finder,
+                        finder,
-                    options.wheel_dir,
+                        options.wheel_dir,
-                    build_options=options.build_options or [],
+                        build_options=options.build_options or [],
-                    global_options=options.global_options or [],
+                        global_options=options.global_options or [],
-                )
+                    )
-                if not wb.build():
+                    if not wb.build():
-                    raise CommandError("Failed to build one or more wheels")
+                        raise CommandError(
-            except PreviousBuildDirError:
+                            "Failed to build one or more wheels"
-                options.no_clean = True
+                        )
-                raise
+                except PreviousBuildDirError:
-            finally:
+                    options.no_clean = True
-                if not options.no_clean:
+                    raise
-                    requirement_set.cleanup_files()
+                finally:
+                    if not options.no_clean:
+                        requirement_set.cleanup_files()

pip/utils/build.py

@@ -0,0 +1,37 @@
+from __future__ import absolute_import
+
+import tempfile
+
+from pip.utils import rmtree
+
+
+class BuildDirectory(object):
+
+    def __init__(self, name=None, delete=None):
+        # If we were not given an explicit directory, and we were not given an
+        # explicit delete option, then we'll default to deleting.
+        if name is None and delete is None:
+            delete = True
+
+        if name is None:
+            name = tempfile.mkdtemp(prefix="pip-build-")
+            # If we were not given an explicit directory, and we were not given
+            # an explicit delete option, then we'll default to deleting.
+            if delete is None:
+                delete = True
+
+        self.name = name
+        self.delete = delete
+
+    def __repr__(self):
+        return "<{} {!r}>".format(self.__class__.__name__, self.name)
+
+    def __enter__(self):
+        return self.name
+
+    def __exit__(self, exc, value, tb):
+        self.cleanup()
+
+    def cleanup(self):
+        if self.delete:
+            rmtree(self.name)

tests/functional/test_install_cleanup.py

@@ -26,12 +26,12 @@ def test_no_clean_option_blocks_cleaning_after_install(script, data):
     """
     Test --no-clean option blocks cleaning after install
     """
-    result = script.pip(
+    build = script.base_path / 'pip-build'
-        'install', '--no-clean', '--no-index',
+    script.pip(
-        '--find-links=%s' % data.find_links, 'simple'
+        'install', '--no-clean', '--no-index', '--build', build,
+        '--find-links=%s' % data.find_links, 'simple',
     )
-    build = script.venv_path / 'build' / 'simple'
+    assert exists(build)
-    assert exists(build), "build/simple should still exist %s" % str(result)
 
 
 def test_cleanup_after_install_editable_from_hg(script, tmpdir):
@@ -157,15 +157,17 @@ def test_cleanup_prevented_upon_build_dir_exception(script, data):
     """
     Test no cleanup occurs after a PreviousBuildDirError
     """
-    build = script.venv_path / 'build' / 'simple'
+    build = script.venv_path / 'build'
-    os.makedirs(build)
+    build_simple = build / 'simple'
-    write_delete_marker_file(script.venv_path / 'build')
+    os.makedirs(build_simple)
-    build.join("setup.py").write("#")
+    write_delete_marker_file(build)
+    build_simple.join("setup.py").write("#")
     result = script.pip(
         'install', '-f', data.find_links, '--no-index', 'simple',
+        '--build', build,
         expect_error=True,
     )
 
     assert result.returncode == PREVIOUS_BUILD_DIR_ERROR
     assert "pip can't proceed" in result.stdout, result.stdout
-    assert exists(build)
+    assert exists(build_simple)

tests/functional/test_wheel.py

@@ -86,11 +86,12 @@ def test_no_clean_option_blocks_cleaning_after_wheel(script, data):
     Test --no-clean option blocks cleaning after wheel build
     """
     script.pip('install', 'wheel')
+    build = script.venv_path / 'build'
     result = script.pip(
-        'wheel', '--no-clean', '--no-index',
+        'wheel', '--no-clean', '--no-index', '--build', build,
         '--find-links=%s' % data.find_links, 'simple',
     )
-    build = script.venv_path / 'build' / 'simple'
+    build = build / 'simple'
     assert exists(build), "build/simple should still exist %s" % str(result)
 
 
@@ -128,6 +129,7 @@ def test_pip_wheel_fail_cause_of_previous_build_dir(script, data):
     # When I call pip trying to install things again
     result = script.pip(
         'wheel', '--no-index', '--find-links=%s' % data.find_links,
+        '--build', script.venv_path / 'build',
         'simple==3.0', expect_error=True,
     )