diff --git a/.gitignore b/.gitignore index dfa41c022..276d24d4d 100644 --- a/.gitignore +++ b/.gitignore @@ -27,13 +27,18 @@ nosetests.xml coverage.xml *.cover tests/data/common_wheels/ +pip-wheel-metadata # Misc *~ .*.sw? +.env/ # For IntelliJ IDEs (basically PyCharm) .idea/ +# For Visual Studio Code +.vscode/ + # Scratch Pad for experiments .scratch/ diff --git a/AUTHORS.txt b/AUTHORS.txt index 5a312d62f..cf6c11036 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -226,6 +226,7 @@ Joseph Long Josh Bronson Josh Hansen Josh Schneier +Julian Berman Julien Demoor jwg4 Jyrki Pulliainen @@ -306,6 +307,7 @@ Nathaniel J. Smith Nehal J Wani Nick Coghlan Nick Stenning +Nick Timkovich Nikhil Benesch Nitesh Sharma Nowell Strite @@ -340,6 +342,7 @@ Phaneendra Chiruvella Phil Freo Phil Pennock Phil Whelan +Philip Jägenstedt Philip Molloy Philippe Ombredanne Pi Delport diff --git a/NEWS.rst b/NEWS.rst index 6ea4be59d..3f98c60ae 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -7,6 +7,46 @@ .. towncrier release notes start +19.0.3 (2019-02-20) +=================== + +Bug Fixes +--------- + +- Fix an ``IndexError`` crash when a legacy build of a wheel fails. (`#6252 `_) +- Fix a regression introduced in 19.0.2 where the filename in a RECORD file + of an installed file would not be updated when installing a wheel. (`#6266 `_) + + +19.0.2 (2019-02-09) +=================== + +Bug Fixes +--------- + +- Fix a crash where PEP 517-based builds using ``--no-cache-dir`` would fail in + some circumstances with an ``AssertionError`` due to not finalizing a build + directory internally. (`#6197 `_) +- Provide a better error message if attempting an editable install of a + directory with a ``pyproject.toml`` but no ``setup.py``. (`#6170 `_) +- The implicit default backend used for projects that provide a ``pyproject.toml`` + file without explicitly specifying ``build-backend`` now behaves more like direct + execution of ``setup.py``, and hence should restore compatibility with projects + that were unable to be installed with ``pip`` 19.0. This raised the minimum + required version of ``setuptools`` for such builds to 40.8.0. (`#6163 `_) +- Allow ``RECORD`` lines with more than three elements, and display a warning. (`#6165 `_) +- ``AdjacentTempDirectory`` fails on unwritable directory instead of locking up the uninstall command. (`#6169 `_) +- Make failed uninstalls roll back more reliably and better at avoiding naming conflicts. (`#6194 `_) +- Ensure the correct wheel file is copied when building PEP 517 distribution is built. (`#6196 `_) +- The Python 2 end of life warning now only shows on CPython, which is the + implementation that has announced end of life plans. (`#6207 `_) + +Improved Documentation +---------------------- + +- Re-write README and documentation index (`#5815 `_) + + 19.0.1 (2019-01-23) =================== diff --git a/README.rst b/README.rst index b9fa08246..660ecc3f7 100644 --- a/README.rst +++ b/README.rst @@ -1,52 +1,46 @@ -pip -=== - -The `PyPA recommended`_ tool for installing Python packages. +pip - The Python Package Installer +================================== .. image:: https://img.shields.io/pypi/v/pip.svg :target: https://pypi.org/project/pip/ -.. image:: https://img.shields.io/travis/pypa/pip/master.svg?label=travis-ci - :target: https://travis-ci.org/pypa/pip - -.. image:: https://img.shields.io/appveyor/ci/pypa/pip.svg?label=appveyor-ci - :target: https://ci.appveyor.com/project/pypa/pip/history - .. image:: https://readthedocs.org/projects/pip/badge/?version=latest :target: https://pip.pypa.io/en/latest -.. image:: https://dev.azure.com/pypa/pip/_apis/build/status/Linux?branchName=master&label=Windows - :target: https://dev.azure.com/pypa/pip/_build/latest?definitionId=6 +pip is the `package installer`_ for Python. You can use pip to install packages from the `Python Package Index`_ and other indexes. -.. image:: https://dev.azure.com/pypa/pip/_apis/build/status/macOS?branchName=master&label=macOS - :target: https://dev.azure.com/pypa/pip/_build/latest?definitionId=7 - -.. image:: https://dev.azure.com/pypa/pip/_apis/build/status/Linux?branchName=master&label=Linux - :target: https://dev.azure.com/pypa/pip/_build/latest?definitionId=4 +Please take a look at our documentation for how to install and use pip: * `Installation`_ -* `Documentation`_ -* `Changelog`_ -* `GitHub Page`_ -* `Issue Tracking`_ -* `User mailing list`_ -* `Dev mailing list`_ +* `Usage`_ +* `Release notes`_ + +If you find bugs, need help, or want to talk to the developers please use our mailing lists or chat rooms: + +* `Issue tracking`_ +* `Discourse channel`_ * `User IRC`_ + +If you want to get involved head over to GitHub to get the source code and feel free to jump on the developer mailing lists and chat rooms: + +* `GitHub page`_ +* `Dev mailing list`_ * `Dev IRC`_ Code of Conduct --------------- Everyone interacting in the pip project's codebases, issue trackers, chat -rooms and mailing lists is expected to follow the `PyPA Code of Conduct`_. +rooms, and mailing lists is expected to follow the `PyPA Code of Conduct`_. -.. _PyPA recommended: https://packaging.python.org/en/latest/current/ +.. _package installer: https://packaging.python.org/en/latest/current/ +.. _Python Package Index: https://pypi.org .. _Installation: https://pip.pypa.io/en/stable/installing.html -.. _Documentation: https://pip.pypa.io/en/stable/ -.. _Changelog: https://pip.pypa.io/en/stable/news.html -.. _GitHub Page: https://github.com/pypa/pip -.. _Issue Tracking: https://github.com/pypa/pip/issues -.. _User mailing list: https://groups.google.com/forum/#!forum/python-virtualenv +.. _Usage: https://pip.pypa.io/en/stable/ +.. _Release notes: https://pip.pypa.io/en/stable/news.html +.. _GitHub page: https://github.com/pypa/pip +.. _Issue tracking: https://github.com/pypa/pip/issues +.. _Discourse channel: https://discuss.python.org/c/packaging .. _Dev mailing list: https://groups.google.com/forum/#!forum/pypa-dev .. _User IRC: https://webchat.freenode.net/?channels=%23pypa .. _Dev IRC: https://webchat.freenode.net/?channels=%23pypa-dev diff --git a/docs/html/index.rst b/docs/html/index.rst index 64bbb79f3..5ce442de0 100644 --- a/docs/html/index.rst +++ b/docs/html/index.rst @@ -1,15 +1,9 @@ -pip -=== +pip - The Python Package Installer +================================== -`User list `_ | -`Dev list `_ | -`GitHub `_ | -`PyPI `_ | -User IRC: #pypa | -Dev IRC: #pypa-dev +pip is the `package installer`_ for Python. You can use pip to install packages from the `Python Package Index`_ and other indexes. -The `PyPA recommended `_ tool -for installing Python packages. +Please take a look at our documentation for how to install and use pip: .. toctree:: :maxdepth: 1 @@ -20,3 +14,34 @@ for installing Python packages. reference/index development/index news + +If you find bugs, need help, or want to talk to the developers please use our mailing lists or chat rooms: + +* `Issue tracking`_ +* `Discourse channel`_ +* `User IRC`_ + +If you want to get involved head over to GitHub to get the source code and feel free to jump on the developer mailing lists and chat rooms: + +* `GitHub page`_ +* `Dev mailing list`_ +* `Dev IRC`_ + +Code of Conduct +--------------- + +Everyone interacting in the pip project's codebases, issue trackers, chat +rooms, and mailing lists is expected to follow the `PyPA Code of Conduct`_. + +.. _package installer: https://packaging.python.org/en/latest/current/ +.. _Python Package Index: https://pypi.org +.. _Installation: https://pip.pypa.io/en/stable/installing.html +.. _Documentation: https://pip.pypa.io/en/stable/ +.. _Changelog: https://pip.pypa.io/en/stable/news.html +.. _GitHub page: https://github.com/pypa/pip +.. _Issue tracking: https://github.com/pypa/pip/issues +.. _Discourse channel: https://discuss.python.org/c/packaging +.. _Dev mailing list: https://groups.google.com/forum/#!forum/pypa-dev +.. _User IRC: https://webchat.freenode.net/?channels=%23pypa +.. _Dev IRC: https://webchat.freenode.net/?channels=%23pypa-dev +.. _PyPA Code of Conduct: https://www.pypa.io/en/latest/code-of-conduct/ diff --git a/docs/html/reference/pip.rst b/docs/html/reference/pip.rst index e3c0f44a4..17b94d728 100644 --- a/docs/html/reference/pip.rst +++ b/docs/html/reference/pip.rst @@ -145,26 +145,28 @@ explicitly manage the build environment. For such workflows, build isolation can be problematic. If this is the case, pip provides a ``--no-build-isolation`` flag to disable build isolation. Users supplying this flag are responsible for ensuring the build environment is managed -appropriately. +appropriately (including ensuring that all required build dependencies are +installed). -By default, pip will continue to use the legacy (``setuptools`` based) build -processing for projects that do not have a ``pyproject.toml`` file. Projects -with a ``pyproject.toml`` file will use a :pep:`517` backend. Projects with -a ``pyproject.toml`` file, but which don't have a ``build-system`` section, +By default, pip will continue to use the legacy (direct ``setup.py`` execution +based) build processing for projects that do not have a ``pyproject.toml`` file. +Projects with a ``pyproject.toml`` file will use a :pep:`517` backend. Projects +with a ``pyproject.toml`` file, but which don't have a ``build-system`` section, will be assumed to have the following backend settings:: [build-system] - requires = ["setuptools>=40.2.0", "wheel"] - build-backend = "setuptools.build_meta" + requires = ["setuptools>=40.8.0", "wheel"] + build-backend = "setuptools.build_meta:__legacy__" .. note:: - ``setuptools`` 40.2.0 is the first version of setuptools with full - :pep:`517` support. - -If a project has ``[build-system]``, but no ``build-backend``, pip will use -``setuptools.build_meta``, but will assume the project requirements include -``setuptools>=40.2.0`` and ``wheel`` (and will report an error if not). + ``setuptools`` 40.8.0 is the first version of setuptools that offers a + :pep:`517` backend that closely mimics directly executing ``setup.py``. + +If a project has ``[build-system]``, but no ``build-backend``, pip will also use +``setuptools.build_meta:__legacy__``, but will expect the project requirements +to include ``setuptools`` and ``wheel`` (and will report an error if the +installed version of ``setuptools`` is not recent enough). If a user wants to explicitly request :pep:`517` handling even though a project doesn't have a ``pyproject.toml`` file, this can be done using the diff --git a/news/6165.bugfix b/news/6165.bugfix deleted file mode 100644 index 2031b40e4..000000000 --- a/news/6165.bugfix +++ /dev/null @@ -1 +0,0 @@ -Allow ``RECORD`` lines with more than three elements, and display a warning. diff --git a/setup.cfg b/setup.cfg index 3dc5c1829..eb870bb5e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,6 +20,7 @@ exclude = _vendor, data select = E,W,F +ignore = W504 [mypy] follow_imports = silent diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index d744cc78a..89830e72b 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -18,8 +18,8 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.ui import open_spinner if MYPY_CHECK_RUNNING: - from typing import Tuple, Set, Iterable, Optional, List # noqa: F401 - from pip._internal.index import PackageFinder # noqa: F401 + from typing import Tuple, Set, Iterable, Optional, List + from pip._internal.index import PackageFinder logger = logging.getLogger(__name__) @@ -192,7 +192,7 @@ class BuildEnvironment(object): args.append('--') args.extend(requirements) with open_spinner(message) as spinner: - call_subprocess(args, show_stdout=False, spinner=spinner) + call_subprocess(args, spinner=spinner) class NoOpBuildEnvironment(BuildEnvironment): diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index eb295c4e7..9379343c2 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -16,8 +16,8 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.wheel import InvalidWheelFilename, Wheel if MYPY_CHECK_RUNNING: - from typing import Optional, Set, List, Any # noqa: F401 - from pip._internal.index import FormatControl # noqa: F401 + from typing import Optional, Set, List, Any + from pip._internal.index import FormatControl logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py index 975f3fe2c..659686f35 100644 --- a/src/pip/_internal/cli/base_command.py +++ b/src/pip/_internal/cli/base_command.py @@ -5,6 +5,7 @@ import logging import logging.config import optparse import os +import platform import sys import traceback @@ -36,10 +37,10 @@ from pip._internal.utils.outdated import pip_version_check from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, List, Tuple, Any # noqa: F401 - from optparse import Values # noqa: F401 - from pip._internal.cache import WheelCache # noqa: F401 - from pip._internal.req.req_set import RequirementSet # noqa: F401 + from typing import Optional, List, Tuple, Any + from optparse import Values + from pip._internal.cache import WheelCache + from pip._internal.req.req_set import RequirementSet __all__ = ['Command'] @@ -145,14 +146,16 @@ class Command(object): gone_in='19.2', ) elif sys.version_info[:2] == (2, 7): - deprecated( - "Python 2.7 will reach the end of its life on January 1st, " - "2020. Please upgrade your Python as Python 2.7 won't be " - "maintained after that date. A future version of pip will " - "drop support for Python 2.7.", - replacement=None, - gone_in=None, + message = ( + "A future version of pip will drop support for Python 2.7." ) + if platform.python_implementation() == "CPython": + message = ( + "Python 2.7 will reach the end of its life on January " + "1st, 2020. Please upgrade your Python as Python 2.7 " + "won't be maintained after that date. " + ) + message + deprecated(message, replacement=None, gone_in=None) # TODO: Try to get these passing down from the command? # without resorting to os.environ to hold these. diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index 5cf5ee970..5a9180a6b 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -24,9 +24,9 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.ui import BAR_TYPES if MYPY_CHECK_RUNNING: - from typing import Any, Callable, Dict, List, Optional, Union # noqa: F401 - from optparse import OptionParser, Values # noqa: F401 - from pip._internal.cli.parser import ConfigOptionParser # noqa: F401 + from typing import Any, Callable, Dict, Optional + from optparse import OptionParser, Values + from pip._internal.cli.parser import ConfigOptionParser def raise_option_error(parser, option, msg): @@ -729,7 +729,7 @@ def _merge_hash(option, opt_str, value, parser): """Given a value spelled "algo:digest", append the digest to a list pointed to in a dict by the algo name.""" if not parser.values.hashes: - parser.values.hashes = {} # type: ignore + parser.values.hashes = {} try: algo, digest = value.split(':', 1) except ValueError: diff --git a/src/pip/_internal/cli/main_parser.py b/src/pip/_internal/cli/main_parser.py index b17c74928..767f35d50 100644 --- a/src/pip/_internal/cli/main_parser.py +++ b/src/pip/_internal/cli/main_parser.py @@ -17,7 +17,7 @@ from pip._internal.utils.misc import get_prog from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Tuple, List # noqa: F401 + from typing import Tuple, List __all__ = ["create_main_parser", "parse_command"] diff --git a/src/pip/_internal/commands/__init__.py b/src/pip/_internal/commands/__init__.py index c7d1da3d9..2e90db34f 100644 --- a/src/pip/_internal/commands/__init__.py +++ b/src/pip/_internal/commands/__init__.py @@ -20,8 +20,8 @@ from pip._internal.commands.wheel import WheelCommand from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import List, Type # noqa: F401 - from pip._internal.cli.base_command import Command # noqa: F401 + from typing import List, Type + from pip._internal.cli.base_command import Command commands_order = [ InstallCommand, diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 1058a56d9..1c244d230 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -359,7 +359,7 @@ class InstallCommand(RequirementCommand): # so we fail here. if build_failures: raise InstallationError( - "Could not build wheels for {} which use" + + "Could not build wheels for {} which use" " PEP 517 and cannot be installed directly".format( ", ".join(r.name for r in build_failures))) diff --git a/src/pip/_internal/commands/show.py b/src/pip/_internal/commands/show.py index f92c9bc6e..a18a9020c 100644 --- a/src/pip/_internal/commands/show.py +++ b/src/pip/_internal/commands/show.py @@ -2,7 +2,7 @@ from __future__ import absolute_import import logging import os -from email.parser import FeedParser # type: ignore +from email.parser import FeedParser from pip._vendor import pkg_resources from pip._vendor.packaging.utils import canonicalize_name diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index fe6df9b75..1b3d41914 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -29,7 +29,7 @@ from pip._internal.utils.misc import ensure_dir, enum from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( # noqa: F401 + from typing import ( Any, Dict, Iterable, List, NewType, Optional, Tuple ) @@ -216,7 +216,7 @@ class Configuration(object): ensure_dir(os.path.dirname(fname)) with open(fname, "w") as f: - parser.write(f) # type: ignore + parser.write(f) # # Private routines diff --git a/src/pip/_internal/download.py b/src/pip/_internal/download.py index 2bbe1762c..3f5bb0d5e 100644 --- a/src/pip/_internal/download.py +++ b/src/pip/_internal/download.py @@ -48,12 +48,12 @@ from pip._internal.utils.ui import DownloadProgressProvider from pip._internal.vcs import vcs if MYPY_CHECK_RUNNING: - from typing import ( # noqa: F401 + from typing import ( Optional, Tuple, Dict, IO, Text, Union ) - from pip._internal.models.link import Link # noqa: F401 - from pip._internal.utils.hashes import Hashes # noqa: F401 - from pip._internal.vcs import AuthInfo # noqa: F401 + from pip._internal.models.link import Link + from pip._internal.utils.hashes import Hashes + from pip._internal.vcs import AuthInfo try: import ssl # noqa @@ -795,7 +795,7 @@ def _copy_dist_from_dir(link_path, location): logger.info('Running setup.py sdist for %s', link_path) with indent_log(): - call_subprocess(sdist_args, cwd=link_path, show_stdout=False) + call_subprocess(sdist_args, cwd=link_path) # unpack sdist into `location` sdist = os.path.join(location, os.listdir(location)[0]) diff --git a/src/pip/_internal/exceptions.py b/src/pip/_internal/exceptions.py index 38ceeea92..7b291a1e4 100644 --- a/src/pip/_internal/exceptions.py +++ b/src/pip/_internal/exceptions.py @@ -8,8 +8,8 @@ from pip._vendor.six import iteritems from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional # noqa: F401 - from pip._internal.req.req_install import InstallRequirement # noqa: F401 + from typing import Optional + from pip._internal.req.req_install import InstallRequirement class PipError(Exception): diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py index 9eda3a351..b6be553b4 100644 --- a/src/pip/_internal/index.py +++ b/src/pip/_internal/index.py @@ -41,15 +41,15 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.wheel import Wheel if MYPY_CHECK_RUNNING: - from logging import Logger # noqa: F401 - from typing import ( # noqa: F401 + from logging import Logger + from typing import ( Tuple, Optional, Any, List, Union, Callable, Set, Sequence, Iterable, MutableMapping ) - from pip._vendor.packaging.version import _BaseVersion # noqa: F401 - from pip._vendor.requests import Response # noqa: F401 - from pip._internal.req import InstallRequirement # noqa: F401 - from pip._internal.download import PipSession # noqa: F401 + from pip._vendor.packaging.version import _BaseVersion + from pip._vendor.requests import Response + from pip._internal.req import InstallRequirement + from pip._internal.download import PipSession SecureOrigin = Tuple[str, str, Optional[str]] BuildTag = Tuple[Any, ...] # either emply tuple or Tuple[int, str] @@ -227,7 +227,7 @@ def _get_html_page(link, session=None): try: resp = _get_html_response(url, session=session) - except _NotHTTP as exc: + except _NotHTTP: logger.debug( 'Skipping page %s because it looks like an archive, and cannot ' 'be checked by HEAD.', link, diff --git a/src/pip/_internal/locations.py b/src/pip/_internal/locations.py index c6e2a3e48..8fd84c3d8 100644 --- a/src/pip/_internal/locations.py +++ b/src/pip/_internal/locations.py @@ -15,7 +15,7 @@ from pip._internal.utils.compat import WINDOWS, expanduser from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any, Union, Dict, List, Optional # noqa: F401 + from typing import Any, Union, Dict, List, Optional # Application Directories diff --git a/src/pip/_internal/models/candidate.py b/src/pip/_internal/models/candidate.py index 4475458ab..b66c3657f 100644 --- a/src/pip/_internal/models/candidate.py +++ b/src/pip/_internal/models/candidate.py @@ -4,9 +4,9 @@ from pip._internal.utils.models import KeyBasedCompareMixin from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from pip._vendor.packaging.version import _BaseVersion # noqa: F401 - from pip._internal.models.link import Link # noqa: F401 - from typing import Any, Union # noqa: F401 + from pip._vendor.packaging.version import _BaseVersion + from pip._internal.models.link import Link + from typing import Any class InstallationCandidate(KeyBasedCompareMixin): diff --git a/src/pip/_internal/models/format_control.py b/src/pip/_internal/models/format_control.py index 971a3914c..53138e48e 100644 --- a/src/pip/_internal/models/format_control.py +++ b/src/pip/_internal/models/format_control.py @@ -3,7 +3,7 @@ from pip._vendor.packaging.utils import canonicalize_name from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, Set, FrozenSet # noqa: F401 + from typing import Optional, Set, FrozenSet class FormatControl(object): diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index ad2f93e19..2f420760b 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -11,8 +11,8 @@ from pip._internal.utils.models import KeyBasedCompareMixin from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, Tuple, Union, Text # noqa: F401 - from pip._internal.index import HTMLPage # noqa: F401 + from typing import Optional, Tuple, Union + from pip._internal.index import HTMLPage class Link(KeyBasedCompareMixin): diff --git a/src/pip/_internal/operations/check.py b/src/pip/_internal/operations/check.py index 0b56eda45..920df5d48 100644 --- a/src/pip/_internal/operations/check.py +++ b/src/pip/_internal/operations/check.py @@ -14,8 +14,8 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING logger = logging.getLogger(__name__) if MYPY_CHECK_RUNNING: - from pip._internal.req.req_install import InstallRequirement # noqa: F401 - from typing import ( # noqa: F401 + from pip._internal.req.req_install import InstallRequirement + from typing import ( Any, Callable, Dict, Optional, Set, Tuple, List ) diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py index 388bb73ab..0c4c76107 100644 --- a/src/pip/_internal/operations/freeze.py +++ b/src/pip/_internal/operations/freeze.py @@ -20,11 +20,11 @@ from pip._internal.utils.misc import ( from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( # noqa: F401 + from typing import ( Iterator, Optional, List, Container, Set, Dict, Tuple, Iterable, Union ) - from pip._internal.cache import WheelCache # noqa: F401 - from pip._vendor.pkg_resources import ( # noqa: F401 + from pip._internal.cache import WheelCache + from pip._vendor.pkg_resources import ( Distribution, Requirement ) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 4f31dd5a6..d1e9896fb 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -22,11 +22,11 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.vcs import vcs if MYPY_CHECK_RUNNING: - from typing import Any, Optional # noqa: F401 - from pip._internal.req.req_install import InstallRequirement # noqa: F401 - from pip._internal.index import PackageFinder # noqa: F401 - from pip._internal.download import PipSession # noqa: F401 - from pip._internal.req.req_tracker import RequirementTracker # noqa: F401 + from typing import Any, Optional + from pip._internal.req.req_install import InstallRequirement + from pip._internal.index import PackageFinder + from pip._internal.download import PipSession + from pip._internal.req.req_tracker import RequirementTracker logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/pep425tags.py b/src/pip/_internal/pep425tags.py index 1e782d1ae..3b68f28d2 100644 --- a/src/pip/_internal/pep425tags.py +++ b/src/pip/_internal/pep425tags.py @@ -15,7 +15,7 @@ from pip._internal.utils.compat import get_extension_suffixes from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( # noqa: F401 + from typing import ( Tuple, Callable, List, Optional, Union, Dict ) diff --git a/src/pip/_internal/pyproject.py b/src/pip/_internal/pyproject.py index d3e1bbe7d..43efbed42 100644 --- a/src/pip/_internal/pyproject.py +++ b/src/pip/_internal/pyproject.py @@ -2,6 +2,7 @@ from __future__ import absolute_import import io import os +import sys from pip._vendor import pytoml, six @@ -9,7 +10,7 @@ from pip._internal.exceptions import InstallationError from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any, Tuple, Optional, List # noqa: F401 + from typing import Any, Tuple, Optional, List def _is_list_of_str(obj): @@ -20,6 +21,17 @@ def _is_list_of_str(obj): ) +def make_pyproject_path(setup_py_dir): + # type: (str) -> str + path = os.path.join(setup_py_dir, 'pyproject.toml') + + # Python2 __file__ should not be unicode + if six.PY2 and isinstance(path, six.text_type): + path = path.encode(sys.getfilesystemencoding()) + + return path + + def load_pyproject_toml( use_pep517, # type: Optional[bool] pyproject_toml, # type: str @@ -99,11 +111,13 @@ def load_pyproject_toml( # section, or the user has no pyproject.toml, but has opted in # explicitly via --use-pep517. # In the absence of any explicit backend specification, we - # assume the setuptools backend, and require wheel and a version - # of setuptools that supports that backend. + # assume the setuptools backend that most closely emulates the + # traditional direct setup.py execution, and require wheel and + # a version of setuptools that supports that backend. + build_system = { - "requires": ["setuptools>=40.2.0", "wheel"], - "build-backend": "setuptools.build_meta", + "requires": ["setuptools>=40.8.0", "wheel"], + "build-backend": "setuptools.build_meta:__legacy__", } # If we're using PEP 517, we have build system information (either @@ -142,7 +156,7 @@ def load_pyproject_toml( # If the user didn't specify a backend, we assume they want to use # the setuptools backend. But we can't be sure they have included # a version of setuptools which supplies the backend, or wheel - # (which is neede by the backend) in their requirements. So we + # (which is needed by the backend) in their requirements. So we # make a note to check that those requirements are present once # we have set up the environment. # This is quite a lot of work to check for a very specific case. But @@ -151,7 +165,7 @@ def load_pyproject_toml( # execute setup.py, but never considered needing to mention the build # tools themselves. The original PEP 518 code had a similar check (but # implemented in a different way). - backend = "setuptools.build_meta" - check = ["setuptools>=40.2.0", "wheel"] + backend = "setuptools.build_meta:__legacy__" + check = ["setuptools>=40.8.0", "wheel"] return (requires, backend, check) diff --git a/src/pip/_internal/req/__init__.py b/src/pip/_internal/req/__init__.py index 8b98f8536..c39f63fa8 100644 --- a/src/pip/_internal/req/__init__.py +++ b/src/pip/_internal/req/__init__.py @@ -9,7 +9,7 @@ from pip._internal.utils.logging import indent_log from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any, List, Sequence # noqa: F401 + from typing import Any, List, Sequence __all__ = [ "RequirementSet", "InstallRequirement", diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index 2171e930c..4e176a31e 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -23,6 +23,7 @@ from pip._internal.download import ( from pip._internal.exceptions import InstallationError from pip._internal.models.index import PyPI, TestPyPI from pip._internal.models.link import Link +from pip._internal.pyproject import make_pyproject_path from pip._internal.req.req_install import InstallRequirement from pip._internal.utils.misc import is_installable_dir from pip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -30,10 +31,10 @@ from pip._internal.vcs import vcs from pip._internal.wheel import Wheel if MYPY_CHECK_RUNNING: - from typing import ( # noqa: F401 + from typing import ( Any, Dict, Optional, Set, Tuple, Union ) - from pip._internal.cache import WheelCache # noqa: F401 + from pip._internal.cache import WheelCache __all__ = [ @@ -77,10 +78,18 @@ def parse_editable(editable_req): if os.path.isdir(url_no_extras): if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): - raise InstallationError( - "Directory %r is not installable. File 'setup.py' not found." % - url_no_extras + msg = ( + 'File "setup.py" not found. Directory cannot be installed ' + 'in editable mode: {}'.format(os.path.abspath(url_no_extras)) ) + pyproject_path = make_pyproject_path(url_no_extras) + if os.path.isfile(pyproject_path): + msg += ( + '\n(A "pyproject.toml" file was found, but editable ' + 'mode currently requires a setup.py based build.)' + ) + raise InstallationError(msg) + # Treating it as code that has already been checked out url_no_extras = path_to_url(url_no_extras) diff --git a/src/pip/_internal/req/req_file.py b/src/pip/_internal/req/req_file.py index 27061a663..1b9512491 100644 --- a/src/pip/_internal/req/req_file.py +++ b/src/pip/_internal/req/req_file.py @@ -22,13 +22,13 @@ from pip._internal.req.constructors import ( from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( # noqa: F401 + from typing import ( Any, Callable, Iterator, List, NoReturn, Optional, Text, Tuple ) - from pip._internal.req import InstallRequirement # noqa: F401 - from pip._internal.cache import WheelCache # noqa: F401 - from pip._internal.index import PackageFinder # noqa: F401 - from pip._internal.download import PipSession # noqa: F401 + from pip._internal.req import InstallRequirement + from pip._internal.cache import WheelCache + from pip._internal.index import PackageFinder + from pip._internal.download import PipSession ReqFileLines = Iterator[Tuple[int, Text]] diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 6786de06c..fd8e3a38a 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -22,7 +22,7 @@ from pip._internal.locations import ( PIP_DELETE_MARKER_FILENAME, running_under_virtualenv, ) from pip._internal.models.link import Link -from pip._internal.pyproject import load_pyproject_toml +from pip._internal.pyproject import load_pyproject_toml, make_pyproject_path from pip._internal.req.req_uninstall import UninstallPathSet from pip._internal.utils.compat import native_str from pip._internal.utils.hashes import Hashes @@ -41,15 +41,15 @@ from pip._internal.vcs import vcs from pip._internal.wheel import move_wheel_files if MYPY_CHECK_RUNNING: - from typing import ( # noqa: F401 + from typing import ( Any, Dict, Iterable, List, Mapping, Optional, Sequence, Union ) - from pip._internal.build_env import BuildEnvironment # noqa: F401 - from pip._internal.cache import WheelCache # noqa: F401 - from pip._internal.index import PackageFinder # noqa: F401 - from pip._vendor.pkg_resources import Distribution # noqa: F401 - from pip._vendor.packaging.specifiers import SpecifierSet # noqa: F401 - from pip._vendor.packaging.markers import Marker # noqa: F401 + from pip._internal.build_env import BuildEnvironment + from pip._internal.cache import WheelCache + from pip._internal.index import PackageFinder + from pip._vendor.pkg_resources import Distribution + from pip._vendor.packaging.specifiers import SpecifierSet + from pip._vendor.packaging.markers import Marker logger = logging.getLogger(__name__) @@ -474,13 +474,7 @@ class InstallRequirement(object): # type: () -> str assert self.source_dir, "No source dir for %s" % self - pp_toml = os.path.join(self.setup_py_dir, 'pyproject.toml') - - # Python2 __file__ should not be unicode - if six.PY2 and isinstance(pp_toml, six.text_type): - pp_toml = pp_toml.encode(sys.getfilesystemencoding()) - - return pp_toml + return make_pyproject_path(self.setup_py_dir) def load_pyproject_toml(self): # type: () -> None @@ -521,7 +515,6 @@ class InstallRequirement(object): cmd, cwd=cwd, extra_environ=extra_environ, - show_stdout=False, spinner=spinner ) self.spin_message = "" @@ -619,7 +612,6 @@ class InstallRequirement(object): call_subprocess( egg_info_cmd + egg_base_option, cwd=self.setup_py_dir, - show_stdout=False, command_desc='python setup.py egg_info') @property @@ -772,7 +764,6 @@ class InstallRequirement(object): list(install_options), cwd=self.setup_py_dir, - show_stdout=False, ) self.install_succeeded = True @@ -957,7 +948,6 @@ class InstallRequirement(object): call_subprocess( install_args + install_options, cwd=self.setup_py_dir, - show_stdout=False, spinner=spinner, ) diff --git a/src/pip/_internal/req/req_set.py b/src/pip/_internal/req/req_set.py index a8b95e288..f5e3fe520 100644 --- a/src/pip/_internal/req/req_set.py +++ b/src/pip/_internal/req/req_set.py @@ -9,8 +9,8 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.wheel import Wheel if MYPY_CHECK_RUNNING: - from typing import Dict, Iterable, List, Optional, Tuple # noqa: F401 - from pip._internal.req.req_install import InstallRequirement # noqa: F401 + from typing import Dict, Iterable, List, Optional, Tuple + from pip._internal.req.req_install import InstallRequirement logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/req/req_tracker.py b/src/pip/_internal/req/req_tracker.py index a64d63a14..e36a3f6b5 100644 --- a/src/pip/_internal/req/req_tracker.py +++ b/src/pip/_internal/req/req_tracker.py @@ -10,10 +10,10 @@ from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from types import TracebackType # noqa: F401 - from typing import Iterator, Optional, Set, Type # noqa: F401 - from pip._internal.req.req_install import InstallRequirement # noqa: F401 - from pip._internal.models.link import Link # noqa: F401 + from types import TracebackType + from typing import Iterator, Optional, Set, Type + from pip._internal.req.req_install import InstallRequirement + from pip._internal.models.link import Link logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/req/req_uninstall.py b/src/pip/_internal/req/req_uninstall.py index 58e6436ef..de167d596 100644 --- a/src/pip/_internal/req/req_uninstall.py +++ b/src/pip/_internal/req/req_uninstall.py @@ -15,16 +15,16 @@ from pip._internal.utils.compat import WINDOWS, cache_from_source, uses_pycache from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import ( FakeFile, ask, dist_in_usersite, dist_is_local, egg_link_path, is_local, - normalize_path, renames, + normalize_path, renames, rmtree, ) -from pip._internal.utils.temp_dir import AdjacentTempDirectory +from pip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( # noqa: F401 + from typing import ( Any, Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple ) - from pip._vendor.pkg_resources import Distribution # noqa: F401 + from pip._vendor.pkg_resources import Distribution logger = logging.getLogger(__name__) @@ -141,7 +141,7 @@ def compress_for_rename(paths): # If all the files we found are in our remaining set of files to # remove, then remove them from the latter set and add a wildcard # for the directory. - if len(all_files - remaining) == 0: + if not (all_files - remaining): remaining.difference_update(all_files) wildcards.add(root + os.sep) @@ -199,6 +199,111 @@ def compress_for_output_listing(paths): return will_remove, will_skip +class StashedUninstallPathSet(object): + """A set of file rename operations to stash files while + tentatively uninstalling them.""" + def __init__(self): + # Mapping from source file root to [Adjacent]TempDirectory + # for files under that directory. + self._save_dirs = {} + # (old path, new path) tuples for each move that may need + # to be undone. + self._moves = [] + + def _get_directory_stash(self, path): + """Stashes a directory. + + Directories are stashed adjacent to their original location if + possible, or else moved/copied into the user's temp dir.""" + + try: + save_dir = AdjacentTempDirectory(path) + save_dir.create() + except OSError: + save_dir = TempDirectory(kind="uninstall") + save_dir.create() + self._save_dirs[os.path.normcase(path)] = save_dir + + return save_dir.path + + def _get_file_stash(self, path): + """Stashes a file. + + If no root has been provided, one will be created for the directory + in the user's temp directory.""" + path = os.path.normcase(path) + head, old_head = os.path.dirname(path), None + save_dir = None + + while head != old_head: + try: + save_dir = self._save_dirs[head] + break + except KeyError: + pass + head, old_head = os.path.dirname(head), head + else: + # Did not find any suitable root + head = os.path.dirname(path) + save_dir = TempDirectory(kind='uninstall') + save_dir.create() + self._save_dirs[head] = save_dir + + relpath = os.path.relpath(path, head) + if relpath and relpath != os.path.curdir: + return os.path.join(save_dir.path, relpath) + return save_dir.path + + def stash(self, path): + """Stashes the directory or file and returns its new location. + """ + if os.path.isdir(path): + new_path = self._get_directory_stash(path) + else: + new_path = self._get_file_stash(path) + + self._moves.append((path, new_path)) + if os.path.isdir(path) and os.path.isdir(new_path): + # If we're moving a directory, we need to + # remove the destination first or else it will be + # moved to inside the existing directory. + # We just created new_path ourselves, so it will + # be removable. + os.rmdir(new_path) + renames(path, new_path) + return new_path + + def commit(self): + """Commits the uninstall by removing stashed files.""" + for _, save_dir in self._save_dirs.items(): + save_dir.cleanup() + self._moves = [] + self._save_dirs = {} + + def rollback(self): + """Undoes the uninstall by moving stashed files back.""" + for p in self._moves: + logging.info("Moving to %s\n from %s", *p) + + for new_path, path in self._moves: + try: + logger.debug('Replacing %s from %s', new_path, path) + if os.path.isfile(new_path): + os.unlink(new_path) + elif os.path.isdir(new_path): + rmtree(new_path) + renames(path, new_path) + except OSError as ex: + logger.error("Failed to restore %s", new_path) + logger.debug("Exception: %s", ex) + + self.commit() + + @property + def can_rollback(self): + return bool(self._moves) + + class UninstallPathSet(object): """A set of file paths to be removed in the uninstallation of a requirement.""" @@ -208,8 +313,7 @@ class UninstallPathSet(object): self._refuse = set() # type: Set[str] self.pth = {} # type: Dict[str, UninstallPthEntries] self.dist = dist - self._save_dirs = [] # type: List[AdjacentTempDirectory] - self._moved_paths = [] # type: List[Tuple[str, str]] + self._moved_paths = StashedUninstallPathSet() def _permitted(self, path): # type: (str) -> bool @@ -250,20 +354,6 @@ class UninstallPathSet(object): else: self._refuse.add(pth_file) - def _stash(self, path): - # type: (str) -> str - best = None - for save_dir in self._save_dirs: - if not path.startswith(save_dir.original + os.sep): - continue - if not best or len(save_dir.original) > len(best.original): - best = save_dir - if best is None: - best = AdjacentTempDirectory(os.path.dirname(path)) - best.create() - self._save_dirs.append(best) - return os.path.join(best.path, os.path.relpath(path, best.original)) - def remove(self, auto_confirm=False, verbose=False): # type: (bool, bool) -> None """Remove paths in ``self.paths`` with confirmation (unless @@ -283,11 +373,14 @@ class UninstallPathSet(object): with indent_log(): if auto_confirm or self._allowed_to_proceed(verbose): - for path in sorted(compact(compress_for_rename(self.paths))): - new_path = self._stash(path) + moved = self._moved_paths + + for_rename = compress_for_rename(self.paths) + + for path in sorted(compact(for_rename)): + moved.stash(path) logger.debug('Removing file or directory %s', path) - self._moved_paths.append((path, new_path)) - renames(path, new_path) + for pth in self.pth.values(): pth.remove() @@ -327,25 +420,21 @@ class UninstallPathSet(object): def rollback(self): # type: () -> None """Rollback the changes previously made by remove().""" - if not self._save_dirs: + if not self._moved_paths.can_rollback: logger.error( "Can't roll back %s; was not uninstalled", self.dist.project_name, ) return logger.info('Rolling back uninstall of %s', self.dist.project_name) - for path, tmp_path in self._moved_paths: - logger.debug('Replacing %s', path) - renames(tmp_path, path) + self._moved_paths.rollback() for pth in self.pth.values(): pth.rollback() def commit(self): # type: () -> None """Remove temporary save dir: rollback will no longer be possible.""" - for save_dir in self._save_dirs: - save_dir.cleanup() - self._moved_paths = [] + self._moved_paths.commit() @classmethod def from_dist(cls, dist): diff --git a/src/pip/_internal/resolve.py b/src/pip/_internal/resolve.py index 33f572f1e..f49667bb8 100644 --- a/src/pip/_internal/resolve.py +++ b/src/pip/_internal/resolve.py @@ -25,15 +25,15 @@ from pip._internal.utils.packaging import check_dist_requires_python from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, DefaultDict, List, Set # noqa: F401 - from pip._internal.download import PipSession # noqa: F401 - from pip._internal.req.req_install import InstallRequirement # noqa: F401 - from pip._internal.index import PackageFinder # noqa: F401 - from pip._internal.req.req_set import RequirementSet # noqa: F401 - from pip._internal.operations.prepare import ( # noqa: F401 + from typing import Optional, DefaultDict, List, Set + from pip._internal.download import PipSession + from pip._internal.req.req_install import InstallRequirement + from pip._internal.index import PackageFinder + from pip._internal.req.req_set import RequirementSet + from pip._internal.operations.prepare import ( DistAbstraction, RequirementPreparer ) - from pip._internal.cache import WheelCache # noqa: F401 + from pip._internal.cache import WheelCache logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/utils/appdirs.py b/src/pip/_internal/utils/appdirs.py index 9af9fa7b5..fb2611104 100644 --- a/src/pip/_internal/utils/appdirs.py +++ b/src/pip/_internal/utils/appdirs.py @@ -13,9 +13,7 @@ from pip._internal.utils.compat import WINDOWS, expanduser from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( # noqa: F401 - List, Union - ) + from typing import List def user_cache_dir(appname): diff --git a/src/pip/_internal/utils/compat.py b/src/pip/_internal/utils/compat.py index 2d8b3bf06..845436e48 100644 --- a/src/pip/_internal/utils/compat.py +++ b/src/pip/_internal/utils/compat.py @@ -14,7 +14,7 @@ from pip._vendor.six import text_type from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Tuple, Text # noqa: F401 + from typing import Tuple, Text try: import ipaddress diff --git a/src/pip/_internal/utils/deprecation.py b/src/pip/_internal/utils/deprecation.py index 0beaf74ba..cd754a15e 100644 --- a/src/pip/_internal/utils/deprecation.py +++ b/src/pip/_internal/utils/deprecation.py @@ -12,7 +12,7 @@ from pip import __version__ as current_version from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any, Optional # noqa: F401 + from typing import Any, Optional class PipDeprecationWarning(Warning): diff --git a/src/pip/_internal/utils/encoding.py b/src/pip/_internal/utils/encoding.py index d36defadb..9861530c9 100644 --- a/src/pip/_internal/utils/encoding.py +++ b/src/pip/_internal/utils/encoding.py @@ -6,7 +6,7 @@ import sys from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import List, Tuple, Text # noqa: F401 + from typing import List, Tuple, Text BOMS = [ (codecs.BOM_UTF8, 'utf8'), diff --git a/src/pip/_internal/utils/glibc.py b/src/pip/_internal/utils/glibc.py index 8a51f6954..5bea655eb 100644 --- a/src/pip/_internal/utils/glibc.py +++ b/src/pip/_internal/utils/glibc.py @@ -7,7 +7,7 @@ import warnings from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, Tuple # noqa: F401 + from typing import Optional, Tuple def glibc_version_string(): diff --git a/src/pip/_internal/utils/hashes.py b/src/pip/_internal/utils/hashes.py index c6df7a187..a7142069f 100644 --- a/src/pip/_internal/utils/hashes.py +++ b/src/pip/_internal/utils/hashes.py @@ -11,14 +11,14 @@ from pip._internal.utils.misc import read_chunks from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( # noqa: F401 + from typing import ( Dict, List, BinaryIO, NoReturn, Iterator ) from pip._vendor.six import PY3 if PY3: - from hashlib import _Hash # noqa: F401 + from hashlib import _Hash else: - from hashlib import _hash as _Hash # noqa: F401 + from hashlib import _hash as _Hash # The recommended hash algo of the moment. Change this whenever the state of diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index 84605ee36..8a5808caf 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -43,13 +43,13 @@ else: from io import StringIO if MYPY_CHECK_RUNNING: - from typing import ( # noqa: F401 + from typing import ( Optional, Tuple, Iterable, List, Match, Union, Any, Mapping, Text, AnyStr, Container ) - from pip._vendor.pkg_resources import Distribution # noqa: F401 - from pip._internal.models.link import Link # noqa: F401 - from pip._internal.utils.ui import SpinnerInterface # noqa: F401 + from pip._vendor.pkg_resources import Distribution + from pip._internal.models.link import Link + from pip._internal.utils.ui import SpinnerInterface __all__ = ['rmtree', 'display_path', 'backup_dir', @@ -650,7 +650,7 @@ def unpack_file( def call_subprocess( cmd, # type: List[str] - show_stdout=True, # type: bool + show_stdout=False, # type: bool cwd=None, # type: Optional[str] on_returncode='raise', # type: str extra_ok_returncodes=None, # type: Optional[Iterable[int]] @@ -677,13 +677,13 @@ def call_subprocess( # # The obvious thing that affects output is the show_stdout= # kwarg. show_stdout=True means, let the subprocess write directly to our - # stdout. Even though it is nominally the default, it is almost never used + # stdout. It is almost never used # inside pip (and should not be used in new code without a very good # reason); as of 2016-02-22 it is only used in a few places inside the VCS # wrapper code. Ideally we should get rid of it entirely, because it # creates a lot of complexity here for a rarely used feature. # - # Most places in pip set show_stdout=False. What this means is: + # Most places in pip use show_stdout=False. What this means is: # - We connect the child stdout to a pipe, which we read. # - By default, we hide the output but show a spinner -- unless the # subprocess exits with an error, in which case we show the output. diff --git a/src/pip/_internal/utils/outdated.py b/src/pip/_internal/utils/outdated.py index 37c47a4a5..f76927611 100644 --- a/src/pip/_internal/utils/outdated.py +++ b/src/pip/_internal/utils/outdated.py @@ -16,9 +16,9 @@ from pip._internal.utils.misc import ensure_dir, get_installed_version from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - import optparse # noqa: F401 - from typing import Any, Dict # noqa: F401 - from pip._internal.download import PipSession # noqa: F401 + import optparse + from typing import Any, Dict + from pip._internal.download import PipSession SELFCHECK_DATE_FMT = "%Y-%m-%dT%H:%M:%SZ" diff --git a/src/pip/_internal/utils/packaging.py b/src/pip/_internal/utils/packaging.py index 7aaf7b5e8..449f3fd6b 100644 --- a/src/pip/_internal/utils/packaging.py +++ b/src/pip/_internal/utils/packaging.py @@ -12,9 +12,9 @@ from pip._internal.utils.misc import display_path from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional # noqa: F401 - from email.message import Message # noqa: F401 - from pip._vendor.pkg_resources import Distribution # noqa: F401 + from typing import Optional + from email.message import Message + from pip._vendor.pkg_resources import Distribution logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/utils/temp_dir.py b/src/pip/_internal/utils/temp_dir.py index e7da91675..2c81ad554 100644 --- a/src/pip/_internal/utils/temp_dir.py +++ b/src/pip/_internal/utils/temp_dir.py @@ -1,5 +1,6 @@ from __future__ import absolute_import +import errno import itertools import logging import os.path @@ -98,7 +99,11 @@ class AdjacentTempDirectory(TempDirectory): """ # The characters that may be used to name the temp directory - LEADING_CHARS = "-~.+=%0123456789" + # We always prepend a ~ and then rotate through these until + # a usable name is found. + # pkg_resources raises a different error for .dist-info folder + # with leading '-' and invalid metadata + LEADING_CHARS = "-~.=%0123456789" def __init__(self, original, delete=None): super(AdjacentTempDirectory, self).__init__(delete=delete) @@ -114,11 +119,17 @@ class AdjacentTempDirectory(TempDirectory): package). """ for i in range(1, len(name)): - if name[i] in cls.LEADING_CHARS: - continue + for candidate in itertools.combinations_with_replacement( + cls.LEADING_CHARS, i - 1): + new_name = '~' + ''.join(candidate) + name[i:] + if new_name != name: + yield new_name + + # If we make it this far, we will have to make a longer name + for i in range(len(cls.LEADING_CHARS)): for candidate in itertools.combinations_with_replacement( cls.LEADING_CHARS, i): - new_name = ''.join(candidate) + name[i:] + new_name = '~' + ''.join(candidate) + name if new_name != name: yield new_name @@ -128,8 +139,10 @@ class AdjacentTempDirectory(TempDirectory): path = os.path.join(root, candidate) try: os.mkdir(path) - except OSError: - pass + except OSError as ex: + # Continue if the name exists already + if ex.errno != errno.EEXIST: + raise else: self.path = os.path.realpath(path) break diff --git a/src/pip/_internal/utils/typing.py b/src/pip/_internal/utils/typing.py index e085cdfeb..10170ce29 100644 --- a/src/pip/_internal/utils/typing.py +++ b/src/pip/_internal/utils/typing.py @@ -21,7 +21,7 @@ In pip, all static-typing related imports should be guarded as follows: from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ... # noqa: F401 + from typing import ... Ref: https://github.com/python/mypy/issues/3216 """ diff --git a/src/pip/_internal/utils/ui.py b/src/pip/_internal/utils/ui.py index 433675d73..822371f19 100644 --- a/src/pip/_internal/utils/ui.py +++ b/src/pip/_internal/utils/ui.py @@ -21,7 +21,7 @@ from pip._internal.utils.misc import format_size from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any, Iterator, IO # noqa: F401 + from typing import Any, Iterator, IO try: from pip._vendor import colorama diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index 9cba76464..41eed89d5 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -16,10 +16,10 @@ from pip._internal.utils.misc import ( from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( # noqa: F401 + from typing import ( Any, Dict, Iterable, List, Mapping, Optional, Text, Tuple, Type ) - from pip._internal.utils.ui import SpinnerInterface # noqa: F401 + from pip._internal.utils.ui import SpinnerInterface AuthInfo = Tuple[Optional[str], Optional[str]] diff --git a/src/pip/_internal/wheel.py b/src/pip/_internal/wheel.py index 03bff0bf9..f6fcbe588 100644 --- a/src/pip/_internal/wheel.py +++ b/src/pip/_internal/wheel.py @@ -41,19 +41,18 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.ui import open_spinner if MYPY_CHECK_RUNNING: - from typing import ( # noqa: F401 - Dict, List, Optional, Sequence, Mapping, Tuple, IO, Text, Any, - Union, Iterable + from typing import ( + Dict, List, Optional, Sequence, Mapping, Tuple, IO, Text, Any, Iterable ) - from pip._vendor.packaging.requirements import Requirement # noqa: F401 - from pip._internal.req.req_install import InstallRequirement # noqa: F401 - from pip._internal.download import PipSession # noqa: F401 - from pip._internal.index import PackageFinder # noqa: F401 - from pip._internal.operations.prepare import ( # noqa: F401 + from pip._vendor.packaging.requirements import Requirement + from pip._internal.req.req_install import InstallRequirement + from pip._internal.download import PipSession + from pip._internal.index import FormatControl, PackageFinder + from pip._internal.operations.prepare import ( RequirementPreparer ) - from pip._internal.cache import WheelCache # noqa: F401 - from pip._internal.pep425tags import Pep425Tag # noqa: F401 + from pip._internal.cache import WheelCache + from pip._internal.pep425tags import Pep425Tag InstalledCSVRow = Tuple[str, ...] @@ -212,12 +211,12 @@ def message_about_scripts_not_on_PATH(scripts): # Format a message msg_lines = [] for parent_dir, scripts in warn_for.items(): - scripts = sorted(scripts) - if len(scripts) == 1: - start_text = "script {} is".format(scripts[0]) + sorted_scripts = sorted(scripts) # type: List[str] + if len(sorted_scripts) == 1: + start_text = "script {} is".format(sorted_scripts[0]) else: start_text = "scripts {} are".format( - ", ".join(scripts[:-1]) + " and " + scripts[-1] + ", ".join(sorted_scripts[:-1]) + " and " + sorted_scripts[-1] ) msg_lines.append( @@ -267,16 +266,23 @@ def get_csv_rows_for_installed( lib_dir, # type: str ): # type: (...) -> List[InstalledCSVRow] + """ + :param installed: A map from archive RECORD path to installation RECORD + path. + """ installed_rows = [] # type: List[InstalledCSVRow] for row in old_csv_rows: if len(row) > 3: logger.warning( 'RECORD line has more than three elements: {}'.format(row) ) - fpath = row[0] - fpath = installed.pop(fpath, fpath) - if fpath in changed: - digest, length = rehash(fpath) + # Make a copy because we are mutating the row. + row = list(row) + old_path = row[0] + new_path = installed.pop(old_path, old_path) + row[0] = new_path + if new_path in changed: + digest, length = rehash(new_path) row[1] = digest row[2] = length installed_rows.append(tuple(row)) @@ -725,6 +731,117 @@ def _contains_egg_info( return bool(_egg_info_re.search(s)) +def should_use_ephemeral_cache( + req, # type: InstallRequirement + format_control, # type: FormatControl + autobuilding, # type: bool + cache_available # type: bool +): + # type: (...) -> Optional[bool] + """ + Return whether to build an InstallRequirement object using the + ephemeral cache. + + :param cache_available: whether a cache directory is available for the + autobuilding=True case. + + :return: True or False to build the requirement with ephem_cache=True + or False, respectively; or None not to build the requirement. + """ + if req.constraint: + return None + if req.is_wheel: + if not autobuilding: + logger.info( + 'Skipping %s, due to already being wheel.', req.name, + ) + return None + if not autobuilding: + return False + + if req.editable or not req.source_dir: + return None + + if req.link and not req.link.is_artifact: + # VCS checkout. Build wheel just for this run. + return True + + if "binary" not in format_control.get_allowed_formats( + canonicalize_name(req.name)): + logger.info( + "Skipping bdist_wheel for %s, due to binaries " + "being disabled for it.", req.name, + ) + return None + + link = req.link + base, ext = link.splitext() + if cache_available and _contains_egg_info(base): + return False + + # Otherwise, build the wheel just for this run using the ephemeral + # cache since we are either in the case of e.g. a local directory, or + # no cache directory is available to use. + return True + + +def format_command( + command_args, # type: List[str] + command_output, # type: str +): + # type: (...) -> str + """ + Format command information for logging. + """ + text = 'Command arguments: {}\n'.format(command_args) + + if not command_output: + text += 'Command output: None' + elif logger.getEffectiveLevel() > logging.DEBUG: + text += 'Command output: [use --verbose to show]' + else: + if not command_output.endswith('\n'): + command_output += '\n' + text += ( + 'Command output:\n{}' + '-----------------------------------------' + ).format(command_output) + + return text + + +def get_legacy_build_wheel_path( + names, # type: List[str] + temp_dir, # type: str + req, # type: InstallRequirement + command_args, # type: List[str] + command_output, # type: str +): + # type: (...) -> Optional[str] + """ + Return the path to the wheel in the temporary build directory. + """ + # Sort for determinism. + names = sorted(names) + if not names: + msg = ( + 'Legacy build of wheel for {!r} created no files.\n' + ).format(req.name) + msg += format_command(command_args, command_output) + logger.warning(msg) + return None + + if len(names) > 1: + msg = ( + 'Legacy build of wheel for {!r} created more than one file.\n' + 'Filenames (choosing first): {}\n' + ).format(req.name, names) + msg += format_command(command_args, command_output) + logger.warning(msg) + + return os.path.join(temp_dir, names[0]) + + class WheelBuilder(object): """Build wheels from a RequirementSet.""" @@ -764,15 +881,14 @@ class WheelBuilder(object): builder = self._build_one_pep517 else: builder = self._build_one_legacy - if builder(req, temp_dir.path, python_tag=python_tag): + wheel_path = builder(req, temp_dir.path, python_tag=python_tag) + if wheel_path is not None: + wheel_name = os.path.basename(wheel_path) + dest_path = os.path.join(output_dir, wheel_name) try: - wheel_name = os.listdir(temp_dir.path)[0] - wheel_path = os.path.join(output_dir, wheel_name) - shutil.move( - os.path.join(temp_dir.path, wheel_name), wheel_path - ) + shutil.move(wheel_path, dest_path) logger.info('Stored in directory: %s', output_dir) - return wheel_path + return dest_path except Exception: pass # Ignore return, we can't do anything else useful. @@ -790,11 +906,15 @@ class WheelBuilder(object): ] + list(self.global_options) def _build_one_pep517(self, req, tempd, python_tag=None): + """Build one InstallRequirement using the PEP 517 build process. + + Returns path to wheel if successfully built. Otherwise, returns None. + """ assert req.metadata_directory is not None try: req.spin_message = 'Building wheel for %s (PEP 517)' % (req.name,) logger.debug('Destination directory: %s', tempd) - wheelname = req.pep517_backend.build_wheel( + wheel_name = req.pep517_backend.build_wheel( tempd, metadata_directory=req.metadata_directory ) @@ -802,17 +922,23 @@ class WheelBuilder(object): # General PEP 517 backends don't necessarily support # a "--python-tag" option, so we rename the wheel # file directly. - newname = replace_python_tag(wheelname, python_tag) + new_name = replace_python_tag(wheel_name, python_tag) os.rename( - os.path.join(tempd, wheelname), - os.path.join(tempd, newname) + os.path.join(tempd, wheel_name), + os.path.join(tempd, new_name) ) - return True + # Reassign to simplify the return at the end of function + wheel_name = new_name except Exception: logger.error('Failed building wheel for %s', req.name) - return False + return None + return os.path.join(tempd, wheel_name) def _build_one_legacy(self, req, tempd, python_tag=None): + """Build one InstallRequirement using the "legacy" build process. + + Returns path to wheel if successfully built. Otherwise, returns None. + """ base_args = self._base_setup_args(req) spin_message = 'Building wheel for %s (setup.py)' % (req.name,) @@ -825,13 +951,21 @@ class WheelBuilder(object): wheel_args += ["--python-tag", python_tag] try: - call_subprocess(wheel_args, cwd=req.setup_py_dir, - show_stdout=False, spinner=spinner) - return True + output = call_subprocess(wheel_args, cwd=req.setup_py_dir, + spinner=spinner) except Exception: spinner.finish("error") logger.error('Failed building wheel for %s', req.name) - return False + return None + names = os.listdir(tempd) + wheel_path = get_legacy_build_wheel_path( + names=names, + temp_dir=tempd, + req=req, + command_args=wheel_args, + command_output=output, + ) + return wheel_path def _clean_one(self, req): base_args = self._base_setup_args(req) @@ -839,7 +973,7 @@ class WheelBuilder(object): logger.info('Running setup.py clean for %s', req.name) clean_args = base_args + ['clean', '--all'] try: - call_subprocess(clean_args, cwd=req.source_dir, show_stdout=False) + call_subprocess(clean_args, cwd=req.source_dir) return True except Exception: logger.error('Failed cleaning build dir for %s', req.name) @@ -858,40 +992,20 @@ class WheelBuilder(object): newly built wheel, in preparation for installation. :return: True if all the wheels built correctly. """ - buildset = [] format_control = self.finder.format_control + # Whether a cache directory is available for autobuilding=True. + cache_available = bool(self._wheel_dir or self.wheel_cache.cache_dir) + for req in requirements: - if req.constraint: + ephem_cache = should_use_ephemeral_cache( + req, format_control=format_control, autobuilding=autobuilding, + cache_available=cache_available, + ) + if ephem_cache is None: continue - if req.is_wheel: - if not autobuilding: - logger.info( - 'Skipping %s, due to already being wheel.', req.name, - ) - elif autobuilding and req.editable: - pass - elif autobuilding and not req.source_dir: - pass - elif autobuilding and req.link and not req.link.is_artifact: - # VCS checkout. Build wheel just for this run. - buildset.append((req, True)) - else: - ephem_cache = False - if autobuilding: - link = req.link - base, ext = link.splitext() - if not _contains_egg_info(base): - # E.g. local directory. Build wheel just for this run. - ephem_cache = True - if "binary" not in format_control.get_allowed_formats( - canonicalize_name(req.name)): - logger.info( - "Skipping bdist_wheel for %s, due to binaries " - "being disabled for it.", req.name, - ) - continue - buildset.append((req, ephem_cache)) + + buildset.append((req, ephem_cache)) if not buildset: return [] diff --git a/tests/conftest.py b/tests/conftest.py index c2d8fc328..e83b06916 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -348,5 +348,5 @@ def in_memory_pip(): @pytest.fixture def deprecated_python(): - """Used to indicate wheither pip deprecated this python version""" + """Used to indicate whether pip deprecated this python version""" return sys.version_info[:2] in [(3, 4), (2, 7)] diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 19794ef55..a42320bbb 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -62,7 +62,7 @@ def test_pep518_refuses_conflicting_requires(script, data): result.returncode != 0 and ('Some build dependencies for %s conflict with PEP 517/518 supported ' 'requirements: setuptools==1.0 is incompatible with ' - 'setuptools>=40.2.0.' % path_to_url(project_dir)) in result.stderr + 'setuptools>=40.8.0.' % path_to_url(project_dir)) in result.stderr ), str(result) @@ -491,13 +491,41 @@ def test_install_from_local_directory_with_no_setup_py(script, data): assert "Neither 'setup.py' nor 'pyproject.toml' found." in result.stderr -def test_editable_install_from_local_directory_with_no_setup_py(script, data): +def test_editable_install__local_dir_no_setup_py( + script, data, deprecated_python): """ - Test installing from a local directory with no 'setup.py'. + Test installing in editable mode from a local directory with no setup.py. """ result = script.pip('install', '-e', data.root, expect_error=True) assert not result.files_created - assert "is not installable. File 'setup.py' not found." in result.stderr + + msg = result.stderr + if deprecated_python: + assert 'File "setup.py" not found. ' in msg + else: + assert msg.startswith('File "setup.py" not found. ') + assert 'pyproject.toml' not in msg + + +def test_editable_install__local_dir_no_setup_py_with_pyproject( + script, deprecated_python): + """ + Test installing in editable mode from a local directory with no setup.py + but that does have pyproject.toml. + """ + local_dir = script.scratch_path.join('temp').mkdir() + pyproject_path = local_dir.join('pyproject.toml') + pyproject_path.write('') + + result = script.pip('install', '-e', local_dir, expect_error=True) + assert not result.files_created + + msg = result.stderr + if deprecated_python: + assert 'File "setup.py" not found. ' in msg + else: + assert msg.startswith('File "setup.py" not found. ') + assert 'A "pyproject.toml" file was found' in msg @pytest.mark.skipif("sys.version_info >= (3,4)") diff --git a/tests/functional/test_pep517.py b/tests/functional/test_pep517.py index a1a45d27b..6235a16e4 100644 --- a/tests/functional/test_pep517.py +++ b/tests/functional/test_pep517.py @@ -123,3 +123,78 @@ def test_pep517_install_with_no_cache_dir(script, tmpdir, data): project_dir, ) result.assert_installed('project', editable=False) + + +def make_pyproject_with_setup(tmpdir, build_system=True, set_backend=True): + project_dir = (tmpdir / 'project').mkdir() + setup_script = ( + 'from setuptools import setup\n' + ) + expect_script_dir_on_path = True + if build_system: + buildsys = { + 'requires': ['setuptools', 'wheel'], + } + if set_backend: + buildsys['build-backend'] = 'setuptools.build_meta' + expect_script_dir_on_path = False + project_data = pytoml.dumps({'build-system': buildsys}) + else: + project_data = '' + + if expect_script_dir_on_path: + setup_script += ( + 'from pep517_test import __version__\n' + ) + else: + setup_script += ( + 'try:\n' + ' import pep517_test\n' + 'except ImportError:\n' + ' pass\n' + 'else:\n' + ' raise RuntimeError("Source dir incorrectly on sys.path")\n' + ) + + setup_script += ( + 'setup(name="pep517_test", version="0.1", packages=["pep517_test"])' + ) + + project_dir.join('pyproject.toml').write(project_data) + project_dir.join('setup.py').write(setup_script) + package_dir = (project_dir / "pep517_test").mkdir() + package_dir.join('__init__.py').write('__version__ = "0.1"') + return project_dir, "pep517_test" + + +def test_no_build_system_section(script, tmpdir, data, common_wheels): + """Check builds with setup.py, pyproject.toml, but no build-system section. + """ + project_dir, name = make_pyproject_with_setup(tmpdir, build_system=False) + result = script.pip( + 'install', '--no-cache-dir', '--no-index', '-f', common_wheels, + project_dir, + ) + result.assert_installed(name, editable=False) + + +def test_no_build_backend_entry(script, tmpdir, data, common_wheels): + """Check builds with setup.py, pyproject.toml, but no build-backend entry. + """ + project_dir, name = make_pyproject_with_setup(tmpdir, set_backend=False) + result = script.pip( + 'install', '--no-cache-dir', '--no-index', '-f', common_wheels, + project_dir, + ) + result.assert_installed(name, editable=False) + + +def test_explicit_setuptools_backend(script, tmpdir, data, common_wheels): + """Check builds with setup.py, pyproject.toml, and a build-backend entry. + """ + project_dir, name = make_pyproject_with_setup(tmpdir) + result = script.pip( + 'install', '--no-cache-dir', '--no-index', '-f', common_wheels, + project_dir, + ) + result.assert_installed(name, editable=False) diff --git a/tests/functional/test_wheel.py b/tests/functional/test_wheel.py index 7a6434505..7ab76c130 100644 --- a/tests/functional/test_wheel.py +++ b/tests/functional/test_wheel.py @@ -14,6 +14,13 @@ def auto_with_wheel(with_wheel): pass +def add_files_to_dist_directory(folder): + (folder / 'dist').makedirs() + (folder / 'dist' / 'a_name-0.0.1.tar.gz').write("hello") + # Not adding a wheel file since that confuses setuptools' backend. + # (folder / 'dist' / 'a_name-0.0.1-py2.py3-none-any.whl').write("hello") + + def test_wheel_exit_status_code_when_no_requirements(script): """ Test wheel exit status code when no requirements specified @@ -210,3 +217,31 @@ def test_pip_wheel_with_user_set_in_config(script, data, common_wheels): '--no-index', '-f', common_wheels ) assert "Successfully built withpyproject" in result.stdout, result.stdout + + +def test_pep517_wheels_are_not_confused_with_other_files(script, tmpdir, data): + """Check correct wheels are copied. (#6196) + """ + pkg_to_wheel = data.src / 'withpyproject' + add_files_to_dist_directory(pkg_to_wheel) + + result = script.pip('wheel', pkg_to_wheel, '-w', script.scratch_path) + assert "Installing build dependencies" in result.stdout, result.stdout + + wheel_file_name = 'withpyproject-0.0.1-py%s-none-any.whl' % pyversion[0] + wheel_file_path = script.scratch / wheel_file_name + assert wheel_file_path in result.files_created, result.stdout + + +def test_legacy_wheels_are_not_confused_with_other_files(script, tmpdir, data): + """Check correct wheels are copied. (#6196) + """ + pkg_to_wheel = data.src / 'simplewheel-1.0' + add_files_to_dist_directory(pkg_to_wheel) + + result = script.pip('wheel', pkg_to_wheel, '-w', script.scratch_path) + assert "Installing build dependencies" not in result.stdout, result.stdout + + wheel_file_name = 'simplewheel-1.0-py%s-none-any.whl' % pyversion[0] + wheel_file_path = script.scratch / wheel_file_name + assert wheel_file_path in result.files_created, result.stdout diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index 5269b8409..837d982b6 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -21,6 +21,10 @@ pyversion = sys.version[:3] pyversion_tuple = sys.version_info +def assert_paths_equal(actual, expected): + os.path.normpath(actual) == os.path.normpath(expected) + + def path_to_url(path): """ Convert a path to URI. The path will be made absolute and diff --git a/tests/lib/configuration_helpers.py b/tests/lib/configuration_helpers.py index bd9ab79b5..d33b8ec09 100644 --- a/tests/lib/configuration_helpers.py +++ b/tests/lib/configuration_helpers.py @@ -10,10 +10,22 @@ import textwrap import pip._internal.configuration from pip._internal.utils.misc import ensure_dir -# This is so that tests don't need to import pip.configuration +# This is so that tests don't need to import pip._internal.configuration. kinds = pip._internal.configuration.kinds +def reset_os_environ(old_environ): + """ + Reset os.environ while preserving the same underlying mapping. + """ + # Preserving the same mapping is preferable to assigning a new mapping + # because the latter has interfered with test isolation by, for example, + # preventing time.tzset() from working in subsequent tests after + # changing os.environ['TZ'] in those tests. + os.environ.clear() + os.environ.update(old_environ) + + class ConfigurationMixin(object): def setup(self): @@ -28,7 +40,7 @@ class ConfigurationMixin(object): for fname in self._files_to_clear: fname.stop() - os.environ = self._old_environ + reset_os_environ(self._old_environ) def patch_configuration(self, variant, di): old = self.configuration._load_config_files diff --git a/tests/lib/options_helpers.py b/tests/lib/options_helpers.py index 3ba46f6a3..a93c40e3a 100644 --- a/tests/lib/options_helpers.py +++ b/tests/lib/options_helpers.py @@ -6,6 +6,7 @@ import os from pip._internal.cli import cmdoptions from pip._internal.cli.base_command import Command from pip._internal.commands import commands_dict +from tests.lib.configuration_helpers import reset_os_environ class FakeCommand(Command): @@ -28,5 +29,5 @@ class AddFakeCommandMixin(object): commands_dict[FakeCommand.name] = FakeCommand def teardown(self): - os.environ = self.environ_before + reset_os_environ(self.environ_before) commands_dict.pop(FakeCommand.name) diff --git a/tests/unit/test_appdirs.py b/tests/unit/test_appdirs.py index 0980cf16a..1ee68ef2f 100644 --- a/tests/unit/test_appdirs.py +++ b/tests/unit/test_appdirs.py @@ -185,11 +185,11 @@ class TestUserDataDir: monkeypatch.setattr(sys, "platform", "darwin") if os.path.isdir('/home/test/Library/Application Support/'): - assert (appdirs.user_data_dir("pip") == - "/home/test/Library/Application Support/pip") + assert (appdirs.user_data_dir("pip") == + "/home/test/Library/Application Support/pip") else: - assert (appdirs.user_data_dir("pip") == - "/home/test/.config/pip") + assert (appdirs.user_data_dir("pip") == + "/home/test/.config/pip") def test_user_data_dir_linux(self, monkeypatch): monkeypatch.setattr(appdirs, "WINDOWS", False) @@ -267,11 +267,11 @@ class TestUserConfigDir: monkeypatch.setattr(sys, "platform", "darwin") if os.path.isdir('/home/test/Library/Application Support/'): - assert (appdirs.user_data_dir("pip") == - "/home/test/Library/Application Support/pip") + assert (appdirs.user_data_dir("pip") == + "/home/test/Library/Application Support/pip") else: - assert (appdirs.user_data_dir("pip") == - "/home/test/.config/pip") + assert (appdirs.user_data_dir("pip") == + "/home/test/.config/pip") def test_user_config_dir_linux(self, monkeypatch): monkeypatch.setattr(appdirs, "WINDOWS", False) diff --git a/tests/unit/test_base_command.py b/tests/unit/test_base_command.py index e25df4a80..f90aaadf2 100644 --- a/tests/unit/test_base_command.py +++ b/tests/unit/test_base_command.py @@ -82,9 +82,7 @@ class Test_base_command_logging(object): def setup(self): self.old_time = time.time time.time = lambda: 1547704837.4 - # Robustify the tests below to the ambient timezone by setting it - # explicitly here. - self.old_tz = getattr(os.environ, 'TZ', None) + self.old_tz = os.environ.get('TZ') os.environ['TZ'] = 'UTC' # time.tzset() is not implemented on some platforms (notably, Windows). if hasattr(time, 'tzset'): diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py index af70f568f..6d72bf623 100644 --- a/tests/unit/test_logging.py +++ b/tests/unit/test_logging.py @@ -34,9 +34,7 @@ class TestIndentingFormatter(object): """ def setup(self): - # Robustify the tests below to the ambient timezone by setting it - # explicitly here. - self.old_tz = getattr(os.environ, 'TZ', None) + self.old_tz = os.environ.get('TZ') os.environ['TZ'] = 'UTC' # time.tzset() is not implemented on some platforms (notably, Windows). if hasattr(time, 'tzset'): diff --git a/tests/unit/test_req_uninstall.py b/tests/unit/test_req_uninstall.py index 076ae05c4..37edb7dfa 100644 --- a/tests/unit/test_req_uninstall.py +++ b/tests/unit/test_req_uninstall.py @@ -5,8 +5,8 @@ from mock import Mock import pip._internal.req.req_uninstall from pip._internal.req.req_uninstall import ( - UninstallPathSet, compact, compress_for_output_listing, - compress_for_rename, uninstallation_paths, + StashedUninstallPathSet, UninstallPathSet, compact, + compress_for_output_listing, compress_for_rename, uninstallation_paths, ) from tests.lib import create_file @@ -175,3 +175,96 @@ class TestUninstallPathSet(object): ups.add(path1) ups.add(path2) assert ups.paths == {path1} + + +class TestStashedUninstallPathSet(object): + WALK_RESULT = [ + ("A", ["B", "C"], ["a.py"]), + ("A/B", ["D"], ["b.py"]), + ("A/B/D", [], ["c.py"]), + ("A/C", [], ["d.py", "e.py"]), + ("A/E", ["F"], ["f.py"]), + ("A/E/F", [], []), + ("A/G", ["H"], ["g.py"]), + ("A/G/H", [], ["h.py"]), + ] + + @classmethod + def mock_walk(cls, root): + for dirname, subdirs, files in cls.WALK_RESULT: + dirname = os.path.sep.join(dirname.split("/")) + if dirname.startswith(root): + yield dirname[len(root) + 1:], subdirs, files + + def test_compress_for_rename(self, monkeypatch): + paths = [os.path.sep.join(p.split("/")) for p in [ + "A/B/b.py", + "A/B/D/c.py", + "A/C/d.py", + "A/E/f.py", + "A/G/g.py", + ]] + + expected_paths = [os.path.sep.join(p.split("/")) for p in [ + "A/B/", # selected everything below A/B + "A/C/d.py", # did not select everything below A/C + "A/E/", # only empty folders remain under A/E + "A/G/g.py", # non-empty folder remains under A/G + ]] + + monkeypatch.setattr('os.walk', self.mock_walk) + + actual_paths = compress_for_rename(paths) + assert set(expected_paths) == set(actual_paths) + + @classmethod + def make_stash(cls, tmpdir, paths): + for dirname, subdirs, files in cls.WALK_RESULT: + root = os.path.join(tmpdir, *dirname.split("/")) + if not os.path.exists(root): + os.mkdir(root) + for d in subdirs: + os.mkdir(os.path.join(root, d)) + for f in files: + with open(os.path.join(root, f), "wb"): + pass + + pathset = StashedUninstallPathSet() + + paths = [os.path.join(tmpdir, *p.split('/')) for p in paths] + stashed_paths = [(p, pathset.stash(p)) for p in paths] + + return pathset, stashed_paths + + def test_stash(self, tmpdir): + pathset, stashed_paths = self.make_stash(tmpdir, [ + "A/B/", "A/C/d.py", "A/E/", "A/G/g.py", + ]) + + for old_path, new_path in stashed_paths: + assert not os.path.exists(old_path) + assert os.path.exists(new_path) + + assert stashed_paths == pathset._moves + + def test_commit(self, tmpdir): + pathset, stashed_paths = self.make_stash(tmpdir, [ + "A/B/", "A/C/d.py", "A/E/", "A/G/g.py", + ]) + + pathset.commit() + + for old_path, new_path in stashed_paths: + assert not os.path.exists(old_path) + assert not os.path.exists(new_path) + + def test_rollback(self, tmpdir): + pathset, stashed_paths = self.make_stash(tmpdir, [ + "A/B/", "A/C/d.py", "A/E/", "A/G/g.py", + ]) + + pathset.rollback() + + for old_path, new_path in stashed_paths: + assert os.path.exists(old_path) + assert not os.path.exists(new_path) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 61d4618f4..2ceebd079 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -553,6 +553,10 @@ class TestTempDirectory(object): "ABC.dist-info", "_+-", "_package", + "A......B", + "AB", + "A", + "2", ]) def test_adjacent_directory_names(self, name): def names(): @@ -566,15 +570,82 @@ class TestTempDirectory(object): # result that works, provided there are many of those # and that shorter names result in totally unique sets, # it's okay to skip part of the test.) - some_names = list(itertools.islice(names(), 10000)) - assert len(some_names) == len(set(some_names)) + some_names = list(itertools.islice(names(), 1000)) + # We should always get at least 1000 names + assert len(some_names) == 1000 - # Ensure original name does not appear - assert not any(n == name for n in names()) + # Ensure original name does not appear early in the set + assert name not in some_names - # Check the first group are correct - assert all(x == y for x, y in - zip(some_names, [c + name[1:] for c in chars])) + if len(name) > 2: + # Names should be at least 90% unique (given the infinite + # range of inputs, and the possibility that generated names + # may already exist on disk anyway, this is a much cheaper + # criteria to enforce than complete uniqueness). + assert len(some_names) > 0.9 * len(set(some_names)) + + # Ensure the first few names are the same length as the original + same_len = list(itertools.takewhile( + lambda x: len(x) == len(name), + some_names + )) + assert len(same_len) > 10 + + # Check the first group are correct + expected_names = ['~' + name[1:]] + expected_names.extend('~' + c + name[2:] for c in chars) + for x, y in zip(some_names, expected_names): + assert x == y + + else: + # All names are going to be longer than our original + assert min(len(x) for x in some_names) > 1 + + # All names are going to be unqiue + assert len(some_names) == len(set(some_names)) + + if len(name) == 2: + # All but the first name are going to end with our original + assert all(x.endswith(name) for x in some_names[1:]) + else: + # All names are going to end with our original + assert all(x.endswith(name) for x in some_names) + + @pytest.mark.parametrize("name", [ + "A", + "ABC", + "ABC.dist-info", + "_+-", + "_package", + ]) + def test_adjacent_directory_exists(self, name, tmpdir): + block_name, expect_name = itertools.islice( + AdjacentTempDirectory._generate_names(name), 2) + + original = os.path.join(tmpdir, name) + blocker = os.path.join(tmpdir, block_name) + + ensure_dir(original) + ensure_dir(blocker) + + with AdjacentTempDirectory(original) as atmp_dir: + assert expect_name == os.path.split(atmp_dir.path)[1] + + def test_adjacent_directory_permission_error(self, monkeypatch): + name = "ABC" + + def raising_mkdir(*args, **kwargs): + raise OSError("Unknown OSError") + + with TempDirectory() as tmp_dir: + original = os.path.join(tmp_dir.path, name) + + ensure_dir(original) + monkeypatch.setattr("os.mkdir", raising_mkdir) + + with pytest.raises(OSError): + with AdjacentTempDirectory(original): + pass class TestGlibc(object): @@ -653,16 +724,27 @@ class TestGetProg(object): assert get_prog() == expected -def test_call_subprocess_works_okay_when_just_given_nothing(): - try: - call_subprocess([sys.executable, '-c', 'print("Hello")']) - except Exception: - assert False, "Expected subprocess call to succeed" +def test_call_subprocess_works__no_keyword_arguments(): + result = call_subprocess( + [sys.executable, '-c', 'print("Hello")'], + ) + assert result.rstrip() == 'Hello' + + +def test_call_subprocess_works__show_stdout_true(): + result = call_subprocess( + [sys.executable, '-c', 'print("Hello")'], + show_stdout=True, + ) + assert result is None def test_call_subprocess_closes_stdin(): with pytest.raises(InstallationError): - call_subprocess([sys.executable, '-c', 'input()']) + call_subprocess( + [sys.executable, '-c', 'input()'], + show_stdout=True, + ) @pytest.mark.parametrize('args, expected', [ diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index 6fe125da8..4f210e2e1 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -10,9 +10,12 @@ from pip._vendor.packaging.requirements import Requirement from pip._internal import pep425tags, wheel from pip._internal.exceptions import InvalidWheelFilename, UnsupportedWheel +from pip._internal.index import FormatControl +from pip._internal.models.link import Link +from pip._internal.req.req_install import InstallRequirement from pip._internal.utils.compat import WINDOWS from pip._internal.utils.misc import unpack_file -from tests.lib import DATA_DIR +from tests.lib import DATA_DIR, assert_paths_equal @pytest.mark.parametrize( @@ -35,6 +38,162 @@ def test_contains_egg_info(s, expected): assert result == expected +def make_test_install_req(base_name=None): + """ + Return an InstallRequirement object for testing purposes. + """ + if base_name is None: + base_name = 'pendulum-2.0.4' + + req = Requirement('pendulum') + link_url = ( + 'https://files.pythonhosted.org/packages/aa/{base_name}.tar.gz' + '#sha256=cf535d36c063575d4752af36df928882b2e0e31541b4482c97d637527' + '85f9fcb' + ).format(base_name=base_name) + link = Link( + url=link_url, + comes_from='https://pypi.org/simple/pendulum/', + requires_python='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + ) + req = InstallRequirement( + req=req, + comes_from=None, + constraint=False, + editable=False, + link=link, + source_dir='/tmp/pip-install-9py5m2z1/pendulum', + ) + + return req + + +@pytest.mark.parametrize( + "base_name, autobuilding, cache_available, expected", + [ + ('pendulum-2.0.4', False, False, False), + # The following cases test autobuilding=True. + # Test _contains_egg_info() returning True. + ('pendulum-2.0.4', True, True, False), + ('pendulum-2.0.4', True, False, True), + # Test _contains_egg_info() returning False. + ('pendulum', True, True, True), + ('pendulum', True, False, True), + ], +) +def test_should_use_ephemeral_cache__issue_6197( + base_name, autobuilding, cache_available, expected, +): + """ + Regression test for: https://github.com/pypa/pip/issues/6197 + """ + req = make_test_install_req(base_name=base_name) + assert not req.is_wheel + assert req.link.is_artifact + + format_control = FormatControl() + ephem_cache = wheel.should_use_ephemeral_cache( + req, format_control=format_control, autobuilding=autobuilding, + cache_available=cache_available, + ) + assert ephem_cache is expected + + +def test_format_command__INFO(caplog): + + caplog.set_level(logging.INFO) + actual = wheel.format_command( + command_args=['arg1', 'arg2'], + command_output='output line 1\noutput line 2\n', + ) + assert actual.splitlines() == [ + "Command arguments: ['arg1', 'arg2']", + 'Command output: [use --verbose to show]', + ] + + +@pytest.mark.parametrize('command_output', [ + # Test trailing newline. + 'output line 1\noutput line 2\n', + # Test no trailing newline. + 'output line 1\noutput line 2', +]) +def test_format_command__DEBUG(caplog, command_output): + caplog.set_level(logging.DEBUG) + actual = wheel.format_command( + command_args=['arg1', 'arg2'], + command_output=command_output, + ) + assert actual.splitlines() == [ + "Command arguments: ['arg1', 'arg2']", + 'Command output:', + 'output line 1', + 'output line 2', + '-----------------------------------------', + ] + + +@pytest.mark.parametrize('log_level', ['DEBUG', 'INFO']) +def test_format_command__empty_output(caplog, log_level): + caplog.set_level(log_level) + actual = wheel.format_command( + command_args=['arg1', 'arg2'], + command_output='', + ) + assert actual.splitlines() == [ + "Command arguments: ['arg1', 'arg2']", + 'Command output: None', + ] + + +def call_get_legacy_build_wheel_path(caplog, names): + req = make_test_install_req() + wheel_path = wheel.get_legacy_build_wheel_path( + names=names, + temp_dir='/tmp/abcd', + req=req, + command_args=['arg1', 'arg2'], + command_output='output line 1\noutput line 2\n', + ) + return wheel_path + + +def test_get_legacy_build_wheel_path(caplog): + actual = call_get_legacy_build_wheel_path(caplog, names=['name']) + assert_paths_equal(actual, '/tmp/abcd/name') + assert not caplog.records + + +def test_get_legacy_build_wheel_path__no_names(caplog): + actual = call_get_legacy_build_wheel_path(caplog, names=[]) + assert actual is None + assert len(caplog.records) == 1 + record = caplog.records[0] + assert record.levelname == 'WARNING' + assert record.message.splitlines() == [ + "Legacy build of wheel for 'pendulum' created no files.", + "Command arguments: ['arg1', 'arg2']", + 'Command output: [use --verbose to show]', + ] + + +def test_get_legacy_build_wheel_path__multiple_names(caplog): + # Deliberately pass the names in non-sorted order. + actual = call_get_legacy_build_wheel_path( + caplog, names=['name2', 'name1'], + ) + assert_paths_equal(actual, '/tmp/abcd/name1') + assert len(caplog.records) == 1 + record = caplog.records[0] + assert record.levelname == 'WARNING' + assert record.message.splitlines() == [ + "Legacy build of wheel for 'pendulum' created more than one file.", + "Filenames (choosing first): ['name1', 'name2']", + "Command arguments: ['arg1', 'arg2']", + 'Command output: [use --verbose to show]', + ] + + @pytest.mark.parametrize("console_scripts", ["pip = pip._internal.main:pip", "pip:pip = pip._internal.main:pip"]) @@ -82,7 +241,9 @@ def call_get_csv_rows_for_installed(tmpdir, text): path = tmpdir.join('temp.txt') path.write(text) - installed = {} + # Test that an installed file appearing in RECORD has its filename + # updated in the new RECORD file. + installed = {'a': 'z'} changed = set() generated = [] lib_dir = '/lib/dir' @@ -104,7 +265,7 @@ def test_get_csv_rows_for_installed(tmpdir, caplog): outrows = call_get_csv_rows_for_installed(tmpdir, text) expected = [ - ('a', 'b', 'c'), + ('z', 'b', 'c'), ('d', 'e', 'f'), ] assert outrows == expected @@ -121,7 +282,7 @@ def test_get_csv_rows_for_installed__long_lines(tmpdir, caplog): outrows = call_get_csv_rows_for_installed(tmpdir, text) expected = [ - ('a', 'b', 'c', 'd'), + ('z', 'b', 'c', 'd'), ('e', 'f', 'g'), ('h', 'i', 'j', 'k'), ] diff --git a/tools/lint-requirements.txt b/tools/lint-requirements.txt index 92ed56ed6..b437a2c4c 100644 --- a/tools/lint-requirements.txt +++ b/tools/lint-requirements.txt @@ -1,2 +1,2 @@ -flake8 == 3.5.0 +flake8 == 3.7.6 isort == 4.3.4 diff --git a/tools/mypy-requirements.txt b/tools/mypy-requirements.txt index 3b774dcb1..ac69c5ad4 100644 --- a/tools/mypy-requirements.txt +++ b/tools/mypy-requirements.txt @@ -1 +1 @@ -mypy == 0.650 +mypy == 0.670 diff --git a/tools/tests-common_wheels-requirements.txt b/tools/tests-common_wheels-requirements.txt index 0a8547bcc..6703d606c 100644 --- a/tools/tests-common_wheels-requirements.txt +++ b/tools/tests-common_wheels-requirements.txt @@ -1,2 +1,9 @@ -setuptools +# Create local setuptools wheel files for testing by: +# 1. Cloning setuptools and checking out the branch of interest +# 2. Running `python3 bootstrap.py` in that directory +# 3. Running `python3 -m pip wheel --no-cache -w /tmp/setuptools_build_meta_legacy/ .` +# 4. Replacing the `setuptools` entry below with a `file:///...` URL +# (Adjust artifact directory used based on preference and operating system) + +setuptools >= 40.8.0 wheel