Merge pull request #11663 from uranusjr/pep-668

news/11381.feature.rst

@@ -0,0 +1,3 @@
+Implement logic to read the ``EXTERNALLY-MANAGED`` file as specified in PEP 668.
+This allows a downstream Python distributor to prevent users from using pip to
+modify the externally managed environment.

src/pip/_internal/commands/install.py

@@ -41,6 +41,7 @@ from pip._internal.utils.distutils_args import parse_distutils_args
 from pip._internal.utils.filesystem import test_writable_dir
 from pip._internal.utils.logging import getLogger
 from pip._internal.utils.misc import (
+    check_externally_managed,
     ensure_dir,
     get_pip_version,
     protect_pip_from_modification_on_windows,
@@ -284,6 +285,20 @@ class InstallCommand(RequirementCommand):
         if options.use_user_site and options.target_dir is not None:
             raise CommandError("Can not combine '--user' and '--target'")
 
+        # Check whether the environment we're installing into is externally
+        # managed, as specified in PEP 668. Specifying --root, --target, or
+        # --prefix disables the check, since there's no reliable way to locate
+        # the EXTERNALLY-MANAGED file for those cases. An exception is also
+        # made specifically for "--dry-run --report" for convenience.
+        installing_into_current_environment = (
+            not (options.dry_run and options.json_report_file)
+            and options.root_path is None
+            and options.target_dir is None
+            and options.prefix_path is None
+        )
+        if installing_into_current_environment:
+            check_externally_managed()
+
         upgrade_strategy = "to-satisfy-only"
         if options.upgrade:
             upgrade_strategy = options.upgrade_strategy

src/pip/_internal/commands/uninstall.py

@@ -14,7 +14,10 @@ from pip._internal.req.constructors import (
     install_req_from_line,
     install_req_from_parsed_requirement,
 )
-from pip._internal.utils.misc import protect_pip_from_modification_on_windows
+from pip._internal.utils.misc import (
+    check_externally_managed,
+    protect_pip_from_modification_on_windows,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -90,6 +93,8 @@ class UninstallCommand(Command, SessionCommandMixin):
                 f'"pip help {self.name}")'
             )
 
+        check_externally_managed()
+
         protect_pip_from_modification_on_windows(
             modifying_pip="pip" in reqs_to_uninstall
         )

src/pip/_internal/exceptions.py

@@ -6,9 +6,14 @@ subpackage and, thus, should not depend on them.
 """
 
 import configparser
+import contextlib
+import locale
+import logging
+import pathlib
 import re
+import sys
 from itertools import chain, groupby, repeat
-from typing import TYPE_CHECKING, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Dict, Iterator, List, Optional, Union
 
 from pip._vendor.requests.models import Request, Response
 from pip._vendor.rich.console import Console, ConsoleOptions, RenderResult
@@ -22,6 +27,8 @@ if TYPE_CHECKING:
     from pip._internal.metadata import BaseDistribution
     from pip._internal.req.req_install import InstallRequirement
 
+logger = logging.getLogger(__name__)
+
 
 #
 # Scaffolding
@@ -658,3 +665,81 @@ class ConfigurationFileCouldNotBeLoaded(ConfigurationError):
             assert self.error is not None
             message_part = f".\n{self.error}\n"
         return f"Configuration file {self.reason}{message_part}"
+
+
+_DEFAULT_EXTERNALLY_MANAGED_ERROR = f"""\
+The Python environment under {sys.prefix} is managed externally, and may not be
+manipulated by the user. Please use specific tooling from the distributor of
+the Python installation to interact with this environment instead.
+"""
+
+
+class ExternallyManagedEnvironment(DiagnosticPipError):
+    """The current environment is externally managed.
+
+    This is raised when the current environment is externally managed, as
+    defined by `PEP 668`_. The ``EXTERNALLY-MANAGED`` configuration is checked
+    and displayed when the error is bubbled up to the user.
+
+    :param error: The error message read from ``EXTERNALLY-MANAGED``.
+    """
+
+    reference = "externally-managed-environment"
+
+    def __init__(self, error: Optional[str]) -> None:
+        if error is None:
+            context = Text(_DEFAULT_EXTERNALLY_MANAGED_ERROR)
+        else:
+            context = Text(error)
+        super().__init__(
+            message="This environment is externally managed",
+            context=context,
+            note_stmt=(
+                "If you believe this is a mistake, please contact your "
+                "Python installation or OS distribution provider."
+            ),
+            hint_stmt=Text("See PEP 668 for the detailed specification."),
+        )
+
+    @staticmethod
+    def _iter_externally_managed_error_keys() -> Iterator[str]:
+        # LC_MESSAGES is in POSIX, but not the C standard. The most common
+        # platform that does not implement this category is Windows, where
+        # using other categories for console message localization is equally
+        # unreliable, so we fall back to the locale-less vendor message. This
+        # can always be re-evaluated when a vendor proposes a new alternative.
+        try:
+            category = locale.LC_MESSAGES
+        except AttributeError:
+            lang: Optional[str] = None
+        else:
+            lang, _ = locale.getlocale(category)
+        if lang is not None:
+            yield f"Error-{lang}"
+            for sep in ("-", "_"):
+                before, found, _ = lang.partition(sep)
+                if not found:
+                    continue
+                yield f"Error-{before}"
+        yield "Error"
+
+    @classmethod
+    def from_config(
+        cls,
+        config: Union[pathlib.Path, str],
+    ) -> "ExternallyManagedEnvironment":
+        parser = configparser.ConfigParser(interpolation=None)
+        try:
+            parser.read(config, encoding="utf-8")
+            section = parser["externally-managed"]
+            for key in cls._iter_externally_managed_error_keys():
+                with contextlib.suppress(KeyError):
+                    return cls(section[key])
+        except KeyError:
+            pass
+        except (OSError, UnicodeDecodeError, configparser.ParsingError):
+            from pip._internal.utils._log import VERBOSE
+
+            exc_info = logger.isEnabledFor(VERBOSE)
+            logger.warning("Failed to read %s", config, exc_info=exc_info)
+        return cls(None)

src/pip/_internal/utils/misc.py

@@ -12,6 +12,7 @@ import posixpath
 import shutil
 import stat
 import sys
+import sysconfig
 import urllib.parse
 from io import StringIO
 from itertools import filterfalse, tee, zip_longest
@@ -38,7 +39,7 @@ from pip._vendor.pyproject_hooks import BuildBackendHookCaller
 from pip._vendor.tenacity import retry, stop_after_delay, wait_fixed
 
 from pip import __version__
-from pip._internal.exceptions import CommandError
+from pip._internal.exceptions import CommandError, ExternallyManagedEnvironment
 from pip._internal.locations import get_major_minor_version
 from pip._internal.utils.compat import WINDOWS
 from pip._internal.utils.virtualenv import running_under_virtualenv
@@ -57,10 +58,10 @@ __all__ = [
     "captured_stdout",
     "ensure_dir",
     "remove_auth_from_url",
+    "check_externally_managed",
     "ConfiguredBuildBackendHookCaller",
 ]
 
-
 logger = logging.getLogger(__name__)
 
 T = TypeVar("T")
@@ -581,6 +582,21 @@ def protect_pip_from_modification_on_windows(modifying_pip: bool) -> None:
         )
 
 
+def check_externally_managed() -> None:
+    """Check whether the current environment is externally managed.
+
+    If the ``EXTERNALLY-MANAGED`` config file is found, the current environment
+    is considered externally managed, and an ExternallyManagedEnvironment is
+    raised.
+    """
+    if running_under_virtualenv():
+        return
+    marker = os.path.join(sysconfig.get_path("stdlib"), "EXTERNALLY-MANAGED")
+    if not os.path.isfile(marker):
+        return
+    raise ExternallyManagedEnvironment.from_config(marker)
+
+
 def is_console_interactive() -> bool:
     """Is this console interactive?"""
     return sys.stdin is not None and sys.stdin.isatty()

tests/functional/test_pep668.py

@@ -0,0 +1,81 @@
+import json
+import pathlib
+import textwrap
+from typing import List
+
+import pytest
+
+from tests.lib import PipTestEnvironment, create_basic_wheel_for_package
+from tests.lib.venv import VirtualEnvironment
+
+
+@pytest.fixture()
+def patch_check_externally_managed(virtualenv: VirtualEnvironment) -> None:
+    # Since the tests are run from a virtual environment, and we can't
+    # guarantee access to the actual stdlib location (where EXTERNALLY-MANAGED
+    # needs to go into), we patch the check to always raise a simple message.
+    virtualenv.sitecustomize = textwrap.dedent(
+        """\
+        from pip._internal.exceptions import ExternallyManagedEnvironment
+        from pip._internal.utils import misc
+
+        def check_externally_managed():
+            raise ExternallyManagedEnvironment("I am externally managed")
+
+        misc.check_externally_managed = check_externally_managed
+        """
+    )
+
+
+@pytest.mark.parametrize(
+    "arguments",
+    [
+        pytest.param(["install"], id="install"),
+        pytest.param(["install", "--user"], id="install-user"),
+        pytest.param(["install", "--dry-run"], id="install-dry-run"),
+        pytest.param(["uninstall", "-y"], id="uninstall"),
+    ],
+)
+@pytest.mark.usefixtures("patch_check_externally_managed")
+def test_fails(script: PipTestEnvironment, arguments: List[str]) -> None:
+    result = script.pip(*arguments, "pip", expect_error=True)
+    assert "I am externally managed" in result.stderr
+
+
+@pytest.mark.parametrize(
+    "arguments",
+    [
+        pytest.param(["install", "--root"], id="install-root"),
+        pytest.param(["install", "--prefix"], id="install-prefix"),
+        pytest.param(["install", "--target"], id="install-target"),
+    ],
+)
+@pytest.mark.usefixtures("patch_check_externally_managed")
+def test_allows_if_out_of_environment(
+    script: PipTestEnvironment,
+    arguments: List[str],
+) -> None:
+    wheel = create_basic_wheel_for_package(script, "foo", "1.0")
+    result = script.pip(*arguments, script.scratch_path, wheel.as_uri())
+    assert "Successfully installed foo-1.0" in result.stdout
+    assert "I am externally managed" not in result.stderr
+
+
+@pytest.mark.usefixtures("patch_check_externally_managed")
+def test_allows_install_dry_run(
+    script: PipTestEnvironment,
+    tmp_path: pathlib.Path,
+) -> None:
+    output = tmp_path.joinpath("out.json")
+    wheel = create_basic_wheel_for_package(script, "foo", "1.0")
+    result = script.pip(
+        "install",
+        "--dry-run",
+        f"--report={output.as_posix()}",
+        wheel.as_uri(),
+        expect_stderr=True,
+    )
+    assert "Would install foo-1.0" in result.stdout
+    assert "I am externally managed" not in result.stderr
+    with output.open(encoding="utf8") as f:
+        assert isinstance(json.load(f), dict)

tests/unit/test_exceptions.py

@@ -1,12 +1,17 @@
 """Tests the presentation style of exceptions."""
 
 import io
+import locale
+import logging
+import pathlib
+import sys
 import textwrap
+from typing import Optional, Tuple
 
 import pytest
 from pip._vendor import rich
 
-from pip._internal.exceptions import DiagnosticPipError
+from pip._internal.exceptions import DiagnosticPipError, ExternallyManagedEnvironment
 
 
 class TestDiagnosticPipErrorCreation:
@@ -472,3 +477,178 @@ class TestDiagnosticPipErrorPresentation_Unicode:
               It broke. :(
             """
         )
+
+
+class TestExternallyManagedEnvironment:
+    default_text = (
+        f"The Python environment under {sys.prefix} is managed externally, "
+        f"and may not be\nmanipulated by the user. Please use specific "
+        f"tooling from the distributor of\nthe Python installation to "
+        f"interact with this environment instead.\n"
+    )
+
+    @pytest.fixture(autouse=True)
+    def patch_locale(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        orig_getlocal = locale.getlocale
+
+        def fake_getlocale(category: int) -> Tuple[Optional[str], Optional[str]]:
+            """Fake getlocale() that always reports zh_Hant for LC_MESSASGES."""
+            result = orig_getlocal(category)
+            if category == getattr(locale, "LC_MESSAGES", None):
+                return "zh_Hant", result[1]
+            return result
+
+        monkeypatch.setattr(locale, "getlocale", fake_getlocale)
+
+    @pytest.fixture()
+    def marker(self, tmp_path: pathlib.Path) -> pathlib.Path:
+        marker = tmp_path.joinpath("EXTERNALLY-MANAGED")
+        marker.touch()
+        return marker
+
+    def test_invalid_config_format(
+        self,
+        caplog: pytest.LogCaptureFixture,
+        marker: pathlib.Path,
+    ) -> None:
+        marker.write_text("invalid", encoding="utf8")
+
+        with caplog.at_level(logging.WARNING, "pip._internal.exceptions"):
+            exc = ExternallyManagedEnvironment.from_config(marker)
+        assert len(caplog.records) == 1
+        assert caplog.records[-1].getMessage() == f"Failed to read {marker}"
+
+        assert str(exc.context) == self.default_text
+
+    @pytest.mark.parametrize(
+        "config",
+        [
+            pytest.param("", id="empty"),
+            pytest.param("[foo]\nblah = blah", id="no-section"),
+            pytest.param("[externally-managed]\nblah = blah", id="no-key"),
+        ],
+    )
+    def test_config_without_key(
+        self,
+        caplog: pytest.LogCaptureFixture,
+        marker: pathlib.Path,
+        config: str,
+    ) -> None:
+        marker.write_text(config, encoding="utf8")
+
+        with caplog.at_level(logging.WARNING, "pip._internal.exceptions"):
+            exc = ExternallyManagedEnvironment.from_config(marker)
+        assert not caplog.records
+        assert str(exc.context) == self.default_text
+
+    @pytest.mark.skipif(
+        sys.platform == "win32",
+        reason="Localization disabled on Windows",
+    )
+    @pytest.mark.parametrize(
+        "config, expected",
+        [
+            pytest.param(
+                """\
+                [externally-managed]
+                Error = 最後
+                Error-en = English
+                Error-zh = 中文
+                Error-zh_Hant = 繁體
+                Error-zh_Hans = 简体
+                """,
+                "繁體",
+                id="full",
+            ),
+            pytest.param(
+                """\
+                [externally-managed]
+                Error = 最後
+                Error-en = English
+                Error-zh = 中文
+                Error-zh_Hans = 简体
+                """,
+                "中文",
+                id="no-variant",
+            ),
+            pytest.param(
+                """\
+                [externally-managed]
+                Error = 最後
+                Error-en = English
+                """,
+                "最後",
+                id="fallback",
+            ),
+        ],
+    )
+    def test_config_canonical(
+        self,
+        caplog: pytest.LogCaptureFixture,
+        marker: pathlib.Path,
+        config: str,
+        expected: str,
+    ) -> None:
+        marker.write_text(
+            textwrap.dedent(config),
+            encoding="utf8",
+        )
+
+        with caplog.at_level(logging.WARNING, "pip._internal.exceptions"):
+            exc = ExternallyManagedEnvironment.from_config(marker)
+        assert not caplog.records
+        assert str(exc.context) == expected
+
+    @pytest.mark.skipif(
+        sys.platform != "win32",
+        reason="Non-Windows should implement localization",
+    )
+    @pytest.mark.parametrize(
+        "config",
+        [
+            pytest.param(
+                """\
+                [externally-managed]
+                Error = 最後
+                Error-en = English
+                Error-zh = 中文
+                Error-zh_Hant = 繁體
+                Error-zh_Hans = 简体
+                """,
+                id="full",
+            ),
+            pytest.param(
+                """\
+                [externally-managed]
+                Error = 最後
+                Error-en = English
+                Error-zh = 中文
+                Error-zh_Hans = 简体
+                """,
+                id="no-variant",
+            ),
+            pytest.param(
+                """\
+                [externally-managed]
+                Error = 最後
+                Error-en = English
+                """,
+                id="fallback",
+            ),
+        ],
+    )
+    def test_config_canonical_no_localization(
+        self,
+        caplog: pytest.LogCaptureFixture,
+        marker: pathlib.Path,
+        config: str,
+    ) -> None:
+        marker.write_text(
+            textwrap.dedent(config),
+            encoding="utf8",
+        )
+
+        with caplog.at_level(logging.WARNING, "pip._internal.exceptions"):
+            exc = ExternallyManagedEnvironment.from_config(marker)
+        assert not caplog.records
+        assert str(exc.context) == "最後"