Implement sysconfig locations and warn on mismatch

src/pip/_internal/exceptions.py

@@ -59,6 +59,21 @@ class NoneMetadataError(PipError):
         )
 
 
+class UserInstallationInvalid(InstallationError):
+    """A --user install is requested on an environment without user site."""
+
+    def __str__(self):
+        # type: () -> str
+        return "User base directory is not specified"
+
+
+class InvalidSchemeCombination(InstallationError):
+    def __str__(self):
+        # type: () -> str
+        before = ", ".join(str(a) for a in self.args[:-1])
+        return f"Cannot set {before} and {self.args[-1]} together"
+
+
 class DistributionNotFound(InstallationError):
     """Raised when a distribution cannot be found to satisfy a requirement"""
 

src/pip/_internal/locations/__init__.py

@@ -1,8 +1,11 @@
+import logging
+import pathlib
+import sysconfig
 from typing import Optional
 
-from pip._internal.models.scheme import Scheme
+from pip._internal.models.scheme import SCHEME_KEYS, Scheme
 
-from . import distutils as _distutils
+from . import _distutils, _sysconfig
 from .base import (
     USER_CACHE_DIR,
     get_major_minor_version,
@@ -24,6 +27,31 @@ __all__ = [
 ]
 
 
+logger = logging.getLogger(__name__)
+
+
+def _default_base(*, user):
+    # type: (bool) -> str
+    if user:
+        base = sysconfig.get_config_var("userbase")
+    else:
+        base = sysconfig.get_config_var("base")
+    assert base is not None
+    return base
+
+
+def _warn_if_mismatch(old, new, *, key):
+    # type: (pathlib.Path, pathlib.Path, str) -> None
+    if old == new:
+        return
+    message = (
+        "Value for %s does not match. Please report this: <URL HERE>"
+        "\ndistutils: %s"
+        "\nsysconfig: %s"
+    )
+    logger.warning(message, key, old, new)
+
+
 def get_scheme(
     dist_name,  # type: str
     user=False,  # type: bool
@@ -33,7 +61,15 @@ def get_scheme(
     prefix=None,  # type: Optional[str]
 ):
     # type: (...) -> Scheme
-    return _distutils.get_scheme(
+    old = _distutils.get_scheme(
+        dist_name,
+        user=user,
+        home=home,
+        root=root,
+        isolated=isolated,
+        prefix=prefix,
+    )
+    new = _sysconfig.get_scheme(
         dist_name,
         user=user,
         home=home,
@@ -42,12 +78,27 @@ def get_scheme(
         prefix=prefix,
     )
 
+    base = prefix or home or _default_base(user=user)
+    for k in SCHEME_KEYS:
+        # Extra join because distutils can return relative paths.
+        old_v = pathlib.Path(base, getattr(old, k))
+        new_v = pathlib.Path(getattr(new, k))
+        _warn_if_mismatch(old_v, new_v, key=f"scheme.{k}")
+
+    return old
+
 
 def get_bin_prefix():
     # type: () -> str
-    return _distutils.get_bin_prefix()
+    old = _distutils.get_bin_prefix()
+    new = _sysconfig.get_bin_prefix()
+    _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="bin_prefix")
+    return old
 
 
 def get_bin_user():
     # type: () -> str
-    return _distutils.get_bin_user()
+    old = _distutils.get_bin_user()
+    new = _sysconfig.get_bin_user()
+    _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="bin_user")
+    return old

src/pip/_internal/locations/_sysconfig.py

@@ -0,0 +1,137 @@
+import distutils.util  # FIXME: For change_root.
+import logging
+import os
+import sys
+import sysconfig
+import typing
+
+from pip._internal.exceptions import (
+    InvalidSchemeCombination,
+    UserInstallationInvalid,
+)
+from pip._internal.models.scheme import SCHEME_KEYS, Scheme
+from pip._internal.utils.virtualenv import running_under_virtualenv
+
+from .base import get_major_minor_version
+
+
+logger = logging.getLogger(__name__)
+
+
+_AVAILABLE_SCHEMES = set(sysconfig.get_scheme_names())
+
+
+def _infer_scheme(variant):
+    # (typing.Literal["home", "prefix", "user"]) -> str
+    """Try to find a scheme for the current platform.
+
+    Unfortunately ``_get_default_scheme()`` is private, so there's no way to
+    ask things like "what is the '_home' scheme on this platform". This tries
+    to answer that with some heuristics while accounting for ad-hoc platforms
+    not covered by CPython's default sysconfig implementation.
+    """
+    # Most schemes are named like this e.g. "posix_home", "nt_user".
+    suffixed = f"{os.name}_{variant}"
+    if suffixed in _AVAILABLE_SCHEMES:
+        return suffixed
+
+    # The user scheme is not available.
+    if variant == "user" and sysconfig.get_config_var("userbase") is None:
+        raise UserInstallationInvalid()
+
+    # On Windows, prefx and home schemes are the same and just called "nt".
+    if os.name in _AVAILABLE_SCHEMES:
+        return os.name
+
+    # Not sure what's happening, some obscure platform that does not fully
+    # implement sysconfig? Just use the POSIX scheme.
+    logger.warning("No %r scheme for %r; fallback to POSIX.", variant, os.name)
+    return f"posix_{variant}"
+
+
+# Update these keys if the user sets a custom home.
+_HOME_KEYS = (
+    "installed_base",
+    "base",
+    "installed_platbase",
+    "platbase",
+    "prefix",
+    "exec_prefix",
+)
+if sysconfig.get_config_var("userbase") is not None:
+    _HOME_KEYS += ("userbase",)
+
+
+def get_scheme(
+    dist_name,  # type: str
+    user=False,  # type: bool
+    home=None,  # type: typing.Optional[str]
+    root=None,  # type: typing.Optional[str]
+    isolated=False,  # type: bool
+    prefix=None,  # type: typing.Optional[str]
+):
+    # type: (...) -> Scheme
+    """
+    Get the "scheme" corresponding to the input parameters.
+
+    :param dist_name: the name of the package to retrieve the scheme for, used
+        in the headers scheme path
+    :param user: indicates to use the "user" scheme
+    :param home: indicates to use the "home" scheme
+    :param root: root under which other directories are re-based
+    :param isolated: ignored, but kept for distutils compatibility (where
+        this controls whether the user-site pydistutils.cfg is honored)
+    :param prefix: indicates to use the "prefix" scheme and provides the
+        base directory for the same
+    """
+    if user and prefix:
+        raise InvalidSchemeCombination("--user", "--prefix")
+    if home and prefix:
+        raise InvalidSchemeCombination("--home", "--prefix")
+
+    if home is not None:
+        scheme = _infer_scheme("home")
+    elif user:
+        scheme = _infer_scheme("user")
+    else:
+        scheme = _infer_scheme("prefix")
+
+    if home is not None:
+        variables = {k: home for k in _HOME_KEYS}
+    elif prefix is not None:
+        variables = {k: prefix for k in _HOME_KEYS}
+    else:
+        variables = {}
+
+    paths = sysconfig.get_paths(scheme=scheme, vars=variables)
+
+    # Special header path for compatibility to distutils.
+    if running_under_virtualenv():
+        base = variables.get("base", sys.prefix)
+        python_xy = f"python{get_major_minor_version()}"
+        paths["include"] = os.path.join(base, "include", "site", python_xy)
+
+    scheme = Scheme(
+        platlib=paths["platlib"],
+        purelib=paths["purelib"],
+        headers=os.path.join(paths["include"], dist_name),
+        scripts=paths["scripts"],
+        data=paths["data"],
+    )
+    if root is not None:
+        for key in SCHEME_KEYS:
+            value = distutils.util.change_root(root, getattr(scheme, key))
+            setattr(scheme, key, value)
+    return scheme
+
+
+def get_bin_prefix():
+    # type: () -> str
+    # Forcing to use /usr/local/bin for standard macOS framework installs.
+    if sys.platform[:6] == "darwin" and sys.prefix[:16] == "/System/Library/":
+        return "/usr/local/bin"
+    return sysconfig.get_path("scripts", scheme=_infer_scheme("prefix"))
+
+
+def get_bin_user():
+    return sysconfig.get_path("scripts", scheme=_infer_scheme("user"))

tests/unit/test_locations.py

@@ -11,7 +11,7 @@ from unittest.mock import Mock
 
 import pytest
 
-from pip._internal.locations import distutils_scheme
+from pip._internal.locations import SCHEME_KEYS, get_scheme
 
 if sys.platform == 'win32':
     pwd = Mock()
@@ -19,6 +19,11 @@ else:
     import pwd
 
 
+def _get_scheme_dict(*args, **kwargs):
+    scheme = get_scheme(*args, **kwargs)
+    return {k: getattr(scheme, k) for k in SCHEME_KEYS}
+
+
 class TestLocations:
     def setup(self):
         self.tempdir = tempfile.mkdtemp()
@@ -83,8 +88,8 @@ class TestDistutilsScheme:
         # root is c:\somewhere\else or /somewhere/else
         root = os.path.normcase(os.path.abspath(
             os.path.join(os.path.sep, 'somewhere', 'else')))
-        norm_scheme = distutils_scheme("example")
-        root_scheme = distutils_scheme("example", root=root)
+        norm_scheme = _get_scheme_dict("example")
+        root_scheme = _get_scheme_dict("example", root=root)
 
         for key, value in norm_scheme.items():
             drive, path = os.path.splitdrive(os.path.abspath(value))
@@ -107,7 +112,7 @@ class TestDistutilsScheme:
             'find_config_files',
             lambda self: [f],
         )
-        scheme = distutils_scheme('example')
+        scheme = _get_scheme_dict('example')
         assert scheme['scripts'] == install_scripts
 
     @pytest.mark.incompatible_with_venv
@@ -129,15 +134,15 @@ class TestDistutilsScheme:
             'find_config_files',
             lambda self: [f],
         )
-        scheme = distutils_scheme('example')
+        scheme = _get_scheme_dict('example')
         assert scheme['platlib'] == install_lib + os.path.sep
         assert scheme['purelib'] == install_lib + os.path.sep
 
     def test_prefix_modifies_appropriately(self):
         prefix = os.path.abspath(os.path.join('somewhere', 'else'))
 
-        normal_scheme = distutils_scheme("example")
-        prefix_scheme = distutils_scheme("example", prefix=prefix)
+        normal_scheme = _get_scheme_dict("example")
+        prefix_scheme = _get_scheme_dict("example", prefix=prefix)
 
         def _calculate_expected(value):
             path = os.path.join(prefix, os.path.relpath(value, sys.prefix))