Merge pull request #11059 from pfmoore/config_settings

Add a UI to set config settings for PEP 517 backends

docs/html/reference/build-system/pyproject-toml.md

@@ -106,6 +106,16 @@ This is considered a stopgap solution until setuptools adds support for
 regular {ref}`deprecation policy <Deprecation Policy>`.
 ```
 
+### Backend Configuration
+
+Build backends have the ability to accept configuration settings, which can
+change the way the build is handled. These settings take the form of a
+series of `key=value` pairs. The user can supply configuration settings
+using the `--config-settings` command line option (which can be supplied
+multiple times, in order to specify multiple settings).
+
+The supplied configuration settings are passed to every backend hook call.
+
 ## Build output
 
 It is the responsibility of the build backend to ensure that the output is

docs/html/reference/requirements-file-format.md

@@ -111,6 +111,7 @@ The options which can be applied to individual requirements are:
 
 - {ref}`--install-option <install_--install-option>`
 - {ref}`--global-option <install_--global-option>`
+- {ref}`--config-settings <install_--config-settings>`
 - `--hash` (for {ref}`Hash-checking mode`)
 
 ## Referring to other requirements files

news/11059.feature.rst

@@ -0,0 +1 @@
+Add a user interface for supplying config settings to build backends.

src/pip/_internal/cli/cmdoptions.py

@@ -801,6 +801,33 @@ no_use_pep517: Any = partial(
     help=SUPPRESS_HELP,
 )
 
+
+def _handle_config_settings(
+    option: Option, opt_str: str, value: str, parser: OptionParser
+) -> None:
+    key, sep, val = value.partition("=")
+    if sep != "=":
+        parser.error(f"Arguments to {opt_str} must be of the form KEY=VAL")  # noqa
+    dest = getattr(parser.values, option.dest)
+    if dest is None:
+        dest = {}
+        setattr(parser.values, option.dest, dest)
+    dest[key] = val
+
+
+config_settings: Callable[..., Option] = partial(
+    Option,
+    "--config-settings",
+    dest="config_settings",
+    type=str,
+    action="callback",
+    callback=_handle_config_settings,
+    metavar="settings",
+    help="Configuration settings to be passed to the PEP 517 build backend. "
+    "Settings take the form KEY=VALUE. Use multiple --config-settings options "
+    "to pass multiple keys to the backend.",
+)
+
 install_options: Callable[..., Option] = partial(
     Option,
     "--install-option",

src/pip/_internal/cli/req_command.py

@@ -325,6 +325,7 @@ class RequirementCommand(IndexGroupCommand):
             install_req_from_req_string,
             isolated=options.isolated_mode,
             use_pep517=use_pep517,
+            config_settings=getattr(options, "config_settings", None),
         )
         suppress_build_failures = cls.determine_build_failure_suppression(options)
         resolver_variant = cls.determine_resolver_variant(options)
@@ -397,6 +398,7 @@ class RequirementCommand(IndexGroupCommand):
                 isolated=options.isolated_mode,
                 use_pep517=options.use_pep517,
                 user_supplied=True,
+                config_settings=getattr(options, "config_settings", None),
             )
             requirements.append(req_to_add)
 
@@ -406,6 +408,7 @@ class RequirementCommand(IndexGroupCommand):
                 user_supplied=True,
                 isolated=options.isolated_mode,
                 use_pep517=options.use_pep517,
+                config_settings=getattr(options, "config_settings", None),
             )
             requirements.append(req_to_add)
 

src/pip/_internal/commands/install.py

@@ -190,6 +190,7 @@ class InstallCommand(RequirementCommand):
         self.cmd_opts.add_option(cmdoptions.use_pep517())
         self.cmd_opts.add_option(cmdoptions.no_use_pep517())
 
+        self.cmd_opts.add_option(cmdoptions.config_settings())
         self.cmd_opts.add_option(cmdoptions.install_options())
         self.cmd_opts.add_option(cmdoptions.global_options())
 

src/pip/_internal/commands/wheel.py

@@ -73,6 +73,7 @@ class WheelCommand(RequirementCommand):
             help="Don't verify if built wheel is valid.",
         )
 
+        self.cmd_opts.add_option(cmdoptions.config_settings())
         self.cmd_opts.add_option(cmdoptions.build_options())
         self.cmd_opts.add_option(cmdoptions.global_options())
 

src/pip/_internal/req/constructors.py

@@ -207,6 +207,7 @@ def install_req_from_editable(
     constraint: bool = False,
     user_supplied: bool = False,
     permit_editable_wheels: bool = False,
+    config_settings: Optional[Dict[str, str]] = None,
 ) -> InstallRequirement:
 
     parts = parse_req_from_editable(editable_req)
@@ -224,6 +225,7 @@ def install_req_from_editable(
         install_options=options.get("install_options", []) if options else [],
         global_options=options.get("global_options", []) if options else [],
         hash_options=options.get("hashes", {}) if options else {},
+        config_settings=config_settings,
         extras=parts.extras,
     )
 
@@ -380,6 +382,7 @@ def install_req_from_line(
     constraint: bool = False,
     line_source: Optional[str] = None,
     user_supplied: bool = False,
+    config_settings: Optional[Dict[str, str]] = None,
 ) -> InstallRequirement:
     """Creates an InstallRequirement from a name, which might be a
     requirement, directory containing 'setup.py', filename, or URL.
@@ -399,6 +402,7 @@ def install_req_from_line(
         install_options=options.get("install_options", []) if options else [],
         global_options=options.get("global_options", []) if options else [],
         hash_options=options.get("hashes", {}) if options else {},
+        config_settings=config_settings,
         constraint=constraint,
         extras=parts.extras,
         user_supplied=user_supplied,
@@ -411,6 +415,7 @@ def install_req_from_req_string(
     isolated: bool = False,
     use_pep517: Optional[bool] = None,
     user_supplied: bool = False,
+    config_settings: Optional[Dict[str, str]] = None,
 ) -> InstallRequirement:
     try:
         req = get_requirement(req_string)
@@ -440,6 +445,7 @@ def install_req_from_req_string(
         isolated=isolated,
         use_pep517=use_pep517,
         user_supplied=user_supplied,
+        config_settings=config_settings,
     )
 
 
@@ -448,6 +454,7 @@ def install_req_from_parsed_requirement(
     isolated: bool = False,
     use_pep517: Optional[bool] = None,
     user_supplied: bool = False,
+    config_settings: Optional[Dict[str, str]] = None,
 ) -> InstallRequirement:
     if parsed_req.is_editable:
         req = install_req_from_editable(
@@ -457,6 +464,7 @@ def install_req_from_parsed_requirement(
             constraint=parsed_req.constraint,
             isolated=isolated,
             user_supplied=user_supplied,
+            config_settings=config_settings,
         )
 
     else:
@@ -469,6 +477,7 @@ def install_req_from_parsed_requirement(
             constraint=parsed_req.constraint,
             line_source=parsed_req.line_source,
             user_supplied=user_supplied,
+            config_settings=config_settings,
         )
     return req
 
@@ -487,4 +496,5 @@ def install_req_from_link_and_ireq(
         install_options=ireq.install_options,
         global_options=ireq.global_options,
         hash_options=ireq.hash_options,
+        config_settings=ireq.config_settings,
     )

src/pip/_internal/req/req_install.py

@@ -46,6 +46,7 @@ from pip._internal.utils.direct_url_helpers import (
 )
 from pip._internal.utils.hashes import Hashes
 from pip._internal.utils.misc import (
+    ConfiguredPep517HookCaller,
     ask_path_exists,
     backup_dir,
     display_path,
@@ -80,6 +81,7 @@ class InstallRequirement:
         install_options: Optional[List[str]] = None,
         global_options: Optional[List[str]] = None,
         hash_options: Optional[Dict[str, List[str]]] = None,
+        config_settings: Optional[Dict[str, str]] = None,
         constraint: bool = False,
         extras: Collection[str] = (),
         user_supplied: bool = False,
@@ -138,6 +140,7 @@ class InstallRequirement:
         self.install_options = install_options if install_options else []
         self.global_options = global_options if global_options else []
         self.hash_options = hash_options if hash_options else {}
+        self.config_settings = config_settings
         # Set to True after successful preparation of this requirement
         self.prepared = False
         # User supplied requirement are explicitly requested for installation
@@ -469,7 +472,8 @@ class InstallRequirement:
         requires, backend, check, backend_path = pyproject_toml_data
         self.requirements_to_check = check
         self.pyproject_requires = requires
-        self.pep517_backend = Pep517HookCaller(
+        self.pep517_backend = ConfiguredPep517HookCaller(
+            self,
             self.unpacked_source_directory,
             backend,
             backend_path=backend_path,

src/pip/_internal/resolution/resolvelib/candidates.py

@@ -69,6 +69,7 @@ def make_install_req_from_link(
             global_options=template.global_options,
             hashes=template.hash_options,
         ),
+        config_settings=template.config_settings,
     )
     ireq.original_link = template.original_link
     ireq.link = link
@@ -92,6 +93,7 @@ def make_install_req_from_editable(
             global_options=template.global_options,
             hashes=template.hash_options,
         ),
+        config_settings=template.config_settings,
     )
 
 
@@ -116,6 +118,7 @@ def _make_install_req_from_dist(
             global_options=template.global_options,
             hashes=template.hash_options,
         ),
+        config_settings=template.config_settings,
     )
     ireq.satisfied_by = dist
     return ireq

src/pip/_internal/utils/misc.py

@@ -21,6 +21,7 @@ from typing import (
     BinaryIO,
     Callable,
     ContextManager,
+    Dict,
     Generator,
     Iterable,
     Iterator,
@@ -33,6 +34,7 @@ from typing import (
     cast,
 )
 
+from pip._vendor.pep517 import Pep517HookCaller
 from pip._vendor.tenacity import retry, stop_after_delay, wait_fixed
 
 from pip import __version__
@@ -55,6 +57,7 @@ __all__ = [
     "captured_stdout",
     "ensure_dir",
     "remove_auth_from_url",
+    "ConfiguredPep517HookCaller",
 ]
 
 
@@ -630,3 +633,91 @@ def partition(
     """
     t1, t2 = tee(iterable)
     return filterfalse(pred, t1), filter(pred, t2)
+
+
+class ConfiguredPep517HookCaller(Pep517HookCaller):
+    def __init__(
+        self,
+        config_holder: Any,
+        source_dir: str,
+        build_backend: str,
+        backend_path: Optional[str] = None,
+        runner: Optional[Callable[..., None]] = None,
+        python_executable: Optional[str] = None,
+    ):
+        super().__init__(
+            source_dir, build_backend, backend_path, runner, python_executable
+        )
+        self.config_holder = config_holder
+
+    def build_wheel(
+        self,
+        wheel_directory: str,
+        config_settings: Optional[Dict[str, str]] = None,
+        metadata_directory: Optional[str] = None,
+    ) -> str:
+        cs = self.config_holder.config_settings
+        return super().build_wheel(
+            wheel_directory, config_settings=cs, metadata_directory=metadata_directory
+        )
+
+    def build_sdist(
+        self, sdist_directory: str, config_settings: Optional[Dict[str, str]] = None
+    ) -> str:
+        cs = self.config_holder.config_settings
+        return super().build_sdist(sdist_directory, config_settings=cs)
+
+    def build_editable(
+        self,
+        wheel_directory: str,
+        config_settings: Optional[Dict[str, str]] = None,
+        metadata_directory: Optional[str] = None,
+    ) -> str:
+        cs = self.config_holder.config_settings
+        return super().build_editable(
+            wheel_directory, config_settings=cs, metadata_directory=metadata_directory
+        )
+
+    def get_requires_for_build_wheel(
+        self, config_settings: Optional[Dict[str, str]] = None
+    ) -> List[str]:
+        cs = self.config_holder.config_settings
+        return super().get_requires_for_build_wheel(config_settings=cs)
+
+    def get_requires_for_build_sdist(
+        self, config_settings: Optional[Dict[str, str]] = None
+    ) -> List[str]:
+        cs = self.config_holder.config_settings
+        return super().get_requires_for_build_sdist(config_settings=cs)
+
+    def get_requires_for_build_editable(
+        self, config_settings: Optional[Dict[str, str]] = None
+    ) -> List[str]:
+        cs = self.config_holder.config_settings
+        return super().get_requires_for_build_editable(config_settings=cs)
+
+    def prepare_metadata_for_build_wheel(
+        self,
+        metadata_directory: str,
+        config_settings: Optional[Dict[str, str]] = None,
+        _allow_fallback: bool = True,
+    ) -> str:
+        cs = self.config_holder.config_settings
+        return super().prepare_metadata_for_build_wheel(
+            metadata_directory=metadata_directory,
+            config_settings=cs,
+            _allow_fallback=_allow_fallback,
+        )
+
+    def prepare_metadata_for_build_editable(
+        self,
+        metadata_directory: str,
+        config_settings: Optional[Dict[str, str]] = None,
+        _allow_fallback: bool = True,
+    ) -> str:
+        cs = self.config_holder.config_settings
+        return super().prepare_metadata_for_build_editable(
+            metadata_directory=metadata_directory,
+            config_settings=cs,
+            _allow_fallback=_allow_fallback,
+        )

tests/functional/test_config_settings.py

@@ -0,0 +1,139 @@
+import json
+from typing import Tuple
+from zipfile import ZipFile
+
+from tests.lib import PipTestEnvironment
+from tests.lib.path import Path
+
+PYPROJECT_TOML = """\
+[build-system]
+requires = []
+build-backend = "dummy_backend:main"
+backend-path = ["backend"]
+"""
+
+BACKEND_SRC = '''
+import csv
+import json
+import os.path
+from zipfile import ZipFile
+import hashlib
+import base64
+import io
+
+WHEEL = """\
+Wheel-Version: 1.0
+Generator: dummy_backend 1.0
+Root-Is-Purelib: true
+Tag: py3-none-any
+"""
+
+METADATA = """\
+Metadata-Version: 2.1
+Name: {project}
+Version: {version}
+Summary: A dummy package
+Author: None
+Author-email: none@example.org
+License: MIT
+"""
+
+def make_wheel(z, project, version, files):
+    record = []
+    def add_file(name, data):
+        data = data.encode("utf-8")
+        z.writestr(name, data)
+        digest = hashlib.sha256(data).digest()
+        hash = base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ASCII")
+        record.append((name, f"sha256={hash}", len(data)))
+    distinfo = f"{project}-{version}.dist-info"
+    add_file(f"{distinfo}/WHEEL", WHEEL)
+    add_file(f"{distinfo}/METADATA", METADATA.format(project=project, version=version))
+    for name, data in files:
+        add_file(name, data)
+    record_name = f"{distinfo}/RECORD"
+    record.append((record_name, "", ""))
+    b = io.BytesIO()
+    rec = io.TextIOWrapper(b, newline="", encoding="utf-8")
+    w = csv.writer(rec)
+    w.writerows(record)
+    z.writestr(record_name, b.getvalue())
+    rec.close()
+
+
+class Backend:
+    def build_wheel(
+        self,
+        wheel_directory,
+        config_settings=None,
+        metadata_directory=None
+    ):
+        if config_settings is None:
+            config_settings = {}
+        w = os.path.join(wheel_directory, "foo-1.0-py3-none-any.whl")
+        with open(w, "wb") as f:
+            with ZipFile(f, "w") as z:
+                make_wheel(
+                    z, "foo", "1.0",
+                    [("config.json", json.dumps(config_settings))]
+                )
+        return "foo-1.0-py3-none-any.whl"
+
+    build_editable = build_wheel
+
+main = Backend()
+'''
+
+
+def make_project(path: Path) -> Tuple[str, str, Path]:
+    name = "foo"
+    version = "1.0"
+    project_dir = path / name
+    backend = project_dir / "backend"
+    backend.mkdir(parents=True)
+    (project_dir / "pyproject.toml").write_text(PYPROJECT_TOML)
+    (backend / "dummy_backend.py").write_text(BACKEND_SRC)
+    return name, version, project_dir
+
+
+def test_backend_sees_config(script: PipTestEnvironment) -> None:
+    name, version, project_dir = make_project(script.scratch_path)
+    script.pip(
+        "wheel",
+        "--config-settings",
+        "FOO=Hello",
+        project_dir,
+    )
+    wheel_file_name = f"{name}-{version}-py3-none-any.whl"
+    wheel_file_path = script.cwd / wheel_file_name
+    with open(wheel_file_path, "rb") as f:
+        with ZipFile(f) as z:
+            output = z.read("config.json")
+            assert json.loads(output) == {"FOO": "Hello"}
+
+
+def test_install_sees_config(script: PipTestEnvironment) -> None:
+    _, _, project_dir = make_project(script.scratch_path)
+    script.pip(
+        "install",
+        "--config-settings",
+        "FOO=Hello",
+        project_dir,
+    )
+    config = script.site_packages_path / "config.json"
+    with open(config, "rb") as f:
+        assert json.load(f) == {"FOO": "Hello"}
+
+
+def test_install_editable_sees_config(script: PipTestEnvironment) -> None:
+    _, _, project_dir = make_project(script.scratch_path)
+    script.pip(
+        "install",
+        "--config-settings",
+        "FOO=Hello",
+        "--editable",
+        project_dir,
+    )
+    config = script.site_packages_path / "config.json"
+    with open(config, "rb") as f:
+        assert json.load(f) == {"FOO": "Hello"}

tests/unit/test_pyproject_config.py

@@ -0,0 +1,44 @@
+import pytest
+
+from pip._internal.commands import create_command
+
+
+@pytest.mark.parametrize(
+    ("command", "expected"),
+    [
+        ("install", True),
+        ("wheel", True),
+        ("freeze", False),
+    ],
+)
+def test_supports_config(command: str, expected: bool) -> None:
+    c = create_command(command)
+    options, _ = c.parse_args([])
+    assert hasattr(options, "config_settings") == expected
+
+
+def test_set_config_value_true() -> None:
+    i = create_command("install")
+    # Invalid argument exits with an error
+    with pytest.raises(SystemExit):
+        options, _ = i.parse_args(["xxx", "--config-settings", "x"])
+
+
+def test_set_config_value() -> None:
+    i = create_command("install")
+    options, _ = i.parse_args(["xxx", "--config-settings", "x=hello"])
+    assert options.config_settings == {"x": "hello"}
+
+
+def test_set_config_empty_value() -> None:
+    i = create_command("install")
+    options, _ = i.parse_args(["xxx", "--config-settings", "x="])
+    assert options.config_settings == {"x": ""}
+
+
+def test_replace_config_value() -> None:
+    i = create_command("install")
+    options, _ = i.parse_args(
+        ["xxx", "--config-settings", "x=hello", "--config-settings", "x=world"]
+    )
+    assert options.config_settings == {"x": "world"}