diff --git a/tests/lib/test_wheel.py b/tests/lib/test_wheel.py new file mode 100644 index 000000000..b5c22ae31 --- /dev/null +++ b/tests/lib/test_wheel.py @@ -0,0 +1,199 @@ +"""Tests for wheel helper. +""" +from email import message_from_string +from functools import partial +from zipfile import ZipFile + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from tests.lib.wheel import ( + _default, + make_metadata_file, + make_wheel, + make_wheel_metadata_file, + message_from_dict, +) + +if MYPY_CHECK_RUNNING: + from email import Message + + +def test_message_from_dict_one_value(): + message = message_from_dict({"a": "1"}) + assert set(message.get_all("a")) == {"1"} + + +def test_message_from_dict_multiple_values(): + message = message_from_dict({"a": ["1", "2"]}) + assert set(message.get_all("a")) == {"1", "2"} + + +def message_from_bytes(contents): + # type: (bytes) -> Message + return message_from_string(contents.decode("utf-8")) + + +default_make_metadata = partial( + make_metadata_file, + name="simple", + value=_default, + version="0.1.0", + updates=_default, + body=_default, +) + + +def default_metadata_checks(f): + assert f.name == "simple-0.1.0.dist-info/METADATA" + message = message_from_bytes(f.contents) + assert message.get_all("Metadata-Version") == ["2.1"] + assert message.get_all("Name") == ["simple"] + assert message.get_all("Version") == ["0.1.0"] + return message + + +def test_make_metadata_file_defaults(): + f = default_make_metadata() + default_metadata_checks(f) + + +def test_make_metadata_file_custom_value(): + f = default_make_metadata(updates={"a": "1"}) + message = default_metadata_checks(f) + assert message.get_all("a") == ["1"] + + +def test_make_metadata_file_custom_value_list(): + f = default_make_metadata(updates={"a": ["1", "2"]}) + message = default_metadata_checks(f) + assert set(message.get_all("a")) == {"1", "2"} + + +def test_make_metadata_file_custom_value_overrides(): + f = default_make_metadata(updates={"Metadata-Version": "2.2"}) + message = message_from_bytes(f.contents) + assert message.get_all("Metadata-Version") == ["2.2"] + + +def test_make_metadata_file_custom_contents(): + value = b"hello" + f = default_make_metadata(value=value) + assert f.contents == value + + +tags = [("py2", "none", "any"), ("py3", "none", "any")] +default_make_wheel_metadata = partial( + make_wheel_metadata_file, + name="simple", + version="0.1.0", + value=_default, + tags=tags, + updates=_default, +) + + +def default_wheel_metadata_checks(f): + assert f.name == "simple-0.1.0.dist-info/WHEEL" + message = message_from_bytes(f.contents) + assert message.get_all("Wheel-Version") == ["1.0"] + assert message.get_all("Generator") == ["pip-test-suite"] + assert message.get_all("Root-Is-Purelib") == ["true"] + assert set(message.get_all("Tag")) == {"py2-none-any", "py3-none-any"} + return message + + +def test_make_wheel_metadata_file_defaults(): + f = default_make_wheel_metadata() + default_wheel_metadata_checks(f) + + +def test_make_wheel_metadata_file_custom_value(): + f = default_make_wheel_metadata(updates={"a": "1"}) + message = default_wheel_metadata_checks(f) + assert message.get_all("a") == ["1"] + + +def test_make_wheel_metadata_file_custom_value_list(): + f = default_make_wheel_metadata(updates={"a": ["1", "2"]}) + message = default_wheel_metadata_checks(f) + assert set(message.get_all("a")) == {"1", "2"} + + +def test_make_wheel_metadata_file_custom_value_override(): + f = default_make_wheel_metadata(updates={"Wheel-Version": "1.1"}) + message = message_from_bytes(f.contents) + assert message.get_all("Wheel-Version") == ["1.1"] + + +def test_make_wheel_metadata_file_custom_contents(): + value = b"hello" + f = default_make_wheel_metadata(value=value) + + assert f.name == "simple-0.1.0.dist-info/WHEEL" + assert f.contents == value + + +def test_make_wheel_metadata_file_no_contents(): + f = default_make_wheel_metadata(value=None) + assert f is None + + +def test_make_wheel_basics(tmpdir): + make_wheel(name="simple", version="0.1.0").save_to_dir(tmpdir) + + expected_wheel_path = tmpdir / "simple-0.1.0-py2.py3-none-any.whl" + assert expected_wheel_path.exists() + + with ZipFile(expected_wheel_path) as z: + names = z.namelist() + assert set(names) == { + "simple-0.1.0.dist-info/METADATA", + "simple-0.1.0.dist-info/RECORD", + "simple-0.1.0.dist-info/WHEEL", + } + + +def test_make_wheel_extra_files(): + with make_wheel( + name="simple", + version="0.1.0", + extra_files={"simple/__init__.py": "a"}, + extra_metadata_files={"LICENSE": "b"}, + extra_data_files={"info.txt": "c"}, + ).as_zipfile() as z: + names = z.namelist() + assert set(names) == { + "simple/__init__.py", + "simple-0.1.0.data/info.txt", + "simple-0.1.0.dist-info/LICENSE", + "simple-0.1.0.dist-info/METADATA", + "simple-0.1.0.dist-info/RECORD", + "simple-0.1.0.dist-info/WHEEL", + } + + assert z.read("simple/__init__.py") == b"a" + assert z.read("simple-0.1.0.dist-info/LICENSE") == b"b" + assert z.read("simple-0.1.0.data/info.txt") == b"c" + + +def test_make_wheel_no_files(): + with make_wheel( + name="simple", + version="0.1.0", + wheel_metadata=None, + metadata=None, + record=None, + ).as_zipfile() as z: + assert not z.namelist() + + +def test_make_wheel_custom_files(): + with make_wheel( + name="simple", + version="0.1.0", + wheel_metadata=b"a", + metadata=b"b", + record=b"c", + ).as_zipfile() as z: + assert z.read("simple-0.1.0.dist-info/WHEEL") == b"a" + assert z.read("simple-0.1.0.dist-info/METADATA") == b"b" + assert z.read("simple-0.1.0.dist-info/RECORD") == b"c" diff --git a/tests/lib/wheel.py b/tests/lib/wheel.py new file mode 100644 index 000000000..8ea458658 --- /dev/null +++ b/tests/lib/wheel.py @@ -0,0 +1,413 @@ +"""Helper for building wheels as would be in test cases. +""" +import itertools +from base64 import urlsafe_b64encode +from collections import namedtuple +from copy import deepcopy +from email.message import Message +from enum import Enum +from functools import partial +from hashlib import sha256 +from io import BytesIO, StringIO +from zipfile import ZipFile + +import csv23 +from pip._vendor.requests.structures import CaseInsensitiveDict +from pip._vendor.six import ensure_binary, ensure_text, iteritems + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from tests.lib.path import Path + +if MYPY_CHECK_RUNNING: + from typing import ( + AnyStr, Callable, Dict, List, Iterable, Optional, Tuple, Sequence, + TypeVar, Union, + ) + + # path, digest, size + RecordLike = Tuple[str, str, str] + RecordCallback = Callable[ + [List["Record"]], Union[str, bytes, List[RecordLike]] + ] + # As would be used in metadata + HeaderValue = Union[str, List[str]] + + +File = namedtuple("File", ["name", "contents"]) +Record = namedtuple("Record", ["path", "digest", "size"]) + + +class Default(Enum): + token = 0 + + +_default = Default.token + + +if MYPY_CHECK_RUNNING: + T = TypeVar("T") + + class Defaulted(Union[Default, T]): + """A type which may be defaulted. + """ + pass + + +def message_from_dict(headers): + # type: (Dict[str, HeaderValue]) -> Message + """Plain key-value pairs are set in the returned message. + + List values are converted into repeated headers in the result. + """ + message = Message() + for name, value in iteritems(headers): + if isinstance(value, list): + for v in value: + message[name] = v + else: + message[name] = value + return message + + +def dist_info_path(name, version, path): + # type: (str, str, str) -> str + return "{}-{}.dist-info/{}".format(name, version, path) + + +def make_metadata_file( + name, # type: str + version, # type: str + value, # type: Defaulted[Optional[AnyStr]] + updates, # type: Defaulted[Dict[str, HeaderValue]] + body, # type: Defaulted[AnyStr] +): + # type: () -> File + if value is None: + return None + + path = dist_info_path(name, version, "METADATA") + + if value is not _default: + return File(path, ensure_binary(value)) + + metadata = CaseInsensitiveDict({ + "Metadata-Version": "2.1", + "Name": name, + "Version": version, + }) + if updates is not _default: + metadata.update(updates) + + message = message_from_dict(metadata) + if body is not _default: + message.set_payload(body) + + return File(path, ensure_binary(message_from_dict(metadata).as_string())) + + +def make_wheel_metadata_file( + name, # type: str + version, # type: str + value, # type: Defaulted[Optional[AnyStr]] + tags, # type: Sequence[Tuple[str, str, str]] + updates, # type: Defaulted[Dict[str, HeaderValue]] +): + # type: (...) -> Optional[File] + if value is None: + return None + + path = dist_info_path(name, version, "WHEEL") + + if value is not _default: + return File(path, ensure_binary(value)) + + metadata = CaseInsensitiveDict({ + "Wheel-Version": "1.0", + "Generator": "pip-test-suite", + "Root-Is-Purelib": "true", + "Tag": ["-".join(parts) for parts in tags], + }) + + if updates is not _default: + metadata.update(updates) + + return File(path, ensure_binary(message_from_dict(metadata).as_string())) + + +def make_entry_points_file( + name, # type: str + version, # type: str + entry_points, # type: Defaulted[Dict[str, List[str]]] + console_scripts, # type: Defaulted[List[str]] +): + # type: (...) -> Optional[File] + if entry_points is _default and console_scripts is _default: + return None + + if entry_points is _default: + entry_points_data = {} + else: + entry_points_data = deepcopy(entry_points) + + if console_scripts is not _default: + entry_points_data["console_scripts"] = console_scripts + + lines = [] + for section, values in iteritems(entry_points_data): + lines.append("[{}]".format(section)) + lines.extend(values) + + return File( + dist_info_path(name, version, "entry_points.txt"), + ensure_binary("\n".join(lines)), + ) + + +def make_files(files): + # type: (Dict[str, AnyStr]) -> List[File] + return [ + File(name, ensure_binary(contents)) + for name, contents in iteritems(files) + ] + + +def make_metadata_files(name, version, files): + # type: (str, str, Dict[str, AnyStr]) -> List[File] + get_path = partial(dist_info_path, name, version) + return [ + File(get_path(name), ensure_binary(contents)) + for name, contents in iteritems(files) + ] + + +def make_data_files(name, version, files): + # type: (str, str, Dict[str, AnyStr]) -> List[File] + data_dir = "{}-{}.data".format(name, version) + return [ + File("{}/{}".format(data_dir, name), ensure_binary(contents)) + for name, contents in iteritems(files) + ] + + +def urlsafe_b64encode_nopad(data): + # type: (bytes) -> str + return urlsafe_b64encode(data).rstrip(b"=").decode("ascii") + + +def digest(contents): + # type: (bytes) -> str + return "sha256={}".format( + urlsafe_b64encode_nopad(sha256(contents).digest()) + ) + + +def record_file_maker_wrapper( + name, # type: str + version, # type: str + files, # type: List[File] + record, # type: Defaulted[Optional[AnyStr]] + record_callback, # type: Defaulted[RecordCallback] +): + # type: (...) -> Iterable[File] + records = [] # type: List[Record] + for file in files: + records.append( + Record( + file.name, digest(file.contents), str(len(file.contents)) + ) + ) + yield file + + if record is None: + return + + record_path = dist_info_path(name, version, "RECORD") + + if record is not _default: + yield File(record_path, ensure_binary(record)) + return + + records.append(Record(record_path, "", "")) + + if record_callback is not _default: + records = record_callback(records) + + with StringIO(newline=u"") as buf: + writer = csv23.writer(buf) + for record in records: + writer.writerow(map(ensure_text, record)) + contents = buf.getvalue().encode("utf-8") + + yield File(record_path, contents) + + +def wheel_name(name, version, pythons, abis, platforms): + # type: (str, str, str, str, str) -> str + stem = "-".join([ + name, + version, + ".".join(pythons), + ".".join(abis), + ".".join(platforms), + ]) + return "{}.whl".format(stem) + + +class WheelBuilder(object): + """A wheel that can be saved or converted to several formats. + """ + + def __init__(self, name, files): + # type: (str, List[File]) -> None + self._name = name + self._files = files + + def save_to_dir(self, path): + # type: (Union[Path, str]) -> str + """Generate wheel file with correct name and save into the provided + directory. + + :returns the wheel file path + """ + path = Path(path) / self._name + path.write_bytes(self.as_bytes()) + return str(path) + + def save_to(self, path): + # type: (Union[Path, str]) -> str + """Generate wheel file, saving to the provided path. Any parent + directories must already exist. + + :returns the wheel file path + """ + path = Path(path) + path.write_bytes(self.as_bytes()) + return str(path) + + def as_bytes(self): + # type: () -> bytes + with BytesIO() as buf: + with ZipFile(buf, "w") as z: + for file in self._files: + z.writestr(file.name, file.contents) + return buf.getvalue() + + def as_zipfile(self): + # type: () -> ZipFile + return ZipFile(BytesIO(self.as_bytes())) + + +def make_wheel( + name, # type: str + version, # type: str + wheel_metadata=_default, # type: Defaulted[Optional[AnyStr]] + wheel_metadata_updates=_default, # type: Defaulted[Dict[str, HeaderValue]] + metadata=_default, # type: Defaulted[Optional[AnyStr]] + metadata_body=_default, # type: Defaulted[AnyStr] + metadata_updates=_default, # type: Defaulted[Dict[str, HeaderValue]] + extra_files=_default, # type: Defaulted[Dict[str, AnyStr]] + extra_metadata_files=_default, # type: Defaulted[Dict[str, AnyStr]] + extra_data_files=_default, # type: Defaulted[Dict[str, AnyStr]] + console_scripts=_default, # type: Defaulted[List[str]] + entry_points=_default, # type: Defaulted[Dict[str, List[str]]] + record=_default, # type: Defaulted[Optional[AnyStr]] + record_callback=_default, # type: Defaulted[RecordCallback] +): + # type: (...) -> WheelBuilder + """ + Helper function for generating test wheels which are compliant by default. + + Examples: + + ``` + # Basic wheel, which will have valid metadata, RECORD, etc + make_wheel(name="foo", version="0.1.0") + # Wheel with custom metadata + make_wheel( + name="foo", + version="0.1.0", + metadata_updates={ + # Overrides default + "Name": "hello", + # Expands into separate Requires-Dist entries + "Requires-Dist": ["a == 1.0", "b == 2.0; sys_platform == 'win32'"], + }, + ) + ``` + + After specifying the wheel, it can be consumed in several ways: + + ``` + # Normal case, valid wheel we want pip to pick up. + make_wheel(...).save_to_dir(tmpdir) + # For a test case, to check that pip validates contents against wheel name. + make_wheel(name="simple", ...).save_to(tmpdir / "notsimple-...") + # In-memory, for unit tests. + z = make_wheel(...).as_zipfile() + ``` + + Below, any unicode value provided for AnyStr will be encoded as utf-8. + + :param name: name of the distribution, propagated to the .dist-info + directory, METADATA, and wheel file name + :param version: version of the distribution, propagated to the .dist-info + directory, METADATA, and wheel file name + :param wheel_metadata: if provided and None, then no WHEEL metadata file + is generated; else if a string then sets the content of the WHEEL file + :param wheel_metadata_updates: override the default WHEEL metadata fields, + ignored if wheel_metadata is provided + :param metadata: if provided and None, then no METADATA file is generated; + else if a string then sets the content of the METADATA file + :param metadata_body: sets the value of the body text in METADATA, ignored + if metadata is provided + :param metadata_updates: override the default METADATA fields, + ignored if metadata is provided + :param extra_files: map from path to file contents for additional files to + be put in the wheel + :param extra_metadata_files: map from path (relative to .dist-info) to file + contents for additional files to be put in the wheel + :param extra_data_files: map from path (relative to .data) to file contents + for additional files to be put in the wheel + :param console_scripts: list of console scripts text to be put into + entry_points.txt - overrides any value set in entry_points + :param entry_points: + :param record: if provided and None, then no RECORD file is generated; + else if a string then sets the content of the RECORD file + :param record_callback: callback function that receives and can edit the + records before they are written to RECORD, ignored if record is + provided + """ + pythons = ["py2", "py3"] + abis = ["none"] + platforms = ["any"] + tags = list(itertools.product(pythons, abis, platforms)) + + possible_files = [ + make_metadata_file( + name, version, metadata, metadata_updates, metadata_body + ), + make_wheel_metadata_file( + name, version, wheel_metadata, tags, wheel_metadata_updates + ), + make_entry_points_file(name, version, entry_points, console_scripts), + ] + + if extra_files is not _default: + possible_files.extend(make_files(extra_files)) + + if extra_metadata_files is not _default: + possible_files.extend( + make_metadata_files(name, version, extra_metadata_files) + ) + + if extra_data_files is not _default: + possible_files.extend(make_data_files(name, version, extra_data_files)) + + actual_files = filter(None, possible_files) + + files_and_record_file = record_file_maker_wrapper( + name, version, actual_files, record, record_callback + ) + wheel_file_name = wheel_name(name, version, pythons, abis, platforms) + + return WheelBuilder(wheel_file_name, files_and_record_file) diff --git a/tools/requirements/tests.txt b/tools/requirements/tests.txt index 747ea321f..136ab9549 100644 --- a/tools/requirements/tests.txt +++ b/tools/requirements/tests.txt @@ -1,4 +1,6 @@ cryptography==2.8 +csv23 +enum34; python_version < '3.4' freezegun mock pretend